Allow non-toplevel modules in python environment (#1042)

environments/python/server.py

@@ -1,12 +1,24 @@
 #!/usr/bin/env python
-
+import importlib
 import logging
-import sys
-import imp
 import os
-import bjoern
+import sys
-from gevent.pywsgi import WSGIServer
+
 from flask import Flask, request, abort, g
+from gevent.pywsgi import WSGIServer
+import bjoern
+
+
+IS_PY2 = (sys.version_info.major == 2)
+
+
+def import_src(path):
+    if IS_PY2:
+        import imp
+        return imp.load_source('mod', path)
+    else:
+        # the imp module is deprecated in Python3. use importlib instead.
+        return importlib.machinery.SourceFileLoader('mod', path).load_module()
 
 
 class FuncApp(Flask):
@@ -34,9 +46,9 @@ class FuncApp(Flask):
         #
         @self.route('/specialize', methods=['POST'])
         def load():
+            self.logger.info('/specialize called')
             # load user function from codepath
-            codepath = '/userfunc/user'
+            self.userfunc = import_src('/userfunc/user').main
-            self.userfunc = (imp.load_source('user', codepath)).main
             return ""
 
         @self.route('/v2/specialize', methods=['POST'])
@@ -44,38 +56,36 @@ class FuncApp(Flask):
             body = request.get_json()
             filepath = body['filepath']
             handler = body['functionName']
+            self.logger.info('/v2/specialize called with  filepath = "{}"   handler = "{}"'.format(filepath, handler))
 
-            # The value of "functionName" is consist of
+            # handler looks like `path.to.module.function`
-            # `<module-name>.<function-name>`.
+            parts = handler.rsplit(".", 1)
-            moduleName, funcName = handler.split(".")
+            if len(handler) == 0:
-
+                # default to main.main if entrypoint wasn't provided
+                moduleName = 'main'
+                funcName = 'main'
+            elif len(parts) == 1:
+                moduleName = 'main'
+                funcName = parts[0]
+            else:
+                moduleName = parts[0]
+                funcName = parts[1]
+            self.logger.debug('moduleName = "{}"    funcName = "{}"'.format(moduleName, funcName))
+            
             # check whether the destination is a directory or a file
             if os.path.isdir(filepath):
                 # add package directory path into module search path
                 sys.path.append(filepath)
 
-                # find module from package path we append previously.
+                self.logger.debug('__package__ = "{}"'.format(__package__))
-                # Python will try to find module from the same name
+                if __package__:
-                # file under the package directory. If search is
+                    mod = importlib.import_module(moduleName, __package__)
-                # successful, the return value is a 3-element tuple;
+                else:
-                # otherwise, an exception "ImportError" is raised.
+                    mod = importlib.import_module(moduleName)
-                # Second parameter of find_module enforces python to
-                # find same name module from the given list of
-                # directories to prevent name confliction with
-                # built-in modules.
-                f, path, desc = imp.find_module(moduleName, [filepath])
 
-                # load module
-                # Return module object is the load is successful;
-                # otherwise, an exception is raised.
-                try:
-                    mod = imp.load_module(moduleName, f, path, desc)
-                finally:
-                    if f:
-                        f.close()
             else:
                 # load source from destination python file
-                mod = imp.load_source(moduleName, filepath)
+                mod = import_src(filepath)
 
             # load user function from module
             self.userfunc = getattr(mod, funcName)

test/test_utils.sh

@@ -463,6 +463,7 @@ run_all_tests() {
         $ROOT/test/tests/test_buildermgr.sh \
         $ROOT/test/tests/test_canary.sh \
         $ROOT/test/tests/test_env_vars.sh \
+        $ROOT/test/tests/test_environments/test_python_env.sh \
         $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \
         $ROOT/test/tests/test_function_test/test_fn_test.sh \
         $ROOT/test/tests/test_function_update.sh \

test/tests/test_environments/python_src/foo.py

@@ -0,0 +1,2 @@
+def bar():
+    return 'THIS_IS_FOO_BAR'

test/tests/test_environments/python_src/main.py

@@ -0,0 +1,5 @@
+def main():
+    return 'THIS_IS_MAIN_MAIN'
+
+def func():
+    return 'THIS_IS_MAIN_FUNC'

test/tests/test_environments/python_src/sub_mod/altmain.py

@@ -0,0 +1,2 @@
+def entrypoint():
+    return 'THIS_IS_ALTMAIN_ENTRYPOINT'

test/tests/test_environments/test_python_env.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+cleanup() {
+    clean_resource_by_id $TEST_ID
+    rm -rf $tmp_dir
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+env_v1api=python-v1-$TEST_ID
+env_v2api=python-v2-$TEST_ID
+fn1=test-python-env-1-$TEST_ID
+fn2=test-python-env-2-$TEST_ID
+fn3=test-python-env-3-$TEST_ID
+fn4=test-python-env-4-$TEST_ID
+fn5=test-python-env-5-$TEST_ID
+pkg=
+
+
+log "Creating v1api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE"
+fission env create \
+    --name $env_v1api \
+    --image $PYTHON_RUNTIME_IMAGE \
+
+log "Creating v2api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE     PYTHON_BUILDER_IMAGE = $PYTHON_BUILDER_IMAGE"
+fission env create \
+    --name $env_v2api \
+    --image $PYTHON_RUNTIME_IMAGE \
+    --builder $PYTHON_BUILDER_IMAGE
+timeout 180s bash -c "wait_for_builder $env_v2api"
+
+log "Creating package ..."
+pushd $ROOT/test/tests/test_environments/python_src/
+zip -r $tmp_dir/src-pkg.zip *
+popd
+pkg=$(fission package create --src $tmp_dir/src-pkg.zip --env $env_v2api | cut -f2 -d' '| tr -d \')
+timeout 60s bash -c "waitBuild $pkg"
+
+
+log "===== 1. test env with v1 api ====="
+fission fn create --name $fn1 --env $env_v1api --code $ROOT/examples/python/hello.py
+fission route create --function $fn1 --url /$fn1 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn1 'Hello, world!'"
+
+
+log "===== 2. test entrypoint = '' ====="
+fission fn create --name $fn2 --env $env_v2api --pkg $pkg
+fission route create --function $fn2 --url /$fn2 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn2 'THIS_IS_MAIN_MAIN'"
+
+
+log "===== 3. test entrypoint = func ====="
+fission fn create --name $fn3 --env $env_v2api --pkg $pkg --entrypoint func
+fission route create --function $fn3 --url /$fn3 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn3 'THIS_IS_MAIN_FUNC'"
+
+
+log "===== 4. test entrypoint = foo.bar ====="
+fission fn create --name $fn4 --env $env_v2api --pkg $pkg --entrypoint foo.bar
+fission route create --function $fn4 --url /$fn4 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn4 'THIS_IS_FOO_BAR'"
+
+
+log "===== 5. test entrypoint = sub_mod.altmain.entrypoint ====="
+fission fn create --name $fn5 --env $env_v2api --pkg $pkg --entrypoint sub_mod.altmain.entrypoint
+fission route create --function $fn5 --url /$fn5 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn5 'THIS_IS_ALTMAIN_ENTRYPOINT'"
+
+
+log "Test PASSED"