From a473335ceb1ccb569f6868cd3e4fb1160978fcdf Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Sun, 2 Jun 2019 00:16:10 -0700 Subject: [PATCH] Allow non-toplevel modules in python environment (#1042) --- environments/python/server.py | 70 ++++++++------ test/test_utils.sh | 1 + .../test_environments/python_src/__init__.py | 0 .../tests/test_environments/python_src/foo.py | 2 + .../test_environments/python_src/main.py | 5 + .../python_src/sub_mod/__init__.py | 0 .../python_src/sub_mod/altmain.py | 2 + .../test_environments/test_python_env.sh | 92 +++++++++++++++++++ 8 files changed, 142 insertions(+), 30 deletions(-) create mode 100644 test/tests/test_environments/python_src/__init__.py create mode 100644 test/tests/test_environments/python_src/foo.py create mode 100644 test/tests/test_environments/python_src/main.py create mode 100644 test/tests/test_environments/python_src/sub_mod/__init__.py create mode 100644 test/tests/test_environments/python_src/sub_mod/altmain.py create mode 100755 test/tests/test_environments/test_python_env.sh diff --git a/environments/python/server.py b/environments/python/server.py index a553dd60..878d69b9 100644 --- a/environments/python/server.py +++ b/environments/python/server.py @@ -1,12 +1,24 @@ #!/usr/bin/env python - +import importlib import logging -import sys -import imp import os -import bjoern -from gevent.pywsgi import WSGIServer +import sys + from flask import Flask, request, abort, g +from gevent.pywsgi import WSGIServer +import bjoern + + +IS_PY2 = (sys.version_info.major == 2) + + +def import_src(path): + if IS_PY2: + import imp + return imp.load_source('mod', path) + else: + # the imp module is deprecated in Python3. use importlib instead. + return importlib.machinery.SourceFileLoader('mod', path).load_module() class FuncApp(Flask): @@ -34,9 +46,9 @@ class FuncApp(Flask): # @self.route('/specialize', methods=['POST']) def load(): + self.logger.info('/specialize called') # load user function from codepath - codepath = '/userfunc/user' - self.userfunc = (imp.load_source('user', codepath)).main + self.userfunc = import_src('/userfunc/user').main return "" @self.route('/v2/specialize', methods=['POST']) @@ -44,38 +56,36 @@ class FuncApp(Flask): body = request.get_json() filepath = body['filepath'] handler = body['functionName'] + self.logger.info('/v2/specialize called with filepath = "{}" handler = "{}"'.format(filepath, handler)) - # The value of "functionName" is consist of - # `.`. - moduleName, funcName = handler.split(".") - + # handler looks like `path.to.module.function` + parts = handler.rsplit(".", 1) + if len(handler) == 0: + # default to main.main if entrypoint wasn't provided + moduleName = 'main' + funcName = 'main' + elif len(parts) == 1: + moduleName = 'main' + funcName = parts[0] + else: + moduleName = parts[0] + funcName = parts[1] + self.logger.debug('moduleName = "{}" funcName = "{}"'.format(moduleName, funcName)) + # check whether the destination is a directory or a file if os.path.isdir(filepath): # add package directory path into module search path sys.path.append(filepath) - # find module from package path we append previously. - # Python will try to find module from the same name - # file under the package directory. If search is - # successful, the return value is a 3-element tuple; - # otherwise, an exception "ImportError" is raised. - # Second parameter of find_module enforces python to - # find same name module from the given list of - # directories to prevent name confliction with - # built-in modules. - f, path, desc = imp.find_module(moduleName, [filepath]) + self.logger.debug('__package__ = "{}"'.format(__package__)) + if __package__: + mod = importlib.import_module(moduleName, __package__) + else: + mod = importlib.import_module(moduleName) - # load module - # Return module object is the load is successful; - # otherwise, an exception is raised. - try: - mod = imp.load_module(moduleName, f, path, desc) - finally: - if f: - f.close() else: # load source from destination python file - mod = imp.load_source(moduleName, filepath) + mod = import_src(filepath) # load user function from module self.userfunc = getattr(mod, funcName) diff --git a/test/test_utils.sh b/test/test_utils.sh index 5b5ffd98..27dc026e 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -463,6 +463,7 @@ run_all_tests() { $ROOT/test/tests/test_buildermgr.sh \ $ROOT/test/tests/test_canary.sh \ $ROOT/test/tests/test_env_vars.sh \ + $ROOT/test/tests/test_environments/test_python_env.sh \ $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \ $ROOT/test/tests/test_function_test/test_fn_test.sh \ $ROOT/test/tests/test_function_update.sh \ diff --git a/test/tests/test_environments/python_src/__init__.py b/test/tests/test_environments/python_src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/tests/test_environments/python_src/foo.py b/test/tests/test_environments/python_src/foo.py new file mode 100644 index 00000000..df6719fc --- /dev/null +++ b/test/tests/test_environments/python_src/foo.py @@ -0,0 +1,2 @@ +def bar(): + return 'THIS_IS_FOO_BAR' diff --git a/test/tests/test_environments/python_src/main.py b/test/tests/test_environments/python_src/main.py new file mode 100644 index 00000000..52e22772 --- /dev/null +++ b/test/tests/test_environments/python_src/main.py @@ -0,0 +1,5 @@ +def main(): + return 'THIS_IS_MAIN_MAIN' + +def func(): + return 'THIS_IS_MAIN_FUNC' diff --git a/test/tests/test_environments/python_src/sub_mod/__init__.py b/test/tests/test_environments/python_src/sub_mod/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/tests/test_environments/python_src/sub_mod/altmain.py b/test/tests/test_environments/python_src/sub_mod/altmain.py new file mode 100644 index 00000000..8112ed6d --- /dev/null +++ b/test/tests/test_environments/python_src/sub_mod/altmain.py @@ -0,0 +1,2 @@ +def entrypoint(): + return 'THIS_IS_ALTMAIN_ENTRYPOINT' diff --git a/test/tests/test_environments/test_python_env.sh b/test/tests/test_environments/test_python_env.sh new file mode 100755 index 00000000..7d62a824 --- /dev/null +++ b/test/tests/test_environments/test_python_env.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -euo pipefail +source $(dirname $0)/../../utils.sh + +TEST_ID=$(generate_test_id) +echo "TEST_ID = $TEST_ID" + +tmp_dir="/tmp/test-$TEST_ID" +mkdir -p $tmp_dir + +ROOT=$(dirname $0)/../../.. + +cleanup() { + clean_resource_by_id $TEST_ID + rm -rf $tmp_dir +} + +if [ -z "${TEST_NOCLEANUP:-}" ]; then + trap cleanup EXIT +else + log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards." +fi + +env_v1api=python-v1-$TEST_ID +env_v2api=python-v2-$TEST_ID +fn1=test-python-env-1-$TEST_ID +fn2=test-python-env-2-$TEST_ID +fn3=test-python-env-3-$TEST_ID +fn4=test-python-env-4-$TEST_ID +fn5=test-python-env-5-$TEST_ID +pkg= + + +log "Creating v1api environment ..." +log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE" +fission env create \ + --name $env_v1api \ + --image $PYTHON_RUNTIME_IMAGE \ + +log "Creating v2api environment ..." +log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE PYTHON_BUILDER_IMAGE = $PYTHON_BUILDER_IMAGE" +fission env create \ + --name $env_v2api \ + --image $PYTHON_RUNTIME_IMAGE \ + --builder $PYTHON_BUILDER_IMAGE +timeout 180s bash -c "wait_for_builder $env_v2api" + +log "Creating package ..." +pushd $ROOT/test/tests/test_environments/python_src/ +zip -r $tmp_dir/src-pkg.zip * +popd +pkg=$(fission package create --src $tmp_dir/src-pkg.zip --env $env_v2api | cut -f2 -d' '| tr -d \') +timeout 60s bash -c "waitBuild $pkg" + + +log "===== 1. test env with v1 api =====" +fission fn create --name $fn1 --env $env_v1api --code $ROOT/examples/python/hello.py +fission route create --function $fn1 --url /$fn1 --method GET +sleep 3 # Waiting for router to catch up +timeout 60 bash -c "test_fn $fn1 'Hello, world!'" + + +log "===== 2. test entrypoint = '' =====" +fission fn create --name $fn2 --env $env_v2api --pkg $pkg +fission route create --function $fn2 --url /$fn2 --method GET +sleep 3 # Waiting for router to catch up +timeout 60 bash -c "test_fn $fn2 'THIS_IS_MAIN_MAIN'" + + +log "===== 3. test entrypoint = func =====" +fission fn create --name $fn3 --env $env_v2api --pkg $pkg --entrypoint func +fission route create --function $fn3 --url /$fn3 --method GET +sleep 3 # Waiting for router to catch up +timeout 60 bash -c "test_fn $fn3 'THIS_IS_MAIN_FUNC'" + + +log "===== 4. test entrypoint = foo.bar =====" +fission fn create --name $fn4 --env $env_v2api --pkg $pkg --entrypoint foo.bar +fission route create --function $fn4 --url /$fn4 --method GET +sleep 3 # Waiting for router to catch up +timeout 60 bash -c "test_fn $fn4 'THIS_IS_FOO_BAR'" + + +log "===== 5. test entrypoint = sub_mod.altmain.entrypoint =====" +fission fn create --name $fn5 --env $env_v2api --pkg $pkg --entrypoint sub_mod.altmain.entrypoint +fission route create --function $fn5 --url /$fn5 --method GET +sleep 3 # Waiting for router to catch up +timeout 60 bash -c "test_fn $fn5 'THIS_IS_ALTMAIN_ENTRYPOINT'" + + +log "Test PASSED"