Allow non-toplevel modules in python environment (#1042)
This commit is contained in:
Soam Vasani
committed by
Ta-Ching Chen
Ta-Ching Chen
parent
93d4f2c77a
commit
a473335ceb
@@ -1,12 +1,24 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import sys
|
||||
import imp
|
||||
import os
|
||||
import bjoern
|
||||
from gevent.pywsgi import WSGIServer
|
||||
import sys
|
||||
|
||||
from flask import Flask, request, abort, g
|
||||
from gevent.pywsgi import WSGIServer
|
||||
import bjoern
|
||||
|
||||
|
||||
IS_PY2 = (sys.version_info.major == 2)
|
||||
|
||||
|
||||
def import_src(path):
|
||||
if IS_PY2:
|
||||
import imp
|
||||
return imp.load_source('mod', path)
|
||||
else:
|
||||
# the imp module is deprecated in Python3. use importlib instead.
|
||||
return importlib.machinery.SourceFileLoader('mod', path).load_module()
|
||||
|
||||
|
||||
class FuncApp(Flask):
|
||||
@@ -34,9 +46,9 @@ class FuncApp(Flask):
|
||||
#
|
||||
@self.route('/specialize', methods=['POST'])
|
||||
def load():
|
||||
self.logger.info('/specialize called')
|
||||
# load user function from codepath
|
||||
codepath = '/userfunc/user'
|
||||
self.userfunc = (imp.load_source('user', codepath)).main
|
||||
self.userfunc = import_src('/userfunc/user').main
|
||||
return ""
|
||||
|
||||
@self.route('/v2/specialize', methods=['POST'])
|
||||
@@ -44,38 +56,36 @@ class FuncApp(Flask):
|
||||
body = request.get_json()
|
||||
filepath = body['filepath']
|
||||
handler = body['functionName']
|
||||
self.logger.info('/v2/specialize called with filepath = "{}" handler = "{}"'.format(filepath, handler))
|
||||
|
||||
# The value of "functionName" is consist of
|
||||
# `<module-name>.<function-name>`.
|
||||
moduleName, funcName = handler.split(".")
|
||||
|
||||
# handler looks like `path.to.module.function`
|
||||
parts = handler.rsplit(".", 1)
|
||||
if len(handler) == 0:
|
||||
# default to main.main if entrypoint wasn't provided
|
||||
moduleName = 'main'
|
||||
funcName = 'main'
|
||||
elif len(parts) == 1:
|
||||
moduleName = 'main'
|
||||
funcName = parts[0]
|
||||
else:
|
||||
moduleName = parts[0]
|
||||
funcName = parts[1]
|
||||
self.logger.debug('moduleName = "{}" funcName = "{}"'.format(moduleName, funcName))
|
||||
|
||||
# check whether the destination is a directory or a file
|
||||
if os.path.isdir(filepath):
|
||||
# add package directory path into module search path
|
||||
sys.path.append(filepath)
|
||||
|
||||
# find module from package path we append previously.
|
||||
# Python will try to find module from the same name
|
||||
# file under the package directory. If search is
|
||||
# successful, the return value is a 3-element tuple;
|
||||
# otherwise, an exception "ImportError" is raised.
|
||||
# Second parameter of find_module enforces python to
|
||||
# find same name module from the given list of
|
||||
# directories to prevent name confliction with
|
||||
# built-in modules.
|
||||
f, path, desc = imp.find_module(moduleName, [filepath])
|
||||
self.logger.debug('__package__ = "{}"'.format(__package__))
|
||||
if __package__:
|
||||
mod = importlib.import_module(moduleName, __package__)
|
||||
else:
|
||||
mod = importlib.import_module(moduleName)
|
||||
|
||||
# load module
|
||||
# Return module object is the load is successful;
|
||||
# otherwise, an exception is raised.
|
||||
try:
|
||||
mod = imp.load_module(moduleName, f, path, desc)
|
||||
finally:
|
||||
if f:
|
||||
f.close()
|
||||
else:
|
||||
# load source from destination python file
|
||||
mod = imp.load_source(moduleName, filepath)
|
||||
mod = import_src(filepath)
|
||||
|
||||
# load user function from module
|
||||
self.userfunc = getattr(mod, funcName)
|
||||
|
||||
@@ -463,6 +463,7 @@ run_all_tests() {
|
||||
$ROOT/test/tests/test_buildermgr.sh \
|
||||
$ROOT/test/tests/test_canary.sh \
|
||||
$ROOT/test/tests/test_env_vars.sh \
|
||||
$ROOT/test/tests/test_environments/test_python_env.sh \
|
||||
$ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \
|
||||
$ROOT/test/tests/test_function_test/test_fn_test.sh \
|
||||
$ROOT/test/tests/test_function_update.sh \
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
def bar():
|
||||
return 'THIS_IS_FOO_BAR'
|
||||
@@ -0,0 +1,5 @@
|
||||
def main():
|
||||
return 'THIS_IS_MAIN_MAIN'
|
||||
|
||||
def func():
|
||||
return 'THIS_IS_MAIN_FUNC'
|
||||
@@ -0,0 +1,2 @@
|
||||
def entrypoint():
|
||||
return 'THIS_IS_ALTMAIN_ENTRYPOINT'
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
source $(dirname $0)/../../utils.sh
|
||||
|
||||
TEST_ID=$(generate_test_id)
|
||||
echo "TEST_ID = $TEST_ID"
|
||||
|
||||
tmp_dir="/tmp/test-$TEST_ID"
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
ROOT=$(dirname $0)/../../..
|
||||
|
||||
cleanup() {
|
||||
clean_resource_by_id $TEST_ID
|
||||
rm -rf $tmp_dir
|
||||
}
|
||||
|
||||
if [ -z "${TEST_NOCLEANUP:-}" ]; then
|
||||
trap cleanup EXIT
|
||||
else
|
||||
log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
|
||||
fi
|
||||
|
||||
env_v1api=python-v1-$TEST_ID
|
||||
env_v2api=python-v2-$TEST_ID
|
||||
fn1=test-python-env-1-$TEST_ID
|
||||
fn2=test-python-env-2-$TEST_ID
|
||||
fn3=test-python-env-3-$TEST_ID
|
||||
fn4=test-python-env-4-$TEST_ID
|
||||
fn5=test-python-env-5-$TEST_ID
|
||||
pkg=
|
||||
|
||||
|
||||
log "Creating v1api environment ..."
|
||||
log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE"
|
||||
fission env create \
|
||||
--name $env_v1api \
|
||||
--image $PYTHON_RUNTIME_IMAGE \
|
||||
|
||||
log "Creating v2api environment ..."
|
||||
log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE PYTHON_BUILDER_IMAGE = $PYTHON_BUILDER_IMAGE"
|
||||
fission env create \
|
||||
--name $env_v2api \
|
||||
--image $PYTHON_RUNTIME_IMAGE \
|
||||
--builder $PYTHON_BUILDER_IMAGE
|
||||
timeout 180s bash -c "wait_for_builder $env_v2api"
|
||||
|
||||
log "Creating package ..."
|
||||
pushd $ROOT/test/tests/test_environments/python_src/
|
||||
zip -r $tmp_dir/src-pkg.zip *
|
||||
popd
|
||||
pkg=$(fission package create --src $tmp_dir/src-pkg.zip --env $env_v2api | cut -f2 -d' '| tr -d \')
|
||||
timeout 60s bash -c "waitBuild $pkg"
|
||||
|
||||
|
||||
log "===== 1. test env with v1 api ====="
|
||||
fission fn create --name $fn1 --env $env_v1api --code $ROOT/examples/python/hello.py
|
||||
fission route create --function $fn1 --url /$fn1 --method GET
|
||||
sleep 3 # Waiting for router to catch up
|
||||
timeout 60 bash -c "test_fn $fn1 'Hello, world!'"
|
||||
|
||||
|
||||
log "===== 2. test entrypoint = '' ====="
|
||||
fission fn create --name $fn2 --env $env_v2api --pkg $pkg
|
||||
fission route create --function $fn2 --url /$fn2 --method GET
|
||||
sleep 3 # Waiting for router to catch up
|
||||
timeout 60 bash -c "test_fn $fn2 'THIS_IS_MAIN_MAIN'"
|
||||
|
||||
|
||||
log "===== 3. test entrypoint = func ====="
|
||||
fission fn create --name $fn3 --env $env_v2api --pkg $pkg --entrypoint func
|
||||
fission route create --function $fn3 --url /$fn3 --method GET
|
||||
sleep 3 # Waiting for router to catch up
|
||||
timeout 60 bash -c "test_fn $fn3 'THIS_IS_MAIN_FUNC'"
|
||||
|
||||
|
||||
log "===== 4. test entrypoint = foo.bar ====="
|
||||
fission fn create --name $fn4 --env $env_v2api --pkg $pkg --entrypoint foo.bar
|
||||
fission route create --function $fn4 --url /$fn4 --method GET
|
||||
sleep 3 # Waiting for router to catch up
|
||||
timeout 60 bash -c "test_fn $fn4 'THIS_IS_FOO_BAR'"
|
||||
|
||||
|
||||
log "===== 5. test entrypoint = sub_mod.altmain.entrypoint ====="
|
||||
fission fn create --name $fn5 --env $env_v2api --pkg $pkg --entrypoint sub_mod.altmain.entrypoint
|
||||
fission route create --function $fn5 --url /$fn5 --method GET
|
||||
sleep 3 # Waiting for router to catch up
|
||||
timeout 60 bash -c "test_fn $fn5 'THIS_IS_ALTMAIN_ENTRYPOINT'"
|
||||
|
||||
|
||||
log "Test PASSED"
|
||||
Reference in New Issue
Block a user