Builder manager bugfixes (#367)

This contains a few different builder manager fixes:

* Update the zip file structure to avoid an extra subdirectory
* Annotate packages with what functions are using them
* Only trigger builds when there is no deployment archive
* Python environment loadpath bugfixes
* Other bugfixes

test/build_and_test.sh

@@ -10,19 +10,22 @@ fi
 
 source $(dirname $0)/test_utils.sh
 
-IMAGE=gcr.io/fission-ci/fission-bundle
-FETCHER_IMAGE=gcr.io/fission-ci/fetcher
-PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env
-PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder
+REPO=gcr.io/fission-ci
+IMAGE=$REPO/fission-bundle
+FETCHER_IMAGE=$REPO/fetcher
 TAG=test
 
 build_and_push_fission_bundle $IMAGE:$TAG
 
 build_and_push_fetcher $FETCHER_IMAGE:$TAG
 
-build_and_push_python_env_runtime $PYTHON_RUNTIME_IMAGE:$TAG
+build_builder
 
-build_and_push_python_env_builder $PYTHON_BUILDER_IMAGE:$TAG
+ENV='python3'
+
+build_and_push_env_runtime $ENV $REPO/$ENV-env:$TAG
+
+build_and_push_env_builder $ENV $REPO/$ENV-env-builder:$TAG
 
 build_fission_cli
 

test/test_utils.sh

@@ -50,10 +50,17 @@ build_and_push_fetcher() {
     popd
 }
 
-build_and_push_python_env_runtime() {
-    image_tag=$1
+build_builder() {
+    pushd $ROOT/builder/cmd
+    ./build.sh
+    popd
+}
 
-    pushd $ROOT/environments/python3/
+build_and_push_env_runtime() {
+    env=$1
+    image_tag=$2
+
+    pushd $ROOT/environments/$env/
     docker build -t $image_tag .
 
     gcloud_login
@@ -62,13 +69,11 @@ build_and_push_python_env_runtime() {
     popd
 }
 
-build_and_push_python_env_builder() {
-    image_tag=$1
+build_and_push_env_builder() {
+    env=$1
+    image_tag=$2
 
-    pushd $ROOT/builder/cmd
-    ./build.sh
-    popd
-    pushd $ROOT/environments/python3/builder
+    pushd $ROOT/environments/$env/builder
     builderDir=${GOPATH}/src/github.com/fission/fission/builder/cmd
     cp ${builderDir}/builder .
 
@@ -80,7 +85,6 @@ build_and_push_python_env_builder() {
     popd
 }
 
-
 build_fission_cli() {
     pushd $ROOT/fission
     go build .
@@ -240,6 +244,7 @@ dump_logs() {
     dump_fission_logs $ns $fns controller
     dump_fission_logs $ns $fns router
     dump_fission_logs $ns $fns poolmgr
+    dump_fission_logs $ns $fns buildermgr
     dump_function_pod_logs $ns $fns
     dump_fission_crds
 }

test/tests/test_buildermgr.sh

@@ -9,8 +9,8 @@ set -euo pipefail
 # 2. package watcher triggers the build if any changes to packages
 
 ROOT=$(dirname $0)/../..
-PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
-PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
+PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python3-env:test
+PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python3-env-builder:test
 
 fn=python-srcbuild-$(date +%s)
 
@@ -23,21 +23,47 @@ checkFunctionResponse() {
     echo $response | grep -i "a: 1 b: {c: 3, d: 4}"
 }
 
+waitBuild() {
+    echo "Waiting for builder manager to finish the build"
+    
+    while true; do
+      kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
+      if [[ $? -eq 0 ]]; then
+          break
+      fi
+    done
+}
+export -f waitBuild
+
+waitEnvBuilder() {
+    echo "Waiting for env builder to catch up"
+
+    while true; do
+      kubectl --namespace fission-builder get pod|grep python|grep Running
+      if [[ $? -eq 0 ]]; then
+          break
+      fi
+    done
+
+    sleep 10
+}
+export -f waitEnvBuilder
+
 echo "Pre-test cleanup"
 fission env delete --name python || true
+kubectl --namespace default get packages|grep -v NAME|awk '{print $1}'|xargs -I@ bash -c 'kubectl --namespace default delete packages @' || true
 
 echo "Creating python env"
 fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
 trap "fission env delete --name python" EXIT
 
-echo "Waiting for env builder to catch up"
-sleep 30
+timeout 180s bash -c waitEnvBuilder
 
 echo "Creating source pacakage"
 zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
 
 echo "Creating function " $fn
-fission fn create --name $fn --env python --src demo-src-pkg.zip --entrypoint "main" --buildcmd "./build.sh"
+fission fn create --name $fn --env python --src demo-src-pkg.zip --entrypoint "user.main" --buildcmd "./build.sh"
 trap "fission fn delete --name $fn" EXIT
 
 echo "Creating route"
@@ -46,15 +72,10 @@ fission route create --function $fn --url /$fn --method GET
 echo "Waiting for router to catch up"
 sleep 3
 
-echo "Doing an HTTP POST on the builder manager's route to start a build"
 pkg=$(kubectl --namespace default get functions $fn -o jsonpath='{.spec.package.packageref.name}')
-echo $pkg
-response=$(curl -X POST $FISSION_URL/proxy/buildermgr/v1/build \
-  -H 'content-type: application/json' \
-  -d "{\"package\": {\"namespace\": \"default\",\"name\": \"$pkg\"}}")
 
-echo "Waiting for builder manager to finish the build triggered by http request"
-sleep 30
+# wait for build to finish at most 60s
+timeout 60s bash -c "waitBuild $pkg"
 
 checkFunctionResponse $fn
 
@@ -62,8 +83,10 @@ echo "Updating function " $fn
 fission fn update --name $fn --src demo-src-pkg.zip
 trap "fission fn delete --name $fn" EXIT
 
-echo "Waiting for builder manager to finish the build triggered by packageWatcher"
-sleep 30
+pkg=$(kubectl --namespace default get functions $fn -o jsonpath='{.spec.package.packageref.name}')
+
+# wait for build to finish at most 60s
+timeout 60s bash -c "waitBuild $pkg"
 
 checkFunctionResponse $fn