Test framework improvement (#1128)

.gitignore

@@ -1,6 +1,12 @@
+# Binaries
 fission-bundle/fission-bundle
 fission/fission
 environments/fetcher/cmd/fetcher
+builder/cmd/builder
+preupgradechecks/pre-upgrade-checks
+
+# Logs
+test/logs/
 
 # Pycharm IDE
 .idea

.travis.yml

@@ -22,7 +22,7 @@ services:
 before_install:
   - sudo apt-get update
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-  - sudo apt-get -y install apache2-utils
+  - sudo apt-get -y install apache2-utils parallel
   - sudo sysctl net.ipv6.conf.all.disable_ipv6=0
 
 install:

test/README.md

@@ -0,0 +1,54 @@
+# Test Framework
+
+## Prerequisite tools
+
+- GNU parallel >= 20161222
+
+If you want to run test on Mac, you also need to install GNU tools for command compatibility.
+With Homebrew, you can get them by:
+
+       brew install coreutils findutils gnu-sed
+
+
+## Run a single test
+
+
+```bash
+cd $REPO/test
+
+# run 'tests/test_node_hello_http.sh'
+./tests/test_node_hello_http.sh
+
+# if you want to preserve the resources that created by test for debug
+export TEST_NOCLEANUP=yes
+./tests/test_node_hello_http.sh
+
+# run 'tests/test_environments/test_go_env.sh' with custom images
+export GO_RUNTIME_IMAGE=my.docker.repo/go-env:test
+./tests/test_environments/test_go_env.sh
+
+```
+
+
+## Run tests in parallel
+
+NOTE: Some tests will consume lots of resources and cause the test fail.
+
+Example: Run tests with 4 concurrent jobs.
+```bash
+cd $REPO/test
+
+export JOBS=4
+./run_test.sh \
+    tests/test_backend_poolmgr.sh \
+    tests/test_node_hello_http.sh \
+    tests/test_pass.sh \
+    tests/test_specs/test_spec.sh
+
+# The test output will be available in logs/
+cat logs/test_backend_poolmgr.sh.log
+
+# The summary report will be saved to logs/_recap
+cat logs/_recap
+```
+

test/init_tools.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Set tools for OS compatibility.
+#
+# Prerequisite on Mac:
+#   brew install coreutils findutils gnu-sed parallel
+
+if [ $(uname -s) == 'Darwin' ]; then
+    if command -v gtimeout >/dev/null; then
+        timeout() { gtimeout "$@"; }
+        export -f timeout
+    else
+        echo '"gtimeout" command not found. Try "brew install coreutils".'
+        exit 1
+    fi
+
+    if command -v gdate >/dev/null; then
+        date() { gdate "$@"; }
+        export -f date
+    else
+        echo '"gdate" command not found. Try "brew install coreutils".'
+        exit 1
+    fi
+
+    if command -v gsed >/dev/null; then
+        sed() { gsed "$@"; }
+        export -f sed
+    else
+        echo '"gsed" command not found. Try "brew install gnu-sed".'
+        exit 1
+    fi
+
+    if command -v greadlink >/dev/null; then
+        readlink() { greadlink "$@"; }
+        export -f readlink
+    else
+        echo '"greadlink" command not found. Try "brew install coreutils".'
+        exit 1
+    fi
+
+    if command -v gtr >/dev/null; then
+        tr() { gtr "$@"; }
+        export -f tr
+    else
+        echo '"gtr" command not found. Try "brew install coreutils".'
+        exit 1
+    fi
+
+    if command -v gxargs >/dev/null; then
+        xargs() { gxargs "$@"; }
+        export -f xargs
+    else
+        echo '"gxargs" command not found. Try "brew install findutils".'
+        exit 1
+    fi
+
+    find_executable() {
+        path=$1; shift
+        find $path -perm +111 -type f "$@"
+    }
+
+else
+    find_executable() {
+        path=$1; shift
+        find $path -executable -type f "$@"
+    }
+fi

test/run_test.sh

@@ -1,11 +1,78 @@
 #!/usr/bin/env bash
-
+#
+# This is a helper script to run test in parallel and collect logs.
+# Usage:
+#       ./run_test.sh                       Run all tests.
+#       ./run_test.sh [test_file ...]       Run specific tests.
+#
+# Environments:
+#       LOG_DIR     Log directory path. (default: $ROOT/test/logs)
+#       JOBS        The number of concurrent jobs. (default: 1)
+#       TIMEOUT     Timeout for each job. (default: 0 (no timeout))
+#
 set -euo pipefail
+source $(dirname $BASH_SOURCE)/init_tools.sh
 
-. $(dirname $0)/test_utils.sh
+ROOT=$(readlink -f $(dirname $0)/..)
+LOG_DIR=${LOG_DIR:-$ROOT/test/logs}
+JOBS=${JOBS:-1}
+TIMEOUT=${TIMEOUT:-0}
 
-FILE=$(pwd)/$1
+main() {
+    if [ $# -eq 0 ]; then
+        args=$(find_executable $ROOT/test/tests -iname 'test_*')
+    else
+        args="$@"
+    fi
 
-FAILURES=0
+    num_skip=0
-run_test ${FILE}
+    mkdir -p $LOG_DIR
-exit $FAILURES
+    test_files=""
+    log_files=""
+    for arg in $args; do
+        if [ ! -f $arg ]; then
+            echo "WARNING: file not found: $arg"
+            continue
+        fi
+
+        absolute_path=$(readlink -f $arg)
+        relative_path=${absolute_path#$ROOT/test/tests/}
+        log_path=$LOG_DIR/${relative_path}.log
+
+        if grep -q "^#test:disabled" $arg; then
+            echo "INFO: the test is marked disabled: $relative_path"
+            num_skip=$((num_skip+1))
+            continue
+        fi
+
+        # make sure the log dir exists.
+        mkdir -p $(dirname $log_path)
+
+        # remove common path for readability
+        test_files="$test_files ${absolute_path#$PWD/}"
+        log_files="$log_files ${log_path#$PWD/}"
+    done
+
+    start_time=$(date +%s)
+    parallel \
+        --joblog - \
+        --jobs $JOBS \
+        --timeout $TIMEOUT \
+        bash -c '{1} > {2} 2>&1' \
+        ::: $test_files :::+ $log_files \
+        | tee $LOG_DIR/_recap \
+        || true
+    end_time=$(date +%s)
+
+    # Get the Exitval in _recap to find if any test failed.
+    num_total=$(cat $LOG_DIR/_recap | wc -l)
+    num_total=$((num_total - 1))    # don't count header
+    num_fail=$(cat $LOG_DIR/_recap | awk 'NR>1 && $7!=0 {print $0}' | wc -l | tr -d ' ')
+    num_pass=$((num_total - num_fail))
+    time=$((end_time - start_time))
+    echo ============================================================
+    echo "PASS: $num_pass    FAIL: $num_fail    SKIP: $num_skip    TIME: ${time}s"
+    return $num_fail
+}
+
+main "$@"

test/run_test_mac.sh

@@ -1,93 +0,0 @@
-#!/usr/bin/env bash
-
-#### Purpose ####
-# The purpose of this script is to enable running some/all of test scripts locally on mac so that 
-# you don't have to always rely on CI cycle for feedback.
-
-#### Usage ####
-# For running a specific test:
-# ./run_test_mac.sh test_spec.sh
-#
-# For running all tests
-# ./run_test_mac.sh
-
-#### Prerequisite ####
-# Need to install following on Mac
-# brew install coreutils  --> For (g)date & (g)timeout equivalents
-# brew install gnu-sed --with-default-names  --> Sed's -i flag does not work without argument on Mac. Check: https://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-bsd-and-linux/22084103#22084103
-# 
-
-#### Caution #### 
-# Some scripts might use additional variables only available during CI cycle such as an image 
-# being built by CI - which you will have to override manually in that script.
-# Some tests known to fail as of now:
-# test_mqtrigger_error.sh, test_mqtrigger.sh (Needs connection to MQ), test_archive_pruner.sh (The package somehow gets created in default package in local setup), test_obj_create_in_diff_ns.sh
-
-
-set -euo pipefail
-
-source $(dirname $0)/test_utils.sh
-
-## Common env parameters
-export FISSION_NAMESPACE=fission
-export FUNCTION_NAMESPACE=fission-function
-export FISSION_ROUTER=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}')
-export FISSION_NATS_STREAMING_URL="http://defaultFissionAuthToken@$(kubectl -n $FISSION_NAMESPACE get svc nats-streaming -o jsonpath='{...ip}:{.spec.ports[0].port}')"
-
-## Parameters used by some specific test cases
-
-export PYTHON_RUNTIME_IMAGE=fission/python-env
-export PYTHON_BUILDER_IMAGE=fission/python-builder
-export GO_RUNTIME_IMAGE=fission/go-env
-export GO_BUILDER_IMAGE=fission/go-builder
-export JVM_RUNTIME_IMAGE=fission/jvm-env
-export JVM_BUILDER_IMAGE=fission/jvm-builder
-
-
-if [ $(uname -s) == 'Darwin' ]
-then
-    # gtimeout needs to be installed separately, do "brew install coreutils".
-    timeout() {
-        gtimeout "$@"
-    }
-    export -f timeout
-
-    # gdate needs to be installed separately, do "brew install coreutils".
-    date() {
-        gdate "$@"
-    }
-    export -f date
-
-    #brew install gnu-sed --with-default-names --> needed for gsed to work
-    sed() {
-      gsed "$@"
-    }
-    export -f sed
-
-    log() {
-        echo "$@"
-    }
-    export -f log
-
-    export FISSION_ROUTER=$(kubectl -n fission get svc router -o jsonpath='{...ip}')
-fi
-
-
-if [[ $# -gt 0 ]]
-then
-  
-  for var in "$@"
-  do
-    test_file=$(find $ROOT/test/tests -iname $var)
-    run_test $test_file
-  done
-
-else 
-
-  test_files=$(find $ROOT/test/tests -iname 'test_*.sh')
-
-  for file in $test_files
-  do
-    run_test ${file}
-  done
-fi 

test/test_utils.sh

@@ -13,22 +13,12 @@ pushd $ROOT_RELPATH
 ROOT=$(pwd)
 popd
 
-export TEST_REPORT=""
+travis_fold_start() {
+    echo -e "travis_fold:start:$1\r\033[33;1m$2\033[0m"
+}
 
-report_msg() {
+travis_fold_end() {
-    TEST_REPORT="$TEST_REPORT\n$1"
+    echo -e "travis_fold:end:$1\r"
-}
-report_test_passed() {
-    report_msg "--- PASSED $1"
-}
-report_test_failed() {
-    report_msg "*** FAILED $1"
-}
-report_test_skipped() {
-    report_msg "### SKIPPED $1"
-}
-show_test_report() {
-    echo -e "------\n$TEST_REPORT\n------"
 }
 
 helm_setup() {
@@ -56,6 +46,7 @@ gcloud_login() {
 
 build_and_push_pre_upgrade_check_image() {
     image_tag=$1
+    travis_fold_start build_and_push_pre_upgrade_check_image $image_tag
 
     pushd $ROOT/preupgradechecks
     ./build.sh
@@ -65,10 +56,12 @@ build_and_push_pre_upgrade_check_image() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_pre_upgrade_check_image
 }
 
 build_and_push_fission_bundle() {
     image_tag=$1
+    travis_fold_start build_and_push_fission_bundle $image_tag
 
     pushd $ROOT/fission-bundle
     ./build.sh
@@ -78,10 +71,12 @@ build_and_push_fission_bundle() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_fission_bundle
 }
 
 build_and_push_fetcher() {
     image_tag=$1
+    travis_fold_start build_and_push_fetcher $image_tag
 
     pushd $ROOT/environments/fetcher/cmd
     ./build.sh
@@ -91,11 +86,13 @@ build_and_push_fetcher() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_fetcher
 }
 
 
 build_and_push_builder() {
     image_tag=$1
+    travis_fold_start build_and_push_builder $image_tag
 
     pushd $ROOT/builder/cmd
     ./build.sh
@@ -105,11 +102,13 @@ build_and_push_builder() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_builder
 }
 
 build_and_push_env_runtime() {
     env=$1
     image_tag=$2
+    travis_fold_start build_and_push_env_runtime.$env $image_tag
 
     pushd $ROOT/environments/$env/
     docker build -q -t $image_tag .
@@ -118,12 +117,14 @@ build_and_push_env_runtime() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_env_runtime.$env
 }
 
 build_and_push_env_builder() {
     env=$1
     image_tag=$2
     builder_image=$3
+    travis_fold_start build_and_push_env_builder.$env $image_tag
 
     pushd $ROOT/environments/$env/builder
 
@@ -133,12 +134,15 @@ build_and_push_env_builder() {
 
     gcloud docker -- push $image_tag
     popd
+    travis_fold_end build_and_push_env_builder.$env
 }
 
 build_fission_cli() {
+    travis_fold_start build_fission_cli "fission cli"
     pushd $ROOT/fission
     go build .
     popd
+    travis_fold_end build_fission_cli
 }
 
 clean_crd_resources() {
@@ -174,6 +178,7 @@ helm_install_fission() {
     routerServiceType=${10}
     serviceType=${11}
     preUpgradeCheckImage=${12}
+    travis_fold_start helm_install_fission "helm install fission id=$id"
 
     ns=f-$id
     fns=f-func-$id
@@ -191,8 +196,7 @@ helm_install_fission() {
         sleep 5
     done
 
-    # only for tests, mv the prefetched prometheus chart to fission-all so helm install fission will install prometheus too
+    helm dependency update $ROOT/charts/fission-all
-    mv $ROOT/test/charts $ROOT/charts/fission-all/
 
     echo "Installing fission"
     helm install		\
@@ -204,6 +208,7 @@ helm_install_fission() {
 	 $ROOT/charts/fission-all
 
     helm list
+    travis_fold_end helm_install_fission
 }
 
 dump_kubernetes_events() {
@@ -398,16 +403,17 @@ dump_all_fission_resources() {
 }
 
 dump_system_info() {
-    echo "--- System Info ---"
+    travis_fold_start dump_system_info "System Info"
     go version
     docker version
     kubectl version
     helm version
-    echo "--- End System Info ---"
+    travis_fold_end dump_system_info
 }
 
 dump_logs() {
     id=$1
+    travis_fold_start dump_logs "dump logs $id"
 
     ns=f-$id
     fns=f-func-$id
@@ -425,13 +431,9 @@ dump_logs() {
     dump_function_pod_logs $ns $fns
     dump_builder_pod_logs $bns
     dump_fission_crds
+    travis_fold_end dump_logs
 }
 
-log() {
-    echo `date +%Y/%m/%d:%H:%M:%S`" $1"
-}
-
-export -f log
 export FAILURES=0
 
 run_all_tests() {
@@ -439,41 +441,79 @@ run_all_tests() {
 
     export FISSION_NAMESPACE=f-$id
     export FUNCTION_NAMESPACE=f-func-$id
+    export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
+    export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
+    export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:test
+    export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test
+    export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test
+    export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test
 
-    test_files=$(find $ROOT/test/tests -iname 'test_*.sh')
+    set +e
+    export TIMEOUT=900  # 15 minutes per test
 
-    for file in $test_files
+    # run tests without newdeploy in parallel.
-    do
+    export JOBS=6
-    run_test ${file}
+    $ROOT/test/run_test.sh \
+        $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \
+        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
+        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \
+        $ROOT/test/tests/recordreplay/test_record_greetings.sh \
+        $ROOT/test/tests/recordreplay/test_record_rv.sh \
+        $ROOT/test/tests/recordreplay/test_recorder_update.sh \
+        $ROOT/test/tests/test_annotations.sh \
+        $ROOT/test/tests/test_archive_pruner.sh \
+        $ROOT/test/tests/test_backend_poolmgr.sh \
+        $ROOT/test/tests/test_buildermgr.sh \
+        $ROOT/test/tests/test_canary.sh \
+        $ROOT/test/tests/test_env_vars.sh \
+        $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \
+        $ROOT/test/tests/test_function_test/test_fn_test.sh \
+        $ROOT/test/tests/test_function_update.sh \
+        $ROOT/test/tests/test_ingress.sh \
+        $ROOT/test/tests/test_internal_routes.sh \
+        $ROOT/test/tests/test_logging/test_function_logs.sh \
+        $ROOT/test/tests/test_node_hello_http.sh \
+        $ROOT/test/tests/test_package_command.sh \
+        $ROOT/test/tests/test_pass.sh \
+        $ROOT/test/tests/test_router_cache_invalidation.sh \
+        $ROOT/test/tests/test_specs/test_spec.sh \
+        $ROOT/test/tests/test_specs/test_spec_multifile.sh
+    FAILURES=$?
+
+    # FIXME: run tests with newdeploy one by one.
+    export JOBS=1
+    $ROOT/test/run_test.sh \
+        $ROOT/test/tests/test_backend_newdeploy.sh \
+        $ROOT/test/tests/test_environments/test_go_env.sh \
+        $ROOT/test/tests/test_environments/test_java_builder.sh \
+        $ROOT/test/tests/test_environments/test_java_env.sh \
+        $ROOT/test/tests/test_fn_update/test_configmap_update.sh \
+        $ROOT/test/tests/test_fn_update/test_env_update.sh \
+        $ROOT/test/tests/test_fn_update/test_nd_pkg_update.sh \
+        $ROOT/test/tests/test_fn_update/test_poolmgr_nd.sh \
+        $ROOT/test/tests/test_fn_update/test_resource_change.sh \
+        $ROOT/test/tests/test_fn_update/test_scale_change.sh \
+        $ROOT/test/tests/test_fn_update/test_secret_update.sh \
+        $ROOT/test/tests/test_obj_create_in_diff_ns.sh \
+        $ROOT/test/tests/test_secret_cfgmap/test_secret_cfgmap.sh
+    FAILURES=$((FAILURES+$?))
+    set -e
+
+    # dump test logs
+    # TODO: the idx does not match seq number in recap.
+    idx=1
+    log_files=$(find $ROOT/test/logs/ -name '*.log')
+    for log_file in $log_files; do
+        test_name=${log_file#$ROOT/test/logs/}
+        travis_fold_start run_test.$idx $test_name
+        echo "========== start $test_name =========="
+        cat $log_file
+        echo "========== end $test_name =========="
+        travis_fold_end run_test.$idx
+        idx=$((idx+1))
     done
 }
 
-run_test() {
-    file=$1
-
-    test_name=${file#${ROOT}/test/tests}
-	test_path=${file}
-
-	if grep "^#test:disabled" ${file}
-	then
-	    report_test_skipped ${test_name}
-	    echo ------- Skipped ${test_name} -------
-	else
-	    echo ------- Running ${test_name} -------
-	    pushd $(dirname ${test_path})
-	    if ${test_path}
-	    then
-		echo [SUCCESS]: ${test_name}
-		report_test_passed ${test_name}
-	    else
-		echo [FAILED]: ${test_name}
-		export FAILURES=$(($FAILURES+1))
-		report_test_failed ${test_name}
-	    fi
-	    popd
-	fi
-}
-
 install_and_test() {
     repo=$1
     image=$2
@@ -508,8 +548,6 @@ install_and_test() {
 
     dump_logs $id
 
-    show_test_report
-
     if [ $FAILURES -ne 0 ]
     then
         # describe each pod in fission ns and function namespace

test/tests/mqtrigger/kafka/test_kafka.sh

@@ -4,22 +4,29 @@
 # Create a function and trigger it using Kafka
 # This test requires Kafka & MQ-Kafka component of Fission installed in the cluster
 set -euo pipefail
+source $(dirname $0)/../../../utils.sh
 set +x
 
-nodeenv="node-kafka"
+TEST_ID=$(generate_test_id)
-goenv="go-kafka"
+echo "TEST_ID = $TEST_ID"
-producerfunc="producer-func"
-consumerfunc="consumer-func"
-consumerfunc2="consumer-func2"
 
-log() {
+tmp_dir="/tmp/test-$TEST_ID"
-    echo $1
+mkdir -p $tmp_dir
-}
+
-export -f log
+nodeenv="node-kafka-$TEST_ID"
+goenv="go-kafka-$TEST_ID"
+producerfunc="producer-func-$TEST_ID"
+consumerfunc="consumer-func-$TEST_ID"
+consumerfunc2="consumer-func2-$TEST_ID"
+mqt="kafkatest-$TEST_ID"
+mqt2="kafkatest2-$TEST_ID"
+topic="testtopic-$TEST_ID"
+resptopic="resptopic-$TEST_ID"
 
 test_mqmessage() {
     echo "Checking for valid response"
 
+    set +e
     while true; do
       response0=$(kubectl -nfission logs -l=messagequeue=kafka)
       echo $response0 | grep -i $1
@@ -28,6 +35,7 @@ test_mqmessage() {
       fi
       sleep 1
     done
+    set -e
 }
 export -f test_mqmessage 
 
@@ -37,6 +45,7 @@ test_fnmessage() {
     # $3: string to look for
     echo "Checking for valid function log"
 
+    set +e
     while true; do
 	response0=$(kubectl -nfission-function logs -l=functionName=$1 -c $2)
 	echo $response0 | grep -i "$3"
@@ -45,12 +54,14 @@ test_fnmessage() {
 	fi
 	sleep 1
     done
+    set -e
 }
 export -f test_fnmessage
 
 waitBuild() {
     log "Waiting for builder manager to finish the build"
     
+    set +e
     while true; do
       kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
       if [[ $? -eq 0 ]]; then
@@ -59,32 +70,33 @@ waitBuild() {
       log "Waiting for build to finish"
       sleep 1
     done
+    set -e
 }
 export -f waitBuild
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name ${goenv} || true
+    clean_resource_by_id $TEST_ID
-    fission env delete --name ${nodeenv} || true
+    rm -rf $tmp_dir
-    fission fn delete --name ${producerfunc} || true
-    fission fn delete --name ${consumerfunc} || true
-    fission fn delete --name ${consumerfunc2} || true
-    fission mqt delete --name kafkatest || true
-    fission mqt delete --name kafkatest2 || true
 }
-export -f cleanup
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
 
 DIR=$(dirname $0)
 
 log "Creating ${nodeenv} environment"
-fission env create --name ${nodeenv} --image fission/node-env
+fission env create --name ${nodeenv} --image ${NODE_RUNTIME_IMAGE}
-trap cleanup EXIT
 
 log "Creating ${goenv} environment"
-fission env create --name ${goenv} --image fission/go-env --builder fission/go-builder
+fission env create --name ${goenv} --image ${GO_RUNTIME_IMAGE} --builder ${GO_BUILDER_IMAGE}
 
 log "Creating package for Kafka producer"
-pushd $DIR/kafka_pub
+cp -r $DIR/kafka_pub $tmp_dir/
+pushd $tmp_dir/kafka_pub
 go mod vendor
 zip -qr kafka.zip * 
 pkgName=$(fission package create --env ${goenv} --src kafka.zip|cut -f2 -d' '| tr -d \')
@@ -96,19 +108,19 @@ timeout 120s bash -c "waitBuild $pkgName"
 log "Package ${pkgName} created"
 
 log "Creating function ${consumerfunc}"
-fission fn create --name ${consumerfunc} --env ${nodeenv} --code hellokafka.js 
+fission fn create --name ${consumerfunc} --env ${nodeenv} --code $DIR/hellokafka.js 
 
 log "Creating function ${consumerfunc2}"
-fission fn create --name ${consumerfunc2} --env ${nodeenv} --code hellokafka.js
+fission fn create --name ${consumerfunc2} --env ${nodeenv} --code $DIR/hellokafka.js
 
 log "Creating function ${producerfunc}"
 fission fn create --name ${producerfunc} --env ${goenv} --pkg ${pkgName} --entrypoint Handler
 
-log "Creating trigger kafkatest"
+log "Creating trigger $mqt"
-fission mqt create --name kafkatest --function ${consumerfunc} --mqtype kafka --topic testtopic --resptopic resptopic
+fission mqt create --name ${mqt} --function ${consumerfunc} --mqtype kafka --topic $topic --resptopic $resptopic
 
-log "Creating trigger kafkatest2"
+log "Creating trigger $mqt2"
-fission mqt create --name kafkatest2 --function ${consumerfunc2} --mqtype kafka --topic resptopic
+fission mqt create --name ${mqt2} --function ${consumerfunc2} --mqtype kafka --topic $resptopic
 
 fission fn test --name ${producerfunc}
 
@@ -122,3 +134,5 @@ log "Testing the header value in ${consumerfunc2}"
 timeout 60 bash -c "test_fnmessage '${consumerfunc2}' '${nodeenv}' 'z-custom-name: Kafka-Header-test'"
 # test if the Fission specific headers are overwritten
 timeout 60 bash -c "test_fnmessage '${consumerfunc2}' '${nodeenv}' 'x-fission-function-name: consumer-func2'"
+
+log "Test PASSED"

test/tests/mqtrigger/nats/test_mqtrigger.sh

@@ -5,21 +5,29 @@
 # 
 
 set -euo pipefail
+source $(dirname $0)/../../../utils.sh
 set +x
 
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
 ROOT=$(dirname $0)/../../..
 DIR=$(dirname $0)
 
 clusterID="fissionMQTrigger"
-topic="foo.bar"
+pubClientID="clientPub-$TEST_ID"
-resptopic="foo.foo"
+subClientID="clientSub-$TEST_ID"
-expectedRespOutput="[foo.foo]: 'Hello, World!'"
+topic="foo.bar$TEST_ID"
+resptopic="foo.foo$TEST_ID"
+expectedRespOutput="[foo.foo$TEST_ID]: 'Hello, World!'"
+
+env=nodejs-$TEST_ID
+fn=hello-$TEST_ID
+mqt=mqt-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
-    fission mqtrigger delete --name $mqt || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -28,18 +36,13 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-log "Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating function"
-fn=hello-$(date +%s)
+fission fn create --name $fn --env $env --code $DIR/main.js --method GET
-fission fn create --name $fn --env nodejs --code $DIR/main.js --method GET
 
 log "Creating message queue trigger"
-mqt=mqt-$(date +%s)
 fission mqtrigger create --name $mqt --function $fn --mqtype "nats-streaming" --topic $topic --resptopic $resptopic
 
 # wait until nats trigger is created
@@ -49,23 +52,18 @@ sleep 5
 # Send a message
 #
 log "Sending message"
-go run $DIR/stan-pub.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id clientPub $topic ""
+go run $DIR/stan-pub.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $pubClientID $topic ""
 
 #
 # Wait for message on response topic 
 #
 log "Waiting for response"
-TIMEOUT=timeout
+response=$(timeout 120s go run $DIR/stan-sub.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $subClientID $resptopic 2>&1)
-if [ $(uname -s) == 'Darwin' ]
-then
-    # If this fails on mac os, do "brew install coreutils".
-    TIMEOUT=gtimeout 
-fi
-response=$($TIMEOUT 120s go run $DIR/stan-sub.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id clientSub $resptopic 2>&1)
 
 if [[ "$response" != "$expectedRespOutput" ]]; then
-    log "$response is not equal to $expectedRespOutput"
+    log "'$response' is not equal to '$expectedRespOutput'"
     exit 1
 fi
 
 log "Subscriber received expected response: $response"
+log "Test PASSED"

test/tests/mqtrigger/nats/test_mqtrigger_error.sh

@@ -2,27 +2,35 @@
 
 #
 # Create a function and trigger it using NATS
-# To run this on Minikube, uncomment line 18
+# To run this on Minikube, uncomment line 24
 
 set -euo pipefail
 set +x
+source $(dirname $0)/../../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../../..
 DIR=$(dirname $0)
 
 clusterID="fissionMQTrigger"
-topic="foo.bar"
+pubClientID="clientPub-$TEST_ID"
-resptopic="foo.foo"
+subClientID="clientSub-$TEST_ID"
-errortopic="foo.error"
+topic="foo.bar$TEST_ID"
+resptopic="foo.foo$TEST_ID"
+errortopic="foo.error$TEST_ID"
 maxretries=1
 # FISSION_NATS_STREAMING_URL="http://defaultFissionAuthToken@$(minikube ip):4222"
-expectedRespOutput="[foo.error]: 'Hello, World!'"
+expectedRespOutput="[foo.error$TEST_ID]: 'Hello, World!'"
+
+env=nodejs-$TEST_ID
+fn=hello-$TEST_ID
+mqt=mqt-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
-    fission mqtrigger delete --name $mqt || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -31,18 +39,13 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-log "Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating function"
-fn=hello-$(date +%s)
+fission fn create --name $fn --env $env --code $DIR/main_error.js --method GET
-fission fn create --name $fn --env nodejs --code $DIR/main_error.js --method GET
 
 log "Creating message queue trigger"
-mqt=mqt-$(date +%s)
 fission mqtrigger create --name $mqt --function $fn --mqtype "nats-streaming" --topic $topic --resptopic $resptopic --errortopic $errortopic --maxretries $maxretries
 log "Updated mqtrigger list"
 fission mqtrigger list
@@ -54,28 +57,20 @@ sleep 5
 # Send a message
 #
 log "Sending message"
-go run $DIR/stan-pub.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id clientPub $topic ""
+go run $DIR/stan-pub.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $pubClientID $topic ""
 
 #
 # Wait for message on error topic
 #
 log "Waiting for response"
-TIMEOUT=timeout
+response=$(go run $DIR/stan-sub.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $subClientID $errortopic 2>&1)
-if [ $(uname -s) == 'Darwin' ]
-then
-    # If this fails on mac os, do "brew install coreutils".
-    TIMEOUT=gtimeout
-fi
-response=$(go run $DIR/stan-sub.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id clientSub $errortopic 2>&1)
 
 log "Subscriber received response: $response"
 
-fission mqtrigger delete --name $mqt
-# kubectl delete functions --all
-
 if [[ "$response" != "$expectedRespOutput" ]]; then
-    log "$response is not equal to $expectedRespOutput"
+    log "'$response' is not equal to '$expectedRespOutput'"
     exit 1
 else
     log "Responses match."
 fi
+log "Test PASSED"

test/tests/recordreplay/test_record_greetings.sh

@@ -7,49 +7,63 @@
 
 set -euo pipefail
 set +x
+source $(dirname $0)/../../utils.sh
 
-ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+cleanup() {
+    log "Cleaning up..."
+    clean_resource_by_id $TEST_ID
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+ROOT=$(dirname $0)/../../..
 DIR=$(dirname $0)
 
-echo "Pre-test cleanup"
+env=python-$TEST_ID
-fission env delete --name python || true
+fn=greetings-$TEST_ID
+recName=rec-$TEST_ID
 
 echo "Creating python env"
-fission env create --name python --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
-# trap "fission env delete --name python" EXIT
 
 echo "Creating function"
-fn=greetings-$(date +%s)
+fission fn create --name $fn --env $env --code $DIR/greetings.py --method GET
-fission fn create --name $fn --env python --code $DIR/greetings.py --method GET
-# trap "fission function delete --name $fn" EXIT
 
 echo "Creating http trigger"
-generated=$(fission route create --function $fn --method POST --url greetings | awk '{print $2}'| tr -d "'")
+generated=$(fission route create --function $fn --method POST --url /$fn | awk '{print $2}'| tr -d "'")
 
 # Wait until trigger is created
 sleep 5
 
 echo "Creating recorder"
-recName="gacrux"
 fission recorder create --name $recName --function $fn
 fission recorder get --name $recName
-# trap "fission recorder delete --name $recName" EXIT
 
 # Wait until recorder is created
 sleep 5
 
 echo "Issuing cURL request:"
-resp=$(curl -X POST "http://$FISSION_ROUTER/greetings" -d "{\"title\":\"Madam\",\"name\":\"Thanh\",\"item\":\"coat\"}")
+resp=$(curl -X POST "http://$FISSION_ROUTER/$fn" -d "{\"title\":\"Madam\",\"name\":\"Thanh\",\"item\":\"coat\"}")
 expectedR="Greetings, Madam Thanh. May I take your coat?"
-recordedStatus="$(fission records view --from 15s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
-expectedS="200OK"
 
-trap "fission recorder delete --name $recName && fission ht delete --name $generated && fission function delete --name $fn && fission env delete --name python" EXIT
+set +o pipefail
+recordedStatus="$(fission records view --from 15s --to 0s -v | grep $fn | awk '{print $4$5}')"
+set -o pipefail
+expectedS="200OK"
 
 if [ "$resp" != "$expectedR" ] || [ "$recordedStatus" != "$expectedS" ]; then
     echo "Response is not equal to expected response."
+    log "expected: status = '$expectedS'  resp = '$expectedR'"
+    log "result:   status = '$recordedStatus'  resp = '$resp'"
     exit 1
 fi
 
 echo "Passed."
-exit 0
+exit 0

test/tests/recordreplay/test_record_rv.sh

@@ -9,87 +9,109 @@
 
 set -euo pipefail
 set +x
+source $(dirname $0)/../../utils.sh
 
-ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+cleanup() {
+    log "Cleaning up..."
+    clean_resource_by_id $TEST_ID
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+ROOT=$(dirname $0)/../../..
 DIR=$(dirname $0)
 expectedR="We'll meet at 9 on Tuesday."
 
-echo "Pre-test cleanup"
+env=python-$TEST_ID
-fission env delete --name python || true
+fn=rv-$TEST_ID
+recName1=rec1-$TEST_ID
+recName2=rec2-$TEST_ID
 
 echo "Creating python env"
-fission env create --name python --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 echo "Creating function"
-fn=rv-$(date +%s)
+fission fn create --name $fn --env $env --code $DIR/rendezvous.py --method GET
-fission fn create --name $fn --env python --code $DIR/rendezvous.py --method GET
 
 echo "Creating trigger A"
-generatedA=$(fission route create --function $fn --method GET --url rvA | awk '{print $2}'| tr -d "'")
+triggerA=$(fission route create --function $fn --method GET --url /$fn-A | awk '{print $2}'| tr -d "'")
+log "triggerA = $triggerA"
 
 echo "Creating trigger B"
-generatedB=$(fission route create --function $fn --method GET --url rvB | awk '{print $2}'| tr -d "'")
+triggerB=$(fission route create --function $fn --method GET --url /$fn-B | awk '{print $2}'| tr -d "'")
+log "triggerB = $triggerB"
 
 # Wait until triggers are created
 sleep 5
 
 echo "Creating recorder by function"
-recName="regulus"
+fission recorder create --name $recName1 --function $fn
-fission recorder create --name $recName --function $fn
+fission recorder get --name $recName1
-fission recorder get --name $recName
 
 # Wait until recorder is created
 sleep 5
 
 echo "Issuing cURL request to urlA:"
-respA=$(curl -X GET "http://$FISSION_ROUTER/rvA?time=9&date=Tuesday")
+respA=$(curl -X GET "http://$FISSION_ROUTER/$fn-A?time=9&date=Tuesday")
-recordedStatusA="$(fission records view --from 5s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+recordedStatusA="$(fission records view --from 5s --to 0s -v | grep $triggerA | awk '{print $4$5}')"
 expectedSA="200OK"
 
 # Separate records
 sleep 5
 
 echo "Issuing cURL request to urlB:"
-respB=$(curl -X GET "http://$FISSION_ROUTER/rvB?time=9&date=Tuesday")
+respB=$(curl -X GET "http://$FISSION_ROUTER/$fn-B?time=9&date=Tuesday")
-recordedStatusB="$(fission records view --from 5s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+recordedStatusB="$(fission records view --from 5s --to 0s -v | grep $triggerB | awk '{print $4$5}')"
 expectedSB="200OK"
 
 if [ "$respA" != "$expectedR" ] || [ "$recordedStatusA" != "$expectedSA" ] || [ "$recordedStatusB" != "$expectedSB" ]; then
     echo "Failed at test case 1."
+    log "expected: statusA = '$expectedSA'  statusB = '$statusB'   respA = '$expectedR'"
+    log "result:   statusA = '$recordedStatusA'  statusB = '$recordedStatusB'   respA = '$respA'"
     exit 1
 fi
 
 echo "Test case 1) Passed."
 
 # Delete first recorder
-fission recorder delete --name $recName
+fission recorder delete --name $recName1
 
 sleep 5
 
 echo "Creating recorder by trigger"
-recName2="regulus2"
+fission recorder create --name $recName2 --trigger $triggerB
-fission recorder create --name $recName2 --trigger $generatedB
 fission recorder get --name $recName2
 
 echo "Issuing cURL request to urlA:"
-respA=$(curl -X GET "http://$FISSION_ROUTER/rvA?time=9&date=Tuesday")
+respA=$(curl -X GET "http://$FISSION_ROUTER/$fn-A?time=9&date=Tuesday")
-recordedStatusA="$(fission records view --from 5s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+# We except there is no records here -> grep will exit 1 -> this script exit 1 because 'pipefail' is set
+# Temporary disable 'pipefail' here
+set +o pipefail
+recordedStatusA="$(fission records view --from 5s --to 0s -v | grep $triggerA | awk '{print $4$5}')"
+set -o pipefail
 expectedSA=""
 
 # Separate records
 sleep 5
 
 echo "Issuing cURL request to urlB:"
-respB=$(curl -X GET "http://$FISSION_ROUTER/rvB?time=9&date=Tuesday")
+respB=$(curl -X GET "http://$FISSION_ROUTER/$fn-B?time=9&date=Tuesday")
-recordedStatusB="$(fission records view --from 5s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+recordedStatusB="$(fission records view --from 5s --to 0s -v | grep $triggerB | awk '{print $4$5}')"
 expectedSB="200OK"
 
 if [ "$respA" != "$expectedR" ] || [ "$recordedStatusA" != "$expectedSA" ] || [ "$recordedStatusB" != "$expectedSB" ]; then
     echo "Failed at test case 2."
+    log "expected: statusA = '$expectedSA'  statusB = '$expectedSB'   respA = '$expectedR'"
+    log "result:   statusA = '$recordedStatusA'  statusB = '$recordedStatusB'   respA = '$respA'"
     exit 1
 fi
 
-trap "fission recorder delete --name $recName2 && fission ht delete --name $generatedA && fission ht delete --name $generatedB && fission fn delete --name $fn && fission env delete --name python" EXIT
-
 echo "All passed."
-exit 0
+exit 0

test/tests/recordreplay/test_recorder_update.sh

@@ -11,28 +11,42 @@
 
 set -euo pipefail
 set +x
+source $(dirname $0)/../../utils.sh
 
-ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+cleanup() {
+    log "Cleaning up..."
+    clean_resource_by_id $TEST_ID
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+ROOT=$(dirname $0)/../../..
 DIR=$(dirname $0)
 
-echo "Pre-test cleanup"
+env=python-$TEST_ID
-fission env delete --name python || true
+fn=rv-$TEST_ID
+recName=rec-$TEST_ID
 
 echo "Creating python env"
-fission env create --name python --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 echo "Creating function"
-fn=rv-$(date +%s)
+fission fn create --name $fn --env $env --code $DIR/rendezvous.py --method GET
-fission fn create --name $fn --env python --code $DIR/rendezvous.py --method GET
 
 echo "Creating http trigger"
-generated=$(fission route create --function $fn --method GET --url rv | awk '{print $2}'| tr -d "'")
+generated=$(fission route create --function $fn --method GET --url /$fn | awk '{print $2}'| tr -d "'")
 
 # Wait until trigger is created
 sleep 5
 
 echo "Creating recorder"
-recName="regulus"
 fission recorder create --name $recName --function $fn
 fission recorder get --name $recName
 
@@ -44,13 +58,17 @@ fission recorder update --name $recName --disable
 sleep 5
 
 echo "Issuing cURL request that should not be recorded:"
-resp=$(curl -X GET "http://$FISSION_ROUTER/rv?time=9&date=Tuesday")
+resp=$(curl -X GET "http://$FISSION_ROUTER/$fn?time=9&date=Tuesday")
-recordedStatus="$(fission records view --from 15s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+set +o pipefail
+recordedStatus="$(fission records view --from 5s --to 0s -v | grep $fn | awk '{print $4$5}')"
+set -o pipefail
 expectedR="We'll meet at 9 on Tuesday."
 expectedS=""
 
 if [ "$resp" != "$expectedR" ] || [ "$recordedStatus" != "$expectedS" ]; then
     echo "Response is not equal to expected response."
+    log "expected: status = '$expectedS'  resp = '$expectedR'"
+    log "result:   status = '$recordedStatus'  resp = '$resp'"
     exit 1
 fi
 
@@ -61,20 +79,24 @@ fission recorder update --name $recName --enable
 sleep 5
 
 echo "Issuing cURL request that should be recorded:"
-resp=$(curl -X GET "http://$FISSION_ROUTER/rv?time=9&date=Tuesday")
+resp=$(curl -X GET "http://$FISSION_ROUTER/$fn?time=9&date=Tuesday")
 expectedR="We'll meet at 9 on Tuesday."
-recordedStatus="$(fission records view --from 15s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+set +o pipefail
+recordedStatus="$(fission records view --from 5s --to 0s -v | grep $fn | awk '{print $4$5}')"
+set -o pipefail
 expectedS="200OK"
 
 if [ "$resp" != "$expectedR" ] || [ "$recordedStatus" != "$expectedS" ]; then
     echo "Response is not equal to expected response."
+    log "expected: status = '$expectedS'  resp = '$expectedR'"
+    log "result:   status = '$recordedStatus'  resp = '$resp'"
     exit 1
 fi
 
 echo "Test case 2) Passed."
 
 # Create new trigger for same function recorded w/ different url
-generated2=$(fission route create --function $fn --method GET --url rv2 | awk '{print $2}'| tr -d "'")
+generated2=$(fission route create --function $fn --method GET --url $fn-2 | awk '{print $2}'| tr -d "'")
 echo "New trigger: $generated2"
 
 # Update recorder to observe new trigger
@@ -82,19 +104,21 @@ fission recorder update --name $recName --trigger $generated2
 fission recorder list
 
 echo "Issuing cURL request that should be recorded:"
-resp=$(curl -X GET "http://$FISSION_ROUTER/rv2?time=9&date=Tuesday")
+resp=$(curl -X GET "http://$FISSION_ROUTER/$fn-2?time=9&date=Tuesday")
 expectedR="We'll meet at 9 on Tuesday."
-recordedStatus="$(fission records view --from 15s --to 0s -v | awk 'FNR == 2 {print $4$5}')"
+set +o pipefail
+recordedStatus="$(fission records view --from 5s --to 0s -v | grep $generated2 | awk '{print $4$5}')"
+set -o pipefail
 expectedS="200OK"
 
 if [ "$resp" != "$expectedR" ] || [ "$recordedStatus" != "$expectedS" ]; then
     echo "Response is not equal to expected response."
+    log "expected: status = '$expectedS'  resp = '$expectedR'"
+    log "result:   status = '$recordedStatus'  resp = '$resp'"
     exit 1
 fi
 
 echo "Test case 3) Passed."
 
-trap "fission recorder delete --name $recName && fission ht delete --name $generated && fission ht delete --name $generated2 && fission function delete --name $fn && fission env delete --name python" EXIT
-
 echo "All passed."
-exit 0
+exit 0

test/tests/test_annotations.sh

@@ -1,21 +1,24 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
 
 # test_annotations.sh - tests whether a user is able to add pod annotations to a Fission environment deployment
 
-TEST_ID=$(date +%s)
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
 ENV=python-${TEST_ID}
-FN=foo-${TEST_ID}
 RESOURCE_NS=default # Change to test-specific namespace once we support namespaced CRDs
 FUNCTION_NS=${FUNCTION_NAMESPACE:-fission-function}
 BUILDER_NS=fission-builder
 LIST_ANNOTATIONS=go-template='{{range $key,$value := .metadata.annotations}}{{$key}}: {{$value}}{{"\n"}}{{end}}'
 
 # fs
-TEST_DIR=/tmp/${TEST_ID}
+ENV_SPEC_FILE=$tmp_dir/${ENV}.yaml
-ENV_SPEC_FILE=${TEST_DIR}/${ENV}.yaml
-FN_FILE=${TEST_DIR}/${FN}.yaml
 
 log_exec() {
     cmd=$@
@@ -25,22 +28,16 @@ log_exec() {
 
 cleanup() {
     log "Cleaning up..."
-    kubectl -n ${RESOURCE_NS} delete environment/${ENV} || true
+    clean_resource_by_id $TEST_ID
-    rm -rf ${TEST_DIR}
+    rm -rf $tmp_dir
-
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-if ! stat ${TEST_DIR} >/dev/null 2>&1 ; then
-    mkdir ${TEST_DIR}
-fi
-
 getPodName() {
     NS=$1
     POD=$2
@@ -57,7 +54,7 @@ getPodName() {
 # https://unix.stackexchange.com/questions/82598/how-do-i-write-a-retry-logic-in-script-to-keep-retrying-to-run-it-upto-5-times/82610
 function retry {
   local n=1
-  local max=5
+  local max=10
   local delay=10 # pods take time to get ready
   while true; do
     "$@" && break || {
@@ -88,9 +85,9 @@ metadata:
 spec:
   builder:
     command: build
-    image: gcr.io/fission-ci/python-env-builder:test
+    image: ${PYTHON_BUILDER_IMAGE}
   runtime:
-    image: gcr.io/fission-ci/python-env:test
+    image: ${PYTHON_RUNTIME_IMAGE}
   version: 2
   poolsize: 1
 EOM

test/tests/test_archive_pruner.sh

@@ -1,32 +1,42 @@
 #!/bin/bash
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 # global variables
+env=python-$TEST_ID
 pkg=""
 http_status=""
 url=""
 
 
 cleanup() {
-    if [ -e "test-deploy-pkg.zip" ]; then
+    clean_resource_by_id $TEST_ID
-        rm -rf test-deploy-pkg.zip test_dir
+    rm -rf $tmp_dir
-    fi
-    if [ -e "/tmp/file" ]; then
-        rm -rf /tmp/file
-    fi
 }
 
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
 create_archive() {
     log "Creating an archive"
-    mkdir test_dir
+    mkdir -p $tmp_dir/archive
-    dd if=/dev/urandom of=test_dir/dynamically_generated_file bs=256k count=1
+    dd if=/dev/urandom of=$tmp_dir/archive/dynamically_generated_file bs=256k count=1
-    printf 'def main():\n    return "Hello, world!"' > test_dir/hello.py
+    printf 'def main():\n    return "Hello, world!"' > $tmp_dir/archive/hello.py
-    zip -jr test-deploy-pkg.zip test_dir/
+    zip -jr $tmp_dir/test-deploy-pkg.zip $tmp_dir/archive/
 }
 
 create_package() {
     log "Creating package"
-    pkg=$(fission package create --deploy "test-deploy-pkg.zip" --env python| cut -f2 -d' '| tr -d \')
+    pkg=$(fission package create --deploy "$tmp_dir/test-deploy-pkg.zip" --env $env| cut -f2 -d' '| tr -d \')
 }
 
 delete_package() {
@@ -36,7 +46,7 @@ delete_package() {
 
 get_archive_url_from_package() {
     log "Getting archive URL from package: $1"
-    url=`kubectl get package $1 -ojsonpath='{.spec.deployment.url}'`
+    url=`kubectl -n default get package $1 -ojsonpath='{.spec.deployment.url}'`
 }
 
 get_archive_from_storage() {
@@ -44,7 +54,7 @@ get_archive_from_storage() {
     controller_ip=$(kubectl -n $FISSION_NAMESPACE get svc controller -o jsonpath='{...ip}')
     controller_proxy_url=`echo $storage_service_url | sed -e "s/storagesvc.$FISSION_NAMESPACE/$controller_ip\/proxy\/storage/"`
     log "controller_proxy_url=$controller_proxy_url"
-    http_status=`curl -sw "%{http_code}" $controller_proxy_url -o /tmp/file`
+    http_status=`curl --retry 5 -sw "%{http_code}" $controller_proxy_url -o /dev/null`
     echo "http_status: $http_status"
 }
 
@@ -56,8 +66,8 @@ get_archive_from_storage() {
 #6. sleep for two minutes
 #7. now verify that both got deleted.
 main() {
-    # trap
+    log "Creating python env"
-    trap cleanup EXIT
+    fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
 
     # create a huge archive
     create_archive
@@ -117,8 +127,7 @@ main() {
         exit 1
     fi
 
-
     log "Test archive pruner PASSED"
 }
 
-main
+main

test/tests/test_backend_newdeploy.sh

@@ -1,13 +1,19 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
 
-ROOT=$(dirname $0)/../.. 
+ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+nodejs_env=nodejs-$TEST_ID
+fn0=nodejs-hello-0-$TEST_ID
+fn1=nodejs-hello-1-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn1 || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -17,19 +23,13 @@ else
 fi
 
 # Create a hello world function in nodejs, test it with an http trigger
-log "NewDeploy ExecutorType: Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+fission env create --name $nodejs_env --image $NODE_RUNTIME_IMAGE --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
-#trap "fission env delete --name nodejs" EXIT
 
 # TODO Imporve test code by reusing common blocks
 
 log "Creating function, testing for cold start with MinScale 0"
-fn0=nodejs-hello-$(date +%N)
+fission fn create --name $fn0 --env $nodejs_env --code $ROOT/examples/nodejs/hello.js --minscale 0 --maxscale 4 --executortype newdeploy
-fission fn create --name $fn0 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 0 --maxscale 4 --executortype newdeploy
-#trap "fission fn delete --name $fn0" EXIT
 
 log "Creating route"
 fission route create --function $fn0 --url /$fn0 --method GET
@@ -38,16 +38,13 @@ log "Waiting for router & newdeploy deployment creation"
 sleep 5
 
 log "Doing an HTTP GET on the function's route"
-response0=$(curl http://$FISSION_ROUTER/$fn0)
+response0=$(curl --retry 5 http://$FISSION_ROUTER/$fn0)
 
 log "Checking for valid response"
 echo $response0 | grep -i hello
 
-
 log "Creating function, testing for warm start with MinScale 1"
-fn1=nodejs-hello-$(date +%N)
+fission fn create --name $fn1 --env $nodejs_env --code $ROOT/examples/nodejs/hello.js --minscale 1 --maxscale 4 --executortype newdeploy
-fission fn create --name $fn1 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 1 --maxscale 4 --executortype newdeploy
-#trap "fission fn delete --name $fn1" EXIT
 
 log "Creating route"
 fission route create --function $fn1 --url /$fn1 --method GET
@@ -56,12 +53,9 @@ log "Waiting for router & newdeploy deployment creation"
 sleep 5
 
 log "Doing an HTTP GET on the function's route"
-response1=$(curl http://$FISSION_ROUTER/$fn1)
+response1=$(curl --retry 5 http://$FISSION_ROUTER/$fn1)
 
 log "Checking for valid response"
 echo $response1 | grep -i hello
 
-# crappy cleanup, improve this later
-kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
-
 log "NewDeploy ExecutorType: All done."

test/tests/test_backend_poolmgr.sh

@@ -1,13 +1,19 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../.. 
 
+env=nodejs-$TEST_ID
+fn=nodejs-hello-$TEST_ID
+
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -16,17 +22,13 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-fn=nodejs-hello-$(date +%N)
 
 # Create a hello world function in nodejs, test it with an http trigger
-log "Poolmgr ExecutorType: Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+fission env create --name $env --image $NODE_RUNTIME_IMAGE --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
 
 log "Creating function"
-fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js --executortype poolmgr
+fission fn create --name $fn --env $env --code $ROOT/examples/nodejs/hello.js --executortype poolmgr
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
@@ -40,7 +42,4 @@ response=$(curl http://$FISSION_ROUTER/$fn)
 log "Checking for valid response"
 echo $response | grep -i hello
 
-# crappy cleanup, improve this later
-kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
-
 log "Poolmgr ExecutorType: All done."

test/tests/test_buildermgr.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 # Create a function with source package in python 
 # to test builder manger functionality. 
@@ -9,10 +16,9 @@ set -euo pipefail
 # 2. package watcher triggers the build if any changes to packages
 
 ROOT=$(dirname $0)/../..
-PYTHON_RUNTIME_IMAGE=${PYTHON_RUNTIME_IMAGE:-gcr.io/fission-ci/python-env:test}
-PYTHON_BUILDER_IMAGE=${PYTHON_BUILDER_IMAGE:-gcr.io/fission-ci/python-env-builder:test}
 
-fn=python-srcbuild-$(date +%s)
+env=python-$TEST_ID
+fn=python-srcbuild-$TEST_ID
 
 checkFunctionResponse() {
     log "Doing an HTTP GET on the function's route"
@@ -31,6 +37,7 @@ waitBuild() {
       if [[ $? -eq 0 ]]; then
           break
       fi
+      sleep 1
     done
 }
 export -f waitBuild
@@ -47,15 +54,15 @@ waitEnvBuilder() {
       if [[ $? -eq 0 ]]; then
           break
       fi
+      sleep 1
     done
 }
 export -f waitEnvBuilder
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name python || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
+    rm -rf $tmp_dir
-    rm demo-src-pkg.zip || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -64,20 +71,16 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-log "Pre-test cleanup"
-fission env delete --name python || true
-kubectl --namespace default get packages|grep -v NAME|awk '{print $1}'|xargs -I@ bash -c 'kubectl --namespace default delete packages @' || true
-
 log "Creating python env"
-fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
 
-timeout 180s bash -c "waitEnvBuilder python"
+timeout 180s bash -c "waitEnvBuilder $env"
 
 log "Creating source pacakage"
-zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
+zip -jr $tmp_dir/demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
 
 log "Creating function " $fn
-fission fn create --name $fn --env python --src demo-src-pkg.zip --entrypoint "user.main" --buildcmd "./build.sh"
+fission fn create --name $fn --env $env --src $tmp_dir/demo-src-pkg.zip --entrypoint "user.main" --buildcmd "./build.sh"
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
@@ -93,7 +96,7 @@ timeout 60s bash -c "waitBuild $pkg"
 checkFunctionResponse $fn
 
 log "Updating function " $fn
-fission fn update --name $fn --src demo-src-pkg.zip
+fission fn update --name $fn --src $tmp_dir/demo-src-pkg.zip
 
 pkg=$(kubectl --namespace default get functions $fn -o jsonpath='{.spec.package.packageref.name}')
 
@@ -102,7 +105,4 @@ timeout 60s bash -c "waitBuild $pkg"
 
 checkFunctionResponse $fn
 
-# crappy cleanup, improve this later
-kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
-
 log "All done."

test/tests/test_canary.sh

@@ -3,52 +3,65 @@
 # has 2 tests to verify the canary deployments - success scenario and a failure scenario
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
-id=""
 ROOT=$(dirname $0)/../..
 
+env=nodejs-$TEST_ID
+fn_v1=fn-v1-$TEST_ID
+fn_v2=fn-v2-$TEST_ID
+fn_v3=fn-v3-$TEST_ID
+route_succ=route-succ-$TEST_ID
+route_fail=route-fail-$TEST_ID
+canary_1=canary-1-$TEST_ID
+canary_2=canary-2-$TEST_ID
+
 cleanup() {
-    fission env delete --name nodejs || true
+    log "Cleaning up..."
-    fission fn delete --name fn-v1 || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name fn-v2 || true
+    rm -rf $tmp_dir
-    fission fn delete --name fn-v3 || true
-    fission ht delete --name route-success || true
-    fission ht delete --name route-fail || true
-    fission canary-config delete --name canary-1 || true
-    fission canary-config delete --name canary-2 || true
 }
 
-trap cleanup EXIT
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
 
 success_scenario() {
     log "Creating nodejs env"
-    fission env create --name nodejs --image fission/node-env --graceperiod 1
+    fission env create --name $env --image $NODE_RUNTIME_IMAGE --graceperiod 1
 
     log "Creating function version-1"
-    fission fn create --name fn-v1 --env nodejs --code $ROOT/examples/nodejs/hello.js
+    fission fn create --name $fn_v1 --env $env --code $ROOT/examples/nodejs/hello.js
 
     log "Creating function version-1"
-    fission fn create --name fn-v2 --env nodejs --code $ROOT/examples/nodejs/hello.js
+    fission fn create --name $fn_v2 --env $env --code $ROOT/examples/nodejs/hello.js
 
     log "Create a route for the version-1 of the function with weight 100% and version-2 with weight 0%"
-    fission route create --name route-success --method GET --url /success --function fn-v1 --weight 100 --function fn-v2 --weight 0
+    fission route create --name $route_succ --method GET --url /$route_succ --function $fn_v1 --weight 100 --function $fn_v2 --weight 0
 
     log "Create a canary config to gradually increment the weight of version-2 by a step of 50 every 1m"
-    fission canary-config create --name canary-1 --newfunction fn-v2 --oldfunction fn-v1 --httptrigger route-success --increment-step 50 --increment-interval 1m --failure-threshold 10
+    fission canary-config create --name $canary_1 --newfunction $fn_v2 --oldfunction $fn_v1 --httptrigger $route_succ --increment-step 50 --increment-interval 1m --failure-threshold 10
 
     sleep 60
 
     log "Fire requests to the route"
-    ab -n 300 -c 1 http://$FISSION_ROUTER/success
+    ab -n 300 -c 1 http://$FISSION_ROUTER/$route_succ
 
     sleep 60
 
     log "verify that version-2 of the function is receiving 100% traffic"
-    weight=`kubectl get httptrigger route-success -o jsonpath='{.spec.functionref.functionweights.fn-v2}'`
+    weight=`kubectl -n default get httptrigger $route_succ -o jsonpath='{.spec.functionref.functionweights.'$fn_v2'}'`
 
     if [ "$weight" != "100" ]; then
-        log "weight of fn-v2 at the end of the test is $weight"
+        log "weight of $fn_v2 at the end of the test is $weight"
-        cleanup
         exit 1
     else
         log "canary success scenario test passed"
@@ -56,32 +69,30 @@ success_scenario() {
 }
 
 failure_scenario() {
-    cp $ROOT/examples/nodejs/hello.js hello_400.js
+    sed 's/200/400/' $ROOT/examples/nodejs/hello.js > $tmp_dir/hello_400.js
-    sed -i 's/200/400/' hello_400.js
 
     log "Creating function version-3"
-    fission fn create --name fn-v3 --env nodejs --code hello_400.js
+    fission fn create --name $fn_v3 --env $env --code $tmp_dir/hello_400.js
 
     log "Create a route for the version-1 of the function with weight 100% and version-3 with weight 0%"
-    fission route create --name route-fail --method GET --url /fail --function fn-v1 --weight 100 --function fn-v3 --weight 0
+    fission route create --name $route_fail --method GET --url /$route_fail --function $fn_v1 --weight 100 --function $fn_v3 --weight 0
     sleep 5
 
     log "Create a canary config to gradually increment the weight of version-2 by a step of 50 every 1m"
-    fission canary-config create --name canary-2 --newfunction fn-v3 --oldfunction fn-v1 --httptrigger route-fail --increment-step 50 --increment-interval 1m --failure-threshold 10
+    fission canary-config create --name $canary_2 --newfunction $fn_v3 --oldfunction $fn_v1 --httptrigger $route_fail --increment-step 50 --increment-interval 1m --failure-threshold 10
 
     sleep 60
 
     log "Fire requests to the route"
-    ab -n 300 -c 1 http://$FISSION_ROUTER/fail
+    ab -n 300 -c 1 http://$FISSION_ROUTER/$route_fail
 
     sleep 60
 
     log "verify that version-3 of the function is receiving 0% traffic because of rollback"
-    weight=`kubectl get httptrigger route-fail -o jsonpath='{.spec.functionref.functionweights.fn-v3}'`
+    weight=`kubectl -n default get httptrigger $route_fail -o jsonpath='{.spec.functionref.functionweights.'$fn_v3'}'`
 
     if [ "$weight" != "0" ]; then
-        log "weight of fn-v3 at the end of the test is $weight"
+        log "weight of 3 at the end of the test is $weight"
-        cleanup
         exit 1
     else
         log "canary failure scenario test passed"
@@ -95,8 +106,6 @@ main() {
     # v3 of a function starts with receiving 0% of the traffic, but because of failure rates crossing the threshold,
     # this test rollbacks the canary deployment to ensure v1 receives 100% of the traffic.
     failure_scenario
-
-    cleanup
 }
 
 main

test/tests/test_env_vars.sh

@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
 
 # test_env_vars.sh - tests whether a user is able to add environment variables to a Fission environment deployment
 
-TEST_ID=$(date +%s)
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
 ENV=python-${TEST_ID}
 FN=foo-${TEST_ID}
 RESOURCE_NS=default # Change to test-specific namespace once we support namespaced CRDs
@@ -12,9 +18,8 @@ FUNCTION_NS=${FUNCTION_NAMESPACE:-fission-function}
 BUILDER_NS=fission-builder
 
 # fs
-TEST_DIR=/tmp/${TEST_ID}
+ENV_SPEC_FILE=${tmp_dir}/${ENV}.yaml
-ENV_SPEC_FILE=${TEST_DIR}/${ENV}.yaml
+FN_FILE=${tmp_dir}/${FN}.yaml
-FN_FILE=${TEST_DIR}/${FN}.yaml
 
 log_exec() {
     cmd=$@
@@ -24,22 +29,16 @@ log_exec() {
 
 cleanup() {
     log "Cleaning up..."
-    kubectl -n ${RESOURCE_NS} delete environment/${ENV} || true
+    clean_resource_by_id $TEST_ID
-    rm -rf ${TEST_DIR}
+    rm -rf $tmp_dir
-
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-if ! stat ${TEST_DIR} >/dev/null 2>&1 ; then
-    mkdir ${TEST_DIR}
-fi
-
 getPodName() {
     NS=$1
     POD=$2
@@ -56,7 +55,7 @@ getPodName() {
 # https://unix.stackexchange.com/questions/82598/how-do-i-write-a-retry-logic-in-script-to-keep-retrying-to-run-it-upto-5-times/82610
 function retry {
   local n=1
-  local max=5
+  local max=10
   local delay=10 # pods take time to get ready
   while true; do
     "$@" && break || {
@@ -83,14 +82,14 @@ metadata:
 spec:
   builder:
     command: build
-    image: gcr.io/fission-ci/python-env-builder:test
+    image: ${PYTHON_BUILDER_IMAGE}
     container:
       env:
       - name: TEST_BUILDER_ENV_KEY
         value: "TEST_BUILDER_ENV_VAR"
 
   runtime:
-    image: gcr.io/fission-ci/python-env:test
+    image: ${PYTHON_RUNTIME_IMAGE}
     container:
       env:
       - name: TEST_RUNTIME_ENV_KEY

test/tests/test_environments/test_go_env.sh

@@ -1,108 +1,112 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 ROOT=$(dirname $0)/../../..
 
 cleanup() {
-    fission fn delete --name hello-go-poolmgr || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name hello-go-nd || true
+    rm -rf $tmp_dir
-    fission env delete --name go || true
-    rm $ROOT/examples/go/vendor-example/vendor.zip || true
 }
 
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+env=go-$TEST_ID
+fn_poolmgr=hello-go-poolmgr-$TEST_ID
+fn_nd=hello-go-nd-$TEST_ID
+
 wait_for_builder() {
+    env=$1
     JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
 
     # wait for tiller ready
+    set +e
     while true; do
-      kubectl --namespace fission-builder get pod -l envName=go -o jsonpath="$JSONPATH" | grep "Ready=True"
+      kubectl --namespace fission-builder get pod -l envName=$env -o jsonpath="$JSONPATH" | grep "Ready=True"
       if [[ $? -eq 0 ]]; then
           break
       fi
-      sleep 5
+      sleep 1
     done
+    set -e
 }
 
 waitBuild() {
     log "Waiting for builder manager to finish the build"
 
+    set +e
     while true; do
       kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
       if [[ $? -eq 0 ]]; then
           break
       fi
-    done
-}
-
-test_fn() {
-    log "Checking for valid response"
-
-    while true; do
-      response0=$(curl http://$FISSION_ROUTER/$1)
-      log $response0 | grep -i $2
-      if [[ $? -eq 0 ]]; then
-        break
-      fi
       sleep 1
     done
+    set -e
 }
 
 export -f wait_for_builder
 export -f waitBuild
-export -f test_fn
 
 cd $ROOT/examples/go
 
-trap cleanup EXIT
-
-GO_RUNTIME_IMAGE=${GO_RUNTIME_IMAGE:-gcr.io/fission-ci/go-env:test}
-GO_BUILDER_IMAGE=${GO_BUILDER_IMAGE:-gcr.io/fission-ci/go-env-builder:test}
-
 log "Creating environment for Golang"
-fission env create --name go --image $GO_RUNTIME_IMAGE --builder $GO_BUILDER_IMAGE --period 5
+fission env create --name $env --image $GO_RUNTIME_IMAGE --builder $GO_BUILDER_IMAGE --period 5
 
-timeout 90 bash -c "wait_for_builder"
+timeout 90 bash -c "wait_for_builder $env"
 
-pkgName=$(fission package create --src hello.go --env go| cut -f2 -d' '| tr -d \')
+pkgName=$(fission package create --src hello.go --env $env| cut -f2 -d' '| tr -d \')
 
 # wait for build to finish at most 90s
 timeout 90 bash -c "waitBuild $pkgName"
 
 log "Creating pool manager & new deployment function for Golang"
-fission fn create --name hello-go-poolmgr --env go --pkg $pkgName --entrypoint Handler
+fission fn create --name $fn_poolmgr --env $env --pkg $pkgName --entrypoint Handler
-fission fn create --name hello-go-nd --env go --pkg $pkgName --entrypoint Handler --executortype newdeploy
+fission fn create --name $fn_nd      --env $env --pkg $pkgName --entrypoint Handler --executortype newdeploy
 
 log "Creating route for new deployment function"
-fission route create --function hello-go-poolmgr --url /hello-go-poolmgr --method GET
+fission route create --function $fn_poolmgr --url /$fn_poolmgr --method GET
-fission route create --function hello-go-nd --url /hello-go-nd --method GET
+fission route create --function $fn_nd      --url /$fn_nd      --method GET
 
 log "Waiting for router & pools to catch up"
 sleep 5
 
 log "Testing pool manager function"
-timeout 60 bash -c "test_fn hello-go-poolmgr 'Hello'"
+timeout 60 bash -c "test_fn $fn_poolmgr 'Hello'"
 
 log "Testing new deployment function"
-timeout 60 bash -c "test_fn hello-go-nd 'Hello'"
+timeout 60 bash -c "test_fn $fn_nd 'Hello'"
 
 # Create zip file without top level directory (vendor-example)
-cd vendor-example && zip -r vendor.zip *
+cd vendor-example && zip -r $tmp_dir/vendor.zip *
 
-pkgName=$(fission package create --src vendor.zip --env go| cut -f2 -d' '| tr -d \')
+pkgName=$(fission package create --src $tmp_dir/vendor.zip --env $env| cut -f2 -d' '| tr -d \')
 
 # wait for build to finish at most 90s
 timeout 90 bash -c "waitBuild $pkgName"
 
 log "Update function package"
-fission fn update --name hello-go-poolmgr --pkg $pkgName
+fission fn update --name $fn_poolmgr --pkg $pkgName
-fission fn update --name hello-go-nd --pkg $pkgName
+fission fn update --name $fn_nd --pkg $pkgName
 
 log "Waiting for router & pools to catch up"
 sleep 5
 
 log "Testing pool manager function with new package"
-timeout 60 bash -c "test_fn hello-go-poolmgr 'vendor'"
+timeout 60 bash -c "test_fn $fn_poolmgr 'Vendor'"
 
 log "Testing new deployment function with new package"
-timeout 60 bash -c "test_fn hello-go-nd 'vendor'"
+timeout 60 bash -c "test_fn $fn_nd 'Vendor'"
+
+log "Test PASSED"

test/tests/test_environments/test_java_builder.sh

@@ -1,35 +1,35 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
 ROOT=$(dirname $0)/../../..
 
-JVM_RUNTIME_IMAGE=${JVM_RUNTIME_IMAGE:-gcr.io/fission-ci/jvm-env:test}
+TEST_ID=$(generate_test_id)
-JVM_BUILDER_IMAGE=${JVM_BUILDER_IMAGE:-gcr.io/fission-ci/jvm-env-builder:test}
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+env=java-$TEST_ID
+fn_p=pbuilderhello-$TEST_ID
+fn_n=nbuilderhello-$TEST_ID
 
 cleanup() {
-    fission fn delete --name pbuilderhello || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name nbuilderhello || true
+    rm -rf $tmp_dir
-    fission env delete --name java || true
-    rm $ROOT/examples/jvm/java/java-src-pkg.zip || true
 }
 
-test_fn() {
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
-    echo "Checking for valid response"
+    trap cleanup EXIT
-
+else
-    while true; do
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
-      response0=$(curl http://$FISSION_ROUTER/$1)
+fi
-      echo $response0 | grep -i $2
-      if [[ $? -eq 0 ]]; then
-        break
-      fi
-      sleep 1
-    done
-}
 
 test_pkg() {
-    echo "Checking for valid response"
+    echo "Checking package for valid response"
 
+    set +e
     while true; do
       response0=$(kubectl get -ndefault package $1 -o=jsonpath='{.status.buildstatus}')
       echo $response0 | grep -i $2
@@ -38,43 +38,43 @@ test_pkg() {
       fi
       sleep 1
     done
+    set -e
 }
 
-export -f test_fn
 export -f test_pkg
 
 cd $ROOT/examples/jvm/java
 
-trap cleanup EXIT
-
 log "Creating zip from source code"
-zip -r java-src-pkg.zip *
+zip -r $tmp_dir/java-src-pkg.zip *
 
 log "Creating Java environment with Java Builder"
-fission env create --name java --image $JVM_RUNTIME_IMAGE --version 2 --keeparchive --builder $JVM_BUILDER_IMAGE
+fission env create --name $env --image $JVM_RUNTIME_IMAGE --version 2 --keeparchive --builder $JVM_BUILDER_IMAGE
 
 log "Creating package from the source archive"
-pkg_name=`fission package create --sourcearchive java-src-pkg.zip --env java|cut -d' ' -f 2|cut -d"'" -f 2`
+pkg_name=`fission package create --sourcearchive $tmp_dir/java-src-pkg.zip --env $env|cut -d' ' -f 2|cut -d"'" -f 2`
 log "Created package $pkg_name"
 
 log "Checking the status of package"
-timeout 300 bash -c "test_pkg $pkg_name 'succeeded'"
+timeout 400 bash -c "test_pkg $pkg_name 'succeeded'"
 
 log "Creating pool manager & new deployment function for Java"
-fission fn create --name nbuilderhello --pkg $pkg_name --env java --entrypoint io.fission.HelloWorld --executortype newdeploy --minscale 1 --maxscale 1
+fission fn create --name $fn_n --pkg $pkg_name --env $env --entrypoint io.fission.HelloWorld --executortype newdeploy --minscale 1 --maxscale 1
-fission fn create --name pbuilderhello --pkg $pkg_name --env java --entrypoint io.fission.HelloWorld
+fission fn create --name $fn_p --pkg $pkg_name --env $env --entrypoint io.fission.HelloWorld
 
 log "Creating route for pool manager function"
-fission route create --function pbuilderhello --url /pbuilderhello --method GET
+fission route create --function $fn_p --url /$fn_p --method GET
 
 log "Creating route for new deployment function"
-fission route create --function nbuilderhello --url /nbuilderhello --method GET
+fission route create --function $fn_n --url /$fn_n --method GET
 
 log "Waiting for router & pools to catch up"
 sleep 5
 
 log "Testing pool manager function"
-timeout 60 bash -c "test_fn pbuilderhello 'Hello'"
+timeout 60 bash -c "test_fn $fn_p 'Hello'"
 
 log "Testing new deployment function"
-timeout 60 bash -c "test_fn nbuilderhello 'Hello'"
+timeout 60 bash -c "test_fn $fn_n 'Hello'"
+
+log "Test PASSED"

test/tests/test_environments/test_java_env.sh

@@ -3,29 +3,26 @@
 #test:disabled
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../../..
 
+env=jvm-$TEST_ID
+fn_n=jvm-hello-n-$TEST_ID
+fn_p=jvm-hello-p-$TEST_ID
+
 cleanup() {
-    fission fn delete --name hellon || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name hellop || true
-    fission env delete --name jvm || true
 }
 
-test_fn() {
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
-    echo "Checking for valid response"
+    trap cleanup EXIT
-
+else
-    while true; do
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
-      response0=$(curl http://$FISSION_ROUTER/$1)
+fi
-      echo $response0 | grep -i $2
-      if [[ $? -eq 0 ]]; then
-        break
-      fi
-      sleep 1
-    done
-}
-
-export -f test_fn
 
 cd $ROOT/examples/jvm/java
 
@@ -34,24 +31,25 @@ log "Creating the jar from application"
 docker run -it --rm  -v "$(pwd)":/usr/src/mymaven -w /usr/src/mymaven maven:3.5-jdk-8 mvn clean package -q
 
 log "Creating environment for Java"
-fission env create --name jvm --image gcr.io/fission-ci/jvm-env:test --version 2 --keeparchive=true
+fission env create --name $env --image $JVM_RUNTIME_IMAGE --version 2 --keeparchive=true
 
 log "Creating pool manager & new deployment function for Java"
-fission fn create --name hellop --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env jvm --entrypoint io.fission.HelloWorld
+fission fn create --name $fn_p --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env $env --entrypoint io.fission.HelloWorld
-fission fn create --name hellon --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env jvm --executortype newdeploy --entrypoint io.fission.HelloWorld
+fission fn create --name $fn_n --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env $env --executortype newdeploy --entrypoint io.fission.HelloWorld
-trap cleanup EXIT
 
 log "Creating route for pool manager function"
-fission route create --function hellop --url /hellop --method GET
+fission route create --name $fn_p --function $fn_p --url /$fn_p --method GET
 
 log "Creating route for new deployment function"
-fission route create --function hellon --url /hellon --method GET
+fission route create --name $fn_n --function $fn_n --url /$fn_n --method GET
 
 log "Waiting for router & pools to catch up"
 sleep 5
 
 log "Testing pool manager function"
-timeout 60 bash -c "test_fn hellop 'Hello'"
+timeout 60 bash -c "test_fn $fn_p 'Hello'"
 
 log "Testing new deployment function"
-timeout 60 bash -c "test_fn hellon 'Hello'"
+timeout 60 bash -c "test_fn $fn_n 'Hello'"
+
+log "Test PASSED"

test/tests/test_fn_update/fnupdate_utils.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-#
-# Common methods used for testing function update
-#
-
-set -euo pipefail
-
-test_fn() {
-    echo "Doing an HTTP GET on the function's route"
-    echo "Checking for valid response"
-
-    while true; do
-      response0=$(curl http://$FISSION_ROUTER/$1)
-      echo $response0 | grep -i $2
-      if [[ $? -eq 0 ]]; then
-        break
-      fi
-      sleep 1
-    done
-}
-export -f test_fn

test/tests/test_fn_update/test_configmap_update.sh

@@ -1,25 +1,26 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 ROOT=$(dirname $0)/../../..
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn_name=hellopy-$(date +%N)
+fn_name=hellopy-$TEST_ID
 
-old_cfgmap=old-cfgmap-$(date +%N)
+old_cfgmap=old-cfgmap-$TEST_ID
-new_cfgmap=new-cfgmap-$(date +%N)
+new_cfgmap=new-cfgmap-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    kubectl delete configmap ${old_cfgmap} -n default || true
+    rm -rf $tmp_dir
-    kubectl delete configmap ${new_cfgmap} -n default || true
-    fission spec destroy || true
-    rm -rf specs || true
-    rm cfgmap.py || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -28,22 +29,25 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-cp ../test_secret_cfgmap/cfgmap.py.template cfgmap.py
+sed "s/{{ FN_CFGMAP }}/${old_cfgmap}/g" \
-sed -i "s/{{ FN_CFGMAP }}/${old_cfgmap}/g" cfgmap.py
+    $(dirname $0)/../test_secret_cfgmap/cfgmap.py.template \
+    > $tmp_dir/cfgmap.py
 
 log "Creating env $env"
-fission env create --name $env --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 log "Creating configmap $old_cfgmap"
 kubectl create configmap ${old_cfgmap} --from-literal=TEST_KEY="TESTVALUE" -n default
 
 log "Creating NewDeploy function spec: $fn_name"
+pushd $tmp_dir
 fission spec init
 fission fn create --spec --name $fn_name --env $env --code cfgmap.py --configmap $old_cfgmap --minscale 1 --maxscale 4 --executortype newdeploy
-fission spec apply ./specs/
+fission spec apply
+popd
 
 log "Creating route"
-fission route create --function ${fn_name} --url /${fn_name} --method GET
+fission route create --name ${fn_name} --function ${fn_name} --url /${fn_name} --method GET
 
 log "Waiting for router to catch up"
 sleep 5
@@ -55,14 +59,18 @@ log "Creating a new cfgmap"
 kubectl create configmap ${new_cfgmap} --from-literal=TEST_KEY="TESTVALUE_NEW" -n default
 
 log "Updating cfgmap and code for the function"
-sed -i "s/${old_cfgmap}/${new_cfgmap}/g" cfgmap.py
+sed -i "s/${old_cfgmap}/${new_cfgmap}/g" $tmp_dir/cfgmap.py
-sed -i "s/${old_cfgmap}/${new_cfgmap}/g" specs/function-$fn_name.yaml
+sed -i "s/${old_cfgmap}/${new_cfgmap}/g" $tmp_dir/specs/function-$fn_name.yaml
 
 log "Applying function changes"
-fission spec apply ./specs/
+pushd $tmp_dir
+fission spec apply
+popd
 
 log "Waiting for changes to take effect"
-sleep 5
+sleep 10
 
 log "Testing function for cfgmap value"
-timeout 60 bash -c "test_fn $fn_name 'TESTVALUE_NEW'"
+timeout 90 bash -c "test_fn $fn_name 'TESTVALUE_NEW'"
+
+log "Test PASSED"

test/tests/test_fn_update/test_env_update.sh

@@ -1,23 +1,22 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../../..
 
-env_old=python-$(date +%N)
+env_old=python-old-$TEST_ID
-env_new=python-$(date +%N)
+env_new=python-new-$TEST_ID
-fn=hellopy-$(date +%N)
+fn=hellopy-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env_old || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
-    fission env delete --name $env_new || true
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
@@ -25,7 +24,7 @@ else
 fi
 
 log "Creating env $env_old"
-fission env create --name $env_old --image fission/python-env
+fission env create --name $env_old --image $PYTHON_RUNTIME_IMAGE
 
 log "Creating function $fn"
 fission fn create --name $fn --env $env_old --code $ROOT/examples/python/hello.py --minscale 1 --maxscale 4 --executortype newdeploy --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
@@ -39,7 +38,7 @@ sleep 5
 timeout 60 bash -c "test_fn $fn 'world'"
 
 log "Creating a new env $env_new"
-fission env create --name $env_new --image fission/python-env
+fission env create --name $env_new --image $PYTHON_RUNTIME_IMAGE
 
 log "Updating function with a new environment"
 fission fn update --name $fn --env $env_new --code $ROOT/examples/python/hello.py --minscale 1 --maxscale 4 --executortype newdeploy --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
@@ -48,3 +47,5 @@ log "Waiting for update to catch up"
 sleep 5
 
 timeout 60 bash -c "test_fn $fn 'world'"
+
+log "Test PASSED"

test/tests/test_fn_update/test_idle_objects_reaper.sh

@@ -1,23 +1,20 @@
 #!/bin/bash
 
-#test:disabled
-
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
-env=python-$(date +%s)
+env=python-$TEST_ID
-fn=hellopython-$(date +%s)
+fn=hellopython-$TEST_ID
 ROOT=$(dirname $0)/../../..
 
 cleanup() {
     log "Cleaning up..."
-    fission fn delete --name ${fn}-nd || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name ${fn}-gpm || true
-    fission env delete --name $env || true
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
@@ -25,7 +22,7 @@ else
 fi
 
 log "Creating Python env $env"
-fission env create --name $env --image fission/python-env --period 5
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --period 5
 
 log "Creating function ${fn}-nd, ${fn}-gpm"
 fission fn create --name ${fn}-nd --env $env --code $ROOT/examples/python/hello.py --minscale 0 --maxscale 2 --executortype newdeploy
@@ -50,14 +47,18 @@ sleep 260
 ndDeployReplicas=$(kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -ojsonpath='{.items[0].spec.replicas}')
 if [ "$ndDeployReplicas" -ne "0" ]
 then
-  log "Failed to reap idle function pod for function ${fn}-nd"
+  log "Failed to reap idle function pod for function ${fn}-nd. replicas should be 0 but got $ndDeployReplicas"
+  kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -o yaml
   exit 1
 fi
 
-gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o name|wc -l)
+set +o pipefail
+gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm | grep Running | wc -l)
+set -o pipefail
 if [ "$gpmNumberOfPod" -ne "0" ]
 then
-  log "Failed to reap idle function pod for function ${fn}-gpm"
+  log "Failed to reap idle function pod for function ${fn}-gpm. replicas should be 0 but got $gpmNumberOfPod"
+  kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o yaml
   exit 1
 fi
 
@@ -70,13 +71,19 @@ timeout 60 bash -c "test_fn ${fn}-gpm 'world'"
 ndDeployReplicas=$(kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -ojsonpath='{.items[0].spec.replicas}')
 if [ "$ndDeployReplicas" -ne "1" ]
 then
-  log "Failed to reap idle function pod for function ${fn}-nd"
+  log "Failed to scale function pod for function ${fn}-nd. replicas should be 1 but got $ndDeployReplicas"
+  kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -o yaml
   exit 1
 fi
 
-gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o name|wc -l)
+set +o pipefail
+gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm | grep Running | wc -l)
+set -o pipefail
 if [ "$gpmNumberOfPod" -ne "1" ]
 then
-  log "Failed to reap idle function pod for function ${fn}-gpm"
+  log "Failed to scale function pod for function ${fn}-gpm. replicas should be 1 but got $gpmNumberOfPod"
+  kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o yaml
   exit 1
 fi
+
+log "Test PASSED"

test/tests/test_fn_update/test_nd_pkg_update.sh

@@ -1,26 +1,18 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-# global variables
+TEST_ID=$(generate_test_id)
-pkg=""
+echo "TEST_ID = $TEST_ID"
-http_status=""
-url=""
 
-source $(dirname $0)/fnupdate_utils.sh
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 cleanup() {
     log "Cleaning up..."
-
+    clean_resource_by_id $TEST_ID
-    if [ -e "test-deploy-pkg.zip" ]; then
+    rm -rf $tmp_dir
-        rm -rf test-deploy-pkg.zip test_dir || true
-    fi
-    if [ -e "/tmp/file" ]; then
-        rm -rf /tmp/file || true
-    fi
-
-    fission env delete --name $env || true
-    fission fn delete --name $fn_name || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -33,19 +25,19 @@ fi
 # Creates a archive, env. with builder and a function and tests for response
 # Then updates archive with a different word and udpates functions to check for new string in response
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn_name=hellopython-$(date +%N)
+fn_name=hellopython-$TEST_ID
 
 log "Creating an archive"
-mkdir test_dir
+mkdir -p $tmp_dir/test_dir
-printf 'def main():\n    return "Hello, world!"' > test_dir/hello.py
+printf 'def main():\n    return "Hello, world!"' > $tmp_dir/test_dir/hello.py
-zip -jr test-deploy-pkg.zip test_dir/
+zip -jr $tmp_dir/test-deploy-pkg.zip $tmp_dir/test_dir/
 
 log "Creating environment"
-fission env create --name $env --image fission/python-env:latest --builder fission/python-builder:latest --mincpu 40 --maxcpu 80 --minmemory 64 --maxmemory 128 --poolsize 2
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE --mincpu 40 --maxcpu 80 --minmemory 64 --maxmemory 128 --poolsize 2
 
 log "Creating functiom"
-fission fn create --name $fn_name --env $env --deploy test-deploy-pkg.zip --entrypoint "hello.main" --executortype newdeploy --minscale 1 --maxscale 4 --targetcpu 50
+fission fn create --name $fn_name --env $env --deploy $tmp_dir/test-deploy-pkg.zip --entrypoint "hello.main" --executortype newdeploy --minscale 1 --maxscale 4 --targetcpu 50
 
 log "Creating route"
 fission route create --function $fn_name --url /$fn_name --method GET
@@ -56,11 +48,11 @@ sleep 5
 timeout 60 bash -c "test_fn $fn_name 'world'"
 
 log "Updating the archive"
-sed -i 's/world/fission/' test_dir/hello.py
+sed -i 's/world/fission/' $tmp_dir/test_dir/hello.py
-zip -jr test-deploy-pkg.zip test_dir/
+zip -jr $tmp_dir/test-deploy-pkg.zip $tmp_dir/test_dir/
 
 log "Updating function with updated package"
-fission fn update --name $fn_name --deploy test-deploy-pkg.zip --entrypoint "hello.main" --executortype newdeploy --minscale 1 --maxscale 4 --targetcpu 50
+fission fn update --name $fn_name --deploy $tmp_dir/test-deploy-pkg.zip --entrypoint "hello.main" --executortype newdeploy --minscale 1 --maxscale 4 --targetcpu 50
 
 log "Waiting for deployment to update"
 sleep 5

test/tests/test_fn_update/test_poolmgr_nd.sh

@@ -1,18 +1,19 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../../..
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn=hellopython-$(date +%N)
+fn=hellopython-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -22,7 +23,7 @@ else
 fi
 
 log "Creating Python env $env"
-fission env create --name $env --image fission/python-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
 
 log "Creating function $fn"
 fission fn create --name $fn --env $env --code $ROOT/examples/python/hello.py
@@ -50,3 +51,4 @@ log "Waiting for router to catch up"
 sleep 5
 
 timeout 60 bash -c "test_fn $fn 'world'"
+log "Test PASSED"

test/tests/test_fn_update/test_resource_change.sh

@@ -1,13 +1,15 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../../..
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn=hellopython-$(date +%N)
+fn=hellopython-$TEST_ID
 
 mincpu1=40
 maxcpu1=140
@@ -21,11 +23,9 @@ maxmem2=768
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
@@ -33,7 +33,7 @@ else
 fi
 
 log "Creating Python env $env"
-fission env create --name $env --image fission/python-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
 
 log "Creating function $fn"
 fission fn create --name $fn --env $env --code $ROOT/examples/python/hello.py --minscale 1 --maxscale 4 --executortype newdeploy --mincpu $mincpu1 --maxcpu $maxcpu1 --minmemory $minmem1 --maxmemory $maxmem1
@@ -87,3 +87,4 @@ then
 fi
 
 timeout 60 bash -c "test_fn $fn 'world'"
+log "Test PASSED"

test/tests/test_fn_update/test_scale_change.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn=hellopython-$(date +%N)
+fn=hellopython-$TEST_ID
 ROOT=$(dirname $0)/../../..
 
 targetMinScale=2
@@ -14,11 +16,9 @@ targetCpuPercent=60
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
 }
 
-cleanup
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
     trap cleanup EXIT
 else
@@ -26,7 +26,7 @@ else
 fi
 
 log "Creating Python env $env"
-fission env create --name $env --image fission/python-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
 
 log "Creating function $fn"
 fission fn create --name $fn --env $env --code $ROOT/examples/python/hello.py --minscale 1 --maxscale 4 --executortype newdeploy --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
@@ -70,3 +70,4 @@ then
 fi
 
 timeout 60 bash -c "test_fn $fn 'world'"
+log "Test PASSED"

test/tests/test_fn_update/test_secret_update.sh

@@ -1,25 +1,26 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-source $(dirname $0)/fnupdate_utils.sh
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 ROOT=$(dirname $0)/../../..
 
-env=python-$(date +%N)
+env=python-$TEST_ID
-fn_name=hellopython-$(date +%N)
+fn_name=hellopython-$TEST_ID
 
-old_secret=old-secret-$(date +%N)
+old_secret=old-secret-$TEST_ID
-new_secret=new-secret-$(date +%N)
+new_secret=new-secret-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    kubectl delete secret ${old_secret} -n default || true
+    rm -rf $tmp_dir
-    kubectl delete secret ${new_secret} -n default || true
-    fission spec destroy || true
-    rm -rf specs || true
-    rm secret.py || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -28,19 +29,22 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-cp $ROOT/test/tests/test_secret_cfgmap/secret.py.template secret.py
+sed "s/{{ FN_SECRET }}/${old_secret}/g" \
-sed -i "s/{{ FN_SECRET }}/${old_secret}/g" secret.py
+    $ROOT/test/tests/test_secret_cfgmap/secret.py.template \
+    > $tmp_dir/secret.py
 
 log "Creating env $env"
-fission env create --name $env --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 log "Creating secret $old_secret"
 kubectl create secret generic ${old_secret} --from-literal=TEST_KEY="TESTVALUE" -n default
 
 log "Creating NewDeploy function spec: $fn_name"
+pushd $tmp_dir
 fission spec init
 fission fn create --spec --name $fn_name --env $env --code secret.py --secret $old_secret --minscale 1 --maxscale 4 --executortype newdeploy
-fission spec apply ./specs/
+fission spec apply
+popd
 
 log "Creating route"
 fission route create --function ${fn_name} --url /${fn_name} --method GET
@@ -55,14 +59,17 @@ log "Creating a new secret"
 kubectl create secret generic ${new_secret} --from-literal=TEST_KEY="TESTVALUE_NEW" -n default
 
 log "Updating secret and code for the function"
-sed -i "s/${old_secret}/${new_secret}/g" secret.py
+sed -i "s/${old_secret}/${new_secret}/g" $tmp_dir/secret.py
-sed -i "s/${old_secret}/${new_secret}/g" specs/function-$fn_name.yaml
+sed -i "s/${old_secret}/${new_secret}/g" $tmp_dir/specs/function-$fn_name.yaml
 
 log "Applying function changes"
-fission spec apply ./specs/
+pushd $tmp_dir
+fission spec apply
+popd
 
 log "Waiting for changes to take effect"
 sleep 5
 
 log "Testing function for secret value"
 timeout 60 bash -c "test_fn $fn_name 'TESTVALUE_NEW'"
+log "Test PASSED"

test/tests/test_function_test/test_fn_test.sh

@@ -4,16 +4,22 @@
 # Disabled because CI Fails for invalid function https://github.com/fission/fission/issues/653
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-env=nodejs-$(date +%N)
+TEST_ID=$(generate_test_id)
-valid_fn_name=hello-$(date +%N)
+echo "TEST_ID = $TEST_ID"
-invalid_fn_name=errhello-$(date +%N)
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+env=nodejs-$TEST_ID
+valid_fn_name=hello-$TEST_ID
+invalid_fn_name=errhello-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name $env || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $valid_fn_name || true
+    rm -rf $tmp_dir
-    fission fn delete --name $invalid_fn_name || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -23,18 +29,18 @@ else
 fi
 
 log "Creating env $env"
-fission env create --name $env --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating valid function $valid_fn_name"
-fission fn create --name $valid_fn_name --env $env --code hello.js
+fission fn create --name $valid_fn_name --env $env --code $(dirname $0)/hello.js
 
 log "Testing valid function $valid_fn_name"
-fission fn test --name $valid_fn_name > /tmp/valid.log
+fission fn test --name $valid_fn_name > $tmp_dir/valid.log
 
 log "---Valid Function logs---"
-cat /tmp/valid.log
+cat $tmp_dir/valid.log
 log "------"
-valid_num=$(grep 'Hello, Fission' /tmp/valid.log | wc -l)
+valid_num=$(grep 'Hello, Fission' $tmp_dir/valid.log | wc -l)
 
 if [ $valid_num -ne 1 ]
 then
@@ -43,26 +49,26 @@ then
 fi
 
 log "Creating function with an error $invalid_fn_name"
-fission fn create --name $invalid_fn_name --env $env --code errhello.js
+fission fn create --name $invalid_fn_name --env $env --code $(dirname $0)/errhello.js
 
 log "Testing invalid function $valid_fn_name"
-fission fn test --name $invalid_fn_name > /tmp/invalid.log
+fission fn test --name $invalid_fn_name > $tmp_dir/invalid.log
 
 for i in {1..10}
 do
-    size=$(wc -c </tmp/invalid.log)
+    size=$(wc -c < $tmp_dir/invalid.log)
     if [ $size == 0 ]
     then
-        fission fn test --name $invalid_fn_name > /tmp/invalid.log
+        fission fn test --name $invalid_fn_name > $tmp_dir/invalid.log
     else
         break
     fi
 done
 
 log "---Invalid Function logs---"
-cat /tmp/invalid.log
+cat $tmp_dir/invalid.log
 log "------"
-invalid_num=$(grep 'SyntaxError' /tmp/invalid.log | wc -l)
+invalid_num=$(grep 'SyntaxError' $tmp_dir/invalid.log | wc -l)
 
 if [ $invalid_num -ne 1 ]
 then

test/tests/test_function_update.sh

@@ -1,16 +1,23 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
 ROOT=$(dirname $0)/../..
 
-fn=nodejs-hello-$(date +%s)
+env=nodejs-$TEST_ID
+fn=nodejs-hello-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
+    rm -rf $tmp_dir
-    rm foo.js || true
-    rm bar.js || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -23,15 +30,12 @@ fi
 # Update it and check it's output, the output should be 
 # different from the previous one.
 
-log "Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating function"
-echo 'module.exports = function(context, callback) { callback(200, "foo!\n"); }' > foo.js
+echo 'module.exports = function(context, callback) { callback(200, "foo!\n"); }' > $tmp_dir/foo.js
-fission fn create --name $fn --env nodejs --code foo.js
+fission fn create --name $fn --env $env --code $tmp_dir/foo.js
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
@@ -52,8 +56,8 @@ echo $response | grep -i foo
 pid=$!
 
 log "Updating function"
-echo 'module.exports = function(context, callback) { callback(200, "bar!\n"); }' > bar.js
+echo 'module.exports = function(context, callback) { callback(200, "bar!\n"); }' > $tmp_dir/bar.js
-fission fn update --name $fn --code bar.js
+fission fn update --name $fn --code $tmp_dir/bar.js
 
 log "Waiting for router to update cache"
 sleep 10
@@ -66,7 +70,4 @@ echo $response | grep -i bar
 
 kill -15 $pid
 
-# crappy cleanup, improve this later
-kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
-
 log "All done."

test/tests/test_ingress.sh

@@ -1,19 +1,28 @@
 #!/bin/bash
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../..
 
-relativeUrl="/itest"
+relativeUrl="/itest-$TEST_ID"
-functionName="hellotest"
+functionName="hellotest-$TEST_ID"
-hostName="test.com"
+hostName="test-$TEST_ID.com"
 
 cleanup() {
-    fission route delete --name $1
+    clean_resource_by_id $TEST_ID
 }
 
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
 log "Creating route for URL $relativeUrl"
 route_name=$(fission route create --url $relativeUrl --function $functionName --createingress| grep trigger| cut -d" " -f 2|cut -d"'" -f 2)
-trap "cleanup $route_name" EXIT
 
 log "Route $route_name created"
 
@@ -42,4 +51,6 @@ if [ $hostName != $actual_host ]
 then
     log "Provided host and host in ingress don't match"
     exit 1
-fi
+fi
+
+log "Test PASSED"

test/tests/test_internal_routes.sh

@@ -6,21 +6,25 @@
 #
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 ROOT=$(dirname $0)/../..
 
-log "Writing functions"
+env=nodejs-$TEST_ID
-f1=f1-$(date +%s)
+f1=f1-$TEST_ID
-f2=f2-$(date +%s)
+f2=f2-$TEST_ID
 log $f1 $f2
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $f1 || true
+    rm -rf $tmp_dir
-    fission fn delete --name $f2 || true
-    rm $f1.js || true
-    rm $f2.js || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -29,23 +33,20 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-log "Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 
 
 for f in $f1 $f2
 do
-    echo "module.exports = function(context, callback) { callback(200, \"$f\n\"); }" > $f.js
+    echo "module.exports = function(context, callback) { callback(200, \"$f\n\"); }" > $tmp_dir/$f.js
 done
 
 log "Creating functions"
 for f in $f1 $f2
 do
-    fission fn create --name $f --env nodejs --code $f.js
+    fission fn create --name $f --env $env --code $tmp_dir/$f.js
 done
 
 log "Waiting for router to catch up"

test/tests/test_logging/test_function_logs.sh

@@ -1,19 +1,23 @@
 #!/bin/bash
 
-
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
-fn=nodejs-logtest-$(date +%N)
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+env=nodejs-$TEST_ID
+fn=nodejs-logtest-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    var=$(fission route list | grep $fn | awk '{print $1;}')
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
+    rm -rf $tmp_dir
-    fission env delete --name nodejs || true
-    log "delete logfile" || true
-    rm "/tmp/logfile" || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -24,16 +28,16 @@ fi
 
 # Create a hello world function in nodejs, test it with an http trigger
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating function"
-fission fn create --name $fn --env nodejs --code log.js
+fission fn create --name $fn --env $env --code $(dirname $0)/log.js
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
 
 log "Waiting for router to catch up"
-sleep 15
+sleep 3
 
 log "Doing 4 HTTP GETs on the function's route"
 for i in 1 2 3 4
@@ -45,18 +49,18 @@ log "Grabbing logs, should have 4 calls in logs"
 
 sleep 60
 
-fission function logs --name $fn --detail > /tmp/logfile
+fission function logs --name $fn --detail > $tmp_dir/logfile
 
-size=$(wc -c </tmp/logfile)
+size=$(wc -c < $tmp_dir/logfile)
 if [ $size == 0 ]
 then
-    fission function logs --name $fn --detail > /tmp/logfile
+    fission function logs --name $fn --detail > $tmp_dir/logfile
 fi
 
 log "---function logs---"
-cat /tmp/logfile
+cat $tmp_dir/logfile
 log "------"
-num=$(grep 'log test' /tmp/logfile | wc -l)
+num=$(grep 'log test' $tmp_dir/logfile | wc -l)
 log $num logs found
 
 if [ $num -ne 4 ]

test/tests/test_node_hello_http.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
 ROOT=$(dirname $0)/../..
 
-fn=nodejs-hello-$(date +%N)
+env=nodejs-$TEST_ID
+fn=nodejs-hello-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name nodejs || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -19,14 +23,11 @@ else
 fi
 
 # Create a hello world function in nodejs, test it with an http trigger
-log "Pre-test cleanup"
-fission env delete --name nodejs || true
-
 log "Creating nodejs env"
-fission env create --name nodejs --image fission/node-env
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
 
 log "Creating function"
-fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js
+fission fn create --name $fn --env $env --code $ROOT/examples/nodejs/hello.js
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
@@ -35,12 +36,9 @@ log "Waiting for router to catch up"
 sleep 3
 
 log "Doing an HTTP GET on the function's route"
-response=$(curl http://$FISSION_ROUTER/$fn)
+response=$(curl --retry 5 http://$FISSION_ROUTER/$fn)
 
 log "Checking for valid response"
 echo $response | grep -i hello
 
-routeid=$(fission route list|grep "$fn"|awk '{print $1}')
-fission route delete --name $routeid || true
-
 log "All done."

test/tests/test_obj_create_in_diff_ns.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
+#test:disabled
 
+# TODO: This test takes too long to run. It should be split into multiple little tests.
 
 # we may not need this to run as a pre-check-in test for every PR. but only once in a while to ensure nothing's broken.
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
 
 id=""
 ROOT=$(dirname $0)/../..

test/tests/test_package_command.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 # Use package command to create packages of type:
 #     1) Multiple source files from a directory
@@ -13,25 +20,27 @@ set -euo pipefail
 # Then create a function to test the packages created by package command are 
 # able to work.
 
+# TODO: seperate to multiple tests
+
 ROOT=$(dirname $0)/../..
-PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
-PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
 
-fn1=python-srcbuild1-$(date +%s)
+env=python-$TEST_ID
-fn2=python-srcbuild2-$(date +%s)
+fn1=python-srcbuild1-$TEST_ID
+fn2=python-srcbuild2-$TEST_ID
 
-fn4=python-deploy4-$(date +%s)
+fn4=python-deploy4-$TEST_ID
-fn5=python-deploy5-$(date +%s)
+fn5=python-deploy5-$TEST_ID
 
 waitBuild() {
     log "Waiting for builder manager to finish the build"
     
     while true; do
-      kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
+      status=$(kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}')
-      if [[ $? -eq 0 ]]; then
+      if (echo $status | grep succeeded); then
-          break
+        break
+      else
+        log "status=$status  Waiting for build to finish"
       fi
-      log "Waiting for build to finish"
       sleep 1
     done
 }
@@ -39,7 +48,7 @@ export -f waitBuild
 
 checkFunctionResponse() {
     log "Doing an HTTP GET on the function's route"
-    response=$(curl http://$FISSION_ROUTER/$1)
+    response=$(curl --retry 5 http://$FISSION_ROUTER/$1)
 
     log "Checking for valid response"
     log $response
@@ -54,24 +63,19 @@ waitEnvBuilder() {
 
     while true; do
       kubectl -n fission-builder get pod -l envName=${env},envResourceVersion=${envRV} \
-        -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' | grep "Ready=True" | grep -i "$1"
+        -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' | grep "Ready=True" | grep -i "$env"
       if [[ $? -eq 0 ]]; then
           break
       fi
+      sleep 1
     done
 }
 export -f waitEnvBuilder
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name python || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn1 || true
+    rm -rf $tmp_dir
-    fission fn delete --name $fn2 || true
-    fission fn delete --name $fn4 || true
-    fission fn delete --name $fn5 || true
-    rm demo-src-pkg.zip || true
-    rm -rf testDir/ || true
-    rm demo-deploy-pkg.zip || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -80,17 +84,14 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-log "Pre-test cleanup"
-fission env delete --name python || true
-
 log "Creating python env"
-fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
 
-timeout 180s bash -c "waitEnvBuilder python"
+timeout 180s bash -c "waitEnvBuilder $env"
 # 1) Multiple source files (multiple inputs, Using * expression, from a directory)
 # Currently only * expression implemented as a test
 pushd $ROOT/examples/python/
-pkg1=$(fission package create --src "sourcepkg/*" --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+pkg1=$(fission package create --src "sourcepkg/*" --env $env --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
 popd
 # wait for build to finish at most 60s
 timeout 60s bash -c "waitBuild $pkg1"
@@ -107,8 +108,8 @@ checkFunctionResponse $fn1 'a: 1 b: {c: 3, d: 4}'
 
 # 2) Source archive file
 log "Creating pacakage with source archive"
-zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
+zip -jr $tmp_dir/demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
-pkg2=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+pkg2=$(fission package create --src $tmp_dir/demo-src-pkg.zip --env $env --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
 
 # wait for build to finish at most 60s
 timeout 60s bash -c "waitBuild $pkg2"
@@ -129,7 +130,7 @@ checkFunctionResponse $fn2 'a: 1 b: {c: 3, d: 4}'
 
 # 4) Deployment files from a directory
 pushd $ROOT/examples/python/
-pkg4=$(fission package create --deploy "multifile/*" --env python| cut -f2 -d' '| tr -d \')
+pkg4=$(fission package create --deploy "multifile/*" --env $env| cut -f2 -d' '| tr -d \')
 popd
 log "Creating function " $fn4
 fission fn create --name $fn4 --pkg $pkg4 --entrypoint "main.main"
@@ -145,15 +146,15 @@ checkFunctionResponse $fn4 'Hello, world!'
 # 5) Deployment archive
 
 log "Creating package with deploy archive"
-mkdir testDir
+mkdir $tmp_dir/deploypkg
-touch testDir/__init__.py
+touch $tmp_dir/deploypkg/__init__.py
-printf 'def main():\n    return "Hello, world!"' > testDir/hello.py
+printf 'def main():\n    return "Hello, world!"' > $tmp_dir/deploypkg/hello.py
-zip -jr demo-deploy-pkg.zip testDir/
+zip -jr $tmp_dir/demo-deploy-pkg.zip $tmp_dir/deploypkg/
-pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \')
+pkg5=$(fission package create --deploy $tmp_dir/demo-deploy-pkg.zip --env $env| cut -f2 -d' '| tr -d \')
 
 
 log "Updating function " $fn5
-fission fn create --name $fn5 --pkg $pkgName --entrypoint "hello.main"
+fission fn create --name $fn5 --pkg $pkg5 --entrypoint "hello.main"
 
 log "Creating route"
 fission route create --function $fn5 --url /$fn5 --method GET
@@ -166,7 +167,4 @@ checkFunctionResponse $fn5 'Hello, world!'
 # 6) Deployment archive from a HTTP location
 # TBD
 
-# crappy cleanup, improve this later
-kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
-
 log "All done."

test/tests/test_pass.sh

@@ -1,6 +1,27 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+cleanup() {
+    log "Cleaning up..."
+    clean_resource_by_id $TEST_ID
+    rm -rf $tmp_dir
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+
 # This doesn't test fission, just the test framework. It ensures we
 # have the right environment, that's all.
 
@@ -9,3 +30,4 @@ log "Test test, please ignore."
 log $FISSION_NATS_STREAMING_URL
 log $FISSION_ROUTER
 which fission
+log "Test PASSED"

test/tests/test_router_cache_invalidation.sh

@@ -1,12 +1,18 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name python || true
+    clean_resource_by_id $TEST_ID
-    fission fn delete --name $fn || true
+    rm -rf $tmp_dir
-    rm -rf testDir-$fn || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -24,24 +30,17 @@ fi
 #    This ensures that router invalidated its cache, made a request to executor to get service for function and retried
 #    the request against this new address.
 
-PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
+env=python-$TEST_ID
-fn=python-func-$(date +%s)
+fn=python-func-$TEST_ID
-
-log "Pre-test cleanup"
-fission env delete --name python || true
 
 log "Creating python env"
-fission env create --name python --image $PYTHON_RUNTIME_IMAGE
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 log "Creating hello.py"
-mkdir testDir-$fn
+printf 'def main():\n    return "Hello, world!"' > $tmp_dir/hello.py
-printf 'def main():\n    return "Hello, world!"' > testDir-$fn/hello.py
 
 log "Creating function " $fn
-fission fn create --name $fn --env python --code testDir-$fn/hello.py
+fission fn create --name $fn --env $env --code $tmp_dir/hello.py
-
-log "rm testDir-$fn"
-rm -rf testDir-$fn
 
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
@@ -69,3 +68,5 @@ if [ "$http_status" -ne "200" ]; then
     log "Something went wrong, http status after deleting function pod is $http_status"
     exit 1
 fi
+
+log "Test PASSED"

test/tests/test_secret_cfgmap/test_secret_cfgmap.sh

@@ -1,30 +1,25 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-ROOT=$(dirname $0)/../..
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
 
-fn=testnormal-$(date +%s)
+tmp_dir="/tmp/test-$TEST_ID"
-fn_secret=testsecret-$(date +%s)
+mkdir -p $tmp_dir
-fn_cfgmap=testcfgmap-$(date +%s)
+
+ROOT=$(dirname $0)/../../..
+
+env=python-$TEST_ID
+fn=testnormal-$TEST_ID
+fn_secret=testsecret-$TEST_ID
+fn_cfgmap=testcfgmap-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission env delete --name python || true
+    clean_resource_by_id $TEST_ID
-    kubectl delete secret -n default ${fn_secret} || true
+    rm -rf $tmp_dir
-    kubectl delete configmap -n default ${fn_cfgmap} || true
-    rm cfgmap.py || true
-    rm secret.py || true
-    # delete functions
-    for f in ${fn_secret} ${fn_cfgmap} ${fn}
-    do
-        fission fn list | grep ${f} | awk '{print $1;}' | xargs -I@ bash -c "fission function delete --name @"
-    done
-    # delete routes
-    for r in ${fn_secret} ${fn_cfgmap} ${fn}
-    do
-        fission route list | grep ${r} | awk '{print $1;}' | xargs -I@ bash -c "fission route delete --name @"
-    done
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -33,11 +28,8 @@ else
     log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
 fi
 
-cp secret.py.template secret.py
+sed "s/{{ FN_SECRET }}/${fn_secret}/g" $(dirname $0)/secret.py.template > $tmp_dir/secret.py
-sed -i "s/{{ FN_SECRET }}/${fn_secret}/g" secret.py
+sed "s/{{ FN_CFGMAP }}/${fn_cfgmap}/g" $(dirname $0)/cfgmap.py.template > $tmp_dir/cfgmap.py
-
-cp cfgmap.py.template cfgmap.py
-sed -i "s/{{ FN_CFGMAP }}/${fn_cfgmap}/g" cfgmap.py
 
 checkFunctionResponse() {
     log "Doing an HTTP GET on the function's route"
@@ -59,17 +51,14 @@ checkFunctionResponse() {
 export -f checkFunctionResponse
 
 # Create a hello world function in nodejs, test it with an http trigger
-log "Pre-test cleanup"
-fission env delete --name python || true
-
 log "Creating python env"
-fission env create --name python --image fission/python-env
+fission env create --name $env --image $PYTHON_RUNTIME_IMAGE
 
 log "Creating secret"
 kubectl create secret generic ${fn_secret} --from-literal=TEST_KEY="TESTVALUE" -n default
 
 log "Creating function with secret"
-fission fn create --name ${fn_secret} --env python --code secret.py --secret ${fn_secret}
+fission fn create --name ${fn_secret} --env $env --code $tmp_dir/secret.py --secret ${fn_secret}
 
 log "Creating route"
 fission route create --function ${fn_secret} --url /${fn_secret} --method GET
@@ -81,7 +70,7 @@ timeout 60 bash -c "checkFunctionResponse ${fn_secret} 'TESTVALUE' 'secret'"
 
 log "Creating function with newdeploy executorType and new secret value"
 kubectl patch secrets ${fn_secret} -p '{"data":{"TEST_KEY":"TkVXVkFMCg=="}}' -n default
-fission fn create --name ${fn_secret}-1 --env python --code secret.py --secret ${fn_secret} --executortype newdeploy
+fission fn create --name ${fn_secret}-1 --env $env --code $tmp_dir/secret.py --secret ${fn_secret} --executortype newdeploy
 
 log "Creating route"
 fission route create --function ${fn_secret}-1 --url /${fn_secret}-1 --method GET
@@ -95,7 +84,7 @@ log "Creating configmap"
 kubectl create configmap ${fn_cfgmap} --from-literal=TEST_KEY="TESTVALUE" -n default
 
 log "creating function with configmap"
-fission fn create --name ${fn_cfgmap} --env python --code cfgmap.py --configmap ${fn_cfgmap}
+fission fn create --name ${fn_cfgmap} --env $env --code $tmp_dir/cfgmap.py --configmap ${fn_cfgmap}
 
 log "Creating route"
 fission route create --function ${fn_cfgmap} --url /${fn_cfgmap} --method GET
@@ -107,7 +96,7 @@ timeout 60 bash -c "checkFunctionResponse ${fn_cfgmap} 'TESTVALUE' 'configmap'"
 
 log "Creating function with newdeploy executorType and new configmap value"
 kubectl patch configmap ${fn_cfgmap} -p '{"data":{"TEST_KEY":"NEWVAL"}}' -n default
-fission fn create --name ${fn_cfgmap}-1 --env python --code cfgmap.py --configmap ${fn_cfgmap} --executortype newdeploy
+fission fn create --name ${fn_cfgmap}-1 --env $env --code $tmp_dir/cfgmap.py --configmap ${fn_cfgmap} --executortype newdeploy
 
 log "Creating route"
 fission route create --function ${fn_cfgmap}-1 --url /${fn_cfgmap}-1 --method GET
@@ -118,7 +107,7 @@ sleep 5
 timeout 60 bash -c "checkFunctionResponse ${fn_cfgmap}-1 'NEWVAL' 'configmap'"
 
 log "testing creating a function without a secret or configmap"
-fission function create --name ${fn} --env python --code empty.py
+fission function create --name ${fn} --env $env --code $(dirname $0)/empty.py
 
 log "Creating route"
 fission route create --function ${fn} --url /${fn} --method GET
@@ -128,3 +117,5 @@ sleep 5
 
 log "HTTP GET on the function's route"
 timeout 60 bash -c "checkFunctionResponse ${fn} 'yes' 'configmap'"
+
+log "Test PASSED"

test/tests/test_specs/test_spec.sh

@@ -1,17 +1,33 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
 
-fn=spec-$(date +%N)
+TEST_ID=$(generate_test_id)
-env=python-$fn
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+env=python-$TEST_ID
+fn=spec-$TEST_ID
 
 cleanup() {
     log "Cleaning up..."
-    fission spec destroy || true
+    clean_resource_by_id $TEST_ID
-    rm -rf specs || true
+    rm -rf $tmp_dir
 }
 
-trap cleanup EXIT
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+cp $ROOT/test/tests/test_specs/hello.py $tmp_dir
+pushd $tmp_dir
 
 # init
 fission spec init
@@ -21,13 +37,13 @@ fission spec init
 [ -f specs/README ]
 [ -f specs/fission-deployment-config.yaml ]
 
-fission env create --spec --name $env --image fission/python-env
+fission env create --spec --name $env --image $PYTHON_RUNTIME_IMAGE
 
 log "create env spec"
 fission spec apply
 
 log "verify env created"
-fission env list | grep python
+fission env list | grep $env
 
 log "generate function spec"
 fission fn create --spec --name $fn --env $env --code hello.py
@@ -40,7 +56,6 @@ grep Function specs/*.yaml
 
 log "Apply specs"
 fission spec apply
-trap "fission fn delete --name $fn" EXIT
 
 log "verify function exists"
 fission fn list | grep $fn
@@ -48,4 +63,6 @@ fission fn list | grep $fn
 sleep 3
 
 log "Test the function"
-fission fn test --name $fn | grep -i hello
+fission fn test --name $fn | grep -i hello
+
+log "Test PASSED"

test/tests/test_specs/test_spec_multifile.sh

@@ -1,27 +1,38 @@
 #!/bin/bash
 
 set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
 
 ROOT=$(dirname $0)/../../..
 
-fn=spec-$(date +%N)
+env=python-$TEST_ID
-env=python-$fn
+fn=spec-$TEST_ID
 
 cleanup() {
-    pushd $ROOT/examples/python
+    log "Cleaning up..."
-    fission spec destroy
+    clean_resource_by_id $TEST_ID
-    rm -rf $ROOT/examples/python/specs
+    rm -rf $tmp_dir
 }
 
-trap cleanup EXIT
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
 
-pushd $ROOT/examples/python
+cp -r $ROOT/examples/python/multifile $tmp_dir/
+pushd $tmp_dir
 
 fission spec init
 
 log "Creating environment spec"
-fission env create --spec --name $env --image fission/python-env --builder fission/python-builder
+fission env create --spec --name $env --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
-fission env list | grep python
 
 log "Creating function spec"
 fission fn create --spec --name $fn --env $env --deploy "multifile/*" --entrypoint main.main
@@ -37,4 +48,6 @@ fission fn test --name $fn | grep -i hello
 
 log "Destroying spec objects"
 fission spec destroy
-popd
+popd
+
+log "Test PASSED"

test/utils.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# Utils for test scripts.
+#
+source $(dirname $BASH_SOURCE)/init_tools.sh
+
+log() {
+    echo `date +%Y/%m/%d:%H:%M:%S`" $@"
+}
+export -f log
+
+generate_test_id() {
+    echo $(cat /dev/urandom | tr -dc 'a-z' | fold -w 8 | head -n 1)
+}
+
+clean_resource_by_id() {
+    test_id=$1
+    KUBECTL="kubectl --namespace default"
+    set +e
+
+    crds=$($KUBECTL get crd | grep "fission.io" | awk '{print $1}')
+    crds="$crds configmaps secrets"
+    for crd in $crds; do
+        $KUBECTL get $crd -o name | grep $test_id | xargs --no-run-if-empty $KUBECTL delete
+    done
+
+    pkg_list=$(fission package list | grep $test_id | awk '{print $1}')
+    for pkg in $pkg_list; do
+        fission package delete --name $pkg
+    done
+
+    route_list=$(fission route list | grep $test_id | awk '{print $1}')
+    for route in $route_list; do
+        fission route delete --name $route
+    done
+    set -e
+}
+
+test_fn() {
+    # Doing an HTTP GET on the function's route
+    # Checking for valid response
+    url="http://$FISSION_ROUTER/$1"
+    expect=$2
+
+    set +e
+    while true; do
+        log "test_fn: call curl"
+        resp=$(curl --silent --show-error "$url")
+        status_code=$?
+        if [ $status_code -ne 0 ]; then
+            log "test_fn: curl failed ($status_code). Retrying ..."
+            sleep 1
+            continue
+        fi
+        if ! (echo $resp | grep "$expect" > /dev/null); then
+            log "test_fn: resp = '$resp'    expect = '$expect'"
+            log "test_fn: expected string not found. Retrying ..."
+            sleep 1
+            continue
+        fi
+        break
+    done
+    set -e
+}
+export -f test_fn
+
+## Common env parameters
+export FISSION_NAMESPACE=${FISSION_NAMESPACE:-fission}
+export FUNCTION_NAMESPACE=${FUNCTION_NAMESPACE:-fission-function}
+
+export FISSION_ROUTER=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}')
+export FISSION_NATS_STREAMING_URL="http://defaultFissionAuthToken@$(kubectl -n $FISSION_NAMESPACE get svc nats-streaming -o jsonpath='{...ip}:{.spec.ports[0].port}')"
+
+## Parameters used by some specific test cases
+export PYTHON_RUNTIME_IMAGE=${PYTHON_RUNTIME_IMAGE:-fission/python-env}
+export PYTHON_BUILDER_IMAGE=${PYTHON_BUILDER_IMAGE:-fission/python-builder}
+export GO_RUNTIME_IMAGE=${GO_RUNTIME_IMAGE:-fission/go-env}
+export GO_BUILDER_IMAGE=${GO_BUILDER_IMAGE:-fission/go-builder}
+export JVM_RUNTIME_IMAGE=${JVM_RUNTIME_IMAGE:-fission/jvm-env}
+export JVM_BUILDER_IMAGE=${JVM_BUILDER_IMAGE:-fission/jvm-builder}
+export NODE_RUNTIME_IMAGE=${NODE_RUNTIME_IMAGE:-fission/node-env}
+