Integration test improvements (#447)

A few improvements to the integration test:

* Search for test files recursively in the test dir
* Output a test report at the end for a quick summary of what tests passed/failed/were skipped
* Add support skipping test files; skip the logging test since it's broken (#446 is tracking this)
* Switch to latest helm version
* Cleanup namespace

hack/travis-kube-setup.sh

@@ -31,7 +31,7 @@ fi
 # Get helm
 if [ ! -f $K8SCLI_DIR/helm ]
 then
-    curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.5.1-linux-amd64.tar.gz
+    curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-linux-amd64.tar.gz
     tar xzvf helm-*.tar.gz
     mv linux-amd64/helm $K8SCLI_DIR/helm
 fi

test/test_utils.sh

@@ -8,7 +8,28 @@
 
 set -euo pipefail
 
-ROOT=$(dirname $0)/..
+ROOT_RELPATH=$(dirname $0)/..
+pushd $ROOT_RELPATH
+ROOT=$(pwd)
+popd
+
+export TEST_REPORT=""
+
+report_msg() {
+    TEST_REPORT="$TEST_REPORT\n$1"
+}
+report_test_passed() {
+    report_msg "--- PASSED $1"
+}
+report_test_failed() {
+    report_msg "*** FAILED $1"
+}
+report_test_skipped() {
+    report_msg "### SKIPPED $1"
+}
+show_test_report() {
+    echo -e "------\n$TEST_REPORT\n------"
+}
 
 helm_setup() {
     helm init
@@ -29,7 +50,7 @@ gcloud_login() {
     then
 	echo $FISSION_CI_SERVICE_ACCOUNT | base64 -d - > $KEY
     fi
-    
+
     gcloud auth activate-service-account --key-file $KEY
 }
 
@@ -41,7 +62,7 @@ build_and_push_fission_bundle() {
     docker build -t $image_tag .
 
     gcloud_login
-    
+
     gcloud docker -- push $image_tag
     popd
 }
@@ -54,7 +75,7 @@ build_and_push_fetcher() {
     docker build -t $image_tag .
 
     gcloud_login
-    
+
     gcloud docker -- push $image_tag
     popd
 }
@@ -94,7 +115,7 @@ build_and_push_env_runtime() {
     docker build -t $image_tag .
 
     gcloud_login
-    
+
     gcloud docker -- push $image_tag
     popd
 }
@@ -109,7 +130,7 @@ build_and_push_env_builder() {
     docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} .
 
     gcloud_login
-    
+
     gcloud docker -- push $image_tag
     popd
 }
@@ -150,7 +171,7 @@ helm_install_fission() {
 
     echo "Deleting old releases"
     helm list -q|xargs helm_uninstall_fission
-    
+
     echo "Installing fission"
     helm install		\
 	 --wait			\
@@ -160,7 +181,7 @@ helm_install_fission() {
 	 --namespace $ns        \
 	 --debug                \
 	 $ROOT/charts/fission-all
-    
+
     helm list
 }
 
@@ -188,15 +209,19 @@ wait_for_services() {
     wait_for_service $id router
 }
 
-helm_uninstall_fission() {
+helm_uninstall_fission() {(set +e
+    id=$1
+
     if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ]
     then
 	echo "Fission uninstallation skipped"
 	return
     fi
     echo "Uninstalling fission"
-    helm delete --purge $1
+    helm delete --purge $id
-}
+
+    kubectl delete ns f-$id
+)}
 export -f helm_uninstall_fission
 
 set_environment() {
@@ -235,7 +260,7 @@ dump_fission_logs() {
     component=$3
 
     echo --- $component logs ---
-    kubectl -n $ns get pod -o name  | grep $component | xargs kubectl -n $ns logs 
+    kubectl -n $ns get pod -o name  | grep $component | xargs kubectl -n $ns logs
     echo --- end $component logs ---
 }
 
@@ -268,7 +293,7 @@ dump_all_fission_resources() {
     ns=$1
 
     echo "--- All objects in the fission namespace $ns ---"
-    kubectl -n $ns get all 
+    kubectl -n $ns get all
     echo "--- End objects in the fission namespace $ns ---"
 }
 
@@ -304,16 +329,31 @@ run_all_tests() {
 
     export FISSION_NAMESPACE=f-$id
     export FUNCTION_NAMESPACE=f-func-$id
-        
+
-    for file in $ROOT/test/tests/test_*.sh
+    test_files=$(find $ROOT/test/tests -iname 'test_*.sh')
+
+    for file in $test_files
     do
-	echo ------- Running $file -------
+	testname=${file#$ROOT/test/tests}
-	if $file
+	testpath=$file
+
+	if grep "^#test:disabled" $file
 	then
-	    echo SUCCESS: $file
+	    report_test_skipped $testname
+	    echo ------- Skipped $testname -------
 	else
-	    echo FAILED: $file
+	    echo ------- Running $testname -------
-	    export FAILURES=$(($FAILURES+1))
+	    pushd $(dirname $testpath)
+	    if $testpath
+	    then
+		echo SUCCESS: $testname
+		report_test_passed $testname
+	    else
+		echo FAILED: $testname
+		export FAILURES=$(($FAILURES+1))
+		report_test_failed $testname
+	    fi
+	    popd
 	fi
     done
 }
@@ -346,6 +386,8 @@ install_and_test() {
 
     dump_logs $id
 
+    show_test_report
+
     if [ $FAILURES -ne 0 ]
     then
 	exit 1

test/tests/test_logging/log.js

@@ -1,8 +1,8 @@
 
 module.exports = async function(context) {
-    console.log("log test log test log test")
+    console.log("log test")
     return {
         status: 200,
-        body: "Log, test!\n"
+        body: "log test\n"
     };
 }

test/tests/test_logging/test_function_logs.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
 
+#test:disabled
+
 set -euo pipefail
 
 ROOT=$(dirname $0)/../..
 
-fn=nodejs-logtest
+fn=nodejs-logtest-$(date +%N)
-
 
 function cleanup {
     echo "Cleanup route"
     var=$(fission route list | grep $fn | awk '{print $1;}')
     fission route delete --name $var
-    fission function delete --name $fn
 }
 
 # Create a hello world function in nodejs, test it with an http trigger
@@ -27,32 +27,32 @@ fission fn create --name $fn --env nodejs --code log.js
 trap "fission fn delete --name $fn" EXIT
 
 echo "Creating route"
-fission route create --function $fn --url /logtest --method GET
+fission route create --function $fn --url /$fn --method GET
+trap cleanup EXIT
 
 echo "Waiting for router to catch up"
 sleep 3
 
-echo "Doing 4 HTTP GET on the function's route"
+echo "Doing 4 HTTP GETs on the function's route"
-curl http://$FISSION_ROUTER/logtest
+for i in 1 2 3 4
-curl http://$FISSION_ROUTER/logtest
+do
-curl http://$FISSION_ROUTER/logtest
+    curl -s http://$FISSION_ROUTER/$fn
-curl http://$FISSION_ROUTER/logtest
+done
-
 
 echo "Grabbing logs, should have 4 calls in logs"
 
 sleep 15
 
-echo "woke up"
+logs=$(fission function logs --name $fn --detail)
-logs=$(fission function logs --name $fn)
+echo "---function logs---"
-num=$(echo "$logs" | grep 'log test' | wc -l)
+echo $logs
-echo $num
+echo "------"
+num=(cat "$logs" | grep 'log test' | wc -l)
+echo $num logs found
 
 if [ $num -ne 4 ]
 then
-    echo "Test Failed"
+    echo "Test Failed: expected 4, found $num logs"
-    trap cleanup EXIT
 fi
-cleanup
  
 echo "All done."