From 80a4bcbcffbe19c1d7ddc1cb268b089abfefbcd9 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Mon, 15 Jan 2018 16:08:38 -0800 Subject: [PATCH] Integration test improvements (#447) A few improvements to the integration test: * Search for test files recursively in the test dir * Output a test report at the end for a quick summary of what tests passed/failed/were skipped * Add support skipping test files; skip the logging test since it's broken (#446 is tracking this) * Switch to latest helm version * Cleanup namespace --- hack/travis-kube-setup.sh | 2 +- test/test_utils.sh | 82 ++++++++++++++----- test/tests/test_logging/log.js | 4 +- test/tests/test_logging/test_function_logs.sh | 34 ++++---- 4 files changed, 82 insertions(+), 40 deletions(-) diff --git a/hack/travis-kube-setup.sh b/hack/travis-kube-setup.sh index 1fc2c862..1e8a6e97 100755 --- a/hack/travis-kube-setup.sh +++ b/hack/travis-kube-setup.sh @@ -31,7 +31,7 @@ fi # Get helm if [ ! -f $K8SCLI_DIR/helm ] then - curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.5.1-linux-amd64.tar.gz + curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-linux-amd64.tar.gz tar xzvf helm-*.tar.gz mv linux-amd64/helm $K8SCLI_DIR/helm fi diff --git a/test/test_utils.sh b/test/test_utils.sh index ffb490a9..9983019d 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -8,7 +8,28 @@ set -euo pipefail -ROOT=$(dirname $0)/.. +ROOT_RELPATH=$(dirname $0)/.. +pushd $ROOT_RELPATH +ROOT=$(pwd) +popd + +export TEST_REPORT="" + +report_msg() { + TEST_REPORT="$TEST_REPORT\n$1" +} +report_test_passed() { + report_msg "--- PASSED $1" +} +report_test_failed() { + report_msg "*** FAILED $1" +} +report_test_skipped() { + report_msg "### SKIPPED $1" +} +show_test_report() { + echo -e "------\n$TEST_REPORT\n------" +} helm_setup() { helm init @@ -29,7 +50,7 @@ gcloud_login() { then echo $FISSION_CI_SERVICE_ACCOUNT | base64 -d - > $KEY fi - + gcloud auth activate-service-account --key-file $KEY } @@ -41,7 +62,7 @@ build_and_push_fission_bundle() { docker build -t $image_tag . gcloud_login - + gcloud docker -- push $image_tag popd } @@ -54,7 +75,7 @@ build_and_push_fetcher() { docker build -t $image_tag . gcloud_login - + gcloud docker -- push $image_tag popd } @@ -94,7 +115,7 @@ build_and_push_env_runtime() { docker build -t $image_tag . gcloud_login - + gcloud docker -- push $image_tag popd } @@ -109,7 +130,7 @@ build_and_push_env_builder() { docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} . gcloud_login - + gcloud docker -- push $image_tag popd } @@ -150,7 +171,7 @@ helm_install_fission() { echo "Deleting old releases" helm list -q|xargs helm_uninstall_fission - + echo "Installing fission" helm install \ --wait \ @@ -160,7 +181,7 @@ helm_install_fission() { --namespace $ns \ --debug \ $ROOT/charts/fission-all - + helm list } @@ -188,15 +209,19 @@ wait_for_services() { wait_for_service $id router } -helm_uninstall_fission() { +helm_uninstall_fission() {(set +e + id=$1 + if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ] then echo "Fission uninstallation skipped" return fi echo "Uninstalling fission" - helm delete --purge $1 -} + helm delete --purge $id + + kubectl delete ns f-$id +)} export -f helm_uninstall_fission set_environment() { @@ -235,7 +260,7 @@ dump_fission_logs() { component=$3 echo --- $component logs --- - kubectl -n $ns get pod -o name | grep $component | xargs kubectl -n $ns logs + kubectl -n $ns get pod -o name | grep $component | xargs kubectl -n $ns logs echo --- end $component logs --- } @@ -268,7 +293,7 @@ dump_all_fission_resources() { ns=$1 echo "--- All objects in the fission namespace $ns ---" - kubectl -n $ns get all + kubectl -n $ns get all echo "--- End objects in the fission namespace $ns ---" } @@ -304,16 +329,31 @@ run_all_tests() { export FISSION_NAMESPACE=f-$id export FUNCTION_NAMESPACE=f-func-$id - - for file in $ROOT/test/tests/test_*.sh + + test_files=$(find $ROOT/test/tests -iname 'test_*.sh') + + for file in $test_files do - echo ------- Running $file ------- - if $file + testname=${file#$ROOT/test/tests} + testpath=$file + + if grep "^#test:disabled" $file then - echo SUCCESS: $file + report_test_skipped $testname + echo ------- Skipped $testname ------- else - echo FAILED: $file - export FAILURES=$(($FAILURES+1)) + echo ------- Running $testname ------- + pushd $(dirname $testpath) + if $testpath + then + echo SUCCESS: $testname + report_test_passed $testname + else + echo FAILED: $testname + export FAILURES=$(($FAILURES+1)) + report_test_failed $testname + fi + popd fi done } @@ -346,6 +386,8 @@ install_and_test() { dump_logs $id + show_test_report + if [ $FAILURES -ne 0 ] then exit 1 diff --git a/test/tests/test_logging/log.js b/test/tests/test_logging/log.js index 6546bdfc..6fc029af 100644 --- a/test/tests/test_logging/log.js +++ b/test/tests/test_logging/log.js @@ -1,8 +1,8 @@ module.exports = async function(context) { - console.log("log test log test log test") + console.log("log test") return { status: 200, - body: "Log, test!\n" + body: "log test\n" }; } diff --git a/test/tests/test_logging/test_function_logs.sh b/test/tests/test_logging/test_function_logs.sh index b605d4ee..59bede20 100755 --- a/test/tests/test_logging/test_function_logs.sh +++ b/test/tests/test_logging/test_function_logs.sh @@ -1,17 +1,17 @@ #!/bin/bash +#test:disabled + set -euo pipefail ROOT=$(dirname $0)/../.. -fn=nodejs-logtest - +fn=nodejs-logtest-$(date +%N) function cleanup { echo "Cleanup route" var=$(fission route list | grep $fn | awk '{print $1;}') fission route delete --name $var - fission function delete --name $fn } # Create a hello world function in nodejs, test it with an http trigger @@ -27,32 +27,32 @@ fission fn create --name $fn --env nodejs --code log.js trap "fission fn delete --name $fn" EXIT echo "Creating route" -fission route create --function $fn --url /logtest --method GET +fission route create --function $fn --url /$fn --method GET +trap cleanup EXIT echo "Waiting for router to catch up" sleep 3 -echo "Doing 4 HTTP GET on the function's route" -curl http://$FISSION_ROUTER/logtest -curl http://$FISSION_ROUTER/logtest -curl http://$FISSION_ROUTER/logtest -curl http://$FISSION_ROUTER/logtest - +echo "Doing 4 HTTP GETs on the function's route" +for i in 1 2 3 4 +do + curl -s http://$FISSION_ROUTER/$fn +done echo "Grabbing logs, should have 4 calls in logs" sleep 15 -echo "woke up" -logs=$(fission function logs --name $fn) -num=$(echo "$logs" | grep 'log test' | wc -l) -echo $num +logs=$(fission function logs --name $fn --detail) +echo "---function logs---" +echo $logs +echo "------" +num=(cat "$logs" | grep 'log test' | wc -l) +echo $num logs found if [ $num -ne 4 ] then - echo "Test Failed" - trap cleanup EXIT + echo "Test Failed: expected 4, found $num logs" fi -cleanup echo "All done."