Improve CI integration test script (#1285)

test/save_cache.sh

@@ -23,6 +23,9 @@ setupCIBuildEnv
 
 mkdir -p ${DOCKER_CACHE_DIR}
 
+# remove old images
+rm -rf ${DOCKER_CACHE_DIR}/*
+
 docker save $(docker history -q $REPO/python-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true
 docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true
 docker save $(docker history -q $REPO/go-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true

test/test_utils.sh

@@ -269,15 +269,15 @@ wait_for_service() {
     ns=f-$id
     while true
     do
-	ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
-	if [ ! -z $ip ]
-	then
-	    break
-	fi
-	echo Waiting for service $svc...
-	sleep 1
+	     ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
+	      if [ ! -z $ip ]; then
+	         break
+	      fi
+	      echo Waiting for service $svc...
+	      sleep 1
     done
 }
+export -f wait_for_service
 
 wait_for_services() {
     id=$1
@@ -286,16 +286,30 @@ wait_for_services() {
     wait_for_service $id router
 
     echo Waiting for service is routable...
-    sleep 10
+    sleep 30
 }
+export -f wait_for_services
+
+check_gitcommit_version() {
+    while true
+    do
+        # ensure we run tests against with the same git commit version of CLI & server
+	      ip=$(fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit")
+	      if [ $? -eq 0 ]; then
+	        break
+	      fi
+	      echo Retrying getting build version from the controller...
+	      sleep 1
+    done
+}
+export -f check_gitcommit_version
 
 helm_uninstall_fission() {(set +e
     id=$1
 
-    if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ]
-    then
-	echo "Fission uninstallation skipped"
-	return
+    if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ]; then
+	    echo "Fission uninstallation skipped"
+	    return
     fi
 
     echo "Uninstalling fission"
@@ -576,12 +590,10 @@ install_and_test() {
 	    exit 1
     fi
 
-    wait_for_services $id
+    timeout 150 bash -c "wait_for_services $id"
+    timeout 120 bash -c "check_gitcommit_version"
     set_environment $id
 
-    # ensure we run tests against with the same git commit version of CLI & server
-    fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit"
-
     run_all_tests $id $imageTag
 
     dump_logs $id