Fission upgrade tests (#605)

Tests for upgrading Fission from the last version to current branch and testing. Currently test is very simple and more tests can be added in future

.travis.yml

@@ -39,6 +39,7 @@ script:
   - hack/verify-govet.sh
   - hack/runtests.sh
   - test/build_and_test.sh
+  - test/upgrade/fission_upgrade_test.sh
 
 notifications:
   slack:

test/build_and_test.sh

@@ -2,6 +2,13 @@
 
 set -euo pipefail
 
+# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
+if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]]
+then
+    echo "Skipping build & test, this is cron job for fission upgrade tests"
+    exit 0
+fi
+
 if [ ! -f ${HOME}/.kube/config ]
 then
     echo "Skipping end to end tests, no cluster credentials"

test/test_utils.sh

@@ -281,6 +281,23 @@ port_forward_services() {
         xargs -I{} kubectl port-forward {} $port:$port -n $ns &
 }
 
+wait_for_service() { 
+    id=$1 
+    svc=$2 
+  
+    ns=f-$id 
+    while true 
+        do 
+        ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}') 
+        if [ ! -z $ip ] 
+        then 
+            break 
+        fi 
+        echo Waiting for service $svc... 
+        sleep 1 
+    done 
+ } 
+
 dump_builder_pod_logs() {
     bns=$1
     builderPods=$(kubectl -n $bns get pod -o name)

test/upgrade/fission_upgrade_test.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
+if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} != "cronNOVALUE" ]]
+then
+    echo "Skipping Fission upgrade tests, not a cron job"
+    exit 0
+fi
+
+if [ ! -f ${HOME}/.kube/config ]
+then
+    echo "Skipping end to end tests, no cluster credentials"
+    exit 0
+fi
+
+ROOT_RELPATH=$(dirname $0)/../..
+pushd $ROOT_RELPATH
+ROOT=$(pwd)
+popd
+
+# This will change for every new release
+CURRENT_VERSION=0.7.0
+
+source $ROOT/test/test_utils.sh
+source $(dirname $0)/fixture_tests.sh
+
+id=$(generate_test_id)
+ns=f-$id
+fns=f-func-$id
+controllerNodeport=31234
+pruneInterval=1
+routerServiceType=LoadBalancer
+
+helmVars=functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType
+
+dump_system_info
+
+timeout 30 bash -c "helm_setup"
+
+echo "Deleting old releases"
+helm list -q|xargs -I@ bash -c "helm_uninstall_fission @"
+
+# deleting ns does take a while after command is issued
+while kubectl get ns| grep "fission-builder"
+do
+    sleep 5
+done
+
+helm install \
+--name $id \
+--wait \
+--timeout 540 \
+--set $helmVars \
+--namespace $ns \
+https://github.com/fission/fission/releases/download/${CURRENT_VERSION}/fission-all-${CURRENT_VERSION}.tgz
+
+mkdir temp && cd temp && curl -Lo fission https://github.com/fission/fission/releases/download/${CURRENT_VERSION}/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/ && cd .. && rm -rf temp
+
+wait_for_service $id "router"
+export FISSION_ROUTER=$(kubectl -n $ns get svc router -o jsonpath='{...ip}')
+
+## Setup - create fixtures for tests
+
+setup_fission_objects
+trap "cleanup_fission_objects $id" EXIT
+
+## Test before upgrade
+
+upgrade_tests
+
+## Build images for Upgrade
+
+REPO=gcr.io/fission-ci
+IMAGE=$REPO/fission-bundle
+FETCHER_IMAGE=$REPO/fetcher
+FLUENTD_IMAGE=gcr.io/fission-ci/fluentd
+BUILDER_IMAGE=$REPO/builder
+TAG=upgrade-test
+PRUNE_INTERVAL=1 # Unit - Minutes; Controls the interval to run archivePruner.
+ROUTER_SERVICE_TYPE=ClusterIP
+
+build_and_push_fission_bundle $IMAGE:$TAG
+
+build_and_push_fetcher $FETCHER_IMAGE:$TAG
+
+build_and_push_fluentd $FLUENTD_IMAGE:$TAG
+
+build_fission_cli
+
+sudo mv $ROOT/fission/fission /usr/local/bin/
+
+## Upgrade 
+
+helmVars=image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType
+
+echo "Upgrading fission"
+helm upgrade	\
+ --wait			\
+ --timeout 540	        \
+ --set $helmVars	\
+ --namespace $ns        \
+ $id $ROOT/charts/fission-all
+
+sleep 10 # Takes a few seconds after upgrade to re-create K8S objects etc.
+
+# No need to wait as upgrade won't create a new LB
+export FISSION_ROUTER=$(kubectl -n $ns get svc router -o jsonpath='{...ip}')
+
+## Tests
+validate_post_upgrade
+upgrade_tests
+
+## Cleanup is done by trap

test/upgrade/fixture_tests.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ROOT_RELPATH=$(dirname $0)/../..
+pushd $ROOT_RELPATH
+ROOT=$(pwd)
+popd
+
+setup_fission_objects() {
+    log "==== Setting up objects for upgrade test ===="
+    log "Creating env, function and route"
+    fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
+
+    fission fn create --name upgradehello --env nodejs --code $ROOT/examples/nodejs/hello.js --executortype poolmgr
+
+    fission route create --function upgradehello --url /upgradehello --method GET
+    log "Waiting for router to catch up"
+    sleep 5
+    log "==== Finished setting up objects for upgrade test ===="
+}
+
+upgrade_tests() {
+    log "==== Tests Start ===="
+    log "Doing an HTTP GET on the function's route"
+    response=$(curl http://$FISSION_ROUTER/upgradehello)
+
+    log "Checking for valid response"
+    echo $response | grep -i hello
+    log "==== Tests End ===="
+}
+
+validate_post_upgrade() {
+    echo `fission -v`
+    echo "Fission environments:"
+    echo `fission env list`
+    echo "Fission Functions:"
+    echo `fission fn list`
+    echo "Fission Routes:"
+    echo `fission route list`
+}
+
+cleanup_fission_objects() {
+    log "==== Cleanup Start ===="
+    log "Input: $1"
+    id=$1
+    echo "Cleaning up objects"
+    fission env delete --name nodejs || true
+    fission fn delete --name upgradehello || true
+
+    echo "Uninstalling fission"
+    helm delete --purge $id
+    kubectl delete ns f-$id || true
+    log "==== Cleanup End ===="
+}