Enable concurrent CI builds (#1236)

Currently, only one CI build can be started at the same time due to we only have only one testing cluster. This PR aims to enable multiple concurrent builds can be triggered to reduce the waiting time for CI builds. The major changes listed below:

1. Each build generate unique images for testing and debugging
2. Check whether the testing cluster is being used by another build.
3. Cache docker image & go mod after build finished
4. Improve dockerfiles for reusing docker build cache

.travis.yml

@@ -4,18 +4,24 @@ dist: trusty
 
 branches:
   only:
-  - master
+    - master
 
 language: go
 
 go:
   - 1.11.6
 
+env:
+  - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on DOCKER_CACHE_DIR=${HOME}/docker/
+
 cache:
   directories:
     - $HOME/google-cloud-sdk/
     - $HOME/k8scli
     - $HOME/tool
+    - $HOME/docker
+    - $HOME/.cache/go-build
+    - $HOME/gopath/pkg/mod
 
 services:
   - docker
@@ -29,24 +35,27 @@ before_install:
 install:
   - hack/travis-kube-setup.sh
 
-env:
-  - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on
-
 before_script:
   - cd ${TRAVIS_BUILD_DIR}
   - go mod download
-  - go mod vendor
   - hack/verify-staticcheck.sh
   - hack/verify-gofmt.sh
   - hack/verify-govet.sh
   - helm lint charts/fission-all/ charts/fission-core/
   - hack/runtests.sh
+  - test/build.sh
 
 script:
   - cd ${TRAVIS_BUILD_DIR}
-  - test/build_and_test.sh
+  - test/test.sh
   - test/upgrade/fission_upgrade_test.sh
 
+before_cache:
+  - test/save_cache.sh # save docker cache
+
+after_script:
+  - kubectl --namespace default delete configmap -l travisID=${TRAVIS_BUILD_ID} # remove in-test configmap to indicate this CI build finished
+
 notifications:
   slack:
     rooms:

cmd/builder/Dockerfile.fission-builder

@@ -4,15 +4,6 @@ RUN apk add --update ca-certificates
 FROM golang:1.11-alpine as fission-builder
 RUN apk add bash ca-certificates git gcc g++ libc-dev
 
-ARG GITCOMMIT=unknown
-# E.g. GITCOMMIT=$(git rev-parse HEAD)
-
-ARG BUILDVERSION=unknown
-# E.g. BUILDVERSION=$(git rev-parse HEAD)
-
-ARG BUILDDATE=unknown
-# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
 ARG GOPKG=github.com/fission/fission
 
 ENV GO111MODULE=on
@@ -29,6 +20,15 @@ COPY ./ ./
 
 WORKDIR /go/src/${GOPKG}/cmd/builder
 
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
     -o /go/bin/builder \
     -gcflags=-trimpath=$GOPATH \

cmd/fetcher/Dockerfile.fission-fetcher

@@ -4,15 +4,6 @@ RUN apk add --update ca-certificates
 FROM golang:1.11-alpine as builder
 RUN apk add bash ca-certificates git gcc g++ libc-dev
 
-ARG GITCOMMIT=unknown
-# E.g. GITCOMMIT=$(git rev-parse HEAD)
-
-ARG BUILDVERSION=unknown
-# E.g. BUILDVERSION=$(git rev-parse HEAD)
-
-ARG BUILDDATE=unknown
-# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
 ARG GOPKG=github.com/fission/fission
 
 ENV GO111MODULE=on
@@ -29,6 +20,15 @@ COPY ./ ./
 
 WORKDIR /go/src/${GOPKG}/cmd/fetcher
 
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
     -o /go/bin/fetcher \
     -gcflags=-trimpath=$GOPATH \

cmd/fission-bundle/Dockerfile.fission-bundle

@@ -4,15 +4,6 @@ RUN apk add --update ca-certificates
 FROM golang:1.11-alpine as builder
 RUN apk add bash ca-certificates git gcc g++ libc-dev
 
-ARG GITCOMMIT=unknown
-# E.g. GITCOMMIT=$(git rev-parse HEAD)
-
-ARG BUILDVERSION=unknown
-# E.g. BUILDVERSION=$(git rev-parse HEAD)
-
-ARG BUILDDATE=unknown
-# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
 ARG GOPKG=github.com/fission/fission
 
 ENV GO111MODULE=on
@@ -29,6 +20,15 @@ COPY ./ ./
 
 WORKDIR /go/src/${GOPKG}/cmd/fission-bundle
 
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
     -o /go/bin/fission-bundle \
     -gcflags=-trimpath=$GOPATH \

cmd/preupgradechecks/Dockerfile.fission-preupgradechecks

@@ -4,15 +4,6 @@ RUN apk add --update ca-certificates
 FROM golang:1.11-alpine as builder
 RUN apk add bash ca-certificates git gcc g++ libc-dev
 
-ARG GITCOMMIT=unknown
-# E.g. GITCOMMIT=$(git rev-parse HEAD)
-
-ARG BUILDVERSION=unknown
-# E.g. BUILDVERSION=$(git rev-parse HEAD)
-
-ARG BUILDDATE=unknown
-# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-
 ARG GOPKG=github.com/fission/fission
 
 ENV GO111MODULE=on
@@ -29,6 +20,15 @@ COPY ./ ./
 
 WORKDIR /go/src/${GOPKG}/cmd/preupgradechecks
 
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
     -o /go/bin/pre-upgrade-checks \
     -gcflags=-trimpath=$GOPATH \

environments/python/Dockerfile

@@ -1,14 +1,15 @@
 FROM alpine:3.5
 
-RUN apk update
+WORKDIR /app
-RUN apk add --no-cache python3 python3-dev build-base libev-dev
+
-RUN pip3 install --upgrade pip
+RUN apk add --no-cache python3 python3-dev build-base libev-dev && \
-RUN rm -r /root/.cache
+    pip3 install --upgrade pip && \
+    rm -r /root/.cache
+
+COPY requirements.txt /app
+RUN pip3 install -r requirements.txt
 
 COPY . /app
-WORKDIR /app
-RUN pip3 install -r requirements.txt
 
 ENTRYPOINT ["python3"]
 CMD ["server.py"]
-

environments/python/Dockerfile-2.7

@@ -1,14 +1,15 @@
 FROM alpine:3.5
 
-RUN apk update
+WORKDIR /app
-RUN apk add --no-cache python python-dev build-base py-pip libev-dev
+
-RUN pip install --upgrade pip
+RUN apk add --no-cache python python-dev build-base py-pip libev-dev && \
-RUN rm -r /root/.cache
+    pip install --upgrade pip && \
+    rm -r /root/.cache
+
+COPY requirements.txt /app
+RUN pip install -r requirements.txt
 
 COPY . /app
-WORKDIR /app
-RUN pip install -r requirements.txt
 
 ENTRYPOINT ["python"]
 CMD ["server.py"]
-

environments/tensorflow-serving/Dockerfile

@@ -10,7 +10,7 @@ ENV APP	   ${GOPATH}/src/github.com/fission/fission/environments/tensorflow-serv
 
 WORKDIR ${APP}
 
-ADD server.go   ${APP}
+ADD server.go ${APP}
 
 RUN go get
 RUN go build -a -o /server server.go

hack/release.sh

@@ -329,8 +329,6 @@ fi
 
 release_environment_check $version $chartsrepo
 
-go mod vendor
-
 # Build release-builder image
 docker build -t fission-release-builder -f $GOPATH/src/github.com/fission/fission/hack/Dockerfile .
 

skaffold.yaml

@@ -5,7 +5,6 @@
 ############## Usage ##############
 # Skaffold CLI should be installed on your machine. 
 # Change <DOCKERHUB_REPO> -> to your Dockerhub username
-# Run `go mod vendor` so that all dependencies are downloaded
 # Run `skaffold run` to build and deploy with helm. 
 # Run`skaffold build` to only build and push images
 ############## Current Limitations ##############

test/build.sh

@@ -17,18 +17,19 @@ fi
 
 source $(dirname $0)/test_utils.sh
 
-REPO=gcr.io/fission-ci
-IMAGE=fission-bundle
-FETCHER_IMAGE=$REPO/fetcher
-BUILDER_IMAGE=$REPO/builder
-TAG=test
-PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes.
-ROUTER_SERVICE_TYPE=LoadBalancer
-SERVICE_TYPE=LoadBalancer
-PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks
-
 dump_system_info
 
+setupCIBuildEnv
+
+load_docker_cache ${DOCKER_CACHE_DIR}/python-env.tar.gz;
+load_docker_cache ${DOCKER_CACHE_DIR}/jvm-env.tar.gz;
+load_docker_cache ${DOCKER_CACHE_DIR}/go-env.tar.gz;
+load_docker_cache ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz;
+
+load_docker_cache ${DOCKER_CACHE_DIR}/python-builder.tar.gz;
+load_docker_cache ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz;
+load_docker_cache ${DOCKER_CACHE_DIR}/go-builder.tar.gz;
+
 build_and_push_fission_bundle $REPO/$IMAGE:$TAG
 
 build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG
@@ -37,7 +38,6 @@ build_and_push_fetcher $FETCHER_IMAGE:$TAG
 
 build_and_push_builder $BUILDER_IMAGE:$TAG
 
-
 build_and_push_env_runtime python $REPO/python-env:$TAG ""
 build_and_push_env_runtime jvm $REPO/jvm-env:$TAG ""
 build_and_push_env_runtime go $REPO/go-env:$TAG "1.12"
@@ -48,5 +48,3 @@ build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG ""
 build_and_push_env_builder go $REPO/go-env-builder:$TAG $BUILDER_IMAGE:$TAG "1.12"
 
 build_fission_cli
-
-install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE

test/save_cache.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
+if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]]
+then
+    echo "Skipping build & test, this is cron job for fission upgrade tests"
+    exit 0
+fi
+
+if [ ! -f ${HOME}/.kube/config ]
+then
+    echo "Skipping end to end tests, no cluster credentials"
+    exit 0
+fi
+
+source $(dirname $0)/test_utils.sh
+
+dump_system_info
+
+setupCIBuildEnv
+
+mkdir -p ${DOCKER_CACHE_DIR}
+
+docker save $(docker history -q $REPO/python-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true
+docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true
+docker save $(docker history -q $REPO/go-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true
+docker save $(docker history -q $REPO/tensorflow-serving-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz || true
+
+docker save $(docker history -q $REPO/python-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/python-builder.tar.gz || true
+docker save $(docker history -q $REPO/jvm-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz || true
+docker save $(docker history -q $REPO/go-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/go-builder.tar.gz || true

test/test.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
+if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]]
+then
+    echo "Skipping build & test, this is cron job for fission upgrade tests"
+    exit 0
+fi
+
+if [ ! -f ${HOME}/.kube/config ]
+then
+    echo "Skipping end to end tests, no cluster credentials"
+    exit 0
+fi
+
+source $(dirname $0)/test_utils.sh
+
+dump_system_info
+
+setupCIBuildEnv
+
+while true; do
+    # ensure that gke cluster is now free for testing
+
+    previous_build_id=$(kubectl --namespace default get configmap in-test --ignore-not-found -o=jsonpath='{.metadata.labels.travisID}')
+
+    build_state=$(curl -s -X GET https://api.travis-ci.org/build/${previous_build_id} \
+        -H "Authorization: token ${TRAVIS_TOKEN}" \
+        -H "Travis-API-Version: 3" | python -c "import sys,json; print json.load(sys.stdin)['state']")
+
+    # If previous build state is not equal to "started" or the previous build id
+    # equals to the current build ID means the previous build is end or restart.
+    # We can remove the configmap safely and start next k8s test safely.
+    if [[ ${TRAVIS_TOKEN} == ${previous_build_id} ]] || [[ $build_state != "started" ]]; then
+        kubectl --namespace default delete configmap -l travisID=${previous_build_id}
+    fi
+
+    created=$(kubectl --namespace default create configmap in-test|grep "created"||true)
+    if [[ -z $created ]]; then
+        echo "Cluster is now in used. Retrying after 15 seconds..."
+        sleep 15
+        continue
+    fi
+    kubectl --namespace default label configmap in-test travisID=${TRAVIS_BUILD_ID}
+    break
+done
+
+install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE

test/test_utils.sh

@@ -53,6 +53,25 @@ getGitCommit() {
     echo $(git rev-parse HEAD)
 }
 
+setupCIBuildEnv() {
+    export REPO=gcr.io/fission-ci
+    export IMAGE=fission-bundle
+    export FETCHER_IMAGE=$REPO/fetcher
+    export BUILDER_IMAGE=$REPO/builder
+    export TAG=test-${TRAVIS_BUILD_ID}
+    export PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes.
+    export ROUTER_SERVICE_TYPE=LoadBalancer
+    export SERVICE_TYPE=LoadBalancer
+    export PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks
+}
+
+load_docker_cache() {
+    cache=$1
+    if [ -f ${cache} ]; then
+        gunzip -c ${cache} | docker load;
+    fi
+}
+
 build_and_push_pre_upgrade_check_image() {
     image_tag=$1
     travis_fold_start build_and_push_pre_upgrade_check_image $image_tag
@@ -448,16 +467,17 @@ export FAILURES=0
 
 run_all_tests() {
     id=$1
+    imageTag=$2
 
     export FISSION_NAMESPACE=f-$id
     export FUNCTION_NAMESPACE=f-func-$id
-    export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
+    export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:${imageTag}
-    export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
+    export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:${imageTag}
-    export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:test
+    export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:${imageTag}
-    export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test
+    export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:${imageTag}
-    export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test
+    export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:${imageTag}
-    export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test
+    export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:${imageTag}
-    export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test
+    export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:${imageTag}
 
     set +e
     export TIMEOUT=900  # 15 minutes per test
@@ -467,8 +487,6 @@ run_all_tests() {
     $ROOT/test/run_test.sh \
         $ROOT/test/tests/test_canary.sh \
         $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \
-        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
-        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \
         $ROOT/test/tests/recordreplay/test_record_greetings.sh \
         $ROOT/test/tests/recordreplay/test_record_rv.sh \
         $ROOT/test/tests/recordreplay/test_recorder_update.sh \
@@ -492,11 +510,13 @@ run_all_tests() {
         $ROOT/test/tests/test_specs/test_spec_multifile.sh \
         $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \
         $ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \
-        $ROOT/test/tests/test_environments/test_go_env.sh
+        $ROOT/test/tests/test_environments/test_go_env.sh \
+        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
+        $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh
     FAILURES=$?
 
     # FIXME: run tests with newdeploy one by one.
-    export JOBS=1
+    export JOBS=2
     $ROOT/test/run_test.sh \
         $ROOT/test/tests/test_backend_newdeploy.sh \
         $ROOT/test/tests/test_environments/test_java_builder.sh \
@@ -562,7 +582,7 @@ install_and_test() {
     # ensure we run tests against with the same git commit version of CLI & server
     fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit"
 
-    run_all_tests $id
+    run_all_tests $id $imageTag
 
     dump_logs $id