From fe8fa94c77508c2effac6743ff3f93cdfb8b856e Mon Sep 17 00:00:00 2001 From: Ta-Ching Chen Date: Mon, 15 Jul 2019 16:33:26 +0800 Subject: [PATCH] Enable concurrent CI builds (#1236) Currently, only one CI build can be started at the same time due to we only have only one testing cluster. This PR aims to enable multiple concurrent builds can be triggered to reduce the waiting time for CI builds. The major changes listed below: 1. Each build generate unique images for testing and debugging 2. Check whether the testing cluster is being used by another build. 3. Cache docker image & go mod after build finished 4. Improve dockerfiles for reusing docker build cache --- .travis.yml | 21 +++++--- cmd/builder/Dockerfile.fission-builder | 18 +++---- cmd/fetcher/Dockerfile.fission-fetcher | 18 +++---- cmd/fission-bundle/Dockerfile.fission-bundle | 18 +++---- .../Dockerfile.fission-preupgradechecks | 18 +++---- environments/python/Dockerfile | 15 +++--- environments/python/Dockerfile-2.7 | 15 +++--- environments/tensorflow-serving/Dockerfile | 2 +- hack/release.sh | 2 - skaffold.yaml | 1 - test/{build_and_test.sh => build.sh} | 24 ++++----- test/save_cache.sh | 33 ++++++++++++ test/test.sh | 50 +++++++++++++++++++ test/test_utils.sh | 44 +++++++++++----- 14 files changed, 194 insertions(+), 85 deletions(-) rename test/{build_and_test.sh => build.sh} (72%) create mode 100755 test/save_cache.sh create mode 100755 test/test.sh diff --git a/.travis.yml b/.travis.yml index c60961c9..86579a33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,18 +4,24 @@ dist: trusty branches: only: - - master + - master language: go go: - 1.11.6 +env: + - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on DOCKER_CACHE_DIR=${HOME}/docker/ + cache: directories: - $HOME/google-cloud-sdk/ - $HOME/k8scli - $HOME/tool + - $HOME/docker + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod services: - docker @@ -29,24 +35,27 @@ before_install: install: - hack/travis-kube-setup.sh -env: - - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on - before_script: - cd ${TRAVIS_BUILD_DIR} - go mod download - - go mod vendor - hack/verify-staticcheck.sh - hack/verify-gofmt.sh - hack/verify-govet.sh - helm lint charts/fission-all/ charts/fission-core/ - hack/runtests.sh + - test/build.sh script: - cd ${TRAVIS_BUILD_DIR} - - test/build_and_test.sh + - test/test.sh - test/upgrade/fission_upgrade_test.sh +before_cache: + - test/save_cache.sh # save docker cache + +after_script: + - kubectl --namespace default delete configmap -l travisID=${TRAVIS_BUILD_ID} # remove in-test configmap to indicate this CI build finished + notifications: slack: rooms: diff --git a/cmd/builder/Dockerfile.fission-builder b/cmd/builder/Dockerfile.fission-builder index f532dc4d..fb9afc4c 100644 --- a/cmd/builder/Dockerfile.fission-builder +++ b/cmd/builder/Dockerfile.fission-builder @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as fission-builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/builder +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/builder \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/fetcher/Dockerfile.fission-fetcher b/cmd/fetcher/Dockerfile.fission-fetcher index 8915b7cb..d1e62967 100644 --- a/cmd/fetcher/Dockerfile.fission-fetcher +++ b/cmd/fetcher/Dockerfile.fission-fetcher @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/fetcher +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/fetcher \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/fission-bundle/Dockerfile.fission-bundle b/cmd/fission-bundle/Dockerfile.fission-bundle index 0e283855..ab928a94 100644 --- a/cmd/fission-bundle/Dockerfile.fission-bundle +++ b/cmd/fission-bundle/Dockerfile.fission-bundle @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/fission-bundle +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/fission-bundle \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks index 4990aec4..b8e9f91e 100644 --- a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks +++ b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/preupgradechecks +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/pre-upgrade-checks \ -gcflags=-trimpath=$GOPATH \ diff --git a/environments/python/Dockerfile b/environments/python/Dockerfile index f4fac883..83370821 100644 --- a/environments/python/Dockerfile +++ b/environments/python/Dockerfile @@ -1,14 +1,15 @@ FROM alpine:3.5 -RUN apk update -RUN apk add --no-cache python3 python3-dev build-base libev-dev -RUN pip3 install --upgrade pip -RUN rm -r /root/.cache +WORKDIR /app + +RUN apk add --no-cache python3 python3-dev build-base libev-dev && \ + pip3 install --upgrade pip && \ + rm -r /root/.cache + +COPY requirements.txt /app +RUN pip3 install -r requirements.txt COPY . /app -WORKDIR /app -RUN pip3 install -r requirements.txt ENTRYPOINT ["python3"] CMD ["server.py"] - diff --git a/environments/python/Dockerfile-2.7 b/environments/python/Dockerfile-2.7 index 87e6455f..ceb60420 100644 --- a/environments/python/Dockerfile-2.7 +++ b/environments/python/Dockerfile-2.7 @@ -1,14 +1,15 @@ FROM alpine:3.5 -RUN apk update -RUN apk add --no-cache python python-dev build-base py-pip libev-dev -RUN pip install --upgrade pip -RUN rm -r /root/.cache +WORKDIR /app + +RUN apk add --no-cache python python-dev build-base py-pip libev-dev && \ + pip install --upgrade pip && \ + rm -r /root/.cache + +COPY requirements.txt /app +RUN pip install -r requirements.txt COPY . /app -WORKDIR /app -RUN pip install -r requirements.txt ENTRYPOINT ["python"] CMD ["server.py"] - diff --git a/environments/tensorflow-serving/Dockerfile b/environments/tensorflow-serving/Dockerfile index ade52016..7fdb210f 100644 --- a/environments/tensorflow-serving/Dockerfile +++ b/environments/tensorflow-serving/Dockerfile @@ -10,7 +10,7 @@ ENV APP ${GOPATH}/src/github.com/fission/fission/environments/tensorflow-serv WORKDIR ${APP} -ADD server.go ${APP} +ADD server.go ${APP} RUN go get RUN go build -a -o /server server.go diff --git a/hack/release.sh b/hack/release.sh index 9b6224be..4a1f0acb 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -329,8 +329,6 @@ fi release_environment_check $version $chartsrepo -go mod vendor - # Build release-builder image docker build -t fission-release-builder -f $GOPATH/src/github.com/fission/fission/hack/Dockerfile . diff --git a/skaffold.yaml b/skaffold.yaml index 8926b022..101b26a6 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -5,7 +5,6 @@ ############## Usage ############## # Skaffold CLI should be installed on your machine. # Change -> to your Dockerhub username -# Run `go mod vendor` so that all dependencies are downloaded # Run `skaffold run` to build and deploy with helm. # Run`skaffold build` to only build and push images ############## Current Limitations ############## diff --git a/test/build_and_test.sh b/test/build.sh similarity index 72% rename from test/build_and_test.sh rename to test/build.sh index 7b6f5ebe..66e03cea 100755 --- a/test/build_and_test.sh +++ b/test/build.sh @@ -17,18 +17,19 @@ fi source $(dirname $0)/test_utils.sh -REPO=gcr.io/fission-ci -IMAGE=fission-bundle -FETCHER_IMAGE=$REPO/fetcher -BUILDER_IMAGE=$REPO/builder -TAG=test -PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes. -ROUTER_SERVICE_TYPE=LoadBalancer -SERVICE_TYPE=LoadBalancer -PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks - dump_system_info +setupCIBuildEnv + +load_docker_cache ${DOCKER_CACHE_DIR}/python-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/jvm-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/go-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz; + +load_docker_cache ${DOCKER_CACHE_DIR}/python-builder.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/go-builder.tar.gz; + build_and_push_fission_bundle $REPO/$IMAGE:$TAG build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG @@ -37,7 +38,6 @@ build_and_push_fetcher $FETCHER_IMAGE:$TAG build_and_push_builder $BUILDER_IMAGE:$TAG - build_and_push_env_runtime python $REPO/python-env:$TAG "" build_and_push_env_runtime jvm $REPO/jvm-env:$TAG "" build_and_push_env_runtime go $REPO/go-env:$TAG "1.12" @@ -48,5 +48,3 @@ build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG "" build_and_push_env_builder go $REPO/go-env-builder:$TAG $BUILDER_IMAGE:$TAG "1.12" build_fission_cli - -install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE diff --git a/test/save_cache.sh b/test/save_cache.sh new file mode 100755 index 00000000..3f39c43c --- /dev/null +++ b/test/save_cache.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +# Unbound variables cause failure, so this readable if block instead of Parameter Expansion +if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]] +then + echo "Skipping build & test, this is cron job for fission upgrade tests" + exit 0 +fi + +if [ ! -f ${HOME}/.kube/config ] +then + echo "Skipping end to end tests, no cluster credentials" + exit 0 +fi + +source $(dirname $0)/test_utils.sh + +dump_system_info + +setupCIBuildEnv + +mkdir -p ${DOCKER_CACHE_DIR} + +docker save $(docker history -q $REPO/python-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true +docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true +docker save $(docker history -q $REPO/go-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true +docker save $(docker history -q $REPO/tensorflow-serving-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz || true + +docker save $(docker history -q $REPO/python-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-builder.tar.gz || true +docker save $(docker history -q $REPO/jvm-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz || true +docker save $(docker history -q $REPO/go-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-builder.tar.gz || true diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 00000000..15e0dca9 --- /dev/null +++ b/test/test.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +# Unbound variables cause failure, so this readable if block instead of Parameter Expansion +if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]] +then + echo "Skipping build & test, this is cron job for fission upgrade tests" + exit 0 +fi + +if [ ! -f ${HOME}/.kube/config ] +then + echo "Skipping end to end tests, no cluster credentials" + exit 0 +fi + +source $(dirname $0)/test_utils.sh + +dump_system_info + +setupCIBuildEnv + +while true; do + # ensure that gke cluster is now free for testing + + previous_build_id=$(kubectl --namespace default get configmap in-test --ignore-not-found -o=jsonpath='{.metadata.labels.travisID}') + + build_state=$(curl -s -X GET https://api.travis-ci.org/build/${previous_build_id} \ + -H "Authorization: token ${TRAVIS_TOKEN}" \ + -H "Travis-API-Version: 3" | python -c "import sys,json; print json.load(sys.stdin)['state']") + + # If previous build state is not equal to "started" or the previous build id + # equals to the current build ID means the previous build is end or restart. + # We can remove the configmap safely and start next k8s test safely. + if [[ ${TRAVIS_TOKEN} == ${previous_build_id} ]] || [[ $build_state != "started" ]]; then + kubectl --namespace default delete configmap -l travisID=${previous_build_id} + fi + + created=$(kubectl --namespace default create configmap in-test|grep "created"||true) + if [[ -z $created ]]; then + echo "Cluster is now in used. Retrying after 15 seconds..." + sleep 15 + continue + fi + kubectl --namespace default label configmap in-test travisID=${TRAVIS_BUILD_ID} + break +done + +install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE diff --git a/test/test_utils.sh b/test/test_utils.sh index 819c043f..47d88164 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -53,6 +53,25 @@ getGitCommit() { echo $(git rev-parse HEAD) } +setupCIBuildEnv() { + export REPO=gcr.io/fission-ci + export IMAGE=fission-bundle + export FETCHER_IMAGE=$REPO/fetcher + export BUILDER_IMAGE=$REPO/builder + export TAG=test-${TRAVIS_BUILD_ID} + export PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes. + export ROUTER_SERVICE_TYPE=LoadBalancer + export SERVICE_TYPE=LoadBalancer + export PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks +} + +load_docker_cache() { + cache=$1 + if [ -f ${cache} ]; then + gunzip -c ${cache} | docker load; + fi +} + build_and_push_pre_upgrade_check_image() { image_tag=$1 travis_fold_start build_and_push_pre_upgrade_check_image $image_tag @@ -448,16 +467,17 @@ export FAILURES=0 run_all_tests() { id=$1 + imageTag=$2 export FISSION_NAMESPACE=f-$id export FUNCTION_NAMESPACE=f-func-$id - export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test - export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test - export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:test - export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test - export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test - export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test - export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test + export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:${imageTag} + export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:${imageTag} + export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:${imageTag} + export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:${imageTag} + export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:${imageTag} + export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:${imageTag} + export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:${imageTag} set +e export TIMEOUT=900 # 15 minutes per test @@ -467,8 +487,6 @@ run_all_tests() { $ROOT/test/run_test.sh \ $ROOT/test/tests/test_canary.sh \ $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \ - $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \ - $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \ $ROOT/test/tests/recordreplay/test_record_greetings.sh \ $ROOT/test/tests/recordreplay/test_record_rv.sh \ $ROOT/test/tests/recordreplay/test_recorder_update.sh \ @@ -492,11 +510,13 @@ run_all_tests() { $ROOT/test/tests/test_specs/test_spec_multifile.sh \ $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \ $ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \ - $ROOT/test/tests/test_environments/test_go_env.sh + $ROOT/test/tests/test_environments/test_go_env.sh \ + $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \ + $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh FAILURES=$? # FIXME: run tests with newdeploy one by one. - export JOBS=1 + export JOBS=2 $ROOT/test/run_test.sh \ $ROOT/test/tests/test_backend_newdeploy.sh \ $ROOT/test/tests/test_environments/test_java_builder.sh \ @@ -562,7 +582,7 @@ install_and_test() { # ensure we run tests against with the same git commit version of CLI & server fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit" - run_all_tests $id + run_all_tests $id $imageTag dump_logs $id