From de9fc05c6d7bad51cc95e657e47e4028c0d39b0f Mon Sep 17 00:00:00 2001 From: Ta-Ching Chen Date: Mon, 15 Jul 2019 16:33:26 +0800 Subject: [PATCH] Enable concurrent CI builds (#1236) Currently, only one CI build can be started at the same time due to we only have only one testing cluster. This PR aims to enable multiple concurrent builds can be triggered to reduce the waiting time for CI builds. The major changes listed below: 1. Each build generate unique images for testing and debugging 2. Check whether the testing cluster is being used by another build. 3. Cache docker image & go mod after build finished 4. Improve dockerfiles for reusing docker build cache --- .travis.yml | 21 +++++--- cmd/builder/Dockerfile.fission-builder | 18 +++---- cmd/fetcher/Dockerfile.fission-fetcher | 18 +++---- cmd/fission-bundle/Dockerfile.fission-bundle | 18 +++---- .../Dockerfile.fission-preupgradechecks | 18 +++---- environments/python/Dockerfile | 15 +++--- environments/python/Dockerfile-2.7 | 15 +++--- environments/tensorflow-serving/Dockerfile | 2 +- hack/release.sh | 2 - skaffold.yaml | 1 - test/{build_and_test.sh => build.sh} | 24 ++++----- test/save_cache.sh | 33 ++++++++++++ test/test.sh | 50 +++++++++++++++++++ test/test_utils.sh | 44 +++++++++++----- 14 files changed, 194 insertions(+), 85 deletions(-) rename test/{build_and_test.sh => build.sh} (72%) create mode 100755 test/save_cache.sh create mode 100755 test/test.sh diff --git a/.travis.yml b/.travis.yml index c60961c9a..86579a338 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,18 +4,24 @@ dist: trusty branches: only: - - master + - master language: go go: - 1.11.6 +env: + - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on DOCKER_CACHE_DIR=${HOME}/docker/ + cache: directories: - $HOME/google-cloud-sdk/ - $HOME/k8scli - $HOME/tool + - $HOME/docker + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod services: - docker @@ -29,24 +35,27 @@ before_install: install: - hack/travis-kube-setup.sh -env: - - KUBECONFIG=${HOME}/.kube/config PATH=$HOME/k8scli:$HOME/tool:$HOME/google-cloud-sdk/bin:${PATH} GO111MODULE=on - before_script: - cd ${TRAVIS_BUILD_DIR} - go mod download - - go mod vendor - hack/verify-staticcheck.sh - hack/verify-gofmt.sh - hack/verify-govet.sh - helm lint charts/fission-all/ charts/fission-core/ - hack/runtests.sh + - test/build.sh script: - cd ${TRAVIS_BUILD_DIR} - - test/build_and_test.sh + - test/test.sh - test/upgrade/fission_upgrade_test.sh +before_cache: + - test/save_cache.sh # save docker cache + +after_script: + - kubectl --namespace default delete configmap -l travisID=${TRAVIS_BUILD_ID} # remove in-test configmap to indicate this CI build finished + notifications: slack: rooms: diff --git a/cmd/builder/Dockerfile.fission-builder b/cmd/builder/Dockerfile.fission-builder index f532dc4dc..fb9afc4ce 100644 --- a/cmd/builder/Dockerfile.fission-builder +++ b/cmd/builder/Dockerfile.fission-builder @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as fission-builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/builder +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/builder \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/fetcher/Dockerfile.fission-fetcher b/cmd/fetcher/Dockerfile.fission-fetcher index 8915b7cb7..d1e629673 100644 --- a/cmd/fetcher/Dockerfile.fission-fetcher +++ b/cmd/fetcher/Dockerfile.fission-fetcher @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/fetcher +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/fetcher \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/fission-bundle/Dockerfile.fission-bundle b/cmd/fission-bundle/Dockerfile.fission-bundle index 0e283855c..ab928a944 100644 --- a/cmd/fission-bundle/Dockerfile.fission-bundle +++ b/cmd/fission-bundle/Dockerfile.fission-bundle @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/fission-bundle +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/fission-bundle \ -gcflags=-trimpath=$GOPATH \ diff --git a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks index 4990aec4a..b8e9f91e5 100644 --- a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks +++ b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks @@ -4,15 +4,6 @@ RUN apk add --update ca-certificates FROM golang:1.11-alpine as builder RUN apk add bash ca-certificates git gcc g++ libc-dev -ARG GITCOMMIT=unknown -# E.g. GITCOMMIT=$(git rev-parse HEAD) - -ARG BUILDVERSION=unknown -# E.g. BUILDVERSION=$(git rev-parse HEAD) - -ARG BUILDDATE=unknown -# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - ARG GOPKG=github.com/fission/fission ENV GO111MODULE=on @@ -29,6 +20,15 @@ COPY ./ ./ WORKDIR /go/src/${GOPKG}/cmd/preupgradechecks +ARG GITCOMMIT=unknown +# E.g. GITCOMMIT=$(git rev-parse HEAD) + +ARG BUILDVERSION=unknown +# E.g. BUILDVERSION=$(git rev-parse HEAD) + +ARG BUILDDATE=unknown +# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -o /go/bin/pre-upgrade-checks \ -gcflags=-trimpath=$GOPATH \ diff --git a/environments/python/Dockerfile b/environments/python/Dockerfile index f4fac8838..833708213 100644 --- a/environments/python/Dockerfile +++ b/environments/python/Dockerfile @@ -1,14 +1,15 @@ FROM alpine:3.5 -RUN apk update -RUN apk add --no-cache python3 python3-dev build-base libev-dev -RUN pip3 install --upgrade pip -RUN rm -r /root/.cache +WORKDIR /app + +RUN apk add --no-cache python3 python3-dev build-base libev-dev && \ + pip3 install --upgrade pip && \ + rm -r /root/.cache + +COPY requirements.txt /app +RUN pip3 install -r requirements.txt COPY . /app -WORKDIR /app -RUN pip3 install -r requirements.txt ENTRYPOINT ["python3"] CMD ["server.py"] - diff --git a/environments/python/Dockerfile-2.7 b/environments/python/Dockerfile-2.7 index 87e6455f2..ceb604207 100644 --- a/environments/python/Dockerfile-2.7 +++ b/environments/python/Dockerfile-2.7 @@ -1,14 +1,15 @@ FROM alpine:3.5 -RUN apk update -RUN apk add --no-cache python python-dev build-base py-pip libev-dev -RUN pip install --upgrade pip -RUN rm -r /root/.cache +WORKDIR /app + +RUN apk add --no-cache python python-dev build-base py-pip libev-dev && \ + pip install --upgrade pip && \ + rm -r /root/.cache + +COPY requirements.txt /app +RUN pip install -r requirements.txt COPY . /app -WORKDIR /app -RUN pip install -r requirements.txt ENTRYPOINT ["python"] CMD ["server.py"] - diff --git a/environments/tensorflow-serving/Dockerfile b/environments/tensorflow-serving/Dockerfile index ade520162..7fdb210f1 100644 --- a/environments/tensorflow-serving/Dockerfile +++ b/environments/tensorflow-serving/Dockerfile @@ -10,7 +10,7 @@ ENV APP ${GOPATH}/src/github.com/fission/fission/environments/tensorflow-serv WORKDIR ${APP} -ADD server.go ${APP} +ADD server.go ${APP} RUN go get RUN go build -a -o /server server.go diff --git a/hack/release.sh b/hack/release.sh index 9b6224bec..4a1f0acbf 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -329,8 +329,6 @@ fi release_environment_check $version $chartsrepo -go mod vendor - # Build release-builder image docker build -t fission-release-builder -f $GOPATH/src/github.com/fission/fission/hack/Dockerfile . diff --git a/skaffold.yaml b/skaffold.yaml index 8926b0220..101b26a6a 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -5,7 +5,6 @@ ############## Usage ############## # Skaffold CLI should be installed on your machine. # Change -> to your Dockerhub username -# Run `go mod vendor` so that all dependencies are downloaded # Run `skaffold run` to build and deploy with helm. # Run`skaffold build` to only build and push images ############## Current Limitations ############## diff --git a/test/build_and_test.sh b/test/build.sh similarity index 72% rename from test/build_and_test.sh rename to test/build.sh index 7b6f5ebed..66e03cea7 100755 --- a/test/build_and_test.sh +++ b/test/build.sh @@ -17,18 +17,19 @@ fi source $(dirname $0)/test_utils.sh -REPO=gcr.io/fission-ci -IMAGE=fission-bundle -FETCHER_IMAGE=$REPO/fetcher -BUILDER_IMAGE=$REPO/builder -TAG=test -PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes. -ROUTER_SERVICE_TYPE=LoadBalancer -SERVICE_TYPE=LoadBalancer -PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks - dump_system_info +setupCIBuildEnv + +load_docker_cache ${DOCKER_CACHE_DIR}/python-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/jvm-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/go-env.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz; + +load_docker_cache ${DOCKER_CACHE_DIR}/python-builder.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz; +load_docker_cache ${DOCKER_CACHE_DIR}/go-builder.tar.gz; + build_and_push_fission_bundle $REPO/$IMAGE:$TAG build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG @@ -37,7 +38,6 @@ build_and_push_fetcher $FETCHER_IMAGE:$TAG build_and_push_builder $BUILDER_IMAGE:$TAG - build_and_push_env_runtime python $REPO/python-env:$TAG "" build_and_push_env_runtime jvm $REPO/jvm-env:$TAG "" build_and_push_env_runtime go $REPO/go-env:$TAG "1.12" @@ -48,5 +48,3 @@ build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG "" build_and_push_env_builder go $REPO/go-env-builder:$TAG $BUILDER_IMAGE:$TAG "1.12" build_fission_cli - -install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE diff --git a/test/save_cache.sh b/test/save_cache.sh new file mode 100755 index 000000000..3f39c43c3 --- /dev/null +++ b/test/save_cache.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +# Unbound variables cause failure, so this readable if block instead of Parameter Expansion +if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]] +then + echo "Skipping build & test, this is cron job for fission upgrade tests" + exit 0 +fi + +if [ ! -f ${HOME}/.kube/config ] +then + echo "Skipping end to end tests, no cluster credentials" + exit 0 +fi + +source $(dirname $0)/test_utils.sh + +dump_system_info + +setupCIBuildEnv + +mkdir -p ${DOCKER_CACHE_DIR} + +docker save $(docker history -q $REPO/python-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true +docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true +docker save $(docker history -q $REPO/go-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true +docker save $(docker history -q $REPO/tensorflow-serving-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz || true + +docker save $(docker history -q $REPO/python-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-builder.tar.gz || true +docker save $(docker history -q $REPO/jvm-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz || true +docker save $(docker history -q $REPO/go-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-builder.tar.gz || true diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 000000000..15e0dca97 --- /dev/null +++ b/test/test.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +# Unbound variables cause failure, so this readable if block instead of Parameter Expansion +if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]] +then + echo "Skipping build & test, this is cron job for fission upgrade tests" + exit 0 +fi + +if [ ! -f ${HOME}/.kube/config ] +then + echo "Skipping end to end tests, no cluster credentials" + exit 0 +fi + +source $(dirname $0)/test_utils.sh + +dump_system_info + +setupCIBuildEnv + +while true; do + # ensure that gke cluster is now free for testing + + previous_build_id=$(kubectl --namespace default get configmap in-test --ignore-not-found -o=jsonpath='{.metadata.labels.travisID}') + + build_state=$(curl -s -X GET https://api.travis-ci.org/build/${previous_build_id} \ + -H "Authorization: token ${TRAVIS_TOKEN}" \ + -H "Travis-API-Version: 3" | python -c "import sys,json; print json.load(sys.stdin)['state']") + + # If previous build state is not equal to "started" or the previous build id + # equals to the current build ID means the previous build is end or restart. + # We can remove the configmap safely and start next k8s test safely. + if [[ ${TRAVIS_TOKEN} == ${previous_build_id} ]] || [[ $build_state != "started" ]]; then + kubectl --namespace default delete configmap -l travisID=${previous_build_id} + fi + + created=$(kubectl --namespace default create configmap in-test|grep "created"||true) + if [[ -z $created ]]; then + echo "Cluster is now in used. Retrying after 15 seconds..." + sleep 15 + continue + fi + kubectl --namespace default label configmap in-test travisID=${TRAVIS_BUILD_ID} + break +done + +install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE diff --git a/test/test_utils.sh b/test/test_utils.sh index 819c043f4..47d88164d 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -53,6 +53,25 @@ getGitCommit() { echo $(git rev-parse HEAD) } +setupCIBuildEnv() { + export REPO=gcr.io/fission-ci + export IMAGE=fission-bundle + export FETCHER_IMAGE=$REPO/fetcher + export BUILDER_IMAGE=$REPO/builder + export TAG=test-${TRAVIS_BUILD_ID} + export PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes. + export ROUTER_SERVICE_TYPE=LoadBalancer + export SERVICE_TYPE=LoadBalancer + export PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks +} + +load_docker_cache() { + cache=$1 + if [ -f ${cache} ]; then + gunzip -c ${cache} | docker load; + fi +} + build_and_push_pre_upgrade_check_image() { image_tag=$1 travis_fold_start build_and_push_pre_upgrade_check_image $image_tag @@ -448,16 +467,17 @@ export FAILURES=0 run_all_tests() { id=$1 + imageTag=$2 export FISSION_NAMESPACE=f-$id export FUNCTION_NAMESPACE=f-func-$id - export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test - export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test - export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:test - export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test - export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test - export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test - export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test + export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:${imageTag} + export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:${imageTag} + export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:${imageTag} + export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:${imageTag} + export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:${imageTag} + export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:${imageTag} + export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:${imageTag} set +e export TIMEOUT=900 # 15 minutes per test @@ -467,8 +487,6 @@ run_all_tests() { $ROOT/test/run_test.sh \ $ROOT/test/tests/test_canary.sh \ $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \ - $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \ - $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \ $ROOT/test/tests/recordreplay/test_record_greetings.sh \ $ROOT/test/tests/recordreplay/test_record_rv.sh \ $ROOT/test/tests/recordreplay/test_recorder_update.sh \ @@ -492,11 +510,13 @@ run_all_tests() { $ROOT/test/tests/test_specs/test_spec_multifile.sh \ $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \ $ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \ - $ROOT/test/tests/test_environments/test_go_env.sh + $ROOT/test/tests/test_environments/test_go_env.sh \ + $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \ + $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh FAILURES=$? # FIXME: run tests with newdeploy one by one. - export JOBS=1 + export JOBS=2 $ROOT/test/run_test.sh \ $ROOT/test/tests/test_backend_newdeploy.sh \ $ROOT/test/tests/test_environments/test_java_builder.sh \ @@ -562,7 +582,7 @@ install_and_test() { # ensure we run tests against with the same git commit version of CLI & server fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit" - run_all_tests $id + run_all_tests $id $imageTag dump_logs $id