Enable concurrent CI builds (#1236)
Currently, only one CI build can be started at the same time due to we only have only one testing cluster. This PR aims to enable multiple concurrent builds can be triggered to reduce the waiting time for CI builds. The major changes listed below: 1. Each build generate unique images for testing and debugging 2. Check whether the testing cluster is being used by another build. 3. Cache docker image & go mod after build finished 4. Improve dockerfiles for reusing docker build cache
This commit is contained in:
Ta-Ching Chen
@@ -17,18 +17,19 @@ fi
|
||||
|
||||
source $(dirname $0)/test_utils.sh
|
||||
|
||||
REPO=gcr.io/fission-ci
|
||||
IMAGE=fission-bundle
|
||||
FETCHER_IMAGE=$REPO/fetcher
|
||||
BUILDER_IMAGE=$REPO/builder
|
||||
TAG=test
|
||||
PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes.
|
||||
ROUTER_SERVICE_TYPE=LoadBalancer
|
||||
SERVICE_TYPE=LoadBalancer
|
||||
PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks
|
||||
|
||||
dump_system_info
|
||||
|
||||
setupCIBuildEnv
|
||||
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/python-env.tar.gz;
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/jvm-env.tar.gz;
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/go-env.tar.gz;
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz;
|
||||
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/python-builder.tar.gz;
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz;
|
||||
load_docker_cache ${DOCKER_CACHE_DIR}/go-builder.tar.gz;
|
||||
|
||||
build_and_push_fission_bundle $REPO/$IMAGE:$TAG
|
||||
|
||||
build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG
|
||||
@@ -37,7 +38,6 @@ build_and_push_fetcher $FETCHER_IMAGE:$TAG
|
||||
|
||||
build_and_push_builder $BUILDER_IMAGE:$TAG
|
||||
|
||||
|
||||
build_and_push_env_runtime python $REPO/python-env:$TAG ""
|
||||
build_and_push_env_runtime jvm $REPO/jvm-env:$TAG ""
|
||||
build_and_push_env_runtime go $REPO/go-env:$TAG "1.12"
|
||||
@@ -48,5 +48,3 @@ build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG ""
|
||||
build_and_push_env_builder go $REPO/go-env-builder:$TAG $BUILDER_IMAGE:$TAG "1.12"
|
||||
|
||||
build_fission_cli
|
||||
|
||||
install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
|
||||
if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]]
|
||||
then
|
||||
echo "Skipping build & test, this is cron job for fission upgrade tests"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -f ${HOME}/.kube/config ]
|
||||
then
|
||||
echo "Skipping end to end tests, no cluster credentials"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source $(dirname $0)/test_utils.sh
|
||||
|
||||
dump_system_info
|
||||
|
||||
setupCIBuildEnv
|
||||
|
||||
mkdir -p ${DOCKER_CACHE_DIR}
|
||||
|
||||
docker save $(docker history -q $REPO/python-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true
|
||||
docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true
|
||||
docker save $(docker history -q $REPO/go-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true
|
||||
docker save $(docker history -q $REPO/tensorflow-serving-env:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz || true
|
||||
|
||||
docker save $(docker history -q $REPO/python-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/python-builder.tar.gz || true
|
||||
docker save $(docker history -q $REPO/jvm-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz || true
|
||||
docker save $(docker history -q $REPO/go-env-builder:$TAG | grep -v '<missing>') | gzip > ${DOCKER_CACHE_DIR}/go-builder.tar.gz || true
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Unbound variables cause failure, so this readable if block instead of Parameter Expansion
|
||||
if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]]
|
||||
then
|
||||
echo "Skipping build & test, this is cron job for fission upgrade tests"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -f ${HOME}/.kube/config ]
|
||||
then
|
||||
echo "Skipping end to end tests, no cluster credentials"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source $(dirname $0)/test_utils.sh
|
||||
|
||||
dump_system_info
|
||||
|
||||
setupCIBuildEnv
|
||||
|
||||
while true; do
|
||||
# ensure that gke cluster is now free for testing
|
||||
|
||||
previous_build_id=$(kubectl --namespace default get configmap in-test --ignore-not-found -o=jsonpath='{.metadata.labels.travisID}')
|
||||
|
||||
build_state=$(curl -s -X GET https://api.travis-ci.org/build/${previous_build_id} \
|
||||
-H "Authorization: token ${TRAVIS_TOKEN}" \
|
||||
-H "Travis-API-Version: 3" | python -c "import sys,json; print json.load(sys.stdin)['state']")
|
||||
|
||||
# If previous build state is not equal to "started" or the previous build id
|
||||
# equals to the current build ID means the previous build is end or restart.
|
||||
# We can remove the configmap safely and start next k8s test safely.
|
||||
if [[ ${TRAVIS_TOKEN} == ${previous_build_id} ]] || [[ $build_state != "started" ]]; then
|
||||
kubectl --namespace default delete configmap -l travisID=${previous_build_id}
|
||||
fi
|
||||
|
||||
created=$(kubectl --namespace default create configmap in-test|grep "created"||true)
|
||||
if [[ -z $created ]]; then
|
||||
echo "Cluster is now in used. Retrying after 15 seconds..."
|
||||
sleep 15
|
||||
continue
|
||||
fi
|
||||
kubectl --namespace default label configmap in-test travisID=${TRAVIS_BUILD_ID}
|
||||
break
|
||||
done
|
||||
|
||||
install_and_test $REPO $IMAGE $TAG $FETCHER_IMAGE $TAG $PRUNE_INTERVAL $ROUTER_SERVICE_TYPE $SERVICE_TYPE $PRE_UPGRADE_CHECK_IMAGE
|
||||
+32
-12
@@ -53,6 +53,25 @@ getGitCommit() {
|
||||
echo $(git rev-parse HEAD)
|
||||
}
|
||||
|
||||
setupCIBuildEnv() {
|
||||
export REPO=gcr.io/fission-ci
|
||||
export IMAGE=fission-bundle
|
||||
export FETCHER_IMAGE=$REPO/fetcher
|
||||
export BUILDER_IMAGE=$REPO/builder
|
||||
export TAG=test-${TRAVIS_BUILD_ID}
|
||||
export PRUNE_INTERVAL=1 # this variable controls the interval to run archivePruner. The unit is in minutes.
|
||||
export ROUTER_SERVICE_TYPE=LoadBalancer
|
||||
export SERVICE_TYPE=LoadBalancer
|
||||
export PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks
|
||||
}
|
||||
|
||||
load_docker_cache() {
|
||||
cache=$1
|
||||
if [ -f ${cache} ]; then
|
||||
gunzip -c ${cache} | docker load;
|
||||
fi
|
||||
}
|
||||
|
||||
build_and_push_pre_upgrade_check_image() {
|
||||
image_tag=$1
|
||||
travis_fold_start build_and_push_pre_upgrade_check_image $image_tag
|
||||
@@ -448,16 +467,17 @@ export FAILURES=0
|
||||
|
||||
run_all_tests() {
|
||||
id=$1
|
||||
imageTag=$2
|
||||
|
||||
export FISSION_NAMESPACE=f-$id
|
||||
export FUNCTION_NAMESPACE=f-func-$id
|
||||
export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
|
||||
export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
|
||||
export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:test
|
||||
export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test
|
||||
export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test
|
||||
export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test
|
||||
export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test
|
||||
export PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:${imageTag}
|
||||
export PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:${imageTag}
|
||||
export GO_RUNTIME_IMAGE=gcr.io/fission-ci/go-env:${imageTag}
|
||||
export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:${imageTag}
|
||||
export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:${imageTag}
|
||||
export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:${imageTag}
|
||||
export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:${imageTag}
|
||||
|
||||
set +e
|
||||
export TIMEOUT=900 # 15 minutes per test
|
||||
@@ -467,8 +487,6 @@ run_all_tests() {
|
||||
$ROOT/test/run_test.sh \
|
||||
$ROOT/test/tests/test_canary.sh \
|
||||
$ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \
|
||||
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
|
||||
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \
|
||||
$ROOT/test/tests/recordreplay/test_record_greetings.sh \
|
||||
$ROOT/test/tests/recordreplay/test_record_rv.sh \
|
||||
$ROOT/test/tests/recordreplay/test_recorder_update.sh \
|
||||
@@ -492,11 +510,13 @@ run_all_tests() {
|
||||
$ROOT/test/tests/test_specs/test_spec_multifile.sh \
|
||||
$ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \
|
||||
$ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \
|
||||
$ROOT/test/tests/test_environments/test_go_env.sh
|
||||
$ROOT/test/tests/test_environments/test_go_env.sh \
|
||||
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
|
||||
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh
|
||||
FAILURES=$?
|
||||
|
||||
# FIXME: run tests with newdeploy one by one.
|
||||
export JOBS=1
|
||||
export JOBS=2
|
||||
$ROOT/test/run_test.sh \
|
||||
$ROOT/test/tests/test_backend_newdeploy.sh \
|
||||
$ROOT/test/tests/test_environments/test_java_builder.sh \
|
||||
@@ -562,7 +582,7 @@ install_and_test() {
|
||||
# ensure we run tests against with the same git commit version of CLI & server
|
||||
fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit"
|
||||
|
||||
run_all_tests $id
|
||||
run_all_tests $id $imageTag
|
||||
|
||||
dump_logs $id
|
||||
|
||||
|
||||
Reference in New Issue
Block a user