diff --git a/.travis.yml b/.travis.yml index 76d4f045..a081345a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ cache: - $HOME/google-cloud-sdk/ - $HOME/k8scli - $HOME/tool - - $HOME/docker - $HOME/.cache/go-build - $HOME/gopath/pkg/mod @@ -28,7 +27,7 @@ services: before_install: - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce +# - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - sudo apt-get -y install apache2-utils parallel realpath - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 @@ -50,9 +49,6 @@ script: - test/test.sh - test/upgrade/fission_upgrade_test.sh -before_cache: - - test/save_cache.sh # save docker cache - after_script: - bash <(curl -s https://codecov.io/bash) - kubectl --namespace default delete configmap -l travisID=${TRAVIS_BUILD_ID} # remove in-test configmap to indicate this CI build finished diff --git a/cmd/builder/Dockerfile.fission-builder b/cmd/builder/Dockerfile.fission-builder index e57ff1c8..f72b10f4 100644 --- a/cmd/builder/Dockerfile.fission-builder +++ b/cmd/builder/Dockerfile.fission-builder @@ -1,7 +1,4 @@ -FROM alpine:3.10 as base -RUN apk add --update ca-certificates - -FROM golang:1.12-alpine as fission-builder +FROM golang:1.12-alpine as godep RUN apk add bash ca-certificates git gcc g++ libc-dev ARG GOPKG=github.com/fission/fission @@ -15,6 +12,11 @@ COPY go.* ./ RUN go mod download +From godep as builder + +ARG GOPKG +WORKDIR /go/src/${GOPKG} + # Copy whole fission directory to work dir COPY ./ ./ @@ -35,8 +37,9 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -asmflags=-trimpath=$GOPATH \ -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}" -FROM base -COPY --from=fission-builder /go/bin/builder / +FROM alpine:3.10 as base +RUN apk add --update ca-certificates +COPY --from=builder /go/bin/builder / EXPOSE 8001 ENTRYPOINT ["/builder"] diff --git a/cmd/fetcher/Dockerfile.fission-fetcher b/cmd/fetcher/Dockerfile.fission-fetcher index 6ac5a87f..3bc391d9 100644 --- a/cmd/fetcher/Dockerfile.fission-fetcher +++ b/cmd/fetcher/Dockerfile.fission-fetcher @@ -1,7 +1,4 @@ -FROM alpine:3.10 as base -RUN apk add --update ca-certificates - -FROM golang:1.12-alpine as builder +FROM golang:1.12-alpine as godep RUN apk add bash ca-certificates git gcc g++ libc-dev ARG GOPKG=github.com/fission/fission @@ -15,6 +12,11 @@ COPY go.* ./ RUN go mod download +From godep as builder + +ARG GOPKG +WORKDIR /go/src/${GOPKG} + # Copy whole fission directory to work dir COPY ./ ./ @@ -35,7 +37,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -asmflags=-trimpath=$GOPATH \ -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}" -FROM base +FROM alpine:3.10 as base +RUN apk add --update ca-certificates COPY --from=builder /go/bin/fetcher / EXPOSE 8000 diff --git a/cmd/fission-bundle/Dockerfile.fission-bundle b/cmd/fission-bundle/Dockerfile.fission-bundle index 66cebab4..978a4fac 100644 --- a/cmd/fission-bundle/Dockerfile.fission-bundle +++ b/cmd/fission-bundle/Dockerfile.fission-bundle @@ -1,7 +1,4 @@ -FROM alpine:3.10 as base -RUN apk add --update ca-certificates - -FROM golang:1.12-alpine as builder +FROM golang:1.12-alpine as godep RUN apk add bash ca-certificates git gcc g++ libc-dev ARG GOPKG=github.com/fission/fission @@ -15,6 +12,11 @@ COPY go.* ./ RUN go mod download +From godep as builder + +ARG GOPKG +WORKDIR /go/src/${GOPKG} + # Copy whole fission directory to work dir COPY ./ ./ @@ -35,7 +37,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -asmflags=-trimpath=$GOPATH \ -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}" -FROM base +FROM alpine:3.10 as base +RUN apk add --update ca-certificates COPY --from=builder /go/bin/fission-bundle / ENTRYPOINT ["/fission-bundle"] diff --git a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks index b8e9f91e..3cd0ef0a 100644 --- a/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks +++ b/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks @@ -1,7 +1,4 @@ -FROM alpine:3.10 as base -RUN apk add --update ca-certificates - -FROM golang:1.11-alpine as builder +FROM golang:1.12-alpine as godep RUN apk add bash ca-certificates git gcc g++ libc-dev ARG GOPKG=github.com/fission/fission @@ -15,6 +12,11 @@ COPY go.* ./ RUN go mod download +From godep as builder + +ARG GOPKG +WORKDIR /go/src/${GOPKG} + # Copy whole fission directory to work dir COPY ./ ./ @@ -35,7 +37,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -asmflags=-trimpath=$GOPATH \ -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}" -FROM base +FROM alpine:3.10 as base +RUN apk add --update ca-certificates COPY --from=builder /go/bin/pre-upgrade-checks / ENTRYPOINT ["/pre-upgrade-checks"] diff --git a/test/build.sh b/test/build.sh index 66e03cea..39fbe684 100755 --- a/test/build.sh +++ b/test/build.sh @@ -21,22 +21,11 @@ dump_system_info setupCIBuildEnv -load_docker_cache ${DOCKER_CACHE_DIR}/python-env.tar.gz; -load_docker_cache ${DOCKER_CACHE_DIR}/jvm-env.tar.gz; -load_docker_cache ${DOCKER_CACHE_DIR}/go-env.tar.gz; -load_docker_cache ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz; - -load_docker_cache ${DOCKER_CACHE_DIR}/python-builder.tar.gz; -load_docker_cache ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz; -load_docker_cache ${DOCKER_CACHE_DIR}/go-builder.tar.gz; - -build_and_push_fission_bundle $REPO/$IMAGE:$TAG - -build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG - -build_and_push_fetcher $FETCHER_IMAGE:$TAG - -build_and_push_builder $BUILDER_IMAGE:$TAG +build_and_push_go_mod_cache_image $REPO/go-mod-image-cache +build_and_push_fission_bundle $REPO/$IMAGE:$TAG $REPO/go-mod-image-cache +build_and_push_pre_upgrade_check_image $PRE_UPGRADE_CHECK_IMAGE:$TAG $REPO/go-mod-image-cache +build_and_push_fetcher $FETCHER_IMAGE:$TAG $REPO/go-mod-image-cache +build_and_push_builder $BUILDER_IMAGE:$TAG $REPO/go-mod-image-cache build_and_push_env_runtime python $REPO/python-env:$TAG "" build_and_push_env_runtime jvm $REPO/jvm-env:$TAG "" diff --git a/test/save_cache.sh b/test/save_cache.sh deleted file mode 100755 index 7447c811..00000000 --- a/test/save_cache.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Unbound variables cause failure, so this readable if block instead of Parameter Expansion -if [[ ${TRAVIS_EVENT_TYPE+NOVALUE} == "cronNOVALUE" ]] -then - echo "Skipping build & test, this is cron job for fission upgrade tests" - exit 0 -fi - -if [ ! -f ${HOME}/.kube/config ] -then - echo "Skipping end to end tests, no cluster credentials" - exit 0 -fi - -source $(dirname $0)/test_utils.sh - -dump_system_info - -setupCIBuildEnv - -mkdir -p ${DOCKER_CACHE_DIR} - -# remove old images -rm -rf ${DOCKER_CACHE_DIR}/* - -docker save $(docker history -q $REPO/python-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-env.tar.gz || true -docker save $(docker history -q $REPO/jvm-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-env.tar.gz || true -docker save $(docker history -q $REPO/go-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-env.tar.gz || true -docker save $(docker history -q $REPO/tensorflow-serving-env:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/tensorflow-serving-env.tar.gz || true - -docker save $(docker history -q $REPO/python-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/python-builder.tar.gz || true -docker save $(docker history -q $REPO/jvm-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/jvm-builder.tar.gz || true -docker save $(docker history -q $REPO/go-env-builder:$TAG | grep -v '') | gzip > ${DOCKER_CACHE_DIR}/go-builder.tar.gz || true diff --git a/test/test_utils.sh b/test/test_utils.sh index 0ba09102..2af521f3 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -65,59 +65,69 @@ setupCIBuildEnv() { export PRE_UPGRADE_CHECK_IMAGE=$REPO/pre-upgrade-checks } -load_docker_cache() { - cache=$1 - if [ -f ${cache} ]; then - gunzip -c ${cache} | docker load; - fi +build_and_push_go_mod_cache_image() { + image_tag=$1 + travis_fold_start go_mod_cache_image $image_tag + + gcloud_login + + gcloud docker -- pull $image_tag + docker build -q -t $image_tag -f $ROOT/cmd/fission-bundle/Dockerfile.fission-bundle --cache-from ${image_tag} --target godep --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . + + gcloud docker -- push $image_tag & + travis_fold_end go_mod_cache_image } build_and_push_pre_upgrade_check_image() { image_tag=$1 + cache_image=$2 travis_fold_start build_and_push_pre_upgrade_check_image $image_tag - docker build -t $image_tag -f $ROOT/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . + docker build -q -t $image_tag -f $ROOT/cmd/preupgradechecks/Dockerfile.fission-preupgradechecks --cache-from ${cache_image} --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . gcloud_login - gcloud docker -- push $image_tag + gcloud docker -- push $image_tag & travis_fold_end build_and_push_pre_upgrade_check_image } build_and_push_fission_bundle() { image_tag=$1 + cache_image=$2 travis_fold_start build_and_push_fission_bundle $image_tag - docker build -q -t $image_tag -f $ROOT/cmd/fission-bundle/Dockerfile.fission-bundle --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . + docker build -q -t $image_tag -f $ROOT/cmd/fission-bundle/Dockerfile.fission-bundle --cache-from ${cache_image} --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . gcloud_login - gcloud docker -- push $image_tag + gcloud docker -- push $image_tag & travis_fold_end build_and_push_fission_bundle } build_and_push_fetcher() { image_tag=$1 + cache_image=$2 travis_fold_start build_and_push_fetcher $image_tag - docker build -q -t $image_tag -f $ROOT/cmd/fetcher/Dockerfile.fission-fetcher --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . + docker build -q -t $image_tag -f $ROOT/cmd/fetcher/Dockerfile.fission-fetcher --cache-from ${cache_image} --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . gcloud_login - gcloud docker -- push $image_tag + gcloud docker -- push $image_tag & travis_fold_end build_and_push_fetcher } build_and_push_builder() { image_tag=$1 + cache_image=$2 travis_fold_start build_and_push_builder $image_tag - docker build -q -t $image_tag -f $ROOT/cmd/builder/Dockerfile.fission-builder --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . + docker build -q -t $image_tag -f $ROOT/cmd/builder/Dockerfile.fission-builder --cache-from ${cache_image} --build-arg GITCOMMIT=$(getGitCommit) --build-arg BUILDDATE=$(getDate) --build-arg BUILDVERSION=$(getVersion) . gcloud_login - gcloud docker -- push $image_tag + gcloud docker -- push $image_tag & travis_fold_end build_and_push_builder } @@ -139,7 +149,7 @@ build_and_push_env_runtime() { gcloud_login - gcloud docker -- push $image_tag + gcloud docker -- push $image_tag & popd travis_fold_end build_and_push_env_runtime.$env } @@ -164,7 +174,7 @@ build_and_push_env_builder() { gcloud_login - gcloud docker -- push ${image_tag} + gcloud docker -- push ${image_tag} & popd travis_fold_end build_and_push_env_builder.$env } @@ -598,12 +608,13 @@ install_and_test() { dump_logs $id - if [ $FAILURES -ne 0 ] - then - # describe each pod in fission ns and function namespace - describe_all_pods $id - exit 1 - fi +# Commented out due to Travis-CI log length limit +# if [ $FAILURES -ne 0 ] +# then +# # describe each pod in fission ns and function namespace +# describe_all_pods $id +# exit 1 +# fi }