Move build process from host to docker container during release process (#682)

This commit is contained in:
Ta-Ching Chen
2018-07-03 21:14:06 -07:00
committed by smruthi2187
parent 9daf97a9ab
commit 44be0f6859
7 changed files with 395 additions and 250 deletions
+8 -2
View File
@@ -1,9 +1,15 @@
FROM microsoft/dotnet:1.1-sdk AS builder
COPY * /proj/
RUN cd /proj && ./project-build.sh
# Build env image
FROM microsoft/dotnet:1.1.0-runtime
WORKDIR /fission-workdir
COPY out .
COPY --from=builder /proj/out .
EXPOSE 8888
ENTRYPOINT ["dotnet"]
CMD ["fission-dotnet.dll"]
CMD ["fission-dotnet.dll"]
+2 -3
View File
@@ -1,6 +1,5 @@
#!/bin/sh
THIS_DIR=$(realpath $(dirname $0))
docker run -v $THIS_DIR:/proj microsoft/dotnet:1.1-sdk sh -c "cd /proj && ./project-build.sh"
# clean up
rm -rf bin obj out
+8 -2
View File
@@ -1,9 +1,15 @@
FROM microsoft/dotnet:2.0.0-sdk AS builder
COPY * /proj/
RUN cd /proj && ./project-build.sh
# Build env image
FROM microsoft/dotnet:2.0-runtime
WORKDIR /fission-workdir
COPY out .
COPY --from=builder /proj/out .
EXPOSE 8888
ENTRYPOINT ["dotnet"]
CMD ["fission-dotnet20.dll"]
CMD ["fission-dotnet20.dll"]
+2 -4
View File
@@ -1,6 +1,4 @@
#!/bin/sh
THIS_DIR=$(realpath $(dirname $0))
docker run -v $THIS_DIR:/proj microsoft/dotnet:2.0.0-sdk sh -c "cd /proj && ./project-build.sh"
# clean up
rm -rf bin obj out
+3
View File
@@ -0,0 +1,3 @@
FROM golang:1.10.2
RUN wget -qO- https://download.docker.com/linux/static/stable/x86_64/docker-17.03.0-ce.tgz | tar xvz -C /usr/local/bin/ --strip 1
RUN wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.9.1-linux-amd64.tar.gz | tar xvz -C /usr/local/bin/ --strip 1
+297
View File
@@ -0,0 +1,297 @@
#!/bin/bash
set -e
#set -x
DIR=$(realpath $(dirname $0))/../
BUILDDIR=$(realpath $DIR)/build
# Build CLI binaries for mac/linux/windows
build_all_cli() {
local version=$1
local date=$2
local gitcommit=$3
build_cli "linux" "linux" $version $date $gitcommit
build_cli "darwin" "osx" $version $date $gitcommit
build_cli "windows" "windows" $version $date $gitcommit
}
# Build cli binary for one OS, and put it in $BUILDDIR/cli/<os>/
build_cli() {
os=$1
osName=$2
local version=$3
local date=$4
local gitcommit=$5
arch="amd64" # parameterize if/when we need to
pushd $DIR/fission
if [ "$osName" == "windows" ]
then
binary=fission-cli-${osName}.exe
else
binary=fission-cli-${osName}
fi
GOOS=$os GOARCH=$arch go build -gcflags=-trimpath=$GOPATH -asmflags=-trimpath=$GOPATH -ldflags "-X github.com/fission/fission.GitCommit=$gitcommit -X github.com/fission/fission.BuildDate=$date -X github.com/fission/fission.Version=$version" -o $binary .
outdir=$BUILDDIR/cli/$osName/
mkdir -p $outdir
mv $binary $outdir
popd
}
# Build fission-bundle image
build_fission_bundle_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/fission-bundle:$version
pushd $DIR/fission-bundle
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/fission-bundle:latest
popd
}
build_fetcher_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/fetcher:$version
pushd $DIR/environments/fetcher/cmd
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/fetcher:latest
popd
}
push_fetcher_image() {
local version=$1
local tag=fission/fetcher:$version
}
build_builder_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/builder:$version
pushd $DIR/builder/cmd
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/builder:latest
popd
}
build_logger_image() {
local version=$1
local tag=fission/fluentd:$version
pushd $DIR/logger/fluentd
docker build -t $tag .
docker tag $tag fission/fluentd:latest
popd
}
push_logger_image() {
local version=$1
local tag=fission/fluentd:$version
}
build_env_image() {
local version=$1
envdir=$2
imgnamebase=$3
imgvariant=$4
if [ -z "$imgvariant" ]
then
# no variant specified, just use the base name
imgname=$imgnamebase
dockerfile="Dockerfile"
else
# variant specified - append variant to image name and assume dockerfile
# exists with same suffix (e.g. image node-env-debian built from Dockerfile-debian)
imgname="$imgname-$imgvariant"
dockerfile="Dockerfile-$imgvariant"
fi
echo "Building $envdir -> $imgname:$version using $dockerfile"
pushd $DIR/environments/$envdir
if [ -f build.sh ]
then
./build.sh
fi
docker build -t fission/$imgname:$version -f $dockerfile .
docker tag fission/$imgname:$version fission/$imgname:latest
popd
}
# Build pre-upgrade-checks image
build_pre_upgrade_checks_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/pre-upgrade-checks:$version
pushd $DIR/preupgradechecks
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/pre-upgrade-checks:latest
popd
}
build_all_envs() {
local version=$1
# call with version, env dir, image name base, image name variant
build_env_image "$version" "nodejs" "node-env" ""
build_env_image "$version" "nodejs" "node-env" "debian"
build_env_image "$version" "binary" "binary-env" ""
build_env_image "$version" "dotnet" "dotnet-env" ""
build_env_image "$version" "dotnet20" "dotnet20-env" ""
build_env_image "$version" "go" "go-env" ""
build_env_image "$version" "perl" "perl-env" ""
build_env_image "$version" "php7" "php-env" ""
build_env_image "$version" "python" "python-env" ""
build_env_image "$version" "python" "python-env" "2.7"
build_env_image "$version" "ruby" "ruby-env" ""
}
build_env_builder_image() {
local version=$1
envdir=$2
imgnamebase=$3
imgvariant=$4
if [ -z "$imgvariant" ]
then
# no variant specified, just use the base name
imgname=$imgnamebase
dockerfile="Dockerfile"
else
# variant specified - append variant to image name and assume dockerfile
# exists with same suffix (e.g. image node-env-debian built from Dockerfile-debian)
imgname="$imgname-$imgvariant"
dockerfile="Dockerfile-$imgvariant"
fi
echo "Building $envdir -> $imgname:$version using $dockerfile"
pushd $DIR/environments/$envdir/builder
docker build -t fission/$imgname:$version -f $dockerfile .
docker tag fission/$imgname:$version fission/$imgname:latest
popd
}
build_all_env_builders() {
local version=$1
# call with version, env dir, image name base, image name variant
build_env_builder_image "$version" "python" "python-builder" ""
build_env_builder_image "$version" "binary" "binary-builder" ""
build_env_builder_image "$version" "go" "go-builder" ""
}
build_charts() {
local version=$1
mkdir -p $BUILDDIR/charts
pushd $DIR/charts
find . -iname *.~?~ | xargs -r rm
for c in fission-all fission-core
do
# https://github.com/kubernetes/helm/issues/1732
helm init --client-only
helm package $c/
mv *.tgz $BUILDDIR/charts/
done
popd
}
build_yamls() {
local version=$1
mkdir -p ${BUILDDIR}/yamls
pushd ${DIR}/charts
find . -iname *.~?~ | xargs rm
for c in fission-all fission-core
do
# for minikube and other environment not support ELB
helm template ${c} -n ${c}-${version} --set serviceType=NodePort,routerServiceType=NodePort > ${c}-${version}-minikube.yaml
# for environments support ELB
helm template ${c} -n ${c}-${version} > ${c}-${version}.yaml
mv *.yaml ${BUILDDIR}/yamls/
done
popd
}
build_all() {
local version=$1
if [ -z "$version" ]
then
echo "Version unspecified"
exit 1
fi
local date=$2
if [ -z "$date" ]
then
echo "Build date unspecified"
exit 1
fi
local gitcommit=$3
if [ -z "gitcommit" ]
then
echo "Git commit unspecified"
exit 1
fi
if [ -e $BUILDDIR ]
then
echo "Removing existing build dir ($BUILDDIR)."
rm -rf $BUILDDIR
fi
mkdir -p $BUILDDIR
build_fission_bundle_image $version $date $gitcommit
build_fetcher_image $version $date $gitcommit
build_builder_image $version $date $gitcommit
build_logger_image $version
build_all_cli $version $date $gitcommit
build_charts $version
build_pre_upgrade_checks_image $version $date $gitcommit
}
version=${VERSION}
date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
gitcommit=$(git rev-parse HEAD)
build_all $version $date $gitcommit
build_all_envs $version
build_all_env_builders $version
build_charts $version
build_yamls $version
+75 -239
View File
@@ -26,61 +26,6 @@ check_clean() {
fi
}
# Build CLI binaries for mac/linux/windows
build_all_cli() {
local version=$1
local date=$2
local gitcommit=$3
build_cli "linux" "linux" $version $date $gitcommit
build_cli "darwin" "osx" $version $date $gitcommit
build_cli "windows" "windows" $version $date $gitcommit
}
# Build cli binary for one OS, and put it in $BUILDDIR/cli/<os>/
build_cli() {
os=$1
osName=$2
local version=$3
local date=$4
local gitcommit=$5
arch="amd64" # parameterize if/when we need to
pushd $DIR/fission
if [ "$osName" == "windows" ]
then
binary=fission-cli-${osName}.exe
else
binary=fission-cli-${osName}
fi
GOOS=$os GOARCH=$arch go build -gcflags=-trimpath=$GOPATH -asmflags=-trimpath=$GOPATH -ldflags "-X github.com/fission/fission.GitCommit=$gitcommit -X github.com/fission/fission.BuildDate=$date -X github.com/fission/fission.Version=$version" -o $binary .
outdir=$BUILDDIR/cli/$osName/
mkdir -p $outdir
mv $binary $outdir
popd
}
# Build fission-bundle image
build_fission_bundle_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/fission-bundle:$version
pushd $DIR/fission-bundle
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/fission-bundle:latest
popd
}
# Push fission-bundle image
push_fission_bundle_image() {
local version=$1
@@ -88,41 +33,12 @@ push_fission_bundle_image() {
docker push $tag
}
build_fetcher_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/fetcher:$version
pushd $DIR/environments/fetcher/cmd
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/fetcher:latest
popd
}
push_fetcher_image() {
local version=$1
local tag=fission/fetcher:$version
docker push $tag
}
build_builder_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/builder:$version
pushd $DIR/builder/cmd
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/builder:latest
popd
}
push_builder_image() {
local version=$1
@@ -130,30 +46,13 @@ push_builder_image() {
docker push $tag
}
build_logger_image() {
local version=$1
local tag=fission/fluentd:$version
pushd $DIR/logger/fluentd
docker build -t $tag .
docker tag $tag fission/fluentd:latest
popd
}
push_logger_image() {
local version=$1
local tag=fission/fluentd:$version
docker push $tag
}
build_and_push_logger_image() {
build_logger_image $1
push_logger_image $1
}
build_and_push_env_image() {
push_env_image() {
local version=$1
envdir=$2
imgnamebase=$3
@@ -163,45 +62,35 @@ build_and_push_env_image() {
then
# no variant specified, just use the base name
imgname=$imgnamebase
dockerfile="Dockerfile"
else
# variant specified - append variant to image name and assume dockerfile
# exists with same suffix (e.g. image node-env-debian built from Dockerfile-debian)
imgname="$imgname-$imgvariant"
dockerfile="Dockerfile-$imgvariant"
fi
echo "Building $envdir -> $imgname:$version using $dockerfile"
pushd $DIR/environments/$envdir
if [ -f build.sh ]
then
./build.sh
fi
docker build -t fission/$imgname:$version -f $dockerfile .
docker tag fission/$imgname:$version fission/$imgname:latest
echo "Pushing $envdir -> $imgname:$version"
docker push fission/$imgname:$version
docker push fission/$imgname:latest
popd
}
build_and_push_all_envs() {
push_all_envs() {
local version=$1
# call with version, env dir, image name base, image name variant
build_and_push_env_image "$version" "nodejs" "node-env" ""
build_and_push_env_image "$version" "nodejs" "node-env" "debian"
build_and_push_env_image "$version" "binary" "binary-env" ""
build_and_push_env_image "$version" "dotnet" "dotnet-env" ""
build_and_push_env_image "$version" "dotnet20" "dotnet20-env" ""
build_and_push_env_image "$version" "go" "go-env" ""
build_and_push_env_image "$version" "perl" "perl-env" ""
build_and_push_env_image "$version" "php7" "php-env" ""
build_and_push_env_image "$version" "python" "python-env" ""
build_and_push_env_image "$version" "python" "python-env" "2.7"
build_and_push_env_image "$version" "ruby" "ruby-env" ""
push_env_image "$version" "nodejs" "node-env" ""
push_env_image "$version" "nodejs" "node-env" "debian"
push_env_image "$version" "binary" "binary-env" ""
push_env_image "$version" "dotnet" "dotnet-env" ""
push_env_image "$version" "dotnet20" "dotnet20-env" ""
push_env_image "$version" "go" "go-env" ""
push_env_image "$version" "perl" "perl-env" ""
push_env_image "$version" "php7" "php-env" ""
push_env_image "$version" "python" "python-env" ""
push_env_image "$version" "python" "python-env" "2.7"
push_env_image "$version" "ruby" "ruby-env" ""
}
build_and_push_env_builder_image() {
push_env_builder_image() {
local version=$1
envdir=$2
imgnamebase=$3
@@ -211,79 +100,24 @@ build_and_push_env_builder_image() {
then
# no variant specified, just use the base name
imgname=$imgnamebase
dockerfile="Dockerfile"
else
# variant specified - append variant to image name and assume dockerfile
# exists with same suffix (e.g. image node-env-debian built from Dockerfile-debian)
imgname="$imgname-$imgvariant"
dockerfile="Dockerfile-$imgvariant"
fi
echo "Building $envdir -> $imgname:$version using $dockerfile"
echo "Pushing $envdir -> $imgname:$version"
pushd $DIR/environments/$envdir/builder
docker build -t fission/$imgname:$version -f $dockerfile .
docker tag fission/$imgname:$version fission/$imgname:latest
docker push fission/$imgname:$version
docker push fission/$imgname:latest
popd
}
build_and_push_all_env_builders() {
push_all_env_builders() {
local version=$1
# call with version, env dir, image name base, image name variant
build_and_push_env_builder_image "$version" "python" "python-builder" ""
build_and_push_env_builder_image "$version" "binary" "binary-builder" ""
build_and_push_env_builder_image "$version" "go" "go-builder" ""
}
build_charts() {
local version=$1
mkdir -p $BUILDDIR/charts
pushd $DIR/charts
find . -iname *.~?~ | xargs rm
for c in fission-all fission-core
do
helm package $c/
mv *.tgz $BUILDDIR/charts/
done
popd
}
build_yamls() {
local version=$1
mkdir -p ${BUILDDIR}/yamls
pushd ${DIR}/charts
find . -iname *.~?~ | xargs rm
for c in fission-all fission-core
do
# for minikube and other environment not support ELB
helm template ${c} -n ${c}-${version} --set serviceType=NodePort,routerServiceType=NodePort > ${c}-${version}-minikube.yaml
# for environments support ELB
helm template ${c} -n ${c}-${version} > ${c}-${version}.yaml
mv *.yaml ${BUILDDIR}/yamls/
done
popd
}
# Build pre-upgrade-checks image
build_pre_upgrade_checks_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/pre-upgrade-checks:$version
pushd $DIR/preupgradechecks
./build.sh $version $date $gitcommit
docker build -t $tag .
docker tag $tag fission/pre-upgrade-checks:latest
popd
push_env_builder_image "$version" "python" "python-builder" ""
push_env_builder_image "$version" "binary" "binary-builder" ""
push_env_builder_image "$version" "go" "go-builder" ""
}
# Push pre-upgrade-checks image
@@ -293,48 +127,6 @@ push_pre_upgrade_checks_image() {
docker push $tag
}
build_all() {
local version=$1
if [ -z "$version" ]
then
echo "Version unspecified"
exit 1
fi
local date=$2
if [ -z "$date" ]
then
echo "Build date unspecified"
exit 1
fi
local gitcommit=$3
if [ -z "gitcommit" ]
then
echo "Git commit unspecified"
exit 1
fi
if [ -e $BUILDDIR ]
then
echo "Removing existing build dir ($BUILDDIR)."
rm -rf $BUILDDIR
fi
mkdir -p $BUILDDIR
build_fission_bundle_image $version $date $gitcommit
build_fetcher_image $version $date $gitcommit
build_builder_image $version $date $gitcommit
build_logger_image $version
build_all_cli $version $date $gitcommit
build_charts $version
build_pre_upgrade_checks_image $version $date $gitcommit
}
push_all() {
local version=$1
push_fission_bundle_image $version
@@ -451,7 +243,17 @@ attach_github_release_yamls() {
--name ${c}-${version}.yaml \
--file $BUILDDIR/yamls/${c}-${version}.yaml
done
}
update_github_charts_repo() {
local version=$1
local chartsrepo=$2
pushd $chartsrepo
cp $BUILDDIR/charts/fission-all-${version}.tgz .
cp $BUILDDIR/charts/fission-core-${version}.tgz .
./index.sh
popd
}
generate_changelog() {
@@ -494,26 +296,60 @@ create_downloads_table () {
}
export -f create_downloads_table
release_environment_check() {
local version=$1
local chartsrepo=$2
check_branch $version
check_clean
if [ ! -f $HOME/.gh-access-token ]
then
echo "Error finding github access token at ${HOME}/.gh-access-token."
exit 1
fi
if [ ! -d $chartsrepo ]
then
echo "Error finding chart repo at $GOPATH/src/github.com/fission/fission-charts"
exit 1
fi
}
export GITHUB_TOKEN=$(cat ~/.gh-access-token)
version=$1
date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
gitcommit=$(git rev-parse HEAD)
chartsrepo=$2
check_branch $version
check_clean
if [ -z $chartsrepo ]
then
chartsrepo="$DIR../fission-charts"
fi
release_environment_check $version $chartsrepo
# Build release-builder image
docker build -t fission-release-builder .
# Build all binaries & container images in docker
# Here we mount docker.sock into container so that docker client can communicate with host docker daemon.
# For more detail please visit https://docs.docker.com/machine/overview/
docker run --rm -v $GOPATH/src:/go/src -v /var/run/docker.sock:/var/run/docker.sock \
-e VERSION=$version -w "/go/src/github.com/fission/fission/hack" fission-release-builder sh -c "./release-build.sh"
build_all $version $date $gitcommit
push_all $version
build_and_push_all_envs $version
build_and_push_all_env_builders $version
build_charts $version
build_yamls $version
push_all_envs $version
push_all_env_builders $version
tag_and_release $version
attach_github_release_cli $version
attach_github_release_charts $version
attach_github_release_yamls $version
update_github_charts_repo $version $chartsrepo
generate_changelog $version
echo "############ DONE #############"
echo "Congratulation, ${version} is ready to ship !!"
echo "Don't forget to push chart repo changes & update CHANGELOG.md"
echo "##############################"