467 lines
10 KiB
Bash
Executable File
467 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
#
|
|
# Test runner. Shell scripts that build fission CLI and server, push a
|
|
# docker image to GCR, deploy it on a cluster, and run tests against
|
|
# that deployment.
|
|
#
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_RELPATH=$(dirname $0)/..
|
|
pushd $ROOT_RELPATH
|
|
ROOT=$(pwd)
|
|
popd
|
|
|
|
export TEST_REPORT=""
|
|
|
|
report_msg() {
|
|
TEST_REPORT="$TEST_REPORT\n$1"
|
|
}
|
|
report_test_passed() {
|
|
report_msg "--- PASSED $1"
|
|
}
|
|
report_test_failed() {
|
|
report_msg "*** FAILED $1"
|
|
}
|
|
report_test_skipped() {
|
|
report_msg "### SKIPPED $1"
|
|
}
|
|
show_test_report() {
|
|
echo -e "------\n$TEST_REPORT\n------"
|
|
}
|
|
|
|
helm_setup() {
|
|
helm init
|
|
# wait for tiller ready
|
|
while true; do
|
|
kubectl --namespace kube-system get pod|grep tiller|grep Running
|
|
if [[ $? -eq 0 ]]; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
export -f helm_setup
|
|
|
|
gcloud_login() {
|
|
KEY=${HOME}/gcloud-service-key.json
|
|
if [ ! -f $KEY ]
|
|
then
|
|
echo $FISSION_CI_SERVICE_ACCOUNT | base64 -d - > $KEY
|
|
fi
|
|
|
|
gcloud auth activate-service-account --key-file $KEY
|
|
}
|
|
|
|
build_and_push_fission_bundle() {
|
|
image_tag=$1
|
|
|
|
pushd $ROOT/fission-bundle
|
|
./build.sh
|
|
docker build -t $image_tag .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
}
|
|
|
|
build_and_push_fetcher() {
|
|
image_tag=$1
|
|
|
|
pushd $ROOT/environments/fetcher/cmd
|
|
./build.sh
|
|
docker build -t $image_tag .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
}
|
|
|
|
|
|
build_and_push_builder() {
|
|
image_tag=$1
|
|
|
|
pushd $ROOT/builder/cmd
|
|
./build.sh
|
|
docker build -t $image_tag .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
}
|
|
|
|
build_and_push_fluentd(){
|
|
image_tag=$1
|
|
|
|
pushd $ROOT/logger/fluentd
|
|
docker build -t $image_tag .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
|
|
}
|
|
|
|
build_and_push_env_runtime() {
|
|
env=$1
|
|
image_tag=$2
|
|
|
|
pushd $ROOT/environments/$env/
|
|
docker build -t $image_tag .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
}
|
|
|
|
build_and_push_env_builder() {
|
|
env=$1
|
|
image_tag=$2
|
|
builder_image=$3
|
|
|
|
pushd $ROOT/environments/$env/builder
|
|
|
|
docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} .
|
|
|
|
gcloud_login
|
|
|
|
gcloud docker -- push $image_tag
|
|
popd
|
|
}
|
|
|
|
build_fission_cli() {
|
|
pushd $ROOT/fission
|
|
go build .
|
|
popd
|
|
}
|
|
|
|
clean_tpr_crd_resources() {
|
|
# clean tpr & crd resources to avoid testing error (ex. no kind "HttptriggerList" is registered for version "fission.io/v1")
|
|
# thirdpartyresources part should be removed after kubernetes test cluster is upgrade to 1.8+
|
|
kubectl --namespace default get thirdpartyresources| grep -v NAME| grep "fission.io"| awk '{print $1}'|xargs -I@ bash -c "kubectl --namespace default delete thirdpartyresources @" || true
|
|
kubectl --namespace default get crd| grep -v NAME| grep "fission.io"| awk '{print $1}'|xargs -I@ bash -c "kubectl --namespace default delete crd @" || true
|
|
}
|
|
|
|
generate_test_id() {
|
|
echo $(date|md5sum|cut -c1-6)
|
|
}
|
|
|
|
helm_install_fission() {
|
|
id=$1
|
|
image=$2
|
|
imageTag=$3
|
|
fetcherImage=$4
|
|
fetcherImageTag=$5
|
|
controllerNodeport=$6
|
|
routerNodeport=$7
|
|
fluentdImage=$8
|
|
fluentdImageTag=$9
|
|
pruneInterval="${10}"
|
|
|
|
ns=f-$id
|
|
fns=f-func-$id
|
|
|
|
helmVars=image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval
|
|
|
|
timeout 30 bash -c "helm_setup"
|
|
|
|
echo "Deleting old releases"
|
|
helm list -q|xargs -I@ bash -c "helm_uninstall_fission @"
|
|
|
|
echo "Installing fission"
|
|
helm install \
|
|
--wait \
|
|
--timeout 540 \
|
|
--name $id \
|
|
--set $helmVars \
|
|
--namespace $ns \
|
|
--debug \
|
|
$ROOT/charts/fission-all
|
|
|
|
helm list
|
|
}
|
|
|
|
wait_for_service() {
|
|
id=$1
|
|
svc=$2
|
|
health_endpoint=$3
|
|
|
|
ns=f-$id
|
|
retry=0
|
|
max_retries=5
|
|
while true
|
|
do
|
|
retry=$((retry+1))
|
|
if ((retry == max_retries)); then
|
|
echo "Waiting for $svc to be routable exceeded max retries. Quitting.."
|
|
exit 1
|
|
fi
|
|
ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
|
|
if [ -z $ip ]; then
|
|
continue
|
|
fi
|
|
echo "IP for $svc : $ip"
|
|
http_status=`curl -sw "%{http_code}" "http://$ip/$health_endpoint"`
|
|
echo "http_status for svc $svc : $http_status"
|
|
if [ "$http_status" -ne "200" ]; then
|
|
echo "Service $svc returned response other than 200. waiting for 200 after backing off for 1 second"
|
|
sleep 1
|
|
else
|
|
break
|
|
fi
|
|
done
|
|
}
|
|
|
|
wait_for_services() {
|
|
id=$1
|
|
|
|
wait_for_service $id controller "healthz"
|
|
wait_for_service $id router "router-healthz"
|
|
|
|
echo "Controller and router services are routable"
|
|
}
|
|
|
|
dump_kubernetes_events() {
|
|
id=$1
|
|
ns=f-$id
|
|
fns=f-func-$id
|
|
echo "--- kubectl events $fns ---"
|
|
kubectl get events -n $fns
|
|
echo "--- end kubectl events $fns ---"
|
|
|
|
echo "--- kubectl events $ns ---"
|
|
kubectl get events -n $ns
|
|
echo "--- end kubectl events $ns ---"
|
|
}
|
|
|
|
dump_tiller_logs() {
|
|
echo "--- tiller logs ---"
|
|
tiller_pod=`kubectl get pods -n kube-system | grep tiller| tr -s " "| cut -d" " -f1`
|
|
kubectl logs $tiller_pod -n kube-system
|
|
echo "--- end tiller logs ---"
|
|
}
|
|
|
|
helm_uninstall_fission() {(set +e
|
|
id=$1
|
|
|
|
if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ]
|
|
then
|
|
echo "Fission uninstallation skipped"
|
|
return
|
|
fi
|
|
|
|
dump_kubernetes_events $id
|
|
dump_tiller_logs
|
|
|
|
echo "Uninstalling fission"
|
|
helm delete --purge $id
|
|
kubectl delete ns f-$id
|
|
)}
|
|
export -f helm_uninstall_fission
|
|
|
|
set_environment() {
|
|
id=$1
|
|
ns=f-$id
|
|
|
|
export FISSION_URL=http://$(kubectl -n $ns get svc controller -o jsonpath='{...ip}')
|
|
export FISSION_ROUTER=$(kubectl -n $ns get svc router -o jsonpath='{...ip}')
|
|
|
|
# set path to include cli
|
|
export PATH=$ROOT/fission:$PATH
|
|
}
|
|
|
|
dump_builder_pod_logs() {
|
|
bns=$1
|
|
builderPods=$(kubectl -n $bns get pod -o name)
|
|
|
|
for p in $builderPods
|
|
do
|
|
echo "--- builder pod logs $p ---"
|
|
containers=$(kubectl -n $bns get $p -o jsonpath={.spec.containers[*].name} --ignore-not-found)
|
|
for c in $containers
|
|
do
|
|
echo "--- builder pod logs $p: container $c ---"
|
|
kubectl -n $bns logs $p $c || true
|
|
echo "--- end builder pod logs $p: container $c ---"
|
|
done
|
|
echo "--- end builder pod logs $p ---"
|
|
done
|
|
|
|
}
|
|
|
|
dump_function_pod_logs() {
|
|
ns=$1
|
|
fns=$2
|
|
|
|
functionPods=$(kubectl -n $fns get pod -o name -l functionName)
|
|
for p in $functionPods
|
|
do
|
|
echo "--- function pod logs $p ---"
|
|
containers=$(kubectl -n $fns get $p -o jsonpath={.spec.containers[*].name} --ignore-not-found)
|
|
for c in $containers
|
|
do
|
|
echo "--- function pod logs $p: container $c ---"
|
|
kubectl -n $fns logs $p $c || true
|
|
echo "--- end function pod logs $p: container $c ---"
|
|
done
|
|
echo "--- end function pod logs $p ---"
|
|
done
|
|
}
|
|
|
|
dump_fission_logs() {
|
|
ns=$1
|
|
fns=$2
|
|
component=$3
|
|
|
|
echo --- $component logs ---
|
|
kubectl -n $ns get pod -o name | grep $component | xargs kubectl -n $ns logs
|
|
echo --- end $component logs ---
|
|
}
|
|
|
|
dump_fission_crd() {
|
|
type=$1
|
|
echo --- All objects of type $type ---
|
|
kubectl --all-namespaces=true get $type -o yaml
|
|
echo --- End objects of type $type ---
|
|
}
|
|
|
|
dump_fission_crds() {
|
|
dump_fission_crd environments.fission.io
|
|
dump_fission_crd functions.fission.io
|
|
dump_fission_crd httptriggers.fission.io
|
|
dump_fission_crd kuberneteswatchtriggers.fission.io
|
|
dump_fission_crd messagequeuetriggers.fission.io
|
|
dump_fission_crd packages.fission.io
|
|
dump_fission_crd timetriggers.fission.io
|
|
}
|
|
|
|
dump_env_pods() {
|
|
fns=$1
|
|
|
|
echo --- All environment pods ---
|
|
kubectl -n $fns get pod -o yaml
|
|
echo --- End environment pods ---
|
|
}
|
|
|
|
dump_all_fission_resources() {
|
|
ns=$1
|
|
|
|
echo "--- All objects in the fission namespace $ns ---"
|
|
kubectl -n $ns get all
|
|
echo "--- End objects in the fission namespace $ns ---"
|
|
}
|
|
|
|
dump_system_info() {
|
|
echo "--- System Info ---"
|
|
go version
|
|
docker version
|
|
kubectl version
|
|
helm version
|
|
echo "--- End System Info ---"
|
|
}
|
|
|
|
dump_logs() {
|
|
id=$1
|
|
|
|
ns=f-$id
|
|
fns=f-func-$id
|
|
bns=fission-builder
|
|
|
|
dump_all_fission_resources $ns
|
|
dump_env_pods $fns
|
|
dump_fission_logs $ns $fns controller
|
|
dump_fission_logs $ns $fns router
|
|
dump_fission_logs $ns $fns buildermgr
|
|
dump_fission_logs $ns $fns executor
|
|
dump_function_pod_logs $ns $fns
|
|
dump_builder_pod_logs $bns
|
|
dump_fission_crds
|
|
}
|
|
|
|
export FAILURES=0
|
|
|
|
run_all_tests() {
|
|
id=$1
|
|
|
|
export FISSION_NAMESPACE=f-$id
|
|
export FUNCTION_NAMESPACE=f-func-$id
|
|
|
|
test_files=$(find $ROOT/test/tests -iname 'test_*.sh')
|
|
|
|
for file in $test_files
|
|
do
|
|
testname=${file#$ROOT/test/tests}
|
|
testpath=$file
|
|
|
|
if grep "^#test:disabled" $file
|
|
then
|
|
report_test_skipped $testname
|
|
echo ------- Skipped $testname -------
|
|
else
|
|
echo ------- Running $testname -------
|
|
pushd $(dirname $testpath)
|
|
if $testpath
|
|
then
|
|
echo SUCCESS: $testname
|
|
report_test_passed $testname
|
|
else
|
|
echo FAILED: $testname
|
|
export FAILURES=$(($FAILURES+1))
|
|
report_test_failed $testname
|
|
fi
|
|
popd
|
|
fi
|
|
done
|
|
}
|
|
|
|
install_and_test() {
|
|
image=$1
|
|
imageTag=$2
|
|
fetcherImage=$3
|
|
fetcherImageTag=$4
|
|
fluentdImage=$5
|
|
fluentdImageTag=$6
|
|
pruneInterval=$7
|
|
|
|
controllerPort=31234
|
|
routerPort=31235
|
|
|
|
clean_tpr_crd_resources
|
|
|
|
id=$(generate_test_id)
|
|
trap "helm_uninstall_fission $id" EXIT
|
|
if ! helm_install_fission $id $image $imageTag $fetcherImage $fetcherImageTag $controllerPort $routerPort $fluentdImage $fluentdImageTag $pruneInterval
|
|
then
|
|
dump_logs $id
|
|
exit 1
|
|
fi
|
|
|
|
wait_for_services $id
|
|
set_environment $id
|
|
|
|
run_all_tests $id
|
|
|
|
dump_logs $id
|
|
|
|
show_test_report
|
|
|
|
if [ $FAILURES -ne 0 ]
|
|
then
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
|
|
# if [ $# -lt 2 ]
|
|
# then
|
|
# echo "Usage: test.sh [image] [imageTag]"
|
|
# exit 1
|
|
# fi
|
|
# install_and_test $1 $2
|