diff --git a/.travis.yml b/.travis.yml index 0f94e3dd..728fcf21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ sudo: required dist: trusty +branches: + only: + - master + language: go go: @@ -18,6 +22,7 @@ services: before_install: - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 install: - go get github.com/Masterminds/glide diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index 8a1a8983..b0b05779 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -129,6 +129,19 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--controllerPort", "8888"] + readinessProbe: + httpGet: + path: "/healthz" + port: 8888 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/healthz" + port: 8888 + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -151,6 +164,19 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] + readinessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -196,6 +222,19 @@ spec: value: "{{ .Values.pullPolicy }}" - name: RUNTIME_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" + readinessProbe: + httpGet: + path: "/healthz" + port: 8888 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/healthz" + port: 8888 + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -535,6 +574,19 @@ spec: volumeMounts: - name: fission-storage mountPath: /fission + readinessProbe: + httpGet: + path: "/healthz" + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/healthz" + port: 8000 + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc volumes: - name: fission-storage diff --git a/charts/fission-core/templates/deployment.yaml b/charts/fission-core/templates/deployment.yaml index 13b25b5e..b3552f02 100644 --- a/charts/fission-core/templates/deployment.yaml +++ b/charts/fission-core/templates/deployment.yaml @@ -129,6 +129,19 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--controllerPort", "8888"] + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -151,6 +164,19 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] + readinessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -194,6 +220,19 @@ spec: value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}" - name: FETCHER_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 35 + periodSeconds: 5 serviceAccount: fission-svc --- diff --git a/common.go b/common.go index 9aa8caca..5c7d444b 100644 --- a/common.go +++ b/common.go @@ -18,9 +18,25 @@ package fission import ( "fmt" + "os" + "os/signal" + "runtime/debug" + "syscall" ) func UrlForFunction(name string) string { prefix := "/fission-function" return fmt.Sprintf("%v/%v", prefix, name) } + +func SetupStackTraceHandler() { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + fmt.Println("Received SIGTERM : Dumping stack trace") + debug.PrintStack() + os.Exit(1) + }() +} diff --git a/controller/api.go b/controller/api.go index 919ff227..00bdd150 100644 --- a/controller/api.go +++ b/controller/api.go @@ -128,8 +128,13 @@ func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request api.respondWithError(w, err) } +func (api *API) HealthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (api *API) Serve(port int) { r := mux.NewRouter() + r.HandleFunc("/healthz", api.HealthHandler).Methods("GET") // Give a useful error message if an older CLI attempts to make a request r.HandleFunc(`/v1/{rest:[a-zA-Z0-9=\-\/]+}`, api.ApiVersionMismatchHandler) r.HandleFunc("/", api.HomeHandler) diff --git a/controller/controller.go b/controller/controller.go index d7cabd70..ba9afd95 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -19,10 +19,14 @@ package controller import ( "log" + "github.com/fission/fission" "github.com/fission/fission/crd" ) func Start(port int) { + // setup a signal handler for SIGTERM + fission.SetupStackTraceHandler() + fc, _, apiExtClient, err := crd.MakeFissionClient() if err != nil { log.Fatalf("Failed to connect to K8s API: %v", err) diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go index 52c20eb1..099e9d64 100644 --- a/environments/fetcher/cmd/main.go +++ b/environments/fetcher/cmd/main.go @@ -10,15 +10,32 @@ import ( "net/http" "net/url" "os" + "os/signal" + "runtime/debug" "strconv" + "syscall" "time" "github.com/fission/fission" "github.com/fission/fission/environments/fetcher" ) +func dumpStackTrace() { + debug.PrintStack() +} + // Usage: fetcher func main() { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + log.Println("Received SIGTERM : Dumping stack trace") + dumpStackTrace() + os.Exit(1) + }() + flag.Usage = fetcherUsage specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup") fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request") @@ -42,7 +59,10 @@ func main() { } } - fetcher := fetcher.MakeFetcher(dir, *secretDir, *configDir) + fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir) + if err != nil { + log.Fatalf("Error making fetcher: %v", err) + } if *specializeOnStart { specializePod(fetcher, fetchPayload, loadPayload) @@ -54,6 +74,8 @@ func main() { mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) + + log.Println("Fetcher ready to receive requests") http.ListenAndServe(":8000", mux) } diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 6ce11576..56d28c93 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -74,14 +74,14 @@ func makeVolumeDir(dirPath string) { } } -func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) *Fetcher { +func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) { makeVolumeDir(sharedVolumePath) makeVolumeDir(sharedSecretPath) makeVolumeDir(sharedConfigPath) fissionClient, kubeClient, _, err := crd.MakeFissionClient() if err != nil { - return nil + return nil, err } return &Fetcher{ sharedVolumePath: sharedVolumePath, @@ -89,7 +89,7 @@ func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigP sharedConfigPath: sharedConfigPath, fissionClient: fissionClient, kubeClient: kubeClient, - } + }, nil } func downloadUrl(url string, localPath string) error { diff --git a/executor/api.go b/executor/api.go index e5cc7a8e..c43a50d4 100644 --- a/executor/api.go +++ b/executor/api.go @@ -99,10 +99,15 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (executor *Executor) Serve(port int) { r := mux.NewRouter() r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST") r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST") + r.HandleFunc("/healthz", executor.healthHandler).Methods("GET") address := fmt.Sprintf(":%v", port) log.Printf("starting executor at port %v", port) ctx, cancel := context.WithCancel(context.Background()) diff --git a/executor/executor.go b/executor/executor.go index fb287dee..213c2bb2 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -18,6 +18,7 @@ package executor import ( "log" + "runtime/debug" "strings" "sync" "time" @@ -181,9 +182,16 @@ func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment return env, nil } +func dumpStackTrace() { + debug.PrintStack() +} + // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types func StartExecutor(fissionNamespace string, functionNamespace string, port int) error { + // setup a signal handler for SIGTERM + fission.SetupStackTraceHandler() + fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() restClient := fissionClient.GetCrdClient() if err != nil { diff --git a/executor/poolmgr/gp.go b/executor/poolmgr/gp.go index bfec9df8..a306942b 100644 --- a/executor/poolmgr/gp.go +++ b/executor/poolmgr/gp.go @@ -519,6 +519,33 @@ func (gp *GenericPool) createPool() error { "-secret-dir", gp.sharedSecretPath, "-cfgmap-dir", gp.sharedCfgMapPath, gp.sharedMountPath}, + ReadinessProbe: &apiv1.Probe{ + InitialDelaySeconds: 1, + PeriodSeconds: 1, + FailureThreshold: 30, + Handler: apiv1.Handler{ + HTTPGet: &apiv1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8000, + }, + }, + }, + }, + LivenessProbe: &apiv1.Probe{ + InitialDelaySeconds: 35, + PeriodSeconds: 5, + Handler: apiv1.Handler{ + HTTPGet: &apiv1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8000, + }, + }, + }, + }, }, }, ServiceAccountName: "fission-fetcher", @@ -528,6 +555,7 @@ func (gp *GenericPool) createPool() error { } depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment) if err != nil { + log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err) return err } gp.deployment = depl diff --git a/router/httpTriggers.go b/router/httpTriggers.go index e7061915..6cc33cbc 100644 --- a/router/httpTriggers.go +++ b/router/httpTriggers.go @@ -89,6 +89,10 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +func routerHealthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter := mux.NewRouter() @@ -150,6 +154,9 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler) } + // Healthz endpoint for the router. + muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET") + return muxRouter } diff --git a/router/router.go b/router/router.go index e9c2fc08..d004a978 100644 --- a/router/router.go +++ b/router/router.go @@ -50,6 +50,7 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/fission/fission" "github.com/fission/fission/crd" executorClient "github.com/fission/fission/executor/client" ) @@ -72,6 +73,9 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv } func Start(port int, executorUrl string) { + // setup a signal handler for SIGTERM + fission.SetupStackTraceHandler() + fmap := makeFunctionServiceMap(time.Minute) fissionClient, _, _, err := crd.MakeFissionClient() diff --git a/storagesvc/storagesvc.go b/storagesvc/storagesvc.go index 4f253f71..f7f2a1f8 100644 --- a/storagesvc/storagesvc.go +++ b/storagesvc/storagesvc.go @@ -25,6 +25,7 @@ import ( "strconv" "time" + "github.com/fission/fission" "github.com/gorilla/handlers" "github.com/gorilla/mux" _ "github.com/graymeta/stow/local" @@ -149,6 +150,10 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request } } +func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func MakeStorageService(storageClient *StowClient, port int) *StorageService { return &StorageService{ storageClient: storageClient, @@ -161,12 +166,16 @@ func (ss *StorageService) Start(port int) { r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST") r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET") r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE") + r.HandleFunc("/healthz", ss.healthHandler).Methods("GET") address := fmt.Sprintf(":%v", port) log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r))) } func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService { + // setup a signal handler for SIGTERM + fission.SetupStackTraceHandler() + // initialize logger log.SetLevel(log.InfoLevel) diff --git a/test/test_utils.sh b/test/test_utils.sh index f660409a..fe9899db 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -59,7 +59,7 @@ build_and_push_fission_bundle() { pushd $ROOT/fission-bundle ./build.sh - docker build -t $image_tag . + docker build -q -t $image_tag . gcloud_login @@ -72,7 +72,7 @@ build_and_push_fetcher() { pushd $ROOT/environments/fetcher/cmd ./build.sh - docker build -t $image_tag . + docker build -q -t $image_tag . gcloud_login @@ -86,7 +86,7 @@ build_and_push_builder() { pushd $ROOT/builder/cmd ./build.sh - docker build -t $image_tag . + docker build -q -t $image_tag . gcloud_login @@ -98,7 +98,7 @@ build_and_push_fluentd(){ image_tag=$1 pushd $ROOT/logger/fluentd - docker build -t $image_tag . + docker build -q -t $image_tag . gcloud_login @@ -112,7 +112,7 @@ build_and_push_env_runtime() { image_tag=$2 pushd $ROOT/environments/$env/ - docker build -t $image_tag . + docker build -q -t $image_tag . gcloud_login @@ -127,7 +127,7 @@ build_and_push_env_builder() { pushd $ROOT/environments/$env/builder - docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} . + docker build -q -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} . gcloud_login @@ -174,14 +174,19 @@ helm_install_fission() { echo "Deleting old releases" helm list -q|xargs -I@ bash -c "helm_uninstall_fission @" + # deleting ns does take a while after command is issued + while `kubectl get ns| grep fission-builder` + do + sleep 5 + done + echo "Installing fission" helm install \ --wait \ - --timeout 600 \ + --timeout 540 \ --name $id \ --set $helmVars \ --namespace $ns \ - --debug \ $ROOT/charts/fission-all helm list @@ -190,30 +195,65 @@ helm_install_fission() { wait_for_service() { id=$1 svc=$2 + health_endpoint=$3 ns=f-$id + retry=0 + max_retries=5 while true do - ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}') - if [ ! -z $ip ] - then - break - fi - echo Waiting for service $svc... - sleep 1 + retry=$((retry+1)) + if ((retry == max_retries)); then + echo "Waiting for $svc to be routable exceeded max retries. Quitting.." + exit 1 + fi + ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}') + if [ -z $ip ]; then + continue + fi + http_status=`curl -sw "%{http_code}" "http://$ip/$health_endpoint"` + echo "http_status for svc $svc : $http_status" + if [ "$http_status" -ne "200" ]; then + echo "Service $svc returned response other than 200. waiting for 200 after backing off for 1 second" + sleep 1 + else + break + fi done } wait_for_services() { id=$1 - wait_for_service $id controller - wait_for_service $id router - - echo Waiting for service is routable... - sleep 10 + echo "\n--- wait for controller and router services to be routable ---" + wait_for_service $id controller "healthz" + wait_for_service $id router "router-healthz" + echo "\n--- end wait for controller and router services to be routable ---" } +dump_kubernetes_events() { + id=$1 + ns=f-$id + fns=f-func-$id + echo "--- kubectl events $fns ---" + kubectl get events -n $fns + echo "--- end kubectl events $fns ---" + + echo "--- kubectl events $ns ---" + kubectl get events -n $ns + echo "--- end kubectl events $ns ---" +} +export -f dump_kubernetes_events + +dump_tiller_logs() { + echo "--- tiller logs ---" + tiller_pod=`kubectl get pods -n kube-system | grep tiller| tr -s " "| cut -d" " -f1` + kubectl logs $tiller_pod --since=30m -n kube-system + echo "--- end tiller logs ---" +} +export -f dump_tiller_logs + + helm_uninstall_fission() {(set +e id=$1 @@ -222,10 +262,10 @@ helm_uninstall_fission() {(set +e echo "Fission uninstallation skipped" return fi + echo "Uninstalling fission" helm delete --purge $id - - kubectl delete ns f-$id + kubectl delete ns f-$id || true )} export -f helm_uninstall_fission @@ -243,7 +283,7 @@ set_environment() { dump_builder_pod_logs() { bns=$1 builderPods=$(kubectl -n $bns get pod -o name) - + for p in $builderPods do echo "--- builder pod logs $p ---" @@ -313,11 +353,30 @@ dump_env_pods() { echo --- End environment pods --- } +describe_pods_ns() { + echo "--- describe pods $1---" + kubectl describe pods -n $1 + echo "--- End describe pods $1 ---" +} + +describe_all_pods() { + id=$1 + ns=f-$id + fns=f-func-$id + bns=fission-builder + + describe_pods_ns $ns + describe_pods_ns $fns + describe_pods_ns $bns +} + dump_all_fission_resources() { ns=$1 echo "--- All objects in the fission namespace $ns ---" - kubectl -n $ns get all + kubectl -n $ns get pods -o wide + echo "" + kubectl -n $ns get svc echo "--- End objects in the fission namespace $ns ---" } @@ -343,11 +402,17 @@ dump_logs() { dump_fission_logs $ns $fns router dump_fission_logs $ns $fns buildermgr dump_fission_logs $ns $fns executor + dump_fission_logs $ns $fns storagesvc dump_function_pod_logs $ns $fns dump_builder_pod_logs $bns dump_fission_crds } +log() { + echo `date +%Y/%m/%d:%H:%M:%S`" $1" +} + +export -f log export FAILURES=0 run_all_tests() { @@ -402,7 +467,9 @@ install_and_test() { trap "helm_uninstall_fission $id" EXIT if ! helm_install_fission $id $image $imageTag $fetcherImage $fetcherImageTag $controllerPort $routerPort $fluentdImage $fluentdImageTag $pruneInterval then - dump_logs $id + describe_all_pods $id + dump_kubernetes_events $id + dump_tiller_logs exit 1 fi @@ -417,6 +484,8 @@ install_and_test() { if [ $FAILURES -ne 0 ] then + # describe each pod in fission ns and function namespace + describe_all_pods $id exit 1 fi } diff --git a/test/tests/test_archive_pruner.sh b/test/tests/test_archive_pruner.sh index cc85ea02..7d7b5f38 100755 --- a/test/tests/test_archive_pruner.sh +++ b/test/tests/test_archive_pruner.sh @@ -16,7 +16,7 @@ cleanup() { } create_archive() { - echo "Creating an archive" + log "Creating an archive" mkdir test_dir dd if=/dev/urandom of=test_dir/dynamically_generated_file bs=256k count=1 printf 'def main():\n return "Hello, world!"' > test_dir/hello.py @@ -24,17 +24,17 @@ create_archive() { } create_package() { - echo "Creating package" + log "Creating package" pkg=$(fission package create --deploy "test-deploy-pkg.zip" --env python| cut -f2 -d' '| tr -d \') } delete_package() { - echo "Deleting package: $1" + log "Deleting package: $1" fission package delete --name $1 } get_archive_url_from_package() { - echo "Getting archive URL from package: $1" + log "Getting archive URL from package: $1" url=`kubectl get package $1 -ojsonpath='{.spec.deployment.url}'` } @@ -42,76 +42,76 @@ get_archive_from_storage() { http_status=`curl -sw "%{http_code}" $1 -o /tmp/file` } -#1. declare trap to cleanup for all the required signals -#2. create an archives with large files such that total size of archive is > 256KB +#1. declare trap to cleanup for EXIT +#2. create an archive with large files such that total size of archive is > 256KB #3. create 2 pkgs referencing those archives #4. delete both the packages #5. verify archives are not recycled . this handles the case where archives are just created but not referenced by pkgs yet. #6. sleep for two minutes -#7. now verify that both get deleted. +#7. now verify that both got deleted. main() { # trap trap cleanup EXIT # create a huge archive create_archive - echo "created archive test-deploy-pkg.zip" + log "created archive test-deploy-pkg.zip" # create packages with the huge archive create_package pkg_1=$pkg get_archive_url_from_package $pkg_1 url_1=$url - echo "pkg: $pkg_1, archive_url : $url_1" + log "pkg: $pkg_1, archive_url : $url_1" create_package pkg_2=$pkg get_archive_url_from_package $pkg_2 url_2=$url - echo "pkg: $pkg_2, archive_url : $url_2" + log "pkg: $pkg_2, archive_url : $url_2" # delete packages delete_package $pkg_1 delete_package $pkg_2 - echo "deleted packages : $pkg_1 $pkg_2" + log "deleted packages : $pkg_1 $pkg_2" # curl on the archive url get_archive_from_storage $url_1 - echo "http_status for $url_1 : $http_status" + log "http_status for $url_1 : $http_status" if [ "$http_status" -ne "200" ]; then - echo "Archive $url_1 absent on storage, while expected to be present" + log "Archive $url_1 absent on storage, while expected to be present" exit 1 fi # curl on the archive url get_archive_from_storage $url_2 - echo "http_status for $url_2 : $http_status" + log "http_status for $url_2 : $http_status" if [ "$http_status" -ne "200" ]; then - echo "Archive $url_2 absent on storage, while expected to be present" + log "Archive $url_2 absent on storage, while expected to be present" exit 1 fi # archivePruner is set to run every minute for test. In production, its set to run every hour. - echo "waiting for packages to get recycled" + log "waiting for packages to get recycled" sleep 120 # curl on the archive url get_archive_from_storage $url_1 - echo "http_status for $url_1 : $http_status" + log "http_status for $url_1 : $http_status" if [ "$http_status" -ne "404" ]; then - echo "Archive $url_1 should have been recycled, but curl returned $http_status, while expected status is 404." + log "Archive $url_1 should have been recycled, but curl returned $http_status, while expected status is 404." exit 1 fi # curl on the archive url get_archive_from_storage $url_2 - echo "http_status for $url_2 : $http_status" + log "http_status for $url_2 : $http_status" if [ "$http_status" -ne "404" ]; then - echo "Archive $url_2 should have been recycled, but curl returned $http_status, while expected status is 404." + log "Archive $url_2 should have been recycled, but curl returned $http_status, while expected status is 404." exit 1 fi - echo "Test archive pruner PASSED" + log "Test archive pruner PASSED" } main \ No newline at end of file diff --git a/test/tests/test_backend_newdeploy.sh b/test/tests/test_backend_newdeploy.sh index 1ec11b70..d889386a 100755 --- a/test/tests/test_backend_newdeploy.sh +++ b/test/tests/test_backend_newdeploy.sh @@ -5,51 +5,51 @@ set -euo pipefail ROOT=$(dirname $0)/../.. # Create a hello world function in nodejs, test it with an http trigger -echo "NewDeploy ExecutorType: Pre-test cleanup" +log "NewDeploy ExecutorType: Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256 trap "fission env delete --name nodejs" EXIT # TODO Imporve test code by reusing common blocks -echo "Creating function, testing for cold start with MinScale 0" +log "Creating function, testing for cold start with MinScale 0" fn0=nodejs-hello-$(date +%N) fission fn create --name $fn0 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 0 --maxscale 4 --executortype newdeploy trap "fission fn delete --name $fn0" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn0 --url /$fn0 --method GET -echo "Waiting for router & newdeploy deployment creation" +log "Waiting for router & newdeploy deployment creation" sleep 5 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response0=$(curl http://$FISSION_ROUTER/$fn0) -echo "Checking for valid response" +log "Checking for valid response" echo $response0 | grep -i hello -echo "Creating function, testing for warm start with MinScale 1" +log "Creating function, testing for warm start with MinScale 1" fn1=nodejs-hello-$(date +%N) fission fn create --name $fn1 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 1 --maxscale 4 --executortype newdeploy trap "fission fn delete --name $fn1" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn1 --url /$fn1 --method GET -echo "Waiting for router & newdeploy deployment creation" +log "Waiting for router & newdeploy deployment creation" sleep 5 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response1=$(curl http://$FISSION_ROUTER/$fn0) -echo "Checking for valid response" +log "Checking for valid response" echo $response1 | grep -i hello # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "NewDeploy ExecutorType: All done." \ No newline at end of file +log "NewDeploy ExecutorType: All done." \ No newline at end of file diff --git a/test/tests/test_backend_poolmgr.sh b/test/tests/test_backend_poolmgr.sh index f6999c20..7f018c13 100755 --- a/test/tests/test_backend_poolmgr.sh +++ b/test/tests/test_backend_poolmgr.sh @@ -7,30 +7,30 @@ ROOT=$(dirname $0)/../.. fn=nodejs-hello-$(date +%N) # Create a hello world function in nodejs, test it with an http trigger -echo "Poolmgr ExecutorType: Pre-test cleanup" +log "Poolmgr ExecutorType: Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256 trap "fission env delete --name nodejs" EXIT -echo "Creating function" +log "Creating function" fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js --executortype poolmgr trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 5 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$fn) -echo "Checking for valid response" +log "Checking for valid response" echo $response | grep -i hello # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "Poolmgr ExecutorType: All done." \ No newline at end of file +log "Poolmgr ExecutorType: All done." \ No newline at end of file diff --git a/test/tests/test_buildermgr.sh b/test/tests/test_buildermgr.sh index 6eba91be..911341c7 100755 --- a/test/tests/test_buildermgr.sh +++ b/test/tests/test_buildermgr.sh @@ -15,16 +15,16 @@ PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test fn=python-srcbuild-$(date +%s) checkFunctionResponse() { - echo "Doing an HTTP GET on the function's route" + log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$1) - echo "Checking for valid response" - echo $response + log "Checking for valid response" + log $response echo $response | grep -i "a: 1 b: {c: 3, d: 4}" } waitBuild() { - echo "Waiting for builder manager to finish the build" + log "Waiting for builder manager to finish the build" while true; do kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded @@ -39,7 +39,7 @@ waitEnvBuilder() { env=$1 envRV=$(kubectl -n default get environments ${env} -o jsonpath='{.metadata.resourceVersion}') - echo "Waiting for env builder to catch up" + log "Waiting for env builder to catch up" while true; do kubectl -n fission-builder get pod -l envName=${env},envResourceVersion=${envRV} \ @@ -51,27 +51,27 @@ waitEnvBuilder() { } export -f waitEnvBuilder -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name python || true kubectl --namespace default get packages|grep -v NAME|awk '{print $1}'|xargs -I@ bash -c 'kubectl --namespace default delete packages @' || true -echo "Creating python env" +log "Creating python env" fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE trap "fission env delete --name python" EXIT timeout 180s bash -c "waitEnvBuilder python" -echo "Creating source pacakage" +log "Creating source pacakage" zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/ -echo "Creating function " $fn +log "Creating function " $fn fission fn create --name $fn --env python --src demo-src-pkg.zip --entrypoint "user.main" --buildcmd "./build.sh" trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 3 pkg=$(kubectl --namespace default get functions $fn -o jsonpath='{.spec.package.packageref.name}') @@ -81,7 +81,7 @@ timeout 60s bash -c "waitBuild $pkg" checkFunctionResponse $fn -echo "Updating function " $fn +log "Updating function " $fn fission fn update --name $fn --src demo-src-pkg.zip trap "fission fn delete --name $fn" EXIT @@ -95,4 +95,4 @@ checkFunctionResponse $fn # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "All done." +log "All done." diff --git a/test/tests/test_function_update.sh b/test/tests/test_function_update.sh index 2a9eeb59..a0351636 100755 --- a/test/tests/test_function_update.sh +++ b/test/tests/test_function_update.sh @@ -10,28 +10,28 @@ fn=nodejs-hello-$(date +%s) # Update it and check it's output, the output should be # different from the previous one. -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env trap "fission env delete --name nodejs" EXIT -echo "Creating function" +log "Creating function" echo 'module.exports = function(context, callback) { callback(200, "foo!\n"); }' > foo.js fission fn create --name $fn --env nodejs --code foo.js trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 10 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$fn) -echo "Checking for valid response" +log "Checking for valid response" echo $response | grep -i foo # Running a background process to keep access the @@ -40,18 +40,18 @@ echo $response | grep -i foo ( watch -n1 curl http://$FISSION_ROUTER/$fn ) > /dev/null 2>&1 & pid=$! -echo "Updating function" +log "Updating function" echo 'module.exports = function(context, callback) { callback(200, "bar!\n"); }' > bar.js fission fn update --name $fn --code bar.js trap "fission fn delete --name $fn" EXIT -echo "Waiting for router to update cache" +log "Waiting for router to update cache" sleep 10 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$fn) -echo "Checking for valid response again" +log "Checking for valid response again" echo $response | grep -i bar kill -15 $pid @@ -59,4 +59,4 @@ kill -15 $pid # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "All done." +log "All done." diff --git a/test/tests/test_internal_routes.sh b/test/tests/test_internal_routes.sh index 3fb6a3b7..7f489a78 100755 --- a/test/tests/test_internal_routes.sh +++ b/test/tests/test_internal_routes.sh @@ -9,38 +9,38 @@ set -euo pipefail ROOT=$(dirname $0)/../.. -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env trap "fission env delete --name nodejs" EXIT -echo "Writing functions" +log "Writing functions" f1=f1-$(date +%s) f2=f2-$(date +%s) -echo $f1 $f2 +log $f1 $f2 for f in $f1 $f2 do echo "module.exports = function(context, callback) { callback(200, \"$f\n\"); }" > $f.js done -echo "Creating functions" +log "Creating functions" for f in $f1 $f2 do fission fn create --name $f --env nodejs --code $f.js trap "fission fn delete --name $f" EXIT done -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 2 -echo "Testing internal routes" +log "Testing internal routes" for f in $f1 $f2 do response=$(curl http://$FISSION_ROUTER/fission-function/$f) echo $response | grep $f done -echo "All done." +log "All done." diff --git a/test/tests/test_logging/test_function_logs.sh b/test/tests/test_logging/test_function_logs.sh index 6c28474a..af7034c9 100755 --- a/test/tests/test_logging/test_function_logs.sh +++ b/test/tests/test_logging/test_function_logs.sh @@ -8,39 +8,39 @@ ROOT=$(dirname $0)/../.. fn=nodejs-logtest-$(date +%N) function cleanup { - echo "Cleanup route" + log "Cleanup route" var=$(fission route list | grep $fn | awk '{print $1;}') fission route delete --name $var - echo "delete logfile" + log "delete logfile" rm "/tmp/logfile" } # Create a hello world function in nodejs, test it with an http trigger -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env trap "fission env delete --name nodejs" EXIT -echo "Creating function" +log "Creating function" fission fn create --name $fn --env nodejs --code log.js trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET trap cleanup EXIT -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 15 -echo "Doing 4 HTTP GETs on the function's route" +log "Doing 4 HTTP GETs on the function's route" for i in 1 2 3 4 do curl -s http://$FISSION_ROUTER/$fn done -echo "Grabbing logs, should have 4 calls in logs" +log "Grabbing logs, should have 4 calls in logs" sleep 15 @@ -52,15 +52,15 @@ then fission function logs --name $fn --detail > /tmp/logfile fi -echo "---function logs---" +log "---function logs---" cat /tmp/logfile -echo "------" +log "------" num=$(grep 'log test' /tmp/logfile | wc -l) -echo $num logs found +log $num logs found if [ $num -ne 4 ] then - echo "Test Failed: expected 4, found $num logs" + log "Test Failed: expected 4, found $num logs" fi - -echo "All done." + +log "All done." diff --git a/test/tests/test_node_hello_http.sh b/test/tests/test_node_hello_http.sh index 9e8caaf1..a89bcb76 100755 --- a/test/tests/test_node_hello_http.sh +++ b/test/tests/test_node_hello_http.sh @@ -7,30 +7,30 @@ ROOT=$(dirname $0)/../.. fn=nodejs-hello-$(date +%N) # Create a hello world function in nodejs, test it with an http trigger -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name nodejs || true -echo "Creating nodejs env" +log "Creating nodejs env" fission env create --name nodejs --image fission/node-env trap "fission env delete --name nodejs" EXIT -echo "Creating function" +log "Creating function" fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 3 -echo "Doing an HTTP GET on the function's route" +log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$fn) -echo "Checking for valid response" +log "Checking for valid response" echo $response | grep -i hello # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "All done." +log "All done." diff --git a/test/tests/test_package_command.sh b/test/tests/test_package_command.sh index 9e1c6ee5..dc8ea52e 100755 --- a/test/tests/test_package_command.sh +++ b/test/tests/test_package_command.sh @@ -14,7 +14,7 @@ PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test fn=python-srcbuild-$(date +%s) waitBuild() { - echo "Waiting for builder manager to finish the build" + log "Waiting for builder manager to finish the build" while true; do kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded @@ -26,11 +26,11 @@ waitBuild() { export -f waitBuild checkFunctionResponse() { - echo "Doing an HTTP GET on the function's route" + log "Doing an HTTP GET on the function's route" response=$(curl http://$FISSION_ROUTER/$1) - echo "Checking for valid response" - echo $response + log "Checking for valid response" + log $response echo $response | grep -i "$2" } @@ -38,7 +38,7 @@ waitEnvBuilder() { env=$1 envRV=$(kubectl -n default get environments ${env} -o jsonpath='{.metadata.resourceVersion}') - echo "Waiting for env builder to catch up" + log "Waiting for env builder to catch up" while true; do kubectl -n fission-builder get pod -l envName=${env},envResourceVersion=${envRV} \ @@ -50,46 +50,46 @@ waitEnvBuilder() { } export -f waitEnvBuilder -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name python || true -echo "Creating python env" +log "Creating python env" fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE trap "fission env delete --name python" EXIT timeout 180s bash -c "waitEnvBuilder python" -echo "Creating pacakage with source archive" +log "Creating pacakage with source archive" zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/ pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \') # wait for build to finish at most 60s timeout 60s bash -c "waitBuild $pkgName" -echo "Creating function " $fn +log "Creating function " $fn fission fn create --name $fn --pkg $pkgName --entrypoint "user.main" trap "fission fn delete --name $fn" EXIT -echo "Creating route" +log "Creating route" fission route create --function $fn --url /$fn --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 3 checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}' -echo "Creating package with deploy archive" +log "Creating package with deploy archive" mkdir testDir touch testDir/__init__.py printf 'def main():\n return "Hello, world!"' > testDir/hello.py zip -jr demo-deploy-pkg.zip testDir/ pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \') -echo "Updating function " $fn +log "Updating function " $fn fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main" trap "fission fn delete --name $fn" EXIT -echo "Waiting for router to update cache" +log "Waiting for router to update cache" sleep 3 checkFunctionResponse $fn 'Hello, world!' @@ -97,4 +97,4 @@ checkFunctionResponse $fn 'Hello, world!' # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger -echo "All done." +log "All done." diff --git a/test/tests/test_pass.sh b/test/tests/test_pass.sh index 94d848c1..a0a3b326 100755 --- a/test/tests/test_pass.sh +++ b/test/tests/test_pass.sh @@ -5,8 +5,8 @@ set -euo pipefail # This doesn't test fission, just the test framework. It ensures we # have the right environment, that's all. -echo "Test test, please ignore." +log "Test test, please ignore." -echo $FISSION_URL -echo $FISSION_ROUTER +log $FISSION_URL +log $FISSION_ROUTER which fission diff --git a/test/tests/test_secret_cfgmap/test_secret_cfgmap.sh b/test/tests/test_secret_cfgmap/test_secret_cfgmap.sh index 20a2933e..4b5dc34f 100755 --- a/test/tests/test_secret_cfgmap/test_secret_cfgmap.sh +++ b/test/tests/test_secret_cfgmap/test_secret_cfgmap.sh @@ -15,7 +15,7 @@ cp cfgmap.py.template cfgmap.py sed -i "s/{{ FN_CFGMAP }}/${fn_cfgmap}/g" cfgmap.py function cleanup { - echo "Cleanup everything" + log "Cleanup everything" kubectl delete secret -n default ${fn_secret} kubectl delete configmap -n default ${fn_cfgmap} fission function delete --name ${fn_secret} @@ -30,84 +30,84 @@ function cleanup { } # Create a hello world function in nodejs, test it with an http trigger -echo "Pre-test cleanup" +log "Pre-test cleanup" fission env delete --name python || true -echo "Creating python env" +log "Creating python env" fission env create --name python --image fission/python-env trap "fission env delete --name python" EXIT -echo "Creating secret" +log "Creating secret" kubectl create secret generic ${fn_secret} --from-literal=TEST_KEY="TESTVALUE" -n default trap "kubectl delete secret ${fn_secret} -n default" EXIT -echo "Creating function with secret" +log "Creating function with secret" fission fn create --name ${fn_secret} --env python --code secret.py --secret ${fn_secret} trap "fission fn delete --name ${fn_secret}" EXIT -echo "Creating route" +log "Creating route" fission route create --function ${fn_secret} --url /${fn_secret} --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 5 -echo "HTTP GET on the function's route" +log "HTTP GET on the function's route" res=$(curl http://${FISSION_ROUTER}/${fn_secret}) val='TESTVALUE' if [[ ${res} != ${val} ]] then - echo "test secret failed" + log "test secret failed" cleanup exit 1 fi -echo "test secret passed" +log "test secret passed" -echo "Creating configmap" +log "Creating configmap" kubectl create configmap ${fn_cfgmap} --from-literal=TEST_KEY=TESTVALUE -n default trap "kubectl delete configmap ${fn_cfgmap} -n default" EXIT -echo "creating function with configmap" +log "creating function with configmap" fission fn create --name ${fn_cfgmap} --env python --code cfgmap.py --configmap ${fn_cfgmap} trap "fission fn delete --name ${fn_cfgmap}" EXIT -echo "Creating route" +log "Creating route" fission route create --function ${fn_cfgmap} --url /${fn_cfgmap} --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 5 -echo "HTTP GET on the function's route" +log "HTTP GET on the function's route" rescfg=$(curl http://${FISSION_ROUTER}/${fn_cfgmap}) if [ ${rescfg} != ${val} ] then - echo "test cfgmap failed" + log "test cfgmap failed" cleanup exit 1 fi -echo "test configmap passed" +log "test configmap passed" -echo "testing creating a function without a secret or configmap" +log "testing creating a function without a secret or configmap" fission function create --name ${fn} --env python --code empty.py trap "fission fn delete --name ${fn}" EXIT -echo "Creating route" +log "Creating route" fission route create --function ${fn} --url /${fn} --method GET -echo "Waiting for router to catch up" +log "Waiting for router to catch up" sleep 5 -echo "HTTP GET on the function's route" +log "HTTP GET on the function's route" resnormal=$(curl http://${FISSION_ROUTER}/${fn}) if [ ${resnormal} != "yes" ] then - echo "test empty failed" + log "test empty failed" cleanup exit 1 fi -echo "test empty passed" +log "test empty passed" -echo "All done." +log "All done." trap "cleanup" EXIT