From 4e445bb3cc2e82aa55fa2e2569ce4f5a624835e0 Mon Sep 17 00:00:00 2001 From: smruthi2187 Date: Mon, 5 Feb 2018 14:17:58 -0800 Subject: [PATCH] All improvements in one commit. --- .travis.yml | 1 + charts/fission-all/templates/deployment.yaml | 36 +++++++++++ charts/fission-core/templates/deployment.yaml | 36 +++++++++++ controller/api.go | 5 ++ environments/fetcher/cmd/main.go | 7 ++- environments/fetcher/fetcher.go | 6 +- executor/api.go | 5 ++ executor/poolmgr/gp.go | 26 ++++++++ router/httpTriggers.go | 7 +++ test/test_utils.sh | 59 +++++++++++++++---- 10 files changed, 172 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f94e3dd..552cd410 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ services: before_install: - sudo apt-get update - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + - sudo sysctl net.ipv6.conf.all.disable_ipv6=0 install: - go get github.com/Masterminds/glide diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index 93cd9a49..ae6b65ba 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -129,6 +129,18 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--controllerPort", "8888"] + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -151,6 +163,18 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] + readinessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -196,6 +220,18 @@ spec: value: "{{ .Values.pullPolicy }}" - name: RUNTIME_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- diff --git a/charts/fission-core/templates/deployment.yaml b/charts/fission-core/templates/deployment.yaml index 13b25b5e..ffcc0096 100644 --- a/charts/fission-core/templates/deployment.yaml +++ b/charts/fission-core/templates/deployment.yaml @@ -129,6 +129,18 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--controllerPort", "8888"] + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -151,6 +163,18 @@ spec: imagePullPolicy: {{ .Values.pullPolicy }} command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] + readinessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/router-healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- @@ -194,6 +218,18 @@ spec: value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}" - name: FETCHER_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" + readinessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 5 + periodSeconds: 2 + livenessProbe: + httpGet: + path: "/healthz" + port: "8888" + initialDelaySeconds: 16 + periodSeconds: 5 serviceAccount: fission-svc --- diff --git a/controller/api.go b/controller/api.go index 919ff227..d890cb18 100644 --- a/controller/api.go +++ b/controller/api.go @@ -128,8 +128,13 @@ func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request api.respondWithError(w, err) } +func (api *API) HealthHandler (w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (api *API) Serve(port int) { r := mux.NewRouter() + r.HandleFunc("/healthz", api.HealthHandler).Methods("GET") // Give a useful error message if an older CLI attempts to make a request r.HandleFunc(`/v1/{rest:[a-zA-Z0-9=\-\/]+}`, api.ApiVersionMismatchHandler) r.HandleFunc("/", api.HomeHandler) diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go index 52c20eb1..a03efa9f 100644 --- a/environments/fetcher/cmd/main.go +++ b/environments/fetcher/cmd/main.go @@ -42,7 +42,10 @@ func main() { } } - fetcher := fetcher.MakeFetcher(dir, *secretDir, *configDir) + fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir) + if err != nil { + log.Fatalf("Error making fetcher: %v", err) + } if *specializeOnStart { specializePod(fetcher, fetchPayload, loadPayload) @@ -54,6 +57,8 @@ func main() { mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) + + log.Println("Fetcher ready to receive requests") http.ListenAndServe(":8000", mux) } diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 6ce11576..56d28c93 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -74,14 +74,14 @@ func makeVolumeDir(dirPath string) { } } -func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) *Fetcher { +func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) { makeVolumeDir(sharedVolumePath) makeVolumeDir(sharedSecretPath) makeVolumeDir(sharedConfigPath) fissionClient, kubeClient, _, err := crd.MakeFissionClient() if err != nil { - return nil + return nil, err } return &Fetcher{ sharedVolumePath: sharedVolumePath, @@ -89,7 +89,7 @@ func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigP sharedConfigPath: sharedConfigPath, fissionClient: fissionClient, kubeClient: kubeClient, - } + }, nil } func downloadUrl(url string, localPath string) error { diff --git a/executor/api.go b/executor/api.go index e5cc7a8e..c43a50d4 100644 --- a/executor/api.go +++ b/executor/api.go @@ -99,10 +99,15 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (executor *Executor) Serve(port int) { r := mux.NewRouter() r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST") r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST") + r.HandleFunc("/healthz", executor.healthHandler).Methods("GET") address := fmt.Sprintf(":%v", port) log.Printf("starting executor at port %v", port) ctx, cancel := context.WithCancel(context.Background()) diff --git a/executor/poolmgr/gp.go b/executor/poolmgr/gp.go index bfec9df8..cf07e3dd 100644 --- a/executor/poolmgr/gp.go +++ b/executor/poolmgr/gp.go @@ -519,6 +519,32 @@ func (gp *GenericPool) createPool() error { "-secret-dir", gp.sharedSecretPath, "-cfgmap-dir", gp.sharedCfgMapPath, gp.sharedMountPath}, + ReadinessProbe: &apiv1.Probe { + InitialDelaySeconds: 5, + PeriodSeconds: 2, + Handler: apiv1.Handler{ + HTTPGet: &apiv1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8000, // TODO : Find out the correct port. + }, + }, + }, + }, + LivenessProbe: &apiv1.Probe { + InitialDelaySeconds: 5, + PeriodSeconds: 5, + Handler: apiv1.Handler{ + HTTPGet: &apiv1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8000, // TODO : Find out the correct port. + }, + }, + }, + }, }, }, ServiceAccountName: "fission-fetcher", diff --git a/router/httpTriggers.go b/router/httpTriggers.go index e7061915..6cc33cbc 100644 --- a/router/httpTriggers.go +++ b/router/httpTriggers.go @@ -89,6 +89,10 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +func routerHealthHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter := mux.NewRouter() @@ -150,6 +154,9 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler) } + // Healthz endpoint for the router. + muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET") + return muxRouter } diff --git a/test/test_utils.sh b/test/test_utils.sh index f660409a..a3a0dfae 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -177,7 +177,7 @@ helm_install_fission() { echo "Installing fission" helm install \ --wait \ - --timeout 600 \ + --timeout 540 \ --name $id \ --set $helmVars \ --namespace $ns \ @@ -190,28 +190,61 @@ helm_install_fission() { wait_for_service() { id=$1 svc=$2 + health_endpoint=$3 ns=f-$id + retry=0 + max_retries=5 while true do - ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}') - if [ ! -z $ip ] - then - break - fi - echo Waiting for service $svc... - sleep 1 + retry=$((retry+1)) + if ((retry == max_retries)); then + echo "Waiting for $svc to be routable exceeded max retries. Quitting.." + exit 1 + fi + ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}') + if [ -z $ip ]; then + continue + fi + echo "IP for $svc : $ip" + http_status=`curl -sw "%{http_code}" "http://$ip/$health_endpoint"` + echo "http_status for svc $svc : $http_status" + if [ "$http_status" -ne "200" ]; then + echo "Service $svc returned response other than 200. waiting for 200 after backing off for 1 second" + sleep 1 + else + break + fi done } wait_for_services() { id=$1 - wait_for_service $id controller - wait_for_service $id router + wait_for_service $id controller "healthz" + wait_for_service $id router "router-healthz" - echo Waiting for service is routable... - sleep 10 + echo "Controller and router services are routable" +} + +dump_kubernetes_events() { + id=$1 + ns=f-$id + fns=f-func-$id + echo "--- kubectl events $fns ---" + kubectl get events -n $fns + echo "--- end kubectl events $fns ---" + + echo "--- kubectl events $ns ---" + kubectl get events -n $ns + echo "--- end kubectl events $ns ---" +} + +dump_tiller_logs() { + echo "--- tiller logs ---" + tiller_pod=`kubectl get pods -n kube-system | grep tiller| tr -s " "| cut -d" " -f1` + kubectl logs $tiller_pod -n kube-system + echo "--- end tiller logs ---" } helm_uninstall_fission() {(set +e @@ -226,6 +259,8 @@ helm_uninstall_fission() {(set +e helm delete --purge $id kubectl delete ns f-$id + dump_kubernetes_events $id + dump_tiller_logs )} export -f helm_uninstall_fission