All improvements in one commit.

.travis.yml

@@ -18,6 +18,7 @@ services:
 before_install:
   - sudo apt-get update
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - sudo sysctl net.ipv6.conf.all.disable_ipv6=0
 
 install:
   - go get github.com/Masterminds/glide

charts/fission-all/templates/deployment.yaml

@@ -129,6 +129,18 @@ spec:
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
         args: ["--controllerPort", "8888"]
+        readinessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+        livenessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---
@@ -151,6 +163,18 @@ spec:
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
         args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
+        readinessProbe:
+          httpGet:
+            path: "/router-healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+         livenessProbe:
+          httpGet:
+            path: "/router-healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---
@@ -196,6 +220,18 @@ spec:
           value: "{{ .Values.pullPolicy }}"
         - name: RUNTIME_IMAGE_PULL_POLICY
           value: "{{ .Values.pullPolicy }}"
+        readinessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+         livenessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---

charts/fission-core/templates/deployment.yaml

@@ -129,6 +129,18 @@ spec:
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
         args: ["--controllerPort", "8888"]
+        readinessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+        livenessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---
@@ -151,6 +163,18 @@ spec:
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
         args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
+        readinessProbe:
+          httpGet:
+            path: "/router-healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+        livenessProbe:
+          httpGet:
+            path: "/router-healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---
@@ -194,6 +218,18 @@ spec:
           value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}"
         - name: FETCHER_IMAGE_PULL_POLICY
           value: "{{ .Values.pullPolicy }}"
+        readinessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 5
+          periodSeconds: 2
+        livenessProbe:
+          httpGet:
+            path: "/healthz"
+            port: "8888"
+          initialDelaySeconds: 16
+          periodSeconds: 5
       serviceAccount: fission-svc
 
 ---

controller/api.go

@@ -128,8 +128,13 @@ func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request
 	api.respondWithError(w, err)
 }
 
+func (api *API) HealthHandler (w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
 func (api *API) Serve(port int) {
 	r := mux.NewRouter()
+	r.HandleFunc("/healthz", api.HealthHandler).Methods("GET")
 	// Give a useful error message if an older CLI attempts to make a request
 	r.HandleFunc(`/v1/{rest:[a-zA-Z0-9=\-\/]+}`, api.ApiVersionMismatchHandler)
 	r.HandleFunc("/", api.HomeHandler)

environments/fetcher/cmd/main.go

@@ -42,7 +42,10 @@ func main() {
 		}
 	}
 
-	fetcher := fetcher.MakeFetcher(dir, *secretDir, *configDir)
+	fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
+	if err != nil {
+		log.Fatalf("Error making fetcher: %v", err)
+	}
 
 	if *specializeOnStart {
 		specializePod(fetcher, fetchPayload, loadPayload)
@@ -54,6 +57,8 @@ func main() {
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
+
+	log.Println("Fetcher ready to receive requests")
 	http.ListenAndServe(":8000", mux)
 }
 

environments/fetcher/fetcher.go

@@ -74,14 +74,14 @@ func makeVolumeDir(dirPath string) {
 	}
 }
 
-func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) *Fetcher {
+func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
 	makeVolumeDir(sharedVolumePath)
 	makeVolumeDir(sharedSecretPath)
 	makeVolumeDir(sharedConfigPath)
 
 	fissionClient, kubeClient, _, err := crd.MakeFissionClient()
 	if err != nil {
-		return nil
+		return nil, err
 	}
 	return &Fetcher{
 		sharedVolumePath: sharedVolumePath,
@@ -89,7 +89,7 @@ func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigP
 		sharedConfigPath: sharedConfigPath,
 		fissionClient:    fissionClient,
 		kubeClient:       kubeClient,
-	}
+	}, nil
 }
 
 func downloadUrl(url string, localPath string) error {

executor/api.go

@@ -99,10 +99,15 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
+func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
 func (executor *Executor) Serve(port int) {
 	r := mux.NewRouter()
 	r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST")
 	r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
+	r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
 	address := fmt.Sprintf(":%v", port)
 	log.Printf("starting executor at port %v", port)
 	ctx, cancel := context.WithCancel(context.Background())

executor/poolmgr/gp.go

@@ -519,6 +519,32 @@ func (gp *GenericPool) createPool() error {
 								"-secret-dir", gp.sharedSecretPath,
 								"-cfgmap-dir", gp.sharedCfgMapPath,
 								gp.sharedMountPath},
+							ReadinessProbe: &apiv1.Probe {
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8000, // TODO : Find out the correct port.
+										},
+									},
+								},
+							},
+							LivenessProbe: &apiv1.Probe {
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       5,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8000, // TODO : Find out the correct port.
+										},
+									},
+								},
+							},
 						},
 					},
 					ServiceAccountName: "fission-fetcher",

router/httpTriggers.go

@@ -89,6 +89,10 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
+func routerHealthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
 func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 	muxRouter := mux.NewRouter()
 
@@ -150,6 +154,9 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
 	}
 
+	// Healthz endpoint for the router.
+	muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET")
+
 	return muxRouter
 }
 

test/test_utils.sh

@@ -177,7 +177,7 @@ helm_install_fission() {
     echo "Installing fission"
     helm install		\
 	 --wait			\
-	 --timeout 600	        \
+	 --timeout 540	        \
 	 --name $id		\
 	 --set $helmVars	\
 	 --namespace $ns        \
@@ -190,28 +190,61 @@ helm_install_fission() {
 wait_for_service() {
     id=$1
     svc=$2
+    health_endpoint=$3
 
     ns=f-$id
+    retry=0
+    max_retries=5
     while true
     do
+        retry=$((retry+1))
+        if ((retry == max_retries)); then
+            echo "Waiting for $svc to be routable exceeded max retries. Quitting.."
+            exit 1
+        fi
         ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
-	if [ ! -z $ip ]
+        if [ -z $ip ]; then
-	then
+            continue
+        fi
+        echo "IP for $svc : $ip"
+        http_status=`curl -sw "%{http_code}" "http://$ip/$health_endpoint"`
+        echo "http_status for svc $svc : $http_status"
+        if [ "$http_status" -ne "200" ]; then
+            echo "Service $svc returned response other than 200. waiting for 200 after backing off for 1 second"
+            sleep 1
+        else
             break
         fi
-	echo Waiting for service $svc...
-	sleep 1
     done
 }
 
 wait_for_services() {
     id=$1
 
-    wait_for_service $id controller
+    wait_for_service $id controller "healthz"
-    wait_for_service $id router
+    wait_for_service $id router "router-healthz"
 
-    echo Waiting for service is routable...
+    echo "Controller and router services are routable"
-    sleep 10
+}
+
+dump_kubernetes_events() {
+    id=$1
+    ns=f-$id
+    fns=f-func-$id
+    echo "--- kubectl events $fns ---"
+    kubectl get events -n $fns
+    echo "--- end kubectl events $fns ---"
+
+    echo "--- kubectl events $ns ---"
+    kubectl get events -n $ns
+    echo "--- end kubectl events $ns ---"
+}
+
+dump_tiller_logs() {
+    echo "--- tiller logs ---"
+    tiller_pod=`kubectl get pods -n kube-system | grep tiller| tr -s " "| cut -d" " -f1`
+    kubectl logs $tiller_pod -n kube-system
+    echo "--- end tiller logs ---"
 }
 
 helm_uninstall_fission() {(set +e
@@ -226,6 +259,8 @@ helm_uninstall_fission() {(set +e
     helm delete --purge $id
 
     kubectl delete ns f-$id
+    dump_kubernetes_events $id
+    dump_tiller_logs
 )}
 export -f helm_uninstall_fission