Added sigHandler to print stackTrace, modified probe params

charts/fission-all/templates/deployment.yaml

@@ -133,13 +133,14 @@ spec:
           httpGet:
             path: "/healthz"
             port: 8888
-          initialDelaySeconds: 5
+          initialDelaySeconds: 1
-          periodSeconds: 2
+          periodSeconds: 1
+          failureThreshold: 30
         livenessProbe:
           httpGet:
             path: "/healthz"
             port: 8888
-          initialDelaySeconds: 16
+          initialDelaySeconds: 35
           periodSeconds: 5
       serviceAccount: fission-svc
 
@@ -167,13 +168,14 @@ spec:
           httpGet:
             path: "/router-healthz"
             port: 8888
-          initialDelaySeconds: 5
+          initialDelaySeconds: 1
-          periodSeconds: 2
+          periodSeconds: 1
+          failureThreshold: 30
         livenessProbe:
           httpGet:
             path: "/router-healthz"
             port: 8888
-          initialDelaySeconds: 16
+          initialDelaySeconds: 35
           periodSeconds: 5
       serviceAccount: fission-svc
 
@@ -224,13 +226,14 @@ spec:
           httpGet:
             path: "/healthz"
             port: 8888
-          initialDelaySeconds: 5
+          initialDelaySeconds: 1
-          periodSeconds: 2
+          periodSeconds: 1
+          failureThreshold: 30
         livenessProbe:
           httpGet:
             path: "/healthz"
             port: 8888
-          initialDelaySeconds: 16
+          initialDelaySeconds: 35
           periodSeconds: 5
       serviceAccount: fission-svc
 
@@ -563,13 +566,14 @@ spec:
           httpGet:
             path: "/healthz"
             port: 8000
-          initialDelaySeconds: 5
+          initialDelaySeconds: 1
-          periodSeconds: 2
+          periodSeconds: 1
+          failureThreshold: 30
         livenessProbe:
           httpGet:
             path: "/healthz"
             port: 8000
-          initialDelaySeconds: 16
+          initialDelaySeconds: 35
           periodSeconds: 5
       serviceAccount: fission-svc
       volumes:

controller/controller.go

@@ -18,11 +18,28 @@ package controller
 
 import (
 	"log"
+	"os/signal"
+	"syscall"
+	"os"
+	"runtime/debug"
 
 	"github.com/fission/fission/crd"
 )
 
+func dumpStackTrace() {
+	debug.PrintStack()
+}
+
 func Start(port int) {
+	// register signal handler for dumping stack trace.
+	c := make(chan os.Signal, 1)
+    	signal.Notify(c, syscall.SIGTERM)
+    	go func() {
+		<-c
+		dumpStackTrace()
+		os.Exit(1)
+    	}()
+
 	fc, _, apiExtClient, err := crd.MakeFissionClient()
 	if err != nil {
 		log.Fatalf("Failed to connect to K8s API: %v", err)

environments/fetcher/cmd/main.go

@@ -12,13 +12,29 @@ import (
 	"os"
 	"strconv"
 	"time"
+	"os/signal"
+	"syscall"
+	"runtime/debug"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/environments/fetcher"
 )
 
+func dumpStackTrace() {
+	debug.PrintStack()
+}
+
 // Usage: fetcher <shared volume path>
 func main() {
+	// register signal handler for dumping stack trace.
+	c := make(chan os.Signal, 1)
+    	signal.Notify(c, syscall.SIGTERM)
+    	go func() {
+		<-c
+		dumpStackTrace()
+		os.Exit(1)
+    	}()
+
 	flag.Usage = fetcherUsage
 	specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
 	fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request")

executor/executor.go

@@ -21,6 +21,10 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"os"
+	"os/signal"
+	"syscall"
+	"runtime/debug"
 
 	"github.com/dchest/uniuri"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -181,9 +185,22 @@ func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment
 	return env, nil
 }
 
+func dumpStackTrace() {
+	debug.PrintStack()
+}
+
 // StartExecutor Starts executor and the executor components such as Poolmgr,
 // deploymgr and potential future executor types
 func StartExecutor(fissionNamespace string, functionNamespace string, port int) error {
+	// register signal handler for dumping stack trace.
+	c := make(chan os.Signal, 1)
+    	signal.Notify(c, syscall.SIGTERM)
+    	go func() {
+		<-c
+		dumpStackTrace()
+		os.Exit(1)
+    	}()
+
 	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
 	restClient := fissionClient.GetCrdClient()
 	if err != nil {

router/router.go

@@ -46,6 +46,9 @@ import (
 	"net/http"
 	"os"
 	"time"
+	"os/signal"
+	"syscall"
+	"runtime/debug"
 
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
@@ -71,7 +74,20 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
 	http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr))
 }
 
+func dumpStackTrace() {
+	debug.PrintStack()
+}
+
 func Start(port int, executorUrl string) {
+	// register signal handler for dumping stack trace.
+	c := make(chan os.Signal, 1)
+    	signal.Notify(c, syscall.SIGTERM)
+    	go func() {
+		<-c
+		dumpStackTrace()
+		os.Exit(1)
+    	}()
+
 	fmap := makeFunctionServiceMap(time.Minute)
 
 	fissionClient, _, _, err := crd.MakeFissionClient()

storagesvc/storagesvc.go

@@ -24,6 +24,9 @@ import (
 	"os"
 	"strconv"
 	"time"
+	"os/signal"
+	"syscall"
+	"runtime/debug"
 
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
@@ -171,7 +174,20 @@ func (ss *StorageService) Start(port int) {
 	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
 }
 
+func dumpStackTrace() {
+	debug.PrintStack()
+}
+
 func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
+	// register signal handler for dumping stack trace.
+	c := make(chan os.Signal, 1)
+    	signal.Notify(c, syscall.SIGTERM)
+    	go func() {
+		<-c
+		dumpStackTrace()
+		os.Exit(1)
+    	}()
+
 	// initialize logger
 	log.SetLevel(log.InfoLevel)
 

test/test_utils.sh

@@ -264,9 +264,7 @@ helm_uninstall_fission() {(set +e
 
     echo "Uninstalling fission"
     helm delete --purge $id
-#    kubectl delete ns f-$id || true
+    kubectl delete ns f-$id || true
-#    kubectl delete ns f-func-$id || true
-#    kubectl delete ns fission-builder || true
 )}
 export -f helm_uninstall_fission
 
@@ -359,6 +357,7 @@ dump_all_fission_resources() {
 
     echo "--- All objects in the fission namespace $ns ---"
     kubectl -n $ns get all
+    kubectl -n $ns get pods -o wide
     echo "--- End objects in the fission namespace $ns ---"
 }