From bce45d6d25f502aabfe5a218bffa7d603a8e59df Mon Sep 17 00:00:00 2001 From: smruthi2187 Date: Wed, 7 Feb 2018 18:31:24 -0800 Subject: [PATCH] Added sigHandler to print stackTrace, modified probe params --- charts/fission-all/templates/deployment.yaml | 28 +++++++++++-------- controller/controller.go | 17 +++++++++++ environments/fetcher/cmd/main.go | 16 +++++++++++ executor/executor.go | 17 +++++++++++ router/router.go | 16 +++++++++++ storagesvc/storagesvc.go | 16 +++++++++++ test/test_utils.sh | 7 ++--- test/tests/test_logging/test_function_logs.sh | 2 +- 8 files changed, 102 insertions(+), 17 deletions(-) diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index 0298c363..4b76065d 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -133,13 +133,14 @@ spec: httpGet: path: "/healthz" port: 8888 - initialDelaySeconds: 5 - periodSeconds: 2 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 livenessProbe: httpGet: path: "/healthz" port: 8888 - initialDelaySeconds: 16 + initialDelaySeconds: 35 periodSeconds: 5 serviceAccount: fission-svc @@ -167,13 +168,14 @@ spec: httpGet: path: "/router-healthz" port: 8888 - initialDelaySeconds: 5 - periodSeconds: 2 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 livenessProbe: httpGet: path: "/router-healthz" port: 8888 - initialDelaySeconds: 16 + initialDelaySeconds: 35 periodSeconds: 5 serviceAccount: fission-svc @@ -224,13 +226,14 @@ spec: httpGet: path: "/healthz" port: 8888 - initialDelaySeconds: 5 - periodSeconds: 2 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 livenessProbe: httpGet: path: "/healthz" port: 8888 - initialDelaySeconds: 16 + initialDelaySeconds: 35 periodSeconds: 5 serviceAccount: fission-svc @@ -563,13 +566,14 @@ spec: httpGet: path: "/healthz" port: 8000 - initialDelaySeconds: 5 - periodSeconds: 2 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 livenessProbe: httpGet: path: "/healthz" port: 8000 - initialDelaySeconds: 16 + initialDelaySeconds: 35 periodSeconds: 5 serviceAccount: fission-svc volumes: diff --git a/controller/controller.go b/controller/controller.go index d7cabd70..c35924f7 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -18,11 +18,28 @@ package controller import ( "log" + "os/signal" + "syscall" + "os" + "runtime/debug" "github.com/fission/fission/crd" ) +func dumpStackTrace() { + debug.PrintStack() +} + func Start(port int) { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + dumpStackTrace() + os.Exit(1) + }() + fc, _, apiExtClient, err := crd.MakeFissionClient() if err != nil { log.Fatalf("Failed to connect to K8s API: %v", err) diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go index a03efa9f..72b8347f 100644 --- a/environments/fetcher/cmd/main.go +++ b/environments/fetcher/cmd/main.go @@ -12,13 +12,29 @@ import ( "os" "strconv" "time" + "os/signal" + "syscall" + "runtime/debug" "github.com/fission/fission" "github.com/fission/fission/environments/fetcher" ) +func dumpStackTrace() { + debug.PrintStack() +} + // Usage: fetcher func main() { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + dumpStackTrace() + os.Exit(1) + }() + flag.Usage = fetcherUsage specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup") fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request") diff --git a/executor/executor.go b/executor/executor.go index fb287dee..c836acfe 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -21,6 +21,10 @@ import ( "strings" "sync" "time" + "os" + "os/signal" + "syscall" + "runtime/debug" "github.com/dchest/uniuri" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -181,9 +185,22 @@ func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment return env, nil } +func dumpStackTrace() { + debug.PrintStack() +} + // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types func StartExecutor(fissionNamespace string, functionNamespace string, port int) error { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + dumpStackTrace() + os.Exit(1) + }() + fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() restClient := fissionClient.GetCrdClient() if err != nil { diff --git a/router/router.go b/router/router.go index e9c2fc08..f7a33dab 100644 --- a/router/router.go +++ b/router/router.go @@ -46,6 +46,9 @@ import ( "net/http" "os" "time" + "os/signal" + "syscall" + "runtime/debug" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -71,7 +74,20 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr)) } +func dumpStackTrace() { + debug.PrintStack() +} + func Start(port int, executorUrl string) { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + dumpStackTrace() + os.Exit(1) + }() + fmap := makeFunctionServiceMap(time.Minute) fissionClient, _, _, err := crd.MakeFissionClient() diff --git a/storagesvc/storagesvc.go b/storagesvc/storagesvc.go index db744d20..aa1ec89a 100644 --- a/storagesvc/storagesvc.go +++ b/storagesvc/storagesvc.go @@ -24,6 +24,9 @@ import ( "os" "strconv" "time" + "os/signal" + "syscall" + "runtime/debug" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -171,7 +174,20 @@ func (ss *StorageService) Start(port int) { log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r))) } +func dumpStackTrace() { + debug.PrintStack() +} + func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService { + // register signal handler for dumping stack trace. + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM) + go func() { + <-c + dumpStackTrace() + os.Exit(1) + }() + // initialize logger log.SetLevel(log.InfoLevel) diff --git a/test/test_utils.sh b/test/test_utils.sh index a422eac7..40c40440 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -264,9 +264,7 @@ helm_uninstall_fission() {(set +e echo "Uninstalling fission" helm delete --purge $id -# kubectl delete ns f-$id || true -# kubectl delete ns f-func-$id || true -# kubectl delete ns fission-builder || true + kubectl delete ns f-$id || true )} export -f helm_uninstall_fission @@ -284,7 +282,7 @@ set_environment() { dump_builder_pod_logs() { bns=$1 builderPods=$(kubectl -n $bns get pod -o name) - + for p in $builderPods do echo "--- builder pod logs $p ---" @@ -359,6 +357,7 @@ dump_all_fission_resources() { echo "--- All objects in the fission namespace $ns ---" kubectl -n $ns get all + kubectl -n $ns get pods -o wide echo "--- End objects in the fission namespace $ns ---" } diff --git a/test/tests/test_logging/test_function_logs.sh b/test/tests/test_logging/test_function_logs.sh index 6c28474a..56fed61c 100755 --- a/test/tests/test_logging/test_function_logs.sh +++ b/test/tests/test_logging/test_function_logs.sh @@ -62,5 +62,5 @@ if [ $num -ne 4 ] then echo "Test Failed: expected 4, found $num logs" fi - + echo "All done."