Capture os signals to gracefully shutdown fission components (#2261)

- Currently, fission components don't handle shutdown signals.
So we don't get any to do the required cleanup before the fission process
exits. Adding signal capture process with cancelling context so
that all dependent processes stop working when the process gets term
signal.
- Set log level to error in otel shutdown function

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

cmd/fetcher/app/server.go

@@ -39,7 +39,7 @@ var (
 	readyToServe uint32
 )
 
-func Run(logger *zap.Logger) {
+func Run(ctx context.Context, logger *zap.Logger) {
 	flag.Usage = fetcherUsage
 	collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "")
 	specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
@@ -62,8 +62,6 @@ func Run(logger *zap.Logger) {
 			}
 		}
 	}
-
-	ctx := context.Background()
 	openTracingEnabled := tracing.TracingEnabled(logger)
 	if openTracingEnabled {
 		go func() {

cmd/fetcher/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/fission/fission/cmd/fetcher/app"
 	"github.com/fission/fission/pkg/utils/loggerfactory"
 	"github.com/fission/fission/pkg/utils/profile"
+	"github.com/fission/fission/pkg/utils/signals"
 )
 
 // Usage: fetcher <shared volume path>
@@ -29,5 +30,6 @@ func main() {
 
 	profile.ProfileIfEnabled(logger)
 
-	app.Run(logger)
+	ctx := signals.SetupSignalHandlerWithContext(logger)
+	app.Run(ctx, logger)
 }

cmd/fission-bundle/main.go

@@ -41,27 +41,28 @@ import (
 	"github.com/fission/fission/pkg/utils/loggerfactory"
 	"github.com/fission/fission/pkg/utils/otel"
 	"github.com/fission/fission/pkg/utils/profile"
+	"github.com/fission/fission/pkg/utils/signals"
 	"github.com/fission/fission/pkg/utils/tracing"
 )
 
-func runController(logger *zap.Logger, port int, openTracingEnabled bool) {
+func runController(ctx context.Context, logger *zap.Logger, port int, openTracingEnabled bool) {
-	controller.Start(logger, port, false, openTracingEnabled)
+	controller.Start(ctx, logger, port, false, openTracingEnabled)
 }
 
-func runRouter(logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) {
+func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) {
-	router.Start(logger, port, executorUrl, openTracingEnabled)
+	router.Start(ctx, logger, port, executorUrl, openTracingEnabled)
 }
 
-func runExecutor(logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error {
+func runExecutor(ctx context.Context, logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error {
-	return executor.StartExecutor(logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled)
+	return executor.StartExecutor(ctx, logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled)
 }
 
-func runKubeWatcher(logger *zap.Logger, routerUrl string) error {
+func runKubeWatcher(ctx context.Context, logger *zap.Logger, routerUrl string) error {
-	return kubewatcher.Start(logger, routerUrl)
+	return kubewatcher.Start(ctx, logger, routerUrl)
 }
 
-func runTimer(logger *zap.Logger, routerUrl string) error {
+func runTimer(ctx context.Context, logger *zap.Logger, routerUrl string) error {
-	return timer.Start(logger, routerUrl)
+	return timer.Start(ctx, logger, routerUrl)
 }
 
 func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error {
@@ -69,20 +70,20 @@ func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error {
 }
 
 // KEDA based MessageQueue Trigger Manager
-func runMQManager(logger *zap.Logger, routerURL string) error {
+func runMQManager(ctx context.Context, logger *zap.Logger, routerURL string) error {
-	return mqt.StartScalerManager(logger, routerURL)
+	return mqt.StartScalerManager(ctx, logger, routerURL)
 }
 
-func runStorageSvc(logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error {
+func runStorageSvc(ctx context.Context, logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error {
-	return storagesvc.Start(logger, storage, port, openTracingEnabled)
+	return storagesvc.Start(ctx, logger, storage, port, openTracingEnabled)
 }
 
-func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
+func runBuilderMgr(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
-	return buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace)
+	return buildermgr.Start(ctx, logger, storageSvcUrl, envBuilderNamespace)
 }
 
-func runLogger() {
+func runLogger(ctx context.Context, logger *zap.Logger) {
-	functionLogger.Start()
+	functionLogger.Start(ctx, logger)
 }
 
 func getPort(logger *zap.Logger, portArg interface{}) int {
@@ -129,10 +130,8 @@ func getServiceName(arguments map[string]interface{}) string {
 }
 
 func exitWithSync(logger *zap.Logger) {
-	err := logger.Sync()
+	// Ignore error, safe to ignore as per https://github.com/uber-go/zap/issues/328
-	if err != nil {
+	_ = logger.Sync()
-		logger.Error("failed to sync log", zap.Error(err))
-	}
 	os.Exit(1)
 }
 
@@ -210,7 +209,8 @@ Options:
 		return
 	}
 
-	ctx := context.Background()
+	ctx := signals.SetupSignalHandlerWithContext(logger)
+
 	openTracingEnabled := tracing.TracingEnabled(logger)
 	if openTracingEnabled {
 		err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments))
@@ -238,21 +238,21 @@ Options:
 
 	if arguments["--controllerPort"] != nil {
 		port := getPort(logger, arguments["--controllerPort"])
-		runController(logger, port, openTracingEnabled)
+		runController(ctx, logger, port, openTracingEnabled)
 		logger.Error("controller exited")
 		return
 	}
 
 	if arguments["--routerPort"] != nil {
 		port := getPort(logger, arguments["--routerPort"])
-		runRouter(logger, port, executorUrl, openTracingEnabled)
+		runRouter(ctx, logger, port, executorUrl, openTracingEnabled)
 		logger.Error("router exited")
 		return
 	}
 
 	if arguments["--executorPort"] != nil {
 		port := getPort(logger, arguments["--executorPort"])
-		err = runExecutor(logger, port, functionNs, envBuilderNs, openTracingEnabled)
+		err = runExecutor(ctx, logger, port, functionNs, envBuilderNs, openTracingEnabled)
 		if err != nil {
 			logger.Error("executor exited", zap.Error(err))
 			return
@@ -260,7 +260,7 @@ Options:
 	}
 
 	if arguments["--kubewatcher"] == true {
-		err = runKubeWatcher(logger, routerUrl)
+		err = runKubeWatcher(ctx, logger, routerUrl)
 		if err != nil {
 			logger.Error("kubewatcher exited", zap.Error(err))
 			return
@@ -268,7 +268,7 @@ Options:
 	}
 
 	if arguments["--timer"] == true {
-		err = runTimer(logger, routerUrl)
+		err = runTimer(ctx, logger, routerUrl)
 		if err != nil {
 			logger.Error("timer exited", zap.Error(err))
 			return
@@ -284,7 +284,7 @@ Options:
 	}
 
 	if arguments["--mqt_keda"] == true {
-		err = runMQManager(logger, routerUrl)
+		err = runMQManager(ctx, logger, routerUrl)
 		if err != nil {
 			logger.Error("mqt scaler manager exited", zap.Error(err))
 			return
@@ -292,7 +292,7 @@ Options:
 	}
 
 	if arguments["--builderMgr"] == true {
-		err = runBuilderMgr(logger, storageSvcUrl, envBuilderNs)
+		err = runBuilderMgr(ctx, logger, storageSvcUrl, envBuilderNs)
 		if err != nil {
 			logger.Error("builder manager exited", zap.Error(err))
 			return
@@ -300,7 +300,7 @@ Options:
 	}
 
 	if arguments["--logger"] == true {
-		runLogger()
+		runLogger(ctx, logger)
 		logger.Error("logger exited")
 		return
 	}
@@ -315,11 +315,13 @@ Options:
 		} else if arguments["--storageType"] == string(storagesvc.StorageTypeLocal) {
 			storage = storagesvc.NewLocalStorage("/fission")
 		}
-		err := runStorageSvc(logger, port, storage, openTracingEnabled)
+		err := runStorageSvc(ctx, logger, port, storage, openTracingEnabled)
 		if err != nil {
 			logger.Error("storage service exited", zap.Error(err))
 			return
 		}
 	}
-	select {}
+
+	<-ctx.Done()
+	logger.Error("exiting")
 }

cmd/preupgradechecks/main.go

@@ -17,13 +17,12 @@ limitations under the License.
 package main
 
 import (
-	"context"
-
 	"github.com/docopt/docopt-go"
 	"go.uber.org/zap"
 
 	"github.com/fission/fission/pkg/info"
 	"github.com/fission/fission/pkg/utils/loggerfactory"
+	"github.com/fission/fission/pkg/utils/signals"
 )
 
 func getStringArgWithDefault(arg interface{}, defaultValue string) string {
@@ -59,7 +58,7 @@ Options:
 			zap.Error(err))
 	}
 
-	ctx := context.Background()
+	ctx := signals.SetupSignalHandlerWithContext(logger)
 	crd := crdBackedClient.GetFunctionCRD(ctx)
 	if crd == nil {
 		logger.Info("nothing to do since CRDs are not present on the cluster")

cmd/reporter/app/cmd_event.go

@@ -16,6 +16,7 @@ limitations under the License.
 package app
 
 import (
+	"context"
 	"log"
 
 	"github.com/spf13/cobra"
@@ -49,7 +50,8 @@ func eventCommandHandler(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	return tracker.Tracker.SendEvent(event)
+	ctx := context.Background()
+	return tracker.Tracker.SendEvent(ctx, event)
 }
 
 //EventCommand reports an event to analytics

pkg/buildermgr/buildermgr.go

@@ -17,6 +17,7 @@ limitations under the License.
 package buildermgr
 
 import (
+	"context"
 	"time"
 
 	"github.com/pkg/errors"
@@ -29,7 +30,7 @@ import (
 )
 
 // Start the buildermgr service.
-func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
+func Start(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
 	bmLogger := logger.Named("builder_manager")
 
 	fissionClient, kubernetesClient, _, _, err := crd.MakeFissionClient()
@@ -56,6 +57,6 @@ func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string)
 	pkgInformer := informerFactory.Core().V1().Packages().Informer()
 	pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
 		kubernetesClient, envBuilderNamespace, storageSvcUrl, &podInformer, &pkgInformer)
-	pkgWatcher.Run()
+	pkgWatcher.Run(ctx)
 	return nil
 }

pkg/buildermgr/pkgwatcher.go

@@ -322,11 +322,10 @@ func (pkgw *packageWatcher) packageInformerHandler() k8sCache.ResourceEventHandl
 	}
 }
 
-func (pkgw *packageWatcher) Run() {
+func (pkgw *packageWatcher) Run(ctx context.Context) {
-	context := context.Background()
+	go (*pkgw.podInformer).Run(ctx.Done())
-	go (*pkgw.podInformer).Run(context.Done())
 	(*pkgw.pkgInformer).AddEventHandler(pkgw.packageInformerHandler())
-	(*pkgw.pkgInformer).Run(context.Done())
+	(*pkgw.pkgInformer).Run(ctx.Done())
 }
 
 // setInitialBuildStatus sets initial build status to a package if it is empty.

pkg/controller/api_test.go

@@ -376,7 +376,8 @@ func TestMain(m *testing.M) {
 
 	panicIf(err)
 
-	go Start(logger, 8888, true, true)
+	ctx := context.Background()
+	go Start(ctx, logger, 8888, true, true)
 
 	time.Sleep(5 * time.Second)
 

pkg/controller/controller.go

@@ -24,7 +24,7 @@ import (
 	"github.com/fission/fission/pkg/crd"
 )
 
-func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) {
+func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) {
 	cLogger := logger.Named("controller")
 
 	fc, kc, apiExtClient, _, err := crd.MakeFissionClient()
@@ -42,12 +42,10 @@ func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled b
 		cLogger.Fatal("error waiting for CRDs", zap.Error(err))
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
 	featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
 	if err != nil {
 		cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err))
 	}
-	defer cancel()
 
 	api, err := MakeAPI(cLogger, featureStatus)
 	if err != nil {

pkg/executor/executor.go

@@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) {
 
 // StartExecutor Starts executor and the executor components such as Poolmgr,
 // deploymgr and potential future executor types
-func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
+func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
 	fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient()
 	if err != nil {
 		return errors.Wrap(err, "failed to get kubernetes client")
@@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 	}
 	cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments()
 	cnmSvcInformer := cnmInformerFactory.Core().V1().Services()
-	ctx := context.Background()
 	cnm, err := container.MakeContainer(
 		ctx, logger,
 		fissionClient, kubernetesClient,
@@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 	if err != nil {
 		return err
 	}
-	go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
+	go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
 	go api.Serve(port, openTracingEnabled)
 	go serveMetric(logger)
 

pkg/executor/executor_test.go

@@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) {
 
 	// create poolmgr
 	port := 9999
-	err = StartExecutor(logger, functionNs, "fission-builder", port, true)
+	ctx := context.Background()
+	err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true)
 	if err != nil {
 		log.Panicf("failed to start poolmgr: %v", err)
 	}

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) {
 	if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok {
 		deploy.logger.Fatal("failed to wait for caches to sync")
 	}
-	go deploy.idleObjectReaper()
+	go deploy.idleObjectReaper(ctx)
 }
 
 // GetTypeName returns the executor type name.
@@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin
 }
 
 // idleObjectReaper reaps objects after certain idle time
-func (deploy *NewDeploy) idleObjectReaper() {
+func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) {
-	ctx := context.Background()
 	pollSleep := 5 * time.Second
 	for {
 		time.Sleep(pollSleep)

pkg/executor/reaper/reaper.go

@@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie
 
 // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
 // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
-func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
+func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
-	ctx := context.Background()
 	for {
 		// some sleep before the next reaper iteration
 		time.Sleep(cleanupRoleBindingInterval)

pkg/kubewatcher/kubewatcher.go

@@ -77,7 +77,7 @@ type (
 	}
 )
 
-func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
+func MakeKubeWatcher(ctx context.Context, logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
 	kw := &KubeWatcher{
 		logger:           logger.Named("kube_watcher"),
 		watches:          make(map[types.UID]watchSubscription),
@@ -85,7 +85,7 @@ func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset,
 		publisher:        publisher,
 		requestChannel:   make(chan *kubeWatcherRequest),
 	}
-	go kw.svc()
+	go kw.svc(ctx)
 	return kw
 }
 
@@ -100,7 +100,7 @@ func (kw *KubeWatcher) Sync(watches []fv1.KubernetesWatchTrigger) error {
 	return resp.error
 }
 
-func (kw *KubeWatcher) svc() {
+func (kw *KubeWatcher) svc(ctx context.Context) {
 	for {
 		req := <-kw.requestChannel
 		switch req.requestType {
@@ -118,7 +118,7 @@ func (kw *KubeWatcher) svc() {
 			// Add new watches
 			for _, w := range req.watches {
 				if _, ok := kw.watches[w.ObjectMeta.UID]; !ok {
-					kw.addWatch(&w) //nolint: errCheck
+					kw.addWatch(ctx, &w) //nolint: errCheck
 				}
 			}
 			req.responseChannel <- &kubeWatcherResponse{error: nil}
@@ -149,7 +149,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error {
 	return err
 }
 
-func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) {
+func createKubernetesWatch(ctx context.Context, kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) {
 	var wi watch.Interface
 	var err error
 	var watchTimeoutSec int64 = 120
@@ -163,22 +163,22 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWa
 	// TODO handle the full list of types
 	switch strings.ToUpper(w.Spec.Type) {
 	case "POD":
-		wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(context.TODO(), listOptions)
+		wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(ctx, listOptions)
 	case "SERVICE":
-		wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(context.TODO(), listOptions)
+		wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(ctx, listOptions)
 	case "REPLICATIONCONTROLLER":
-		wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(context.TODO(), listOptions)
+		wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(ctx, listOptions)
 	case "JOB":
-		wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(context.TODO(), listOptions)
+		wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(ctx, listOptions)
 	default:
 		err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type))
 	}
 	return wi, err
 }
 
-func (kw *KubeWatcher) addWatch(w *fv1.KubernetesWatchTrigger) error {
+func (kw *KubeWatcher) addWatch(ctx context.Context, w *fv1.KubernetesWatchTrigger) error {
 	kw.logger.Info("adding watch", zap.String("name", w.ObjectMeta.Name), zap.Any("function", w.Spec.FunctionReference))
-	ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
+	ws, err := MakeWatchSubscription(ctx, kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
 	if err != nil {
 		return err
 	}
@@ -198,7 +198,7 @@ func (kw *KubeWatcher) removeWatch(w *fv1.KubernetesWatchTrigger) error {
 	return nil
 }
 
-func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
+func MakeWatchSubscription(ctx context.Context, logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
 	var stopped int32 = 0
 	ws := &watchSubscription{
 		logger:              logger.Named("watch_subscription"),
@@ -210,16 +210,16 @@ func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, ku
 		lastResourceVersion: "",
 	}
 
-	err := ws.restartWatch()
+	err := ws.restartWatch(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	go ws.eventDispatchLoop()
+	go ws.eventDispatchLoop(ctx)
 	return ws, nil
 }
 
-func (ws *watchSubscription) restartWatch() error {
+func (ws *watchSubscription) restartWatch(ctx context.Context) error {
 	retries := 60
 	for {
 		ws.logger.Info("(re)starting watch",
@@ -227,7 +227,7 @@ func (ws *watchSubscription) restartWatch() error {
 			zap.String("namespace", ws.watch.Spec.Namespace),
 			zap.String("type", ws.watch.Spec.Type),
 			zap.String("last_resource_version", ws.lastResourceVersion))
-		wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
+		wi, err := createKubernetesWatch(ctx, ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
 		if err != nil {
 			retries--
 			if retries > 0 {
@@ -250,7 +250,7 @@ func getResourceVersion(obj runtime.Object) (string, error) {
 	return m.GetResourceVersion(), nil
 }
 
-func (ws *watchSubscription) eventDispatchLoop() {
+func (ws *watchSubscription) eventDispatchLoop(ctx context.Context) {
 	ws.logger.Info("listening to watch", zap.String("name", ws.watch.ObjectMeta.Name))
 	for {
 		// check watchSubscription is stopped or not before waiting for event
@@ -269,7 +269,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
 			} else {
 				// watch closed due to timeout, restart it.
 				ws.logger.Warn("watch timed out - restarting", zap.String("watch_name", ws.watch.ObjectMeta.Name))
-				err := ws.restartWatch()
+				err := ws.restartWatch(ctx)
 				if err != nil {
 					ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name))
 				}
@@ -283,7 +283,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
 			// Start from the beginning to get around "too old resource version"
 			ws.lastResourceVersion = ""
 			time.Sleep(time.Second)
-			err := ws.restartWatch()
+			err := ws.restartWatch(ctx)
 			if err != nil {
 				ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name))
 			}

pkg/kubewatcher/main.go

@@ -17,6 +17,8 @@ limitations under the License.
 package kubewatcher
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
@@ -24,7 +26,7 @@ import (
 	"github.com/fission/fission/pkg/publisher"
 )
 
-func Start(logger *zap.Logger, routerUrl string) error {
+func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error {
 	fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
 	if err != nil {
 		return errors.Wrap(err, "failed to get fission or kubernetes client")
@@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error {
 	}
 
 	poster := publisher.MakeWebhookPublisher(logger, routerUrl)
-	kubeWatch := MakeKubeWatcher(logger, kubeClient, poster)
+	kubeWatch := MakeKubeWatcher(ctx, logger, kubeClient, poster)
 	MakeWatchSync(logger, fissionClient, kubeWatch)
 
 	return nil

pkg/logger/logger.go

@@ -17,6 +17,7 @@ limitations under the License.
 package logger
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -32,7 +33,6 @@ import (
 	fv1 "github.com/fission/fission/pkg/apis/core/v1"
 	"github.com/fission/fission/pkg/crd"
 	"github.com/fission/fission/pkg/utils"
-	"github.com/fission/fission/pkg/utils/loggerfactory"
 )
 
 var nodeName = os.Getenv("NODE_NAME")
@@ -157,10 +157,7 @@ func symlinkReaper(zapLogger *zap.Logger) {
 	}
 }
 
-func Start() {
+func Start(ctx context.Context, logger *zap.Logger) {
-	logger := loggerfactory.GetLogger()
-	defer logger.Sync()
-
 	if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) {
 		logger.Info("symlink path not exist, create it",
 			zap.String("fissionSymlinkPath", fissionSymlinkPath))
@@ -177,6 +174,6 @@ func Start() {
 	informerFactory := k8sInformers.NewSharedInformerFactory(kubernetesClient, time.Minute*30)
 	podInformer := informerFactory.Core().V1().Pods().Informer()
 	podInformer.AddEventHandler(podInformerHandlers(logger))
-	podInformer.Run(make(chan struct{}))
+	podInformer.Run(ctx.Done())
-	logger.Fatal("Stop watching pod changes")
+	logger.Error("Stop watching pod changes")
 }

pkg/mqtrigger/scalermanager.go

@@ -149,7 +149,7 @@ func mqTriggerEventHandlers(logger *zap.Logger, kubeClient *kubernetes.Clientset
 
 // StartScalerManager watches for changes in MessageQueueTrigger and,
 // Based on changes, it Creates, Updates and Deletes Objects of Kind ScaledObjects, AuthenticationTriggers and Deployments
-func StartScalerManager(logger *zap.Logger, routerURL string) error {
+func StartScalerManager(ctx context.Context, logger *zap.Logger, routerURL string) error {
 	fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
 	if err != nil {
 		return err
@@ -161,7 +161,7 @@ func StartScalerManager(logger *zap.Logger, routerURL string) error {
 	informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30)
 	mqTriggerInformer := informerFactory.Core().V1().MessageQueueTriggers().Informer()
 	mqTriggerInformer.AddEventHandler(mqTriggerEventHandlers(logger, kubeClient, routerURL))
-	mqTriggerInformer.Run(context.Background().Done())
+	mqTriggerInformer.Run(ctx.Done())
 	return nil
 }
 

pkg/router/router.go

@@ -126,7 +126,7 @@ func serveMetric(logger *zap.Logger) {
 }
 
 // Start starts a router
-func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
+func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
 	fmap := makeFunctionServiceMap(logger, time.Minute)
 
 	fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
@@ -258,10 +258,8 @@ func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled
 	logger.Info("starting router", zap.Int("port", port))
 
 	tracer := otel.Tracer("router")
-	ctx, span := tracer.Start(context.Background(), "router/Start")
+	ctx, span := tracer.Start(ctx, "router/Start")
 	defer span.End()
 
-	ctxWithCancel, cancel := context.WithCancel(ctx)
+	serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled)
-	defer cancel()
-	serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled)
 }

pkg/storagesvc/archivePruner.go

@@ -75,13 +75,13 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) {
 // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
 // and not the archives that are referenced by them, leaving the archives as orphans.
 // getOrphanArchives reaps the orphaned archives.
-func (pruner *ArchivePruner) getOrphanArchives() {
+func (pruner *ArchivePruner) getOrphanArchives(ctx context.Context) {
 	pruner.logger.Debug("getting orphan archives")
 	archivesRefByPkgs := make([]string, 0)
 	var archiveID string
 
 	// get all pkgs from kubernetes
-	pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
+	pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
 		return
@@ -137,12 +137,12 @@ func (pruner *ArchivePruner) getOrphanArchives() {
 // Start starts a go routine that listens to a channel for archive IDs that need to deleted.
 // Also wakes up at regular intervals to make a list of archive IDs that need to be reaped
 // and sends them over to the channel for deletion
-func (pruner *ArchivePruner) Start() {
+func (pruner *ArchivePruner) Start(ctx context.Context) {
 	ticker := time.NewTicker(pruner.pruneInterval * time.Minute)
 	go pruner.pruneArchives()
 	for range ticker.C {
 		// This method fetches unused archive IDs and sends them to archiveChannel for deletion
 		// silencing the errors, hoping they go away in next iteration.
-		pruner.getOrphanArchives()
+		pruner.getOrphanArchives(ctx)
 	}
 }

pkg/storagesvc/client/storagesvc_test.go

@@ -76,7 +76,7 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource {
 	return resource
 }
 
-func startS3StorageService(endpoint, bucketName, subDir string) {
+func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) {
 	// testID := uniuri.NewLen(8)
 	port := 8081
 
@@ -94,7 +94,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) {
 	os.Setenv("STORAGE_S3_REGION", minioRegion)
 
 	storage := storagesvc.NewS3Storage()
-	_ = storagesvc.Start(logger, storage, port, true)
+	_ = storagesvc.Start(ctx, logger, storage, port, true)
 }
 
 func TestS3StorageService(t *testing.T) {
@@ -135,7 +135,7 @@ func TestS3StorageService(t *testing.T) {
 	// Start storagesvc
 	bucketName := "test-s3-service"
 	subDir := "x/y/z"
-	startS3StorageService(endpoint, bucketName, subDir)
+	startS3StorageService(context.Background(), endpoint, bucketName, subDir)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
@@ -211,7 +211,7 @@ func TestLocalStorageService(t *testing.T) {
 	localPath := fmt.Sprintf("/tmp/%v", testID)
 	_ = os.Mkdir(localPath, os.ModePerm)
 	storage := storagesvc.NewLocalStorage(localPath)
-	_ = storagesvc.Start(logger, storage, port, true)
+	_ = storagesvc.Start(context.Background(), logger, storage, port, true)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))

pkg/storagesvc/storagesvc.go

@@ -17,6 +17,7 @@ limitations under the License.
 package storagesvc
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -221,7 +222,7 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) {
 }
 
 // Start runs storage service
-func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error {
+func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error {
 	enablePruner := true
 	// create a storage client
 	storageClient, err := MakeStowClient(logger, storage)
@@ -244,7 +245,7 @@ func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled boo
 		if err != nil {
 			return errors.Wrap(err, "Error creating archivePruner")
 		}
-		go pruner.Start()
+		go pruner.Start(ctx)
 	}
 
 	logger.Info("storage service started")

pkg/timer/main.go

@@ -17,6 +17,8 @@ limitations under the License.
 package timer
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
@@ -24,7 +26,7 @@ import (
 	"github.com/fission/fission/pkg/publisher"
 )
 
-func Start(logger *zap.Logger, routerUrl string) error {
+func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error {
 	fissionClient, _, _, _, err := crd.MakeFissionClient()
 	if err != nil {
 		return errors.Wrap(err, "failed to get fission or kubernetes client")
@@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error {
 	}
 
 	poster := publisher.MakeWebhookPublisher(logger, routerUrl)
-	MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster))
+	MakeTimerSync(ctx, logger, fissionClient, MakeTimer(logger, poster))
 
 	return nil
 }

pkg/timer/timerSync.go

@@ -35,19 +35,19 @@ type (
 	}
 )
 
-func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
+func MakeTimerSync(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
 	ws := &TimerSync{
 		logger:        logger.Named("timer_sync"),
 		fissionClient: fissionClient,
 		timer:         timer,
 	}
-	go ws.syncSvc()
+	go ws.syncSvc(ctx)
 	return ws
 }
 
-func (ws *TimerSync) syncSvc() {
+func (ws *TimerSync) syncSvc(ctx context.Context) {
 	for {
-		triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
+		triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
 		if err != nil {
 			if utils.IsNetworkError(err) {
 				ws.logger.Info("encountered a network error - will retry", zap.Error(err))

pkg/tracker/tracker.go

@@ -54,7 +54,7 @@ func init() {
 	Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: id.String()}
 }
 
-func (t *tracker) SendEvent(e Event) error {
+func (t *tracker) SendEvent(ctx context.Context, e Event) error {
 	if t.gaPropertyID == "" {
 		return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set")
 	}
@@ -81,7 +81,7 @@ func (t *tracker) SendEvent(e Event) error {
 	}
 
 	buf := bytes.NewBufferString(v.Encode())
-	req, err := http.NewRequest("POST", GA_API_URL, buf)
+	req, err := http.NewRequestWithContext(ctx, "POST", GA_API_URL, buf)
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 	req.Header.Add("User-Agent", "ga-tracker/1.0")
 	if err != nil {

pkg/utils/otel/provider.go

@@ -216,11 +216,11 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
 	return func(ctx context.Context) {
 		err := tracerProvider.Shutdown(ctx)
 		if err != nil && logger != nil {
-			logger.Fatal("error shutting down trace provider", zap.Error(err))
+			logger.Error("error shutting down trace provider", zap.Error(err))
 		}
 		if traceExporter != nil {
 			if err = traceExporter.Shutdown(ctx); err != nil && logger != nil {
-				logger.Fatal("error shutting down trace exporter", zap.Error(err))
+				logger.Error("error shutting down trace exporter", zap.Error(err))
 			}
 		}
 	}, nil

pkg/utils/signals/signals.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2021 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package signals
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go.uber.org/zap"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+
+func SetupSignalHandlerWithContext(logger *zap.Logger) context.Context {
+	var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}
+
+	close(onlyOneSignalHandler) // panics when called twice
+
+	ctx, cancel := context.WithCancel(context.Background())
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, shutdownSignals...)
+	go func() {
+		signal := <-c
+		logger.Info("Received signal", zap.String("signal", signal.String()))
+		cancel()
+		<-c
+		panic("multiple signals received")
+	}()
+
+	return ctx
+}