From c5ae216a4e0ceb5d9425601ef0d87da7d056090c Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Thu, 11 Nov 2021 17:02:11 +0530 Subject: [PATCH] Capture os signals to gracefully shutdown fission components (#2261) - Currently, fission components don't handle shutdown signals. So we don't get any to do the required cleanup before the fission process exits. Adding signal capture process with cancelling context so that all dependent processes stop working when the process gets term signal. - Set log level to error in otel shutdown function Signed-off-by: Sanket Sudake --- cmd/fetcher/app/server.go | 4 +- cmd/fetcher/main.go | 4 +- cmd/fission-bundle/main.go | 68 ++++++++++--------- cmd/preupgradechecks/main.go | 5 +- cmd/reporter/app/cmd_event.go | 4 +- pkg/buildermgr/buildermgr.go | 5 +- pkg/buildermgr/pkgwatcher.go | 7 +- pkg/controller/api_test.go | 3 +- pkg/controller/controller.go | 4 +- pkg/executor/executor.go | 5 +- pkg/executor/executor_test.go | 3 +- .../executortype/newdeploy/newdeploymgr.go | 5 +- pkg/executor/reaper/reaper.go | 3 +- pkg/kubewatcher/kubewatcher.go | 38 +++++------ pkg/kubewatcher/main.go | 6 +- pkg/logger/logger.go | 11 ++- pkg/mqtrigger/scalermanager.go | 4 +- pkg/router/router.go | 8 +-- pkg/storagesvc/archivePruner.go | 8 +-- pkg/storagesvc/client/storagesvc_test.go | 8 +-- pkg/storagesvc/storagesvc.go | 5 +- pkg/timer/main.go | 6 +- pkg/timer/timerSync.go | 8 +-- pkg/tracker/tracker.go | 4 +- pkg/utils/otel/provider.go | 4 +- pkg/utils/signals/signals.go | 46 +++++++++++++ 26 files changed, 161 insertions(+), 115 deletions(-) create mode 100644 pkg/utils/signals/signals.go diff --git a/cmd/fetcher/app/server.go b/cmd/fetcher/app/server.go index d2927f791..d17f5f593 100644 --- a/cmd/fetcher/app/server.go +++ b/cmd/fetcher/app/server.go @@ -39,7 +39,7 @@ var ( readyToServe uint32 ) -func Run(logger *zap.Logger) { +func Run(ctx context.Context, logger *zap.Logger) { flag.Usage = fetcherUsage collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "") specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup") @@ -62,8 +62,6 @@ func Run(logger *zap.Logger) { } } } - - ctx := context.Background() openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { go func() { diff --git a/cmd/fetcher/main.go b/cmd/fetcher/main.go index fb98d69a4..25f8b8cb1 100644 --- a/cmd/fetcher/main.go +++ b/cmd/fetcher/main.go @@ -20,6 +20,7 @@ import ( "github.com/fission/fission/cmd/fetcher/app" "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" ) // Usage: fetcher @@ -29,5 +30,6 @@ func main() { profile.ProfileIfEnabled(logger) - app.Run(logger) + ctx := signals.SetupSignalHandlerWithContext(logger) + app.Run(ctx, logger) } diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 55d82139e..5f089594f 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -41,27 +41,28 @@ import ( "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/otel" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" "github.com/fission/fission/pkg/utils/tracing" ) -func runController(logger *zap.Logger, port int, openTracingEnabled bool) { - controller.Start(logger, port, false, openTracingEnabled) +func runController(ctx context.Context, logger *zap.Logger, port int, openTracingEnabled bool) { + controller.Start(ctx, logger, port, false, openTracingEnabled) } -func runRouter(logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { - router.Start(logger, port, executorUrl, openTracingEnabled) +func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { + router.Start(ctx, logger, port, executorUrl, openTracingEnabled) } -func runExecutor(logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { - return executor.StartExecutor(logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) +func runExecutor(ctx context.Context, logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { + return executor.StartExecutor(ctx, logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) } -func runKubeWatcher(logger *zap.Logger, routerUrl string) error { - return kubewatcher.Start(logger, routerUrl) +func runKubeWatcher(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return kubewatcher.Start(ctx, logger, routerUrl) } -func runTimer(logger *zap.Logger, routerUrl string) error { - return timer.Start(logger, routerUrl) +func runTimer(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return timer.Start(ctx, logger, routerUrl) } func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { @@ -69,20 +70,20 @@ func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { } // KEDA based MessageQueue Trigger Manager -func runMQManager(logger *zap.Logger, routerURL string) error { - return mqt.StartScalerManager(logger, routerURL) +func runMQManager(ctx context.Context, logger *zap.Logger, routerURL string) error { + return mqt.StartScalerManager(ctx, logger, routerURL) } -func runStorageSvc(logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { - return storagesvc.Start(logger, storage, port, openTracingEnabled) +func runStorageSvc(ctx context.Context, logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { + return storagesvc.Start(ctx, logger, storage, port, openTracingEnabled) } -func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { - return buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace) +func runBuilderMgr(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { + return buildermgr.Start(ctx, logger, storageSvcUrl, envBuilderNamespace) } -func runLogger() { - functionLogger.Start() +func runLogger(ctx context.Context, logger *zap.Logger) { + functionLogger.Start(ctx, logger) } func getPort(logger *zap.Logger, portArg interface{}) int { @@ -129,10 +130,8 @@ func getServiceName(arguments map[string]interface{}) string { } func exitWithSync(logger *zap.Logger) { - err := logger.Sync() - if err != nil { - logger.Error("failed to sync log", zap.Error(err)) - } + // Ignore error, safe to ignore as per https://github.com/uber-go/zap/issues/328 + _ = logger.Sync() os.Exit(1) } @@ -210,7 +209,8 @@ Options: return } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) + openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments)) @@ -238,21 +238,21 @@ Options: if arguments["--controllerPort"] != nil { port := getPort(logger, arguments["--controllerPort"]) - runController(logger, port, openTracingEnabled) + runController(ctx, logger, port, openTracingEnabled) logger.Error("controller exited") return } if arguments["--routerPort"] != nil { port := getPort(logger, arguments["--routerPort"]) - runRouter(logger, port, executorUrl, openTracingEnabled) + runRouter(ctx, logger, port, executorUrl, openTracingEnabled) logger.Error("router exited") return } if arguments["--executorPort"] != nil { port := getPort(logger, arguments["--executorPort"]) - err = runExecutor(logger, port, functionNs, envBuilderNs, openTracingEnabled) + err = runExecutor(ctx, logger, port, functionNs, envBuilderNs, openTracingEnabled) if err != nil { logger.Error("executor exited", zap.Error(err)) return @@ -260,7 +260,7 @@ Options: } if arguments["--kubewatcher"] == true { - err = runKubeWatcher(logger, routerUrl) + err = runKubeWatcher(ctx, logger, routerUrl) if err != nil { logger.Error("kubewatcher exited", zap.Error(err)) return @@ -268,7 +268,7 @@ Options: } if arguments["--timer"] == true { - err = runTimer(logger, routerUrl) + err = runTimer(ctx, logger, routerUrl) if err != nil { logger.Error("timer exited", zap.Error(err)) return @@ -284,7 +284,7 @@ Options: } if arguments["--mqt_keda"] == true { - err = runMQManager(logger, routerUrl) + err = runMQManager(ctx, logger, routerUrl) if err != nil { logger.Error("mqt scaler manager exited", zap.Error(err)) return @@ -292,7 +292,7 @@ Options: } if arguments["--builderMgr"] == true { - err = runBuilderMgr(logger, storageSvcUrl, envBuilderNs) + err = runBuilderMgr(ctx, logger, storageSvcUrl, envBuilderNs) if err != nil { logger.Error("builder manager exited", zap.Error(err)) return @@ -300,7 +300,7 @@ Options: } if arguments["--logger"] == true { - runLogger() + runLogger(ctx, logger) logger.Error("logger exited") return } @@ -315,11 +315,13 @@ Options: } else if arguments["--storageType"] == string(storagesvc.StorageTypeLocal) { storage = storagesvc.NewLocalStorage("/fission") } - err := runStorageSvc(logger, port, storage, openTracingEnabled) + err := runStorageSvc(ctx, logger, port, storage, openTracingEnabled) if err != nil { logger.Error("storage service exited", zap.Error(err)) return } } - select {} + + <-ctx.Done() + logger.Error("exiting") } diff --git a/cmd/preupgradechecks/main.go b/cmd/preupgradechecks/main.go index 35bb28875..fb714441d 100644 --- a/cmd/preupgradechecks/main.go +++ b/cmd/preupgradechecks/main.go @@ -17,13 +17,12 @@ limitations under the License. package main import ( - "context" - "github.com/docopt/docopt-go" "go.uber.org/zap" "github.com/fission/fission/pkg/info" "github.com/fission/fission/pkg/utils/loggerfactory" + "github.com/fission/fission/pkg/utils/signals" ) func getStringArgWithDefault(arg interface{}, defaultValue string) string { @@ -59,7 +58,7 @@ Options: zap.Error(err)) } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) crd := crdBackedClient.GetFunctionCRD(ctx) if crd == nil { logger.Info("nothing to do since CRDs are not present on the cluster") diff --git a/cmd/reporter/app/cmd_event.go b/cmd/reporter/app/cmd_event.go index 431f35c64..44c336121 100644 --- a/cmd/reporter/app/cmd_event.go +++ b/cmd/reporter/app/cmd_event.go @@ -16,6 +16,7 @@ limitations under the License. package app import ( + "context" "log" "github.com/spf13/cobra" @@ -49,7 +50,8 @@ func eventCommandHandler(cmd *cobra.Command, args []string) error { return err } - return tracker.Tracker.SendEvent(event) + ctx := context.Background() + return tracker.Tracker.SendEvent(ctx, event) } //EventCommand reports an event to analytics diff --git a/pkg/buildermgr/buildermgr.go b/pkg/buildermgr/buildermgr.go index 6e4a9f4c9..33adcb5a7 100644 --- a/pkg/buildermgr/buildermgr.go +++ b/pkg/buildermgr/buildermgr.go @@ -17,6 +17,7 @@ limitations under the License. package buildermgr import ( + "context" "time" "github.com/pkg/errors" @@ -29,7 +30,7 @@ import ( ) // Start the buildermgr service. -func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { +func Start(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { bmLogger := logger.Named("builder_manager") fissionClient, kubernetesClient, _, _, err := crd.MakeFissionClient() @@ -56,6 +57,6 @@ func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) pkgInformer := informerFactory.Core().V1().Packages().Informer() pkgWatcher := makePackageWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl, &podInformer, &pkgInformer) - pkgWatcher.Run() + pkgWatcher.Run(ctx) return nil } diff --git a/pkg/buildermgr/pkgwatcher.go b/pkg/buildermgr/pkgwatcher.go index 3d87e451d..fd0eb4638 100644 --- a/pkg/buildermgr/pkgwatcher.go +++ b/pkg/buildermgr/pkgwatcher.go @@ -322,11 +322,10 @@ func (pkgw *packageWatcher) packageInformerHandler() k8sCache.ResourceEventHandl } } -func (pkgw *packageWatcher) Run() { - context := context.Background() - go (*pkgw.podInformer).Run(context.Done()) +func (pkgw *packageWatcher) Run(ctx context.Context) { + go (*pkgw.podInformer).Run(ctx.Done()) (*pkgw.pkgInformer).AddEventHandler(pkgw.packageInformerHandler()) - (*pkgw.pkgInformer).Run(context.Done()) + (*pkgw.pkgInformer).Run(ctx.Done()) } // setInitialBuildStatus sets initial build status to a package if it is empty. diff --git a/pkg/controller/api_test.go b/pkg/controller/api_test.go index 975953a25..c0bbca3eb 100644 --- a/pkg/controller/api_test.go +++ b/pkg/controller/api_test.go @@ -376,7 +376,8 @@ func TestMain(m *testing.M) { panicIf(err) - go Start(logger, 8888, true, true) + ctx := context.Background() + go Start(ctx, logger, 8888, true, true) time.Sleep(5 * time.Second) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e7a5b20c3..83bebf8b6 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,7 +24,7 @@ import ( "github.com/fission/fission/pkg/crd" ) -func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { cLogger := logger.Named("controller") fc, kc, apiExtClient, _, err := crd.MakeFissionClient() @@ -42,12 +42,10 @@ func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled b cLogger.Fatal("error waiting for CRDs", zap.Error(err)) } - ctx, cancel := context.WithCancel(context.Background()) featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) if err != nil { cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err)) } - defer cancel() api, err := MakeAPI(cLogger, featureStatus) if err != nil { diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 34603be29..6145d5d5e 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) { // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types -func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { +func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get kubernetes client") @@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames } cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments() cnmSvcInformer := cnmInformerFactory.Core().V1().Services() - ctx := context.Background() cnm, err := container.MakeContainer( ctx, logger, fissionClient, kubernetesClient, @@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames if err != nil { return err } - go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) + go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) go api.Serve(port, openTracingEnabled) go serveMetric(logger) diff --git a/pkg/executor/executor_test.go b/pkg/executor/executor_test.go index b40cc3ec0..9c3638007 100644 --- a/pkg/executor/executor_test.go +++ b/pkg/executor/executor_test.go @@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) { // create poolmgr port := 9999 - err = StartExecutor(logger, functionNs, "fission-builder", port, true) + ctx := context.Background() + err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true) if err != nil { log.Panicf("failed to start poolmgr: %v", err) } diff --git a/pkg/executor/executortype/newdeploy/newdeploymgr.go b/pkg/executor/executortype/newdeploy/newdeploymgr.go index aea0f014e..2908ac593 100644 --- a/pkg/executor/executortype/newdeploy/newdeploymgr.go +++ b/pkg/executor/executortype/newdeploy/newdeploymgr.go @@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) { if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok { deploy.logger.Fatal("failed to wait for caches to sync") } - go deploy.idleObjectReaper() + go deploy.idleObjectReaper(ctx) } // GetTypeName returns the executor type name. @@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin } // idleObjectReaper reaps objects after certain idle time -func (deploy *NewDeploy) idleObjectReaper() { - ctx := context.Background() +func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) { pollSleep := 5 * time.Second for { time.Sleep(pollSleep) diff --git a/pkg/executor/reaper/reaper.go b/pkg/executor/reaper/reaper.go index 335871c15..0b8f54962 100644 --- a/pkg/executor/reaper/reaper.go +++ b/pkg/executor/reaper/reaper.go @@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object. -func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { - ctx := context.Background() +func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { for { // some sleep before the next reaper iteration time.Sleep(cleanupRoleBindingInterval) diff --git a/pkg/kubewatcher/kubewatcher.go b/pkg/kubewatcher/kubewatcher.go index 9d1725ef6..a7ddaacfb 100644 --- a/pkg/kubewatcher/kubewatcher.go +++ b/pkg/kubewatcher/kubewatcher.go @@ -77,7 +77,7 @@ type ( } ) -func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { +func MakeKubeWatcher(ctx context.Context, logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ logger: logger.Named("kube_watcher"), watches: make(map[types.UID]watchSubscription), @@ -85,7 +85,7 @@ func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher: publisher, requestChannel: make(chan *kubeWatcherRequest), } - go kw.svc() + go kw.svc(ctx) return kw } @@ -100,7 +100,7 @@ func (kw *KubeWatcher) Sync(watches []fv1.KubernetesWatchTrigger) error { return resp.error } -func (kw *KubeWatcher) svc() { +func (kw *KubeWatcher) svc(ctx context.Context) { for { req := <-kw.requestChannel switch req.requestType { @@ -118,7 +118,7 @@ func (kw *KubeWatcher) svc() { // Add new watches for _, w := range req.watches { if _, ok := kw.watches[w.ObjectMeta.UID]; !ok { - kw.addWatch(&w) //nolint: errCheck + kw.addWatch(ctx, &w) //nolint: errCheck } } req.responseChannel <- &kubeWatcherResponse{error: nil} @@ -149,7 +149,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error { return err } -func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { +func createKubernetesWatch(ctx context.Context, kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { var wi watch.Interface var err error var watchTimeoutSec int64 = 120 @@ -163,22 +163,22 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWa // TODO handle the full list of types switch strings.ToUpper(w.Spec.Type) { case "POD": - wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(ctx, listOptions) case "SERVICE": - wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(ctx, listOptions) case "REPLICATIONCONTROLLER": - wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(ctx, listOptions) case "JOB": - wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(ctx, listOptions) default: err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)) } return wi, err } -func (kw *KubeWatcher) addWatch(w *fv1.KubernetesWatchTrigger) error { +func (kw *KubeWatcher) addWatch(ctx context.Context, w *fv1.KubernetesWatchTrigger) error { kw.logger.Info("adding watch", zap.String("name", w.ObjectMeta.Name), zap.Any("function", w.Spec.FunctionReference)) - ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) + ws, err := MakeWatchSubscription(ctx, kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) if err != nil { return err } @@ -198,7 +198,7 @@ func (kw *KubeWatcher) removeWatch(w *fv1.KubernetesWatchTrigger) error { return nil } -func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(ctx context.Context, logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ logger: logger.Named("watch_subscription"), @@ -210,16 +210,16 @@ func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, ku lastResourceVersion: "", } - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { return nil, err } - go ws.eventDispatchLoop() + go ws.eventDispatchLoop(ctx) return ws, nil } -func (ws *watchSubscription) restartWatch() error { +func (ws *watchSubscription) restartWatch(ctx context.Context) error { retries := 60 for { ws.logger.Info("(re)starting watch", @@ -227,7 +227,7 @@ func (ws *watchSubscription) restartWatch() error { zap.String("namespace", ws.watch.Spec.Namespace), zap.String("type", ws.watch.Spec.Type), zap.String("last_resource_version", ws.lastResourceVersion)) - wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) + wi, err := createKubernetesWatch(ctx, ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- if retries > 0 { @@ -250,7 +250,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { return m.GetResourceVersion(), nil } -func (ws *watchSubscription) eventDispatchLoop() { +func (ws *watchSubscription) eventDispatchLoop(ctx context.Context) { ws.logger.Info("listening to watch", zap.String("name", ws.watch.ObjectMeta.Name)) for { // check watchSubscription is stopped or not before waiting for event @@ -269,7 +269,7 @@ func (ws *watchSubscription) eventDispatchLoop() { } else { // watch closed due to timeout, restart it. ws.logger.Warn("watch timed out - restarting", zap.String("watch_name", ws.watch.ObjectMeta.Name)) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } @@ -283,7 +283,7 @@ func (ws *watchSubscription) eventDispatchLoop() { // Start from the beginning to get around "too old resource version" ws.lastResourceVersion = "" time.Sleep(time.Second) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } diff --git a/pkg/kubewatcher/main.go b/pkg/kubewatcher/main.go index 4a62dc638..629014823 100644 --- a/pkg/kubewatcher/main.go +++ b/pkg/kubewatcher/main.go @@ -17,6 +17,8 @@ limitations under the License. package kubewatcher import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - kubeWatch := MakeKubeWatcher(logger, kubeClient, poster) + kubeWatch := MakeKubeWatcher(ctx, logger, kubeClient, poster) MakeWatchSync(logger, fissionClient, kubeWatch) return nil diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 423fedd1e..843519e76 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -17,6 +17,7 @@ limitations under the License. package logger import ( + "context" "fmt" "log" "os" @@ -32,7 +33,6 @@ import ( fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/crd" "github.com/fission/fission/pkg/utils" - "github.com/fission/fission/pkg/utils/loggerfactory" ) var nodeName = os.Getenv("NODE_NAME") @@ -157,10 +157,7 @@ func symlinkReaper(zapLogger *zap.Logger) { } } -func Start() { - logger := loggerfactory.GetLogger() - defer logger.Sync() - +func Start(ctx context.Context, logger *zap.Logger) { if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) { logger.Info("symlink path not exist, create it", zap.String("fissionSymlinkPath", fissionSymlinkPath)) @@ -177,6 +174,6 @@ func Start() { informerFactory := k8sInformers.NewSharedInformerFactory(kubernetesClient, time.Minute*30) podInformer := informerFactory.Core().V1().Pods().Informer() podInformer.AddEventHandler(podInformerHandlers(logger)) - podInformer.Run(make(chan struct{})) - logger.Fatal("Stop watching pod changes") + podInformer.Run(ctx.Done()) + logger.Error("Stop watching pod changes") } diff --git a/pkg/mqtrigger/scalermanager.go b/pkg/mqtrigger/scalermanager.go index ed27ece8c..85283b5ae 100644 --- a/pkg/mqtrigger/scalermanager.go +++ b/pkg/mqtrigger/scalermanager.go @@ -149,7 +149,7 @@ func mqTriggerEventHandlers(logger *zap.Logger, kubeClient *kubernetes.Clientset // StartScalerManager watches for changes in MessageQueueTrigger and, // Based on changes, it Creates, Updates and Deletes Objects of Kind ScaledObjects, AuthenticationTriggers and Deployments -func StartScalerManager(logger *zap.Logger, routerURL string) error { +func StartScalerManager(ctx context.Context, logger *zap.Logger, routerURL string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return err @@ -161,7 +161,7 @@ func StartScalerManager(logger *zap.Logger, routerURL string) error { informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30) mqTriggerInformer := informerFactory.Core().V1().MessageQueueTriggers().Informer() mqTriggerInformer.AddEventHandler(mqTriggerEventHandlers(logger, kubeClient, routerURL)) - mqTriggerInformer.Run(context.Background().Done()) + mqTriggerInformer.Run(ctx.Done()) return nil } diff --git a/pkg/router/router.go b/pkg/router/router.go index 42957694a..79b57de82 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -126,7 +126,7 @@ func serveMetric(logger *zap.Logger) { } // Start starts a router -func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { fmap := makeFunctionServiceMap(logger, time.Minute) fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() @@ -258,10 +258,8 @@ func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled logger.Info("starting router", zap.Int("port", port)) tracer := otel.Tracer("router") - ctx, span := tracer.Start(context.Background(), "router/Start") + ctx, span := tracer.Start(ctx, "router/Start") defer span.End() - ctxWithCancel, cancel := context.WithCancel(ctx) - defer cancel() - serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) + serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) } diff --git a/pkg/storagesvc/archivePruner.go b/pkg/storagesvc/archivePruner.go index 87bcd5055..8293c78c8 100644 --- a/pkg/storagesvc/archivePruner.go +++ b/pkg/storagesvc/archivePruner.go @@ -75,13 +75,13 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) { // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes // and not the archives that are referenced by them, leaving the archives as orphans. // getOrphanArchives reaps the orphaned archives. -func (pruner *ArchivePruner) getOrphanArchives() { +func (pruner *ArchivePruner) getOrphanArchives(ctx context.Context) { pruner.logger.Debug("getting orphan archives") archivesRefByPkgs := make([]string, 0) var archiveID string // get all pkgs from kubernetes - pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { pruner.logger.Error("error getting package list from kubernetes", zap.Error(err)) return @@ -137,12 +137,12 @@ func (pruner *ArchivePruner) getOrphanArchives() { // Start starts a go routine that listens to a channel for archive IDs that need to deleted. // Also wakes up at regular intervals to make a list of archive IDs that need to be reaped // and sends them over to the channel for deletion -func (pruner *ArchivePruner) Start() { +func (pruner *ArchivePruner) Start(ctx context.Context) { ticker := time.NewTicker(pruner.pruneInterval * time.Minute) go pruner.pruneArchives() for range ticker.C { // This method fetches unused archive IDs and sends them to archiveChannel for deletion // silencing the errors, hoping they go away in next iteration. - pruner.getOrphanArchives() + pruner.getOrphanArchives(ctx) } } diff --git a/pkg/storagesvc/client/storagesvc_test.go b/pkg/storagesvc/client/storagesvc_test.go index 4304245ae..d44d66f7d 100644 --- a/pkg/storagesvc/client/storagesvc_test.go +++ b/pkg/storagesvc/client/storagesvc_test.go @@ -76,7 +76,7 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource { return resource } -func startS3StorageService(endpoint, bucketName, subDir string) { +func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) { // testID := uniuri.NewLen(8) port := 8081 @@ -94,7 +94,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) { os.Setenv("STORAGE_S3_REGION", minioRegion) storage := storagesvc.NewS3Storage() - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(ctx, logger, storage, port, true) } func TestS3StorageService(t *testing.T) { @@ -135,7 +135,7 @@ func TestS3StorageService(t *testing.T) { // Start storagesvc bucketName := "test-s3-service" subDir := "x/y/z" - startS3StorageService(endpoint, bucketName, subDir) + startS3StorageService(context.Background(), endpoint, bucketName, subDir) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081)) @@ -211,7 +211,7 @@ func TestLocalStorageService(t *testing.T) { localPath := fmt.Sprintf("/tmp/%v", testID) _ = os.Mkdir(localPath, os.ModePerm) storage := storagesvc.NewLocalStorage(localPath) - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(context.Background(), logger, storage, port, true) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", port)) diff --git a/pkg/storagesvc/storagesvc.go b/pkg/storagesvc/storagesvc.go index 642eddbd4..8fadd452c 100644 --- a/pkg/storagesvc/storagesvc.go +++ b/pkg/storagesvc/storagesvc.go @@ -17,6 +17,7 @@ limitations under the License. package storagesvc import ( + "context" "encoding/json" "fmt" "net/http" @@ -221,7 +222,7 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) { } // Start runs storage service -func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { +func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { enablePruner := true // create a storage client storageClient, err := MakeStowClient(logger, storage) @@ -244,7 +245,7 @@ func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled boo if err != nil { return errors.Wrap(err, "Error creating archivePruner") } - go pruner.Start() + go pruner.Start(ctx) } logger.Info("storage service started") diff --git a/pkg/timer/main.go b/pkg/timer/main.go index 64da005dd..d9df1e192 100644 --- a/pkg/timer/main.go +++ b/pkg/timer/main.go @@ -17,6 +17,8 @@ limitations under the License. package timer import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, _, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster)) + MakeTimerSync(ctx, logger, fissionClient, MakeTimer(logger, poster)) return nil } diff --git a/pkg/timer/timerSync.go b/pkg/timer/timerSync.go index 2a7b7446a..547c2ae42 100644 --- a/pkg/timer/timerSync.go +++ b/pkg/timer/timerSync.go @@ -35,19 +35,19 @@ type ( } ) -func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { +func MakeTimerSync(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ logger: logger.Named("timer_sync"), fissionClient: fissionClient, timer: timer, } - go ws.syncSvc() + go ws.syncSvc(ctx) return ws } -func (ws *TimerSync) syncSvc() { +func (ws *TimerSync) syncSvc(ctx context.Context) { for { - triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { if utils.IsNetworkError(err) { ws.logger.Info("encountered a network error - will retry", zap.Error(err)) diff --git a/pkg/tracker/tracker.go b/pkg/tracker/tracker.go index 2a4979bab..9a9351ca5 100644 --- a/pkg/tracker/tracker.go +++ b/pkg/tracker/tracker.go @@ -54,7 +54,7 @@ func init() { Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: id.String()} } -func (t *tracker) SendEvent(e Event) error { +func (t *tracker) SendEvent(ctx context.Context, e Event) error { if t.gaPropertyID == "" { return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set") } @@ -81,7 +81,7 @@ func (t *tracker) SendEvent(e Event) error { } buf := bytes.NewBufferString(v.Encode()) - req, err := http.NewRequest("POST", GA_API_URL, buf) + req, err := http.NewRequestWithContext(ctx, "POST", GA_API_URL, buf) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("User-Agent", "ga-tracker/1.0") if err != nil { diff --git a/pkg/utils/otel/provider.go b/pkg/utils/otel/provider.go index 9b7adcf71..40c7044ac 100644 --- a/pkg/utils/otel/provider.go +++ b/pkg/utils/otel/provider.go @@ -216,11 +216,11 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) ( return func(ctx context.Context) { err := tracerProvider.Shutdown(ctx) if err != nil && logger != nil { - logger.Fatal("error shutting down trace provider", zap.Error(err)) + logger.Error("error shutting down trace provider", zap.Error(err)) } if traceExporter != nil { if err = traceExporter.Shutdown(ctx); err != nil && logger != nil { - logger.Fatal("error shutting down trace exporter", zap.Error(err)) + logger.Error("error shutting down trace exporter", zap.Error(err)) } } }, nil diff --git a/pkg/utils/signals/signals.go b/pkg/utils/signals/signals.go new file mode 100644 index 000000000..17cf53f0c --- /dev/null +++ b/pkg/utils/signals/signals.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package signals + +import ( + "context" + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" +) + +var onlyOneSignalHandler = make(chan struct{}) + +func SetupSignalHandlerWithContext(logger *zap.Logger) context.Context { + var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + + close(onlyOneSignalHandler) // panics when called twice + + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + signal := <-c + logger.Info("Received signal", zap.String("signal", signal.String())) + cancel() + <-c + panic("multiple signals received") + }() + + return ctx +}