From 59e876062f6263d20c79f84bc2d90a9fcb452ba2 Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Thu, 11 Nov 2021 17:02:11 +0530 Subject: [PATCH] Capture os signals to gracefully shutdown fission components (#2261) - Currently, fission components don't handle shutdown signals. So we don't get any to do the required cleanup before the fission process exits. Adding signal capture process with cancelling context so that all dependent processes stop working when the process gets term signal. - Set log level to error in otel shutdown function Signed-off-by: Sanket Sudake --- cmd/fetcher/app/server.go | 4 +- cmd/fetcher/main.go | 4 +- cmd/fission-bundle/main.go | 68 ++++++++++--------- cmd/preupgradechecks/main.go | 5 +- cmd/reporter/app/cmd_event.go | 4 +- pkg/buildermgr/buildermgr.go | 5 +- pkg/buildermgr/pkgwatcher.go | 7 +- pkg/controller/api_test.go | 3 +- pkg/controller/controller.go | 4 +- pkg/executor/executor.go | 5 +- pkg/executor/executor_test.go | 3 +- .../executortype/newdeploy/newdeploymgr.go | 5 +- pkg/executor/reaper/reaper.go | 3 +- pkg/kubewatcher/kubewatcher.go | 38 +++++------ pkg/kubewatcher/main.go | 6 +- pkg/logger/logger.go | 11 ++- pkg/mqtrigger/scalermanager.go | 4 +- pkg/router/router.go | 8 +-- pkg/storagesvc/archivePruner.go | 8 +-- pkg/storagesvc/client/storagesvc_test.go | 8 +-- pkg/storagesvc/storagesvc.go | 5 +- pkg/timer/main.go | 6 +- pkg/timer/timerSync.go | 8 +-- pkg/tracker/tracker.go | 4 +- pkg/utils/otel/provider.go | 4 +- pkg/utils/signals/signals.go | 46 +++++++++++++ 26 files changed, 161 insertions(+), 115 deletions(-) create mode 100644 pkg/utils/signals/signals.go diff --git a/cmd/fetcher/app/server.go b/cmd/fetcher/app/server.go index d2927f79..d17f5f59 100644 --- a/cmd/fetcher/app/server.go +++ b/cmd/fetcher/app/server.go @@ -39,7 +39,7 @@ var ( readyToServe uint32 ) -func Run(logger *zap.Logger) { +func Run(ctx context.Context, logger *zap.Logger) { flag.Usage = fetcherUsage collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "") specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup") @@ -62,8 +62,6 @@ func Run(logger *zap.Logger) { } } } - - ctx := context.Background() openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { go func() { diff --git a/cmd/fetcher/main.go b/cmd/fetcher/main.go index fb98d69a..25f8b8cb 100644 --- a/cmd/fetcher/main.go +++ b/cmd/fetcher/main.go @@ -20,6 +20,7 @@ import ( "github.com/fission/fission/cmd/fetcher/app" "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" ) // Usage: fetcher @@ -29,5 +30,6 @@ func main() { profile.ProfileIfEnabled(logger) - app.Run(logger) + ctx := signals.SetupSignalHandlerWithContext(logger) + app.Run(ctx, logger) } diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 55d82139..5f089594 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -41,27 +41,28 @@ import ( "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/otel" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" "github.com/fission/fission/pkg/utils/tracing" ) -func runController(logger *zap.Logger, port int, openTracingEnabled bool) { - controller.Start(logger, port, false, openTracingEnabled) +func runController(ctx context.Context, logger *zap.Logger, port int, openTracingEnabled bool) { + controller.Start(ctx, logger, port, false, openTracingEnabled) } -func runRouter(logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { - router.Start(logger, port, executorUrl, openTracingEnabled) +func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { + router.Start(ctx, logger, port, executorUrl, openTracingEnabled) } -func runExecutor(logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { - return executor.StartExecutor(logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) +func runExecutor(ctx context.Context, logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { + return executor.StartExecutor(ctx, logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) } -func runKubeWatcher(logger *zap.Logger, routerUrl string) error { - return kubewatcher.Start(logger, routerUrl) +func runKubeWatcher(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return kubewatcher.Start(ctx, logger, routerUrl) } -func runTimer(logger *zap.Logger, routerUrl string) error { - return timer.Start(logger, routerUrl) +func runTimer(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return timer.Start(ctx, logger, routerUrl) } func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { @@ -69,20 +70,20 @@ func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { } // KEDA based MessageQueue Trigger Manager -func runMQManager(logger *zap.Logger, routerURL string) error { - return mqt.StartScalerManager(logger, routerURL) +func runMQManager(ctx context.Context, logger *zap.Logger, routerURL string) error { + return mqt.StartScalerManager(ctx, logger, routerURL) } -func runStorageSvc(logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { - return storagesvc.Start(logger, storage, port, openTracingEnabled) +func runStorageSvc(ctx context.Context, logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { + return storagesvc.Start(ctx, logger, storage, port, openTracingEnabled) } -func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { - return buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace) +func runBuilderMgr(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { + return buildermgr.Start(ctx, logger, storageSvcUrl, envBuilderNamespace) } -func runLogger() { - functionLogger.Start() +func runLogger(ctx context.Context, logger *zap.Logger) { + functionLogger.Start(ctx, logger) } func getPort(logger *zap.Logger, portArg interface{}) int { @@ -129,10 +130,8 @@ func getServiceName(arguments map[string]interface{}) string { } func exitWithSync(logger *zap.Logger) { - err := logger.Sync() - if err != nil { - logger.Error("failed to sync log", zap.Error(err)) - } + // Ignore error, safe to ignore as per https://github.com/uber-go/zap/issues/328 + _ = logger.Sync() os.Exit(1) } @@ -210,7 +209,8 @@ Options: return } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) + openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments)) @@ -238,21 +238,21 @@ Options: if arguments["--controllerPort"] != nil { port := getPort(logger, arguments["--controllerPort"]) - runController(logger, port, openTracingEnabled) + runController(ctx, logger, port, openTracingEnabled) logger.Error("controller exited") return } if arguments["--routerPort"] != nil { port := getPort(logger, arguments["--routerPort"]) - runRouter(logger, port, executorUrl, openTracingEnabled) + runRouter(ctx, logger, port, executorUrl, openTracingEnabled) logger.Error("router exited") return } if arguments["--executorPort"] != nil { port := getPort(logger, arguments["--executorPort"]) - err = runExecutor(logger, port, functionNs, envBuilderNs, openTracingEnabled) + err = runExecutor(ctx, logger, port, functionNs, envBuilderNs, openTracingEnabled) if err != nil { logger.Error("executor exited", zap.Error(err)) return @@ -260,7 +260,7 @@ Options: } if arguments["--kubewatcher"] == true { - err = runKubeWatcher(logger, routerUrl) + err = runKubeWatcher(ctx, logger, routerUrl) if err != nil { logger.Error("kubewatcher exited", zap.Error(err)) return @@ -268,7 +268,7 @@ Options: } if arguments["--timer"] == true { - err = runTimer(logger, routerUrl) + err = runTimer(ctx, logger, routerUrl) if err != nil { logger.Error("timer exited", zap.Error(err)) return @@ -284,7 +284,7 @@ Options: } if arguments["--mqt_keda"] == true { - err = runMQManager(logger, routerUrl) + err = runMQManager(ctx, logger, routerUrl) if err != nil { logger.Error("mqt scaler manager exited", zap.Error(err)) return @@ -292,7 +292,7 @@ Options: } if arguments["--builderMgr"] == true { - err = runBuilderMgr(logger, storageSvcUrl, envBuilderNs) + err = runBuilderMgr(ctx, logger, storageSvcUrl, envBuilderNs) if err != nil { logger.Error("builder manager exited", zap.Error(err)) return @@ -300,7 +300,7 @@ Options: } if arguments["--logger"] == true { - runLogger() + runLogger(ctx, logger) logger.Error("logger exited") return } @@ -315,11 +315,13 @@ Options: } else if arguments["--storageType"] == string(storagesvc.StorageTypeLocal) { storage = storagesvc.NewLocalStorage("/fission") } - err := runStorageSvc(logger, port, storage, openTracingEnabled) + err := runStorageSvc(ctx, logger, port, storage, openTracingEnabled) if err != nil { logger.Error("storage service exited", zap.Error(err)) return } } - select {} + + <-ctx.Done() + logger.Error("exiting") } diff --git a/cmd/preupgradechecks/main.go b/cmd/preupgradechecks/main.go index 35bb2887..fb714441 100644 --- a/cmd/preupgradechecks/main.go +++ b/cmd/preupgradechecks/main.go @@ -17,13 +17,12 @@ limitations under the License. package main import ( - "context" - "github.com/docopt/docopt-go" "go.uber.org/zap" "github.com/fission/fission/pkg/info" "github.com/fission/fission/pkg/utils/loggerfactory" + "github.com/fission/fission/pkg/utils/signals" ) func getStringArgWithDefault(arg interface{}, defaultValue string) string { @@ -59,7 +58,7 @@ Options: zap.Error(err)) } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) crd := crdBackedClient.GetFunctionCRD(ctx) if crd == nil { logger.Info("nothing to do since CRDs are not present on the cluster") diff --git a/cmd/reporter/app/cmd_event.go b/cmd/reporter/app/cmd_event.go index 431f35c6..44c33612 100644 --- a/cmd/reporter/app/cmd_event.go +++ b/cmd/reporter/app/cmd_event.go @@ -16,6 +16,7 @@ limitations under the License. package app import ( + "context" "log" "github.com/spf13/cobra" @@ -49,7 +50,8 @@ func eventCommandHandler(cmd *cobra.Command, args []string) error { return err } - return tracker.Tracker.SendEvent(event) + ctx := context.Background() + return tracker.Tracker.SendEvent(ctx, event) } //EventCommand reports an event to analytics diff --git a/pkg/buildermgr/buildermgr.go b/pkg/buildermgr/buildermgr.go index 6e4a9f4c..33adcb5a 100644 --- a/pkg/buildermgr/buildermgr.go +++ b/pkg/buildermgr/buildermgr.go @@ -17,6 +17,7 @@ limitations under the License. package buildermgr import ( + "context" "time" "github.com/pkg/errors" @@ -29,7 +30,7 @@ import ( ) // Start the buildermgr service. -func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { +func Start(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { bmLogger := logger.Named("builder_manager") fissionClient, kubernetesClient, _, _, err := crd.MakeFissionClient() @@ -56,6 +57,6 @@ func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) pkgInformer := informerFactory.Core().V1().Packages().Informer() pkgWatcher := makePackageWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl, &podInformer, &pkgInformer) - pkgWatcher.Run() + pkgWatcher.Run(ctx) return nil } diff --git a/pkg/buildermgr/pkgwatcher.go b/pkg/buildermgr/pkgwatcher.go index 3d87e451..fd0eb463 100644 --- a/pkg/buildermgr/pkgwatcher.go +++ b/pkg/buildermgr/pkgwatcher.go @@ -322,11 +322,10 @@ func (pkgw *packageWatcher) packageInformerHandler() k8sCache.ResourceEventHandl } } -func (pkgw *packageWatcher) Run() { - context := context.Background() - go (*pkgw.podInformer).Run(context.Done()) +func (pkgw *packageWatcher) Run(ctx context.Context) { + go (*pkgw.podInformer).Run(ctx.Done()) (*pkgw.pkgInformer).AddEventHandler(pkgw.packageInformerHandler()) - (*pkgw.pkgInformer).Run(context.Done()) + (*pkgw.pkgInformer).Run(ctx.Done()) } // setInitialBuildStatus sets initial build status to a package if it is empty. diff --git a/pkg/controller/api_test.go b/pkg/controller/api_test.go index 975953a2..c0bbca3e 100644 --- a/pkg/controller/api_test.go +++ b/pkg/controller/api_test.go @@ -376,7 +376,8 @@ func TestMain(m *testing.M) { panicIf(err) - go Start(logger, 8888, true, true) + ctx := context.Background() + go Start(ctx, logger, 8888, true, true) time.Sleep(5 * time.Second) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e7a5b20c..83bebf8b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,7 +24,7 @@ import ( "github.com/fission/fission/pkg/crd" ) -func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { cLogger := logger.Named("controller") fc, kc, apiExtClient, _, err := crd.MakeFissionClient() @@ -42,12 +42,10 @@ func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled b cLogger.Fatal("error waiting for CRDs", zap.Error(err)) } - ctx, cancel := context.WithCancel(context.Background()) featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) if err != nil { cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err)) } - defer cancel() api, err := MakeAPI(cLogger, featureStatus) if err != nil { diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 34603be2..6145d5d5 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) { // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types -func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { +func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get kubernetes client") @@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames } cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments() cnmSvcInformer := cnmInformerFactory.Core().V1().Services() - ctx := context.Background() cnm, err := container.MakeContainer( ctx, logger, fissionClient, kubernetesClient, @@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames if err != nil { return err } - go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) + go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) go api.Serve(port, openTracingEnabled) go serveMetric(logger) diff --git a/pkg/executor/executor_test.go b/pkg/executor/executor_test.go index b40cc3ec..9c363800 100644 --- a/pkg/executor/executor_test.go +++ b/pkg/executor/executor_test.go @@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) { // create poolmgr port := 9999 - err = StartExecutor(logger, functionNs, "fission-builder", port, true) + ctx := context.Background() + err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true) if err != nil { log.Panicf("failed to start poolmgr: %v", err) } diff --git a/pkg/executor/executortype/newdeploy/newdeploymgr.go b/pkg/executor/executortype/newdeploy/newdeploymgr.go index aea0f014..2908ac59 100644 --- a/pkg/executor/executortype/newdeploy/newdeploymgr.go +++ b/pkg/executor/executortype/newdeploy/newdeploymgr.go @@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) { if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok { deploy.logger.Fatal("failed to wait for caches to sync") } - go deploy.idleObjectReaper() + go deploy.idleObjectReaper(ctx) } // GetTypeName returns the executor type name. @@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin } // idleObjectReaper reaps objects after certain idle time -func (deploy *NewDeploy) idleObjectReaper() { - ctx := context.Background() +func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) { pollSleep := 5 * time.Second for { time.Sleep(pollSleep) diff --git a/pkg/executor/reaper/reaper.go b/pkg/executor/reaper/reaper.go index 335871c1..0b8f5496 100644 --- a/pkg/executor/reaper/reaper.go +++ b/pkg/executor/reaper/reaper.go @@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object. -func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { - ctx := context.Background() +func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { for { // some sleep before the next reaper iteration time.Sleep(cleanupRoleBindingInterval) diff --git a/pkg/kubewatcher/kubewatcher.go b/pkg/kubewatcher/kubewatcher.go index 9d1725ef..a7ddaacf 100644 --- a/pkg/kubewatcher/kubewatcher.go +++ b/pkg/kubewatcher/kubewatcher.go @@ -77,7 +77,7 @@ type ( } ) -func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { +func MakeKubeWatcher(ctx context.Context, logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ logger: logger.Named("kube_watcher"), watches: make(map[types.UID]watchSubscription), @@ -85,7 +85,7 @@ func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher: publisher, requestChannel: make(chan *kubeWatcherRequest), } - go kw.svc() + go kw.svc(ctx) return kw } @@ -100,7 +100,7 @@ func (kw *KubeWatcher) Sync(watches []fv1.KubernetesWatchTrigger) error { return resp.error } -func (kw *KubeWatcher) svc() { +func (kw *KubeWatcher) svc(ctx context.Context) { for { req := <-kw.requestChannel switch req.requestType { @@ -118,7 +118,7 @@ func (kw *KubeWatcher) svc() { // Add new watches for _, w := range req.watches { if _, ok := kw.watches[w.ObjectMeta.UID]; !ok { - kw.addWatch(&w) //nolint: errCheck + kw.addWatch(ctx, &w) //nolint: errCheck } } req.responseChannel <- &kubeWatcherResponse{error: nil} @@ -149,7 +149,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error { return err } -func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { +func createKubernetesWatch(ctx context.Context, kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { var wi watch.Interface var err error var watchTimeoutSec int64 = 120 @@ -163,22 +163,22 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWa // TODO handle the full list of types switch strings.ToUpper(w.Spec.Type) { case "POD": - wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(ctx, listOptions) case "SERVICE": - wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(ctx, listOptions) case "REPLICATIONCONTROLLER": - wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(ctx, listOptions) case "JOB": - wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(ctx, listOptions) default: err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)) } return wi, err } -func (kw *KubeWatcher) addWatch(w *fv1.KubernetesWatchTrigger) error { +func (kw *KubeWatcher) addWatch(ctx context.Context, w *fv1.KubernetesWatchTrigger) error { kw.logger.Info("adding watch", zap.String("name", w.ObjectMeta.Name), zap.Any("function", w.Spec.FunctionReference)) - ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) + ws, err := MakeWatchSubscription(ctx, kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) if err != nil { return err } @@ -198,7 +198,7 @@ func (kw *KubeWatcher) removeWatch(w *fv1.KubernetesWatchTrigger) error { return nil } -func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(ctx context.Context, logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ logger: logger.Named("watch_subscription"), @@ -210,16 +210,16 @@ func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, ku lastResourceVersion: "", } - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { return nil, err } - go ws.eventDispatchLoop() + go ws.eventDispatchLoop(ctx) return ws, nil } -func (ws *watchSubscription) restartWatch() error { +func (ws *watchSubscription) restartWatch(ctx context.Context) error { retries := 60 for { ws.logger.Info("(re)starting watch", @@ -227,7 +227,7 @@ func (ws *watchSubscription) restartWatch() error { zap.String("namespace", ws.watch.Spec.Namespace), zap.String("type", ws.watch.Spec.Type), zap.String("last_resource_version", ws.lastResourceVersion)) - wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) + wi, err := createKubernetesWatch(ctx, ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- if retries > 0 { @@ -250,7 +250,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { return m.GetResourceVersion(), nil } -func (ws *watchSubscription) eventDispatchLoop() { +func (ws *watchSubscription) eventDispatchLoop(ctx context.Context) { ws.logger.Info("listening to watch", zap.String("name", ws.watch.ObjectMeta.Name)) for { // check watchSubscription is stopped or not before waiting for event @@ -269,7 +269,7 @@ func (ws *watchSubscription) eventDispatchLoop() { } else { // watch closed due to timeout, restart it. ws.logger.Warn("watch timed out - restarting", zap.String("watch_name", ws.watch.ObjectMeta.Name)) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } @@ -283,7 +283,7 @@ func (ws *watchSubscription) eventDispatchLoop() { // Start from the beginning to get around "too old resource version" ws.lastResourceVersion = "" time.Sleep(time.Second) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } diff --git a/pkg/kubewatcher/main.go b/pkg/kubewatcher/main.go index 4a62dc63..62901482 100644 --- a/pkg/kubewatcher/main.go +++ b/pkg/kubewatcher/main.go @@ -17,6 +17,8 @@ limitations under the License. package kubewatcher import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - kubeWatch := MakeKubeWatcher(logger, kubeClient, poster) + kubeWatch := MakeKubeWatcher(ctx, logger, kubeClient, poster) MakeWatchSync(logger, fissionClient, kubeWatch) return nil diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 423fedd1..843519e7 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -17,6 +17,7 @@ limitations under the License. package logger import ( + "context" "fmt" "log" "os" @@ -32,7 +33,6 @@ import ( fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/crd" "github.com/fission/fission/pkg/utils" - "github.com/fission/fission/pkg/utils/loggerfactory" ) var nodeName = os.Getenv("NODE_NAME") @@ -157,10 +157,7 @@ func symlinkReaper(zapLogger *zap.Logger) { } } -func Start() { - logger := loggerfactory.GetLogger() - defer logger.Sync() - +func Start(ctx context.Context, logger *zap.Logger) { if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) { logger.Info("symlink path not exist, create it", zap.String("fissionSymlinkPath", fissionSymlinkPath)) @@ -177,6 +174,6 @@ func Start() { informerFactory := k8sInformers.NewSharedInformerFactory(kubernetesClient, time.Minute*30) podInformer := informerFactory.Core().V1().Pods().Informer() podInformer.AddEventHandler(podInformerHandlers(logger)) - podInformer.Run(make(chan struct{})) - logger.Fatal("Stop watching pod changes") + podInformer.Run(ctx.Done()) + logger.Error("Stop watching pod changes") } diff --git a/pkg/mqtrigger/scalermanager.go b/pkg/mqtrigger/scalermanager.go index ed27ece8..85283b5a 100644 --- a/pkg/mqtrigger/scalermanager.go +++ b/pkg/mqtrigger/scalermanager.go @@ -149,7 +149,7 @@ func mqTriggerEventHandlers(logger *zap.Logger, kubeClient *kubernetes.Clientset // StartScalerManager watches for changes in MessageQueueTrigger and, // Based on changes, it Creates, Updates and Deletes Objects of Kind ScaledObjects, AuthenticationTriggers and Deployments -func StartScalerManager(logger *zap.Logger, routerURL string) error { +func StartScalerManager(ctx context.Context, logger *zap.Logger, routerURL string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return err @@ -161,7 +161,7 @@ func StartScalerManager(logger *zap.Logger, routerURL string) error { informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30) mqTriggerInformer := informerFactory.Core().V1().MessageQueueTriggers().Informer() mqTriggerInformer.AddEventHandler(mqTriggerEventHandlers(logger, kubeClient, routerURL)) - mqTriggerInformer.Run(context.Background().Done()) + mqTriggerInformer.Run(ctx.Done()) return nil } diff --git a/pkg/router/router.go b/pkg/router/router.go index 42957694..79b57de8 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -126,7 +126,7 @@ func serveMetric(logger *zap.Logger) { } // Start starts a router -func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { fmap := makeFunctionServiceMap(logger, time.Minute) fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() @@ -258,10 +258,8 @@ func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled logger.Info("starting router", zap.Int("port", port)) tracer := otel.Tracer("router") - ctx, span := tracer.Start(context.Background(), "router/Start") + ctx, span := tracer.Start(ctx, "router/Start") defer span.End() - ctxWithCancel, cancel := context.WithCancel(ctx) - defer cancel() - serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) + serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) } diff --git a/pkg/storagesvc/archivePruner.go b/pkg/storagesvc/archivePruner.go index 87bcd505..8293c78c 100644 --- a/pkg/storagesvc/archivePruner.go +++ b/pkg/storagesvc/archivePruner.go @@ -75,13 +75,13 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) { // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes // and not the archives that are referenced by them, leaving the archives as orphans. // getOrphanArchives reaps the orphaned archives. -func (pruner *ArchivePruner) getOrphanArchives() { +func (pruner *ArchivePruner) getOrphanArchives(ctx context.Context) { pruner.logger.Debug("getting orphan archives") archivesRefByPkgs := make([]string, 0) var archiveID string // get all pkgs from kubernetes - pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { pruner.logger.Error("error getting package list from kubernetes", zap.Error(err)) return @@ -137,12 +137,12 @@ func (pruner *ArchivePruner) getOrphanArchives() { // Start starts a go routine that listens to a channel for archive IDs that need to deleted. // Also wakes up at regular intervals to make a list of archive IDs that need to be reaped // and sends them over to the channel for deletion -func (pruner *ArchivePruner) Start() { +func (pruner *ArchivePruner) Start(ctx context.Context) { ticker := time.NewTicker(pruner.pruneInterval * time.Minute) go pruner.pruneArchives() for range ticker.C { // This method fetches unused archive IDs and sends them to archiveChannel for deletion // silencing the errors, hoping they go away in next iteration. - pruner.getOrphanArchives() + pruner.getOrphanArchives(ctx) } } diff --git a/pkg/storagesvc/client/storagesvc_test.go b/pkg/storagesvc/client/storagesvc_test.go index 4304245a..d44d66f7 100644 --- a/pkg/storagesvc/client/storagesvc_test.go +++ b/pkg/storagesvc/client/storagesvc_test.go @@ -76,7 +76,7 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource { return resource } -func startS3StorageService(endpoint, bucketName, subDir string) { +func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) { // testID := uniuri.NewLen(8) port := 8081 @@ -94,7 +94,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) { os.Setenv("STORAGE_S3_REGION", minioRegion) storage := storagesvc.NewS3Storage() - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(ctx, logger, storage, port, true) } func TestS3StorageService(t *testing.T) { @@ -135,7 +135,7 @@ func TestS3StorageService(t *testing.T) { // Start storagesvc bucketName := "test-s3-service" subDir := "x/y/z" - startS3StorageService(endpoint, bucketName, subDir) + startS3StorageService(context.Background(), endpoint, bucketName, subDir) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081)) @@ -211,7 +211,7 @@ func TestLocalStorageService(t *testing.T) { localPath := fmt.Sprintf("/tmp/%v", testID) _ = os.Mkdir(localPath, os.ModePerm) storage := storagesvc.NewLocalStorage(localPath) - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(context.Background(), logger, storage, port, true) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", port)) diff --git a/pkg/storagesvc/storagesvc.go b/pkg/storagesvc/storagesvc.go index 642eddbd..8fadd452 100644 --- a/pkg/storagesvc/storagesvc.go +++ b/pkg/storagesvc/storagesvc.go @@ -17,6 +17,7 @@ limitations under the License. package storagesvc import ( + "context" "encoding/json" "fmt" "net/http" @@ -221,7 +222,7 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) { } // Start runs storage service -func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { +func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { enablePruner := true // create a storage client storageClient, err := MakeStowClient(logger, storage) @@ -244,7 +245,7 @@ func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled boo if err != nil { return errors.Wrap(err, "Error creating archivePruner") } - go pruner.Start() + go pruner.Start(ctx) } logger.Info("storage service started") diff --git a/pkg/timer/main.go b/pkg/timer/main.go index 64da005d..d9df1e19 100644 --- a/pkg/timer/main.go +++ b/pkg/timer/main.go @@ -17,6 +17,8 @@ limitations under the License. package timer import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, _, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster)) + MakeTimerSync(ctx, logger, fissionClient, MakeTimer(logger, poster)) return nil } diff --git a/pkg/timer/timerSync.go b/pkg/timer/timerSync.go index 2a7b7446..547c2ae4 100644 --- a/pkg/timer/timerSync.go +++ b/pkg/timer/timerSync.go @@ -35,19 +35,19 @@ type ( } ) -func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { +func MakeTimerSync(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ logger: logger.Named("timer_sync"), fissionClient: fissionClient, timer: timer, } - go ws.syncSvc() + go ws.syncSvc(ctx) return ws } -func (ws *TimerSync) syncSvc() { +func (ws *TimerSync) syncSvc(ctx context.Context) { for { - triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { if utils.IsNetworkError(err) { ws.logger.Info("encountered a network error - will retry", zap.Error(err)) diff --git a/pkg/tracker/tracker.go b/pkg/tracker/tracker.go index 2a4979ba..9a9351ca 100644 --- a/pkg/tracker/tracker.go +++ b/pkg/tracker/tracker.go @@ -54,7 +54,7 @@ func init() { Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: id.String()} } -func (t *tracker) SendEvent(e Event) error { +func (t *tracker) SendEvent(ctx context.Context, e Event) error { if t.gaPropertyID == "" { return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set") } @@ -81,7 +81,7 @@ func (t *tracker) SendEvent(e Event) error { } buf := bytes.NewBufferString(v.Encode()) - req, err := http.NewRequest("POST", GA_API_URL, buf) + req, err := http.NewRequestWithContext(ctx, "POST", GA_API_URL, buf) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("User-Agent", "ga-tracker/1.0") if err != nil { diff --git a/pkg/utils/otel/provider.go b/pkg/utils/otel/provider.go index 9b7adcf7..40c7044a 100644 --- a/pkg/utils/otel/provider.go +++ b/pkg/utils/otel/provider.go @@ -216,11 +216,11 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) ( return func(ctx context.Context) { err := tracerProvider.Shutdown(ctx) if err != nil && logger != nil { - logger.Fatal("error shutting down trace provider", zap.Error(err)) + logger.Error("error shutting down trace provider", zap.Error(err)) } if traceExporter != nil { if err = traceExporter.Shutdown(ctx); err != nil && logger != nil { - logger.Fatal("error shutting down trace exporter", zap.Error(err)) + logger.Error("error shutting down trace exporter", zap.Error(err)) } } }, nil diff --git a/pkg/utils/signals/signals.go b/pkg/utils/signals/signals.go new file mode 100644 index 00000000..17cf53f0 --- /dev/null +++ b/pkg/utils/signals/signals.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package signals + +import ( + "context" + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" +) + +var onlyOneSignalHandler = make(chan struct{}) + +func SetupSignalHandlerWithContext(logger *zap.Logger) context.Context { + var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + + close(onlyOneSignalHandler) // panics when called twice + + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + signal := <-c + logger.Info("Received signal", zap.String("signal", signal.String())) + cancel() + <-c + panic("multiple signals received") + }() + + return ctx +}