diff --git a/cmd/fetcher/app/server.go b/cmd/fetcher/app/server.go index d2927f79..d17f5f59 100644 --- a/cmd/fetcher/app/server.go +++ b/cmd/fetcher/app/server.go @@ -39,7 +39,7 @@ var ( readyToServe uint32 ) -func Run(logger *zap.Logger) { +func Run(ctx context.Context, logger *zap.Logger) { flag.Usage = fetcherUsage collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "") specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup") @@ -62,8 +62,6 @@ func Run(logger *zap.Logger) { } } } - - ctx := context.Background() openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { go func() { diff --git a/cmd/fetcher/main.go b/cmd/fetcher/main.go index fb98d69a..25f8b8cb 100644 --- a/cmd/fetcher/main.go +++ b/cmd/fetcher/main.go @@ -20,6 +20,7 @@ import ( "github.com/fission/fission/cmd/fetcher/app" "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" ) // Usage: fetcher @@ -29,5 +30,6 @@ func main() { profile.ProfileIfEnabled(logger) - app.Run(logger) + ctx := signals.SetupSignalHandlerWithContext(logger) + app.Run(ctx, logger) } diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 55d82139..5f089594 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -41,27 +41,28 @@ import ( "github.com/fission/fission/pkg/utils/loggerfactory" "github.com/fission/fission/pkg/utils/otel" "github.com/fission/fission/pkg/utils/profile" + "github.com/fission/fission/pkg/utils/signals" "github.com/fission/fission/pkg/utils/tracing" ) -func runController(logger *zap.Logger, port int, openTracingEnabled bool) { - controller.Start(logger, port, false, openTracingEnabled) +func runController(ctx context.Context, logger *zap.Logger, port int, openTracingEnabled bool) { + controller.Start(ctx, logger, port, false, openTracingEnabled) } -func runRouter(logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { - router.Start(logger, port, executorUrl, openTracingEnabled) +func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) { + router.Start(ctx, logger, port, executorUrl, openTracingEnabled) } -func runExecutor(logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { - return executor.StartExecutor(logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) +func runExecutor(ctx context.Context, logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error { + return executor.StartExecutor(ctx, logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled) } -func runKubeWatcher(logger *zap.Logger, routerUrl string) error { - return kubewatcher.Start(logger, routerUrl) +func runKubeWatcher(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return kubewatcher.Start(ctx, logger, routerUrl) } -func runTimer(logger *zap.Logger, routerUrl string) error { - return timer.Start(logger, routerUrl) +func runTimer(ctx context.Context, logger *zap.Logger, routerUrl string) error { + return timer.Start(ctx, logger, routerUrl) } func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { @@ -69,20 +70,20 @@ func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error { } // KEDA based MessageQueue Trigger Manager -func runMQManager(logger *zap.Logger, routerURL string) error { - return mqt.StartScalerManager(logger, routerURL) +func runMQManager(ctx context.Context, logger *zap.Logger, routerURL string) error { + return mqt.StartScalerManager(ctx, logger, routerURL) } -func runStorageSvc(logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { - return storagesvc.Start(logger, storage, port, openTracingEnabled) +func runStorageSvc(ctx context.Context, logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error { + return storagesvc.Start(ctx, logger, storage, port, openTracingEnabled) } -func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { - return buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace) +func runBuilderMgr(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { + return buildermgr.Start(ctx, logger, storageSvcUrl, envBuilderNamespace) } -func runLogger() { - functionLogger.Start() +func runLogger(ctx context.Context, logger *zap.Logger) { + functionLogger.Start(ctx, logger) } func getPort(logger *zap.Logger, portArg interface{}) int { @@ -129,10 +130,8 @@ func getServiceName(arguments map[string]interface{}) string { } func exitWithSync(logger *zap.Logger) { - err := logger.Sync() - if err != nil { - logger.Error("failed to sync log", zap.Error(err)) - } + // Ignore error, safe to ignore as per https://github.com/uber-go/zap/issues/328 + _ = logger.Sync() os.Exit(1) } @@ -210,7 +209,8 @@ Options: return } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) + openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments)) @@ -238,21 +238,21 @@ Options: if arguments["--controllerPort"] != nil { port := getPort(logger, arguments["--controllerPort"]) - runController(logger, port, openTracingEnabled) + runController(ctx, logger, port, openTracingEnabled) logger.Error("controller exited") return } if arguments["--routerPort"] != nil { port := getPort(logger, arguments["--routerPort"]) - runRouter(logger, port, executorUrl, openTracingEnabled) + runRouter(ctx, logger, port, executorUrl, openTracingEnabled) logger.Error("router exited") return } if arguments["--executorPort"] != nil { port := getPort(logger, arguments["--executorPort"]) - err = runExecutor(logger, port, functionNs, envBuilderNs, openTracingEnabled) + err = runExecutor(ctx, logger, port, functionNs, envBuilderNs, openTracingEnabled) if err != nil { logger.Error("executor exited", zap.Error(err)) return @@ -260,7 +260,7 @@ Options: } if arguments["--kubewatcher"] == true { - err = runKubeWatcher(logger, routerUrl) + err = runKubeWatcher(ctx, logger, routerUrl) if err != nil { logger.Error("kubewatcher exited", zap.Error(err)) return @@ -268,7 +268,7 @@ Options: } if arguments["--timer"] == true { - err = runTimer(logger, routerUrl) + err = runTimer(ctx, logger, routerUrl) if err != nil { logger.Error("timer exited", zap.Error(err)) return @@ -284,7 +284,7 @@ Options: } if arguments["--mqt_keda"] == true { - err = runMQManager(logger, routerUrl) + err = runMQManager(ctx, logger, routerUrl) if err != nil { logger.Error("mqt scaler manager exited", zap.Error(err)) return @@ -292,7 +292,7 @@ Options: } if arguments["--builderMgr"] == true { - err = runBuilderMgr(logger, storageSvcUrl, envBuilderNs) + err = runBuilderMgr(ctx, logger, storageSvcUrl, envBuilderNs) if err != nil { logger.Error("builder manager exited", zap.Error(err)) return @@ -300,7 +300,7 @@ Options: } if arguments["--logger"] == true { - runLogger() + runLogger(ctx, logger) logger.Error("logger exited") return } @@ -315,11 +315,13 @@ Options: } else if arguments["--storageType"] == string(storagesvc.StorageTypeLocal) { storage = storagesvc.NewLocalStorage("/fission") } - err := runStorageSvc(logger, port, storage, openTracingEnabled) + err := runStorageSvc(ctx, logger, port, storage, openTracingEnabled) if err != nil { logger.Error("storage service exited", zap.Error(err)) return } } - select {} + + <-ctx.Done() + logger.Error("exiting") } diff --git a/cmd/preupgradechecks/main.go b/cmd/preupgradechecks/main.go index 35bb2887..fb714441 100644 --- a/cmd/preupgradechecks/main.go +++ b/cmd/preupgradechecks/main.go @@ -17,13 +17,12 @@ limitations under the License. package main import ( - "context" - "github.com/docopt/docopt-go" "go.uber.org/zap" "github.com/fission/fission/pkg/info" "github.com/fission/fission/pkg/utils/loggerfactory" + "github.com/fission/fission/pkg/utils/signals" ) func getStringArgWithDefault(arg interface{}, defaultValue string) string { @@ -59,7 +58,7 @@ Options: zap.Error(err)) } - ctx := context.Background() + ctx := signals.SetupSignalHandlerWithContext(logger) crd := crdBackedClient.GetFunctionCRD(ctx) if crd == nil { logger.Info("nothing to do since CRDs are not present on the cluster") diff --git a/cmd/reporter/app/cmd_event.go b/cmd/reporter/app/cmd_event.go index 431f35c6..44c33612 100644 --- a/cmd/reporter/app/cmd_event.go +++ b/cmd/reporter/app/cmd_event.go @@ -16,6 +16,7 @@ limitations under the License. package app import ( + "context" "log" "github.com/spf13/cobra" @@ -49,7 +50,8 @@ func eventCommandHandler(cmd *cobra.Command, args []string) error { return err } - return tracker.Tracker.SendEvent(event) + ctx := context.Background() + return tracker.Tracker.SendEvent(ctx, event) } //EventCommand reports an event to analytics diff --git a/pkg/buildermgr/buildermgr.go b/pkg/buildermgr/buildermgr.go index 6e4a9f4c..33adcb5a 100644 --- a/pkg/buildermgr/buildermgr.go +++ b/pkg/buildermgr/buildermgr.go @@ -17,6 +17,7 @@ limitations under the License. package buildermgr import ( + "context" "time" "github.com/pkg/errors" @@ -29,7 +30,7 @@ import ( ) // Start the buildermgr service. -func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { +func Start(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { bmLogger := logger.Named("builder_manager") fissionClient, kubernetesClient, _, _, err := crd.MakeFissionClient() @@ -56,6 +57,6 @@ func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) pkgInformer := informerFactory.Core().V1().Packages().Informer() pkgWatcher := makePackageWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl, &podInformer, &pkgInformer) - pkgWatcher.Run() + pkgWatcher.Run(ctx) return nil } diff --git a/pkg/buildermgr/pkgwatcher.go b/pkg/buildermgr/pkgwatcher.go index 3d87e451..fd0eb463 100644 --- a/pkg/buildermgr/pkgwatcher.go +++ b/pkg/buildermgr/pkgwatcher.go @@ -322,11 +322,10 @@ func (pkgw *packageWatcher) packageInformerHandler() k8sCache.ResourceEventHandl } } -func (pkgw *packageWatcher) Run() { - context := context.Background() - go (*pkgw.podInformer).Run(context.Done()) +func (pkgw *packageWatcher) Run(ctx context.Context) { + go (*pkgw.podInformer).Run(ctx.Done()) (*pkgw.pkgInformer).AddEventHandler(pkgw.packageInformerHandler()) - (*pkgw.pkgInformer).Run(context.Done()) + (*pkgw.pkgInformer).Run(ctx.Done()) } // setInitialBuildStatus sets initial build status to a package if it is empty. diff --git a/pkg/controller/api_test.go b/pkg/controller/api_test.go index 975953a2..c0bbca3e 100644 --- a/pkg/controller/api_test.go +++ b/pkg/controller/api_test.go @@ -376,7 +376,8 @@ func TestMain(m *testing.M) { panicIf(err) - go Start(logger, 8888, true, true) + ctx := context.Background() + go Start(ctx, logger, 8888, true, true) time.Sleep(5 * time.Second) diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e7a5b20c..83bebf8b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,7 +24,7 @@ import ( "github.com/fission/fission/pkg/crd" ) -func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) { cLogger := logger.Named("controller") fc, kc, apiExtClient, _, err := crd.MakeFissionClient() @@ -42,12 +42,10 @@ func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled b cLogger.Fatal("error waiting for CRDs", zap.Error(err)) } - ctx, cancel := context.WithCancel(context.Background()) featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) if err != nil { cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err)) } - defer cancel() api, err := MakeAPI(cLogger, featureStatus) if err != nil { diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 34603be2..6145d5d5 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) { // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types -func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { +func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error { fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get kubernetes client") @@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames } cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments() cnmSvcInformer := cnmInformerFactory.Core().V1().Services() - ctx := context.Background() cnm, err := container.MakeContainer( ctx, logger, fissionClient, kubernetesClient, @@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames if err != nil { return err } - go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) + go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) go api.Serve(port, openTracingEnabled) go serveMetric(logger) diff --git a/pkg/executor/executor_test.go b/pkg/executor/executor_test.go index b40cc3ec..9c363800 100644 --- a/pkg/executor/executor_test.go +++ b/pkg/executor/executor_test.go @@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) { // create poolmgr port := 9999 - err = StartExecutor(logger, functionNs, "fission-builder", port, true) + ctx := context.Background() + err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true) if err != nil { log.Panicf("failed to start poolmgr: %v", err) } diff --git a/pkg/executor/executortype/newdeploy/newdeploymgr.go b/pkg/executor/executortype/newdeploy/newdeploymgr.go index aea0f014..2908ac59 100644 --- a/pkg/executor/executortype/newdeploy/newdeploymgr.go +++ b/pkg/executor/executortype/newdeploy/newdeploymgr.go @@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) { if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok { deploy.logger.Fatal("failed to wait for caches to sync") } - go deploy.idleObjectReaper() + go deploy.idleObjectReaper(ctx) } // GetTypeName returns the executor type name. @@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin } // idleObjectReaper reaps objects after certain idle time -func (deploy *NewDeploy) idleObjectReaper() { - ctx := context.Background() +func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) { pollSleep := 5 * time.Second for { time.Sleep(pollSleep) diff --git a/pkg/executor/reaper/reaper.go b/pkg/executor/reaper/reaper.go index 335871c1..0b8f5496 100644 --- a/pkg/executor/reaper/reaper.go +++ b/pkg/executor/reaper/reaper.go @@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object. -func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { - ctx := context.Background() +func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { for { // some sleep before the next reaper iteration time.Sleep(cleanupRoleBindingInterval) diff --git a/pkg/kubewatcher/kubewatcher.go b/pkg/kubewatcher/kubewatcher.go index 9d1725ef..a7ddaacf 100644 --- a/pkg/kubewatcher/kubewatcher.go +++ b/pkg/kubewatcher/kubewatcher.go @@ -77,7 +77,7 @@ type ( } ) -func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { +func MakeKubeWatcher(ctx context.Context, logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ logger: logger.Named("kube_watcher"), watches: make(map[types.UID]watchSubscription), @@ -85,7 +85,7 @@ func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher: publisher, requestChannel: make(chan *kubeWatcherRequest), } - go kw.svc() + go kw.svc(ctx) return kw } @@ -100,7 +100,7 @@ func (kw *KubeWatcher) Sync(watches []fv1.KubernetesWatchTrigger) error { return resp.error } -func (kw *KubeWatcher) svc() { +func (kw *KubeWatcher) svc(ctx context.Context) { for { req := <-kw.requestChannel switch req.requestType { @@ -118,7 +118,7 @@ func (kw *KubeWatcher) svc() { // Add new watches for _, w := range req.watches { if _, ok := kw.watches[w.ObjectMeta.UID]; !ok { - kw.addWatch(&w) //nolint: errCheck + kw.addWatch(ctx, &w) //nolint: errCheck } } req.responseChannel <- &kubeWatcherResponse{error: nil} @@ -149,7 +149,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error { return err } -func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { +func createKubernetesWatch(ctx context.Context, kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) { var wi watch.Interface var err error var watchTimeoutSec int64 = 120 @@ -163,22 +163,22 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWa // TODO handle the full list of types switch strings.ToUpper(w.Spec.Type) { case "POD": - wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(ctx, listOptions) case "SERVICE": - wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(ctx, listOptions) case "REPLICATIONCONTROLLER": - wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(ctx, listOptions) case "JOB": - wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(context.TODO(), listOptions) + wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(ctx, listOptions) default: err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)) } return wi, err } -func (kw *KubeWatcher) addWatch(w *fv1.KubernetesWatchTrigger) error { +func (kw *KubeWatcher) addWatch(ctx context.Context, w *fv1.KubernetesWatchTrigger) error { kw.logger.Info("adding watch", zap.String("name", w.ObjectMeta.Name), zap.Any("function", w.Spec.FunctionReference)) - ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) + ws, err := MakeWatchSubscription(ctx, kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) if err != nil { return err } @@ -198,7 +198,7 @@ func (kw *KubeWatcher) removeWatch(w *fv1.KubernetesWatchTrigger) error { return nil } -func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(ctx context.Context, logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ logger: logger.Named("watch_subscription"), @@ -210,16 +210,16 @@ func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, ku lastResourceVersion: "", } - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { return nil, err } - go ws.eventDispatchLoop() + go ws.eventDispatchLoop(ctx) return ws, nil } -func (ws *watchSubscription) restartWatch() error { +func (ws *watchSubscription) restartWatch(ctx context.Context) error { retries := 60 for { ws.logger.Info("(re)starting watch", @@ -227,7 +227,7 @@ func (ws *watchSubscription) restartWatch() error { zap.String("namespace", ws.watch.Spec.Namespace), zap.String("type", ws.watch.Spec.Type), zap.String("last_resource_version", ws.lastResourceVersion)) - wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) + wi, err := createKubernetesWatch(ctx, ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- if retries > 0 { @@ -250,7 +250,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { return m.GetResourceVersion(), nil } -func (ws *watchSubscription) eventDispatchLoop() { +func (ws *watchSubscription) eventDispatchLoop(ctx context.Context) { ws.logger.Info("listening to watch", zap.String("name", ws.watch.ObjectMeta.Name)) for { // check watchSubscription is stopped or not before waiting for event @@ -269,7 +269,7 @@ func (ws *watchSubscription) eventDispatchLoop() { } else { // watch closed due to timeout, restart it. ws.logger.Warn("watch timed out - restarting", zap.String("watch_name", ws.watch.ObjectMeta.Name)) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } @@ -283,7 +283,7 @@ func (ws *watchSubscription) eventDispatchLoop() { // Start from the beginning to get around "too old resource version" ws.lastResourceVersion = "" time.Sleep(time.Second) - err := ws.restartWatch() + err := ws.restartWatch(ctx) if err != nil { ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name)) } diff --git a/pkg/kubewatcher/main.go b/pkg/kubewatcher/main.go index 4a62dc63..62901482 100644 --- a/pkg/kubewatcher/main.go +++ b/pkg/kubewatcher/main.go @@ -17,6 +17,8 @@ limitations under the License. package kubewatcher import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - kubeWatch := MakeKubeWatcher(logger, kubeClient, poster) + kubeWatch := MakeKubeWatcher(ctx, logger, kubeClient, poster) MakeWatchSync(logger, fissionClient, kubeWatch) return nil diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 423fedd1..843519e7 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -17,6 +17,7 @@ limitations under the License. package logger import ( + "context" "fmt" "log" "os" @@ -32,7 +33,6 @@ import ( fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/crd" "github.com/fission/fission/pkg/utils" - "github.com/fission/fission/pkg/utils/loggerfactory" ) var nodeName = os.Getenv("NODE_NAME") @@ -157,10 +157,7 @@ func symlinkReaper(zapLogger *zap.Logger) { } } -func Start() { - logger := loggerfactory.GetLogger() - defer logger.Sync() - +func Start(ctx context.Context, logger *zap.Logger) { if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) { logger.Info("symlink path not exist, create it", zap.String("fissionSymlinkPath", fissionSymlinkPath)) @@ -177,6 +174,6 @@ func Start() { informerFactory := k8sInformers.NewSharedInformerFactory(kubernetesClient, time.Minute*30) podInformer := informerFactory.Core().V1().Pods().Informer() podInformer.AddEventHandler(podInformerHandlers(logger)) - podInformer.Run(make(chan struct{})) - logger.Fatal("Stop watching pod changes") + podInformer.Run(ctx.Done()) + logger.Error("Stop watching pod changes") } diff --git a/pkg/mqtrigger/scalermanager.go b/pkg/mqtrigger/scalermanager.go index ed27ece8..85283b5a 100644 --- a/pkg/mqtrigger/scalermanager.go +++ b/pkg/mqtrigger/scalermanager.go @@ -149,7 +149,7 @@ func mqTriggerEventHandlers(logger *zap.Logger, kubeClient *kubernetes.Clientset // StartScalerManager watches for changes in MessageQueueTrigger and, // Based on changes, it Creates, Updates and Deletes Objects of Kind ScaledObjects, AuthenticationTriggers and Deployments -func StartScalerManager(logger *zap.Logger, routerURL string) error { +func StartScalerManager(ctx context.Context, logger *zap.Logger, routerURL string) error { fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() if err != nil { return err @@ -161,7 +161,7 @@ func StartScalerManager(logger *zap.Logger, routerURL string) error { informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30) mqTriggerInformer := informerFactory.Core().V1().MessageQueueTriggers().Informer() mqTriggerInformer.AddEventHandler(mqTriggerEventHandlers(logger, kubeClient, routerURL)) - mqTriggerInformer.Run(context.Background().Done()) + mqTriggerInformer.Run(ctx.Done()) return nil } diff --git a/pkg/router/router.go b/pkg/router/router.go index 42957694..79b57de8 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -126,7 +126,7 @@ func serveMetric(logger *zap.Logger) { } // Start starts a router -func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { +func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) { fmap := makeFunctionServiceMap(logger, time.Minute) fissionClient, kubeClient, _, _, err := crd.MakeFissionClient() @@ -258,10 +258,8 @@ func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled logger.Info("starting router", zap.Int("port", port)) tracer := otel.Tracer("router") - ctx, span := tracer.Start(context.Background(), "router/Start") + ctx, span := tracer.Start(ctx, "router/Start") defer span.End() - ctxWithCancel, cancel := context.WithCancel(ctx) - defer cancel() - serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) + serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled) } diff --git a/pkg/storagesvc/archivePruner.go b/pkg/storagesvc/archivePruner.go index 87bcd505..8293c78c 100644 --- a/pkg/storagesvc/archivePruner.go +++ b/pkg/storagesvc/archivePruner.go @@ -75,13 +75,13 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) { // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes // and not the archives that are referenced by them, leaving the archives as orphans. // getOrphanArchives reaps the orphaned archives. -func (pruner *ArchivePruner) getOrphanArchives() { +func (pruner *ArchivePruner) getOrphanArchives(ctx context.Context) { pruner.logger.Debug("getting orphan archives") archivesRefByPkgs := make([]string, 0) var archiveID string // get all pkgs from kubernetes - pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { pruner.logger.Error("error getting package list from kubernetes", zap.Error(err)) return @@ -137,12 +137,12 @@ func (pruner *ArchivePruner) getOrphanArchives() { // Start starts a go routine that listens to a channel for archive IDs that need to deleted. // Also wakes up at regular intervals to make a list of archive IDs that need to be reaped // and sends them over to the channel for deletion -func (pruner *ArchivePruner) Start() { +func (pruner *ArchivePruner) Start(ctx context.Context) { ticker := time.NewTicker(pruner.pruneInterval * time.Minute) go pruner.pruneArchives() for range ticker.C { // This method fetches unused archive IDs and sends them to archiveChannel for deletion // silencing the errors, hoping they go away in next iteration. - pruner.getOrphanArchives() + pruner.getOrphanArchives(ctx) } } diff --git a/pkg/storagesvc/client/storagesvc_test.go b/pkg/storagesvc/client/storagesvc_test.go index 4304245a..d44d66f7 100644 --- a/pkg/storagesvc/client/storagesvc_test.go +++ b/pkg/storagesvc/client/storagesvc_test.go @@ -76,7 +76,7 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource { return resource } -func startS3StorageService(endpoint, bucketName, subDir string) { +func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) { // testID := uniuri.NewLen(8) port := 8081 @@ -94,7 +94,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) { os.Setenv("STORAGE_S3_REGION", minioRegion) storage := storagesvc.NewS3Storage() - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(ctx, logger, storage, port, true) } func TestS3StorageService(t *testing.T) { @@ -135,7 +135,7 @@ func TestS3StorageService(t *testing.T) { // Start storagesvc bucketName := "test-s3-service" subDir := "x/y/z" - startS3StorageService(endpoint, bucketName, subDir) + startS3StorageService(context.Background(), endpoint, bucketName, subDir) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081)) @@ -211,7 +211,7 @@ func TestLocalStorageService(t *testing.T) { localPath := fmt.Sprintf("/tmp/%v", testID) _ = os.Mkdir(localPath, os.ModePerm) storage := storagesvc.NewLocalStorage(localPath) - _ = storagesvc.Start(logger, storage, port, true) + _ = storagesvc.Start(context.Background(), logger, storage, port, true) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", port)) diff --git a/pkg/storagesvc/storagesvc.go b/pkg/storagesvc/storagesvc.go index 642eddbd..8fadd452 100644 --- a/pkg/storagesvc/storagesvc.go +++ b/pkg/storagesvc/storagesvc.go @@ -17,6 +17,7 @@ limitations under the License. package storagesvc import ( + "context" "encoding/json" "fmt" "net/http" @@ -221,7 +222,7 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) { } // Start runs storage service -func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { +func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error { enablePruner := true // create a storage client storageClient, err := MakeStowClient(logger, storage) @@ -244,7 +245,7 @@ func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled boo if err != nil { return errors.Wrap(err, "Error creating archivePruner") } - go pruner.Start() + go pruner.Start(ctx) } logger.Info("storage service started") diff --git a/pkg/timer/main.go b/pkg/timer/main.go index 64da005d..d9df1e19 100644 --- a/pkg/timer/main.go +++ b/pkg/timer/main.go @@ -17,6 +17,8 @@ limitations under the License. package timer import ( + "context" + "github.com/pkg/errors" "go.uber.org/zap" @@ -24,7 +26,7 @@ import ( "github.com/fission/fission/pkg/publisher" ) -func Start(logger *zap.Logger, routerUrl string) error { +func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error { fissionClient, _, _, _, err := crd.MakeFissionClient() if err != nil { return errors.Wrap(err, "failed to get fission or kubernetes client") @@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error { } poster := publisher.MakeWebhookPublisher(logger, routerUrl) - MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster)) + MakeTimerSync(ctx, logger, fissionClient, MakeTimer(logger, poster)) return nil } diff --git a/pkg/timer/timerSync.go b/pkg/timer/timerSync.go index 2a7b7446..547c2ae4 100644 --- a/pkg/timer/timerSync.go +++ b/pkg/timer/timerSync.go @@ -35,19 +35,19 @@ type ( } ) -func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { +func MakeTimerSync(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ logger: logger.Named("timer_sync"), fissionClient: fissionClient, timer: timer, } - go ws.syncSvc() + go ws.syncSvc(ctx) return ws } -func (ws *TimerSync) syncSvc() { +func (ws *TimerSync) syncSvc(ctx context.Context) { for { - triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { if utils.IsNetworkError(err) { ws.logger.Info("encountered a network error - will retry", zap.Error(err)) diff --git a/pkg/tracker/tracker.go b/pkg/tracker/tracker.go index 2a4979ba..9a9351ca 100644 --- a/pkg/tracker/tracker.go +++ b/pkg/tracker/tracker.go @@ -54,7 +54,7 @@ func init() { Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: id.String()} } -func (t *tracker) SendEvent(e Event) error { +func (t *tracker) SendEvent(ctx context.Context, e Event) error { if t.gaPropertyID == "" { return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set") } @@ -81,7 +81,7 @@ func (t *tracker) SendEvent(e Event) error { } buf := bytes.NewBufferString(v.Encode()) - req, err := http.NewRequest("POST", GA_API_URL, buf) + req, err := http.NewRequestWithContext(ctx, "POST", GA_API_URL, buf) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("User-Agent", "ga-tracker/1.0") if err != nil { diff --git a/pkg/utils/otel/provider.go b/pkg/utils/otel/provider.go index 9b7adcf7..40c7044a 100644 --- a/pkg/utils/otel/provider.go +++ b/pkg/utils/otel/provider.go @@ -216,11 +216,11 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) ( return func(ctx context.Context) { err := tracerProvider.Shutdown(ctx) if err != nil && logger != nil { - logger.Fatal("error shutting down trace provider", zap.Error(err)) + logger.Error("error shutting down trace provider", zap.Error(err)) } if traceExporter != nil { if err = traceExporter.Shutdown(ctx); err != nil && logger != nil { - logger.Fatal("error shutting down trace exporter", zap.Error(err)) + logger.Error("error shutting down trace exporter", zap.Error(err)) } } }, nil diff --git a/pkg/utils/signals/signals.go b/pkg/utils/signals/signals.go new file mode 100644 index 00000000..17cf53f0 --- /dev/null +++ b/pkg/utils/signals/signals.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package signals + +import ( + "context" + "os" + "os/signal" + "syscall" + + "go.uber.org/zap" +) + +var onlyOneSignalHandler = make(chan struct{}) + +func SetupSignalHandlerWithContext(logger *zap.Logger) context.Context { + var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} + + close(onlyOneSignalHandler) // panics when called twice + + ctx, cancel := context.WithCancel(context.Background()) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + signal := <-c + logger.Info("Received signal", zap.String("signal", signal.String())) + cancel() + <-c + panic("multiple signals received") + }() + + return ctx +}