Capture os signals to gracefully shutdown fission components (#2261)

- Currently, fission components don't handle shutdown signals.
So we don't get any to do the required cleanup before the fission process
exits. Adding signal capture process with cancelling context so
that all dependent processes stop working when the process gets term
signal.
- Set log level to error in otel shutdown function

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/executor.go

@@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) {
 
 // StartExecutor Starts executor and the executor components such as Poolmgr,
 // deploymgr and potential future executor types
-func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
+func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
 	fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient()
 	if err != nil {
 		return errors.Wrap(err, "failed to get kubernetes client")
@@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 	}
 	cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments()
 	cnmSvcInformer := cnmInformerFactory.Core().V1().Services()
-	ctx := context.Background()
 	cnm, err := container.MakeContainer(
 		ctx, logger,
 		fissionClient, kubernetesClient,
@@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 	if err != nil {
 		return err
 	}
-	go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
+	go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
 	go api.Serve(port, openTracingEnabled)
 	go serveMetric(logger)
 

pkg/executor/executor_test.go

@@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) {
 
 	// create poolmgr
 	port := 9999
-	err = StartExecutor(logger, functionNs, "fission-builder", port, true)
+	ctx := context.Background()
+	err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true)
 	if err != nil {
 		log.Panicf("failed to start poolmgr: %v", err)
 	}

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) {
 	if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok {
 		deploy.logger.Fatal("failed to wait for caches to sync")
 	}
-	go deploy.idleObjectReaper()
+	go deploy.idleObjectReaper(ctx)
 }
 
 // GetTypeName returns the executor type name.
@@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin
 }
 
 // idleObjectReaper reaps objects after certain idle time
-func (deploy *NewDeploy) idleObjectReaper() {
-	ctx := context.Background()
+func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) {
 	pollSleep := 5 * time.Second
 	for {
 		time.Sleep(pollSleep)

pkg/executor/reaper/reaper.go

@@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie
 
 // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
 // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
-func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
-	ctx := context.Background()
+func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
 	for {
 		// some sleep before the next reaper iteration
 		time.Sleep(cleanupRoleBindingInterval)