Pods immediately terminate for idletimeout in new deployment and container executer type (#2459)

* added waitgroup to handle multiple goroutine scenario
* changes for cache service
* changes to get function service from cache by UID

pkg/executor/executortype/container/containermgr.go

@@ -34,6 +34,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	appsinformers "k8s.io/client-go/informers/apps/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
@@ -141,7 +142,7 @@ func (caaf *Container) Run(ctx context.Context) {
 	if ok := k8sCache.WaitForCacheSync(ctx.Done(), caaf.deplListerSynced, caaf.svcListerSynced); !ok {
 		caaf.logger.Fatal("failed to wait for caches to sync")
 	}
-	go caaf.idleObjectReaper()
+	go caaf.idleObjectReaper(ctx)
 }
 
 // GetTypeName returns the executor type name.
@@ -168,7 +169,7 @@ func (caaf *Container) GetFuncSvc(ctx context.Context, fn *fv1.Function) (*fscac
 // GetFuncSvcFromCache returns a function service from cache; error otherwise.
 func (caaf *Container) GetFuncSvcFromCache(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
 	otelUtils.SpanTrackEvent(ctx, "GetFuncSvcFromCache", otelUtils.GetAttributesForFunction(fn)...)
-	return caaf.fsCache.GetByFunction(&fn.ObjectMeta)
+	return caaf.fsCache.GetByFunctionUID(fn.UID)
 }
 
 // DeleteFuncSvcFromCache deletes a function service from cache.
@@ -703,73 +704,71 @@ func (caaf *Container) updateStatus(fn *fv1.Function, err error, message string)
 }
 
 // idleObjectReaper reaps objects after certain idle time
-func (caaf *Container) idleObjectReaper() {
+func (caaf *Container) idleObjectReaper(ctx context.Context) {
-	ctx := context.Background()
+	// calling function doIdleObjectReaper() repeatedly at given interval of time
+	wait.UntilWithContext(ctx, caaf.doIdleObjectReaper, time.Second*5)
+}
 
-	pollSleep := 5 * time.Second
+func (caaf *Container) doIdleObjectReaper(ctx context.Context) {
-	for {
+	funcSvcs, err := caaf.fsCache.ListOld(time.Second * 5)
-		time.Sleep(pollSleep)
+	if err != nil {
+		caaf.logger.Error("error reaping idle pods", zap.Error(err))
+		return
+	}
 
-		funcSvcs, err := caaf.fsCache.ListOld(pollSleep)
+	for i := range funcSvcs {
-		if err != nil {
+		fsvc := funcSvcs[i]
-			caaf.logger.Error("error reaping idle pods", zap.Error(err))
+
+		if fsvc.Executor != fv1.ExecutorTypeContainer {
 			continue
 		}
 
-		for i := range funcSvcs {
+		fn, err := caaf.fissionClient.CoreV1().Functions(fsvc.Function.Namespace).Get(ctx, fsvc.Function.Name, metav1.GetOptions{})
-			fsvc := funcSvcs[i]
+		if err != nil {
-
+			// CaaF manager handles the function delete event and clean cache/kubeobjs itself,
-			if fsvc.Executor != fv1.ExecutorTypeContainer {
+			// so we ignore the not found error for functions with CaaF executor type here.
+			if k8sErrs.IsNotFound(err) && fsvc.Executor == fv1.ExecutorTypeContainer {
 				continue
 			}
-
+			caaf.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
-			fn, err := caaf.fissionClient.CoreV1().Functions(fsvc.Function.Namespace).Get(ctx, fsvc.Function.Name, metav1.GetOptions{})
+			continue
-			if err != nil {
-				// CaaF manager handles the function delete event and clean cache/kubeobjs itself,
-				// so we ignore the not found error for functions with CaaF executor type here.
-				if k8sErrs.IsNotFound(err) && fsvc.Executor == fv1.ExecutorTypeContainer {
-					continue
-				}
-				caaf.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
-				continue
-			}
-
-			idlePodReapTime := caaf.defaultIdlePodReapTime
-			if fn.Spec.IdleTimeout != nil {
-				idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
-			}
-
-			if time.Since(fsvc.Atime) < idlePodReapTime {
-				continue
-			}
-
-			go func() {
-				deployObj := getDeploymentObj(fsvc.KubernetesObjects)
-				if deployObj == nil {
-					caaf.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-					return
-				}
-
-				currentDeploy, err := caaf.kubernetesClient.AppsV1().
-					Deployments(deployObj.Namespace).Get(ctx, deployObj.Name, metav1.GetOptions{})
-				if err != nil {
-					caaf.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-					return
-				}
-
-				minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
-
-				// do nothing if the current replicas is already lower than minScale
-				if *currentDeploy.Spec.Replicas <= minScale {
-					return
-				}
-
-				err = caaf.scaleDeployment(ctx, deployObj.Namespace, deployObj.Name, minScale)
-				if err != nil {
-					caaf.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-				}
-			}()
 		}
+
+		idlePodReapTime := caaf.defaultIdlePodReapTime
+		if fn.Spec.IdleTimeout != nil {
+			idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
+		}
+
+		if time.Since(fsvc.Atime) < idlePodReapTime {
+			continue
+		}
+
+		go func() {
+			deployObj := getDeploymentObj(fsvc.KubernetesObjects)
+			if deployObj == nil {
+				caaf.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				return
+			}
+
+			currentDeploy, err := caaf.kubernetesClient.AppsV1().
+				Deployments(deployObj.Namespace).Get(ctx, deployObj.Name, metav1.GetOptions{})
+			if err != nil {
+				caaf.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				return
+			}
+
+			minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+
+			// do nothing if the current replicas is already lower than minScale
+			if *currentDeploy.Spec.Replicas <= minScale {
+				return
+			}
+
+			err = caaf.scaleDeployment(ctx, deployObj.Namespace, deployObj.Name, minScale)
+			if err != nil {
+				caaf.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+			}
+		}()
 	}
 }
 

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -35,6 +35,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	appsinformers "k8s.io/client-go/informers/apps/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
@@ -169,7 +170,7 @@ func (deploy *NewDeploy) GetFuncSvc(ctx context.Context, fn *fv1.Function) (*fsc
 // GetFuncSvcFromCache returns a function service from cache; error otherwise.
 func (deploy *NewDeploy) GetFuncSvcFromCache(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
 	otelUtils.SpanTrackEvent(ctx, "GetFuncSvcFromCache")
-	return deploy.fsCache.GetByFunction(&fn.ObjectMeta)
+	return deploy.fsCache.GetByFunctionUID(fn.UID)
 }
 
 // DeleteFuncSvcFromCache deletes a function service from cache.
@@ -593,6 +594,7 @@ func (deploy *NewDeploy) updateFunction(ctx context.Context, oldFn *fv1.Function
 	if oldFn.Spec.Environment != newFn.Spec.Environment ||
 		oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef ||
 		oldFn.Spec.Package.FunctionName != newFn.Spec.Package.FunctionName {
+		deploy.logger.Debug("deployment changed", zap.String("msg", "deployment changed"))
 		deployChanged = true
 	}
 
@@ -766,88 +768,87 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin
 
 // idleObjectReaper reaps objects after certain idle time
 func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) {
-	pollSleep := 5 * time.Second
+	// calling function doIdleObjectReaper() repeatedly at given interval of time
-	for {
+	wait.UntilWithContext(ctx, deploy.doIdleObjectReaper, time.Second*5)
-		time.Sleep(pollSleep)
+}
 
-		envs, err := deploy.fissionClient.CoreV1().Environments(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+func (deploy *NewDeploy) doIdleObjectReaper(ctx context.Context) {
-		if err != nil {
+	envs, err := deploy.fissionClient.CoreV1().Environments(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
-			deploy.logger.Fatal("failed to get environment list", zap.Error(err))
+	if err != nil {
-		}
+		deploy.logger.Fatal("failed to get environment list", zap.Error(err))
+	}
 
-		envList := make(map[k8sTypes.UID]struct{})
+	envList := make(map[k8sTypes.UID]struct{})
-		for _, env := range envs.Items {
+	for _, env := range envs.Items {
-			envList[env.ObjectMeta.UID] = struct{}{}
+		envList[env.ObjectMeta.UID] = struct{}{}
-		}
+	}
 
-		funcSvcs, err := deploy.fsCache.ListOld(pollSleep)
+	funcSvcs, err := deploy.fsCache.ListOld(time.Second * 5)
-		if err != nil {
+	if err != nil {
-			deploy.logger.Error("error reaping idle pods", zap.Error(err))
+		deploy.logger.Error("error reaping idle pods", zap.Error(err))
+		return
+	}
+
+	for i := range funcSvcs {
+		fsvc := funcSvcs[i]
+		if fsvc.Executor != fv1.ExecutorTypeNewdeploy {
 			continue
 		}
 
-		for i := range funcSvcs {
+		// For function with the environment that no longer exists, executor
-			fsvc := funcSvcs[i]
+		// scales down the deployment as usual and prints log to notify user.
-
+		if _, ok := envList[fsvc.Environment.ObjectMeta.UID]; !ok {
-			if fsvc.Executor != fv1.ExecutorTypeNewdeploy {
+			deploy.logger.Warn("function environment no longer exists",
-				continue
+				zap.String("environment", fsvc.Environment.ObjectMeta.Name),
-			}
+				zap.String("function", fsvc.Name))
-
-			// For function with the environment that no longer exists, executor
-			// scales down the deployment as usual and prints log to notify user.
-			if _, ok := envList[fsvc.Environment.ObjectMeta.UID]; !ok {
-				deploy.logger.Warn("function environment no longer exists",
-					zap.String("environment", fsvc.Environment.ObjectMeta.Name),
-					zap.String("function", fsvc.Name))
-			}
-
-			fn, err := deploy.fissionClient.CoreV1().Functions(fsvc.Function.Namespace).Get(ctx, fsvc.Function.Name, metav1.GetOptions{})
-			if err != nil {
-				// Newdeploy manager handles the function delete event and clean cache/kubeobjs itself,
-				// so we ignore the not found error for functions with newdeploy executor type here.
-				if k8sErrs.IsNotFound(err) && fsvc.Executor == fv1.ExecutorTypeNewdeploy {
-					continue
-				}
-				deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
-				continue
-			}
-
-			idlePodReapTime := deploy.defaultIdlePodReapTime
-			if fn.Spec.IdleTimeout != nil {
-				idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
-			}
-
-			if time.Since(fsvc.Atime) < idlePodReapTime {
-				continue
-			}
-
-			go func() {
-				deployObj := getDeploymentObj(fsvc.KubernetesObjects)
-				if deployObj == nil {
-					deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-					return
-				}
-
-				currentDeploy, err := deploy.kubernetesClient.AppsV1().
-					Deployments(deployObj.Namespace).Get(ctx, deployObj.Name, metav1.GetOptions{})
-				if err != nil {
-					deploy.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-					return
-				}
-
-				minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
-
-				// do nothing if the current replicas is already lower than minScale
-				if *currentDeploy.Spec.Replicas <= minScale {
-					return
-				}
-
-				err = deploy.scaleDeployment(ctx, deployObj.Namespace, deployObj.Name, minScale)
-				if err != nil {
-					deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
-				}
-			}()
 		}
+
+		fn, err := deploy.fissionClient.CoreV1().Functions(fsvc.Function.Namespace).Get(ctx, fsvc.Function.Name, metav1.GetOptions{})
+		if err != nil {
+			// Newdeploy manager handles the function delete event and clean cache/kubeobjs itself,
+			// so we ignore the not found error for functions with newdeploy executor type here.
+			if k8sErrs.IsNotFound(err) && fsvc.Executor == fv1.ExecutorTypeNewdeploy {
+				continue
+			}
+			deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
+			continue
+		}
+
+		idlePodReapTime := deploy.defaultIdlePodReapTime
+		if fn.Spec.IdleTimeout != nil {
+			idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
+		}
+
+		if time.Since(fsvc.Atime) < idlePodReapTime {
+			continue
+		}
+
+		go func() {
+			deployObj := getDeploymentObj(fsvc.KubernetesObjects)
+			if deployObj == nil {
+				deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				return
+			}
+
+			currentDeploy, err := deploy.kubernetesClient.AppsV1().
+				Deployments(deployObj.Namespace).Get(ctx, deployObj.Name, metav1.GetOptions{})
+			if err != nil {
+				deploy.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				return
+			}
+
+			minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+
+			// do nothing if the current replicas is already lower than minScale
+			if *currentDeploy.Spec.Replicas <= minScale {
+				return
+			}
+
+			err = deploy.scaleDeployment(ctx, deployObj.Namespace, deployObj.Name, minScale)
+			if err != nil {
+				deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+			}
+		}()
 	}
 }
 

pkg/executor/executortype/poolmgr/gpm.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
 	appsinformers "k8s.io/client-go/informers/apps/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
@@ -169,7 +170,7 @@ func (gpm *GenericPoolManager) Run(ctx context.Context) {
 	gpm.poolPodC.InjectGpm(gpm)
 	go gpm.WebsocketStartEventChecker(gpm.kubernetesClient)
 	go gpm.NoActiveConnectionEventChecker(gpm.kubernetesClient)
-	go gpm.idleObjectReaper()
+	go gpm.idleObjectReaper(ctx)
 	go gpm.poolPodC.Run(ctx.Done())
 }
 
@@ -569,95 +570,93 @@ func (gpm *GenericPoolManager) getFunctionEnv(ctx context.Context, fn *fv1.Funct
 }
 
 // idleObjectReaper reaps objects after certain idle time
-func (gpm *GenericPoolManager) idleObjectReaper() {
+func (gpm *GenericPoolManager) idleObjectReaper(ctx context.Context) {
-	ctx := context.Background()
+	// calling function doIdleObjectReaper() repeatedly at given interval of time
-	pollSleep := 5 * time.Second
+	wait.UntilWithContext(ctx, gpm.doIdleObjectReaper, time.Second*5)
+}
 
-	for {
+func (gpm *GenericPoolManager) doIdleObjectReaper(ctx context.Context) {
-		time.Sleep(pollSleep)
+	envs, err := gpm.fissionClient.CoreV1().Environments(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		gpm.logger.Error("failed to get environment list", zap.Error(err))
+		return
+	}
 
-		envs, err := gpm.fissionClient.CoreV1().Environments(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+	envList := make(map[k8sTypes.UID]struct{})
-		if err != nil {
+	for _, env := range envs.Items {
-			gpm.logger.Error("failed to get environment list", zap.Error(err))
+		envList[env.ObjectMeta.UID] = struct{}{}
+	}
+
+	fns, err := gpm.fissionClient.CoreV1().Functions(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		gpm.logger.Error("failed to get environment list", zap.Error(err))
+		return
+	}
+
+	fnList := make(map[k8sTypes.UID]fv1.Function)
+	for i, fn := range fns.Items {
+		fnList[fn.ObjectMeta.UID] = fns.Items[i]
+	}
+
+	funcSvcs, err := gpm.fsCache.ListOldForPool(time.Second * 5)
+	if err != nil {
+		gpm.logger.Error("error reaping idle pods", zap.Error(err))
+		return
+	}
+
+	for i := range funcSvcs {
+		fsvc := funcSvcs[i]
+
+		if fsvc.Executor != fv1.ExecutorTypePoolmgr {
 			continue
 		}
 
-		envList := make(map[k8sTypes.UID]struct{})
+		if _, ok := gpm.fsCache.WebsocketFsvc.Load(fsvc.Name); ok {
-		for _, env := range envs.Items {
+			continue
-			envList[env.ObjectMeta.UID] = struct{}{}
+		}
+		// For function with the environment that no longer exists, executor
+		// cleanups the idle pod as usual and prints log to notify user.
+		if _, ok := envList[fsvc.Environment.ObjectMeta.UID]; !ok {
+			gpm.logger.Warn("function environment no longer exists",
+				zap.String("environment", fsvc.Environment.ObjectMeta.Name),
+				zap.String("function", fsvc.Name))
 		}
 
-		fns, err := gpm.fissionClient.CoreV1().Functions(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+		if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fv1.AllowedFunctionsPerContainerInfinite {
-		if err != nil {
-			gpm.logger.Error("failed to get environment list", zap.Error(err))
 			continue
 		}
 
-		fnList := make(map[k8sTypes.UID]fv1.Function)
+		idlePodReapTime := gpm.defaultIdlePodReapTime
-		for i, fn := range fns.Items {
+		if fn, ok := fnList[fsvc.Function.UID]; ok {
-			fnList[fn.ObjectMeta.UID] = fns.Items[i]
+			if fn.Spec.IdleTimeout != nil {
+				idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
+			}
 		}
 
-		funcSvcs, err := gpm.fsCache.ListOldForPool(pollSleep)
+		if time.Since(fsvc.Atime) < idlePodReapTime {
-		if err != nil {
-			gpm.logger.Error("error reaping idle pods", zap.Error(err))
 			continue
 		}
 
-		for i := range funcSvcs {
+		go func() {
-			fsvc := funcSvcs[i]
+			deleted, err := gpm.fsCache.DeleteOldPoolCache(ctx, fsvc, idlePodReapTime)
-
+			if err != nil {
-			if fsvc.Executor != fv1.ExecutorTypePoolmgr {
+				gpm.logger.Error("error deleting Kubernetes objects for function service",
-				continue
+					zap.Error(err),
+					zap.Any("service", fsvc))
 			}
-
+			if deleted {
-			if _, ok := gpm.fsCache.WebsocketFsvc.Load(fsvc.Name); ok {
+				for i := range fsvc.KubernetesObjects {
-				continue
+					gpm.logger.Info("release idle function resources",
-			}
+						zap.String("function", fsvc.Function.Name),
-			// For function with the environment that no longer exists, executor
+						zap.String("address", fsvc.Address),
-			// cleanups the idle pod as usual and prints log to notify user.
+						zap.String("executor", string(fsvc.Executor)),
-			if _, ok := envList[fsvc.Environment.ObjectMeta.UID]; !ok {
+						zap.String("pod", fsvc.Name),
-				gpm.logger.Warn("function environment no longer exists",
+					)
-					zap.String("environment", fsvc.Environment.ObjectMeta.Name),
+					reaper.CleanupKubeObject(ctx, gpm.logger, gpm.kubernetesClient, &fsvc.KubernetesObjects[i])
-					zap.String("function", fsvc.Name))
+					time.Sleep(50 * time.Millisecond)
-			}
-
-			if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fv1.AllowedFunctionsPerContainerInfinite {
-				continue
-			}
-
-			idlePodReapTime := gpm.defaultIdlePodReapTime
-			if fn, ok := fnList[fsvc.Function.UID]; ok {
-				if fn.Spec.IdleTimeout != nil {
-					idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
 				}
 			}
-
+		}()
-			if time.Since(fsvc.Atime) < idlePodReapTime {
-				continue
-			}
-
-			go func() {
-				deleted, err := gpm.fsCache.DeleteOldPoolCache(ctx, fsvc, idlePodReapTime)
-				if err != nil {
-					gpm.logger.Error("error deleting Kubernetes objects for function service",
-						zap.Error(err),
-						zap.Any("service", fsvc))
-				}
-				if deleted {
-					for i := range fsvc.KubernetesObjects {
-						gpm.logger.Info("release idle function resources",
-							zap.String("function", fsvc.Function.Name),
-							zap.String("address", fsvc.Address),
-							zap.String("executor", string(fsvc.Executor)),
-							zap.String("pod", fsvc.Name),
-						)
-						reaper.CleanupKubeObject(ctx, gpm.logger, gpm.kubernetesClient, &fsvc.KubernetesObjects[i])
-						time.Sleep(50 * time.Millisecond)
-					}
-				}
-			}()
-		}
 	}
 }
 

pkg/executor/fscache/functionServiceCache.go

@@ -130,10 +130,16 @@ func (fsc *FunctionServiceCache) service() {
 			resp.error = fsc._touchByAddress(req.address)
 		case LISTOLD:
 			// get svcs idle for > req.age
-			fscs := fsc.byFunction.Copy()
+			fscs := fsc.byFunctionUID.Copy()
 			funcObjects := make([]*FuncSvc, 0)
 			for _, funcSvc := range fscs {
-				fsvc := funcSvc.(*FuncSvc)
+				mI := funcSvc.(metav1.ObjectMeta)
+				fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&mI))
+				if err != nil {
+					fsc.logger.Error("error while getting service", zap.Any("error", err))
+					return
+				}
+				fsvc := fsvcI.(*FuncSvc)
 				if time.Since(fsvc.Atime) > req.age {
 					funcObjects = append(funcObjects, fsvc)
 				}