Fix poolmanager terminates running function pod periodically (#1435)

The pool manager keeps terminating function pod periodically even there are
traffic to the function. The root cause is that executor, poolmgr, newdeploy
manage their own functionServiceCache separately. And when router taps a
function, executor updates the access time of the function service entry in its
own cache without notifying executor types to do the update as well. Hence,
the access time of function service entry in poolmanager cache never gets updated.
Due to the access time never gets updated, the idle pod reaper in poolmanager
then thinks the function pod is in idle state and recycle it.

This PR removes the cache in executor itself, and when router tries to tap a function,
executor will call executor type to tap the function and update access time.

pkg/executor/executor.go

@@ -32,9 +32,10 @@ import (
 	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
 	"github.com/fission/fission/pkg/crd"
 	"github.com/fission/fission/pkg/executor/cms"
+	"github.com/fission/fission/pkg/executor/executortype"
+	"github.com/fission/fission/pkg/executor/executortype/newdeploy"
+	"github.com/fission/fission/pkg/executor/executortype/poolmgr"
 	"github.com/fission/fission/pkg/executor/fscache"
-	"github.com/fission/fission/pkg/executor/newdeploy"
-	"github.com/fission/fission/pkg/executor/poolmgr"
 	"github.com/fission/fission/pkg/executor/reaper"
 	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
 )
@@ -43,12 +44,10 @@ type (
 	Executor struct {
 		logger *zap.Logger
 
-		gpm *poolmgr.GenericPoolManager
-		ndm *newdeploy.NewDeploy
-		cms *cms.ConfigSecretController
+		executorTypes map[fv1.ExecutorType]executortype.ExecutorType
+		cms           *cms.ConfigSecretController
 
 		fissionClient *crd.FissionClient
-		fsCache       *fscache.FunctionServiceCache
 
 		requestChan chan *createFuncServiceRequest
 		fsCreateWg  map[string]*sync.WaitGroup
@@ -64,21 +63,20 @@ type (
 	}
 )
 
-func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, cms *cms.ConfigSecretController, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
+func MakeExecutor(logger *zap.Logger, cms *cms.ConfigSecretController,
+	fissionClient *crd.FissionClient, types map[fv1.ExecutorType]executortype.ExecutorType) (*Executor, error) {
 	executor := &Executor{
 		logger:        logger.Named("executor"),
-		gpm:           gpm,
-		ndm:           ndm,
 		cms:           cms,
 		fissionClient: fissionClient,
-		fsCache:       fsCache,
+		executorTypes: types,
 
 		requestChan: make(chan *createFuncServiceRequest),
 		fsCreateWg:  make(map[string]*sync.WaitGroup),
 	}
 	go executor.serveCreateFuncServices()
 
-	return executor
+	return executor, nil
 }
 
 // All non-cached function service requests go through this goroutine
@@ -115,6 +113,7 @@ func (executor *Executor) serveCreateFuncServices() {
 				buffer := 10 // add some buffer time for specialization
 				fnSpecializationTimeoutContext, cancel := context.WithTimeout(context.Background(),
 					time.Duration(req.function.Spec.InvokeStrategy.ExecutionStrategy.SpecializationTimeout+buffer)*time.Second)
+				defer cancel()
 
 				fsvc, err := executor.createServiceForFunction(fnSpecializationTimeoutContext, req.function)
 				req.respChan <- &createFuncServiceResponse{
@@ -122,8 +121,6 @@ func (executor *Executor) serveCreateFuncServices() {
 					err:     err,
 				}
 				delete(executor.fsCreateWg, crd.CacheKey(fnMetadata))
-
-				cancel()
 				wg.Done()
 			}()
 		} else {
@@ -134,7 +131,7 @@ func (executor *Executor) serveCreateFuncServices() {
 				wg.Wait()
 
 				// get the function service from the cache
-				fsvc, err := executor.fsCache.GetByFunction(fnMetadata)
+				fsvc, err := executor.getFunctionServiceFromCache(req.function)
 
 				// fsCache return error when the entry does not exist/expire.
 				// It normally happened if there are multiple requests are
@@ -155,18 +152,13 @@ func (executor *Executor) createServiceForFunction(ctx context.Context, fn *fv1.
 		zap.String("function_name", fn.Metadata.Name),
 		zap.String("function_namespace", fn.Metadata.Namespace))
 
-	executorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
-
-	var fsvc *fscache.FuncSvc
-	var fsvcErr error
-
-	switch executorType {
-	case fv1.ExecutorTypeNewdeploy:
-		fsvc, fsvcErr = executor.ndm.GetFuncSvc(ctx, fn)
-	default:
-		fsvc, fsvcErr = executor.gpm.GetFuncSvc(ctx, fn)
+	t := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+	e, ok := executor.executorTypes[t]
+	if !ok {
+		return nil, errors.Errorf("Unknown executor type '%v'", t)
 	}
 
+	fsvc, fsvcErr := e.GetFuncSvc(ctx, fn)
 	if fsvcErr != nil {
 		e := "error creating service for function"
 		executor.logger.Error(e,
@@ -174,25 +166,18 @@ func (executor *Executor) createServiceForFunction(ctx context.Context, fn *fv1.
 			zap.String("function_name", fn.Metadata.Name),
 			zap.String("function_namespace", fn.Metadata.Namespace))
 		fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", fn.Metadata.Name, e))
-	} else if fsvc != nil {
-		_, err := executor.fsCache.Add(*fsvc)
-		if err != nil {
-			return nil, err
-		}
 	}
 
-	executor.fsCache.IncreaseColdStarts(fn.Metadata.Name, string(fn.Metadata.UID))
-
 	return fsvc, fsvcErr
 }
 
-// isValidAddress invokes isValidService or isValidPod depending on the type of executor
-func (executor *Executor) isValidAddress(fsvc *fscache.FuncSvc) bool {
-	if fsvc.Executor == fscache.NEWDEPLOY {
-		return executor.ndm.IsValid(fsvc)
-	} else {
-		return executor.gpm.IsValid(fsvc)
+func (executor *Executor) getFunctionServiceFromCache(fn *fv1.Function) (*fscache.FuncSvc, error) {
+	t := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+	e, ok := executor.executorTypes[t]
+	if !ok {
+		return nil, errors.Errorf("Unknown executor type '%v'", t)
 	}
+	return e.GetFuncSvcFromCache(fn)
 }
 
 func serveMetric(logger *zap.Logger) {
@@ -223,7 +208,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 	}
 
 	restClient := fissionClient.GetCrdClient()
-	fsCache := fscache.MakeFunctionServiceCache(logger)
 
 	poolID := strings.ToLower(uniuri.NewLen(8))
 	reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID)
@@ -239,9 +223,16 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
 		fissionClient, kubernetesClient, restClient,
 		functionNamespace, fetcherConfig, poolID)
 
-	cms := cms.MakeConfigSecretController(logger, fissionClient, kubernetesClient, ndm, gpm)
+	executorTypes := make(map[fv1.ExecutorType]executortype.ExecutorType)
+	executorTypes[gpm.GetTypeName()] = gpm
+	executorTypes[ndm.GetTypeName()] = ndm
 
-	api := MakeExecutor(logger, gpm, ndm, cms, fissionClient, fsCache)
+	cms := cms.MakeConfigSecretController(logger, fissionClient, kubernetesClient, executorTypes)
+
+	api, err := MakeExecutor(logger, cms, fissionClient, executorTypes)
+	if err != nil {
+		return err
+	}
 
 	go api.Serve(port)
 	go serveMetric(logger)