Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Function-level idle timeout setting (#1538)

Browse Source 
This PR allows users to specify the idle timeout setting at Function-level.
Fix #1050
This commit is contained in:
Ta-Ching Chen
2020-02-21 02:01:26 +08:00
committed by GitHub
parent cc14044b6b
commit fc5711ce7c
10 changed files with 99 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/executortype/newdeploy/newdeploymgr.go
+36 -23
View File
@@ -74,7 +74,7 @@ type (
envStore k8sCache.Store
envController k8sCache.Controller
idlePodReapTime time.Duration
defaultIdlePodReapTime time.Duration
}
)
@@ -112,7 +112,7 @@ func MakeNewDeploy(
runtimeImagePullPolicy: utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")),
useIstio: enableIstio,
idlePodReapTime: 2 * time.Minute,
defaultIdlePodReapTime: 2 * time.Minute,
}
if nd.crdClient != nil {
@@ -769,7 +769,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin
// idleObjectReaper reaps objects after certain idle time
func (deploy *NewDeploy) idleObjectReaper() {
pollSleep := time.Duration(deploy.idlePodReapTime)
pollSleep := 5 * time.Second
for {
time.Sleep(pollSleep)
@@ -783,13 +783,15 @@ func (deploy *NewDeploy) idleObjectReaper() {
envList[env.ObjectMeta.UID] = struct{}{}
}
funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
funcSvcs, err := deploy.fsCache.ListOld(pollSleep)
if err != nil {
deploy.logger.Error("error reaping idle pods", zap.Error(err))
continue
}
for _, fsvc := range funcSvcs {
for i := range funcSvcs {
fsvc := funcSvcs[i]
if fsvc.Executor != fv1.ExecutorTypeNewdeploy {
continue
}
@@ -813,30 +815,41 @@ func (deploy *NewDeploy) idleObjectReaper() {
continue
}
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil {
deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
idlePodReapTime := deploy.defaultIdlePodReapTime
if fn.Spec.IdleTimeout != nil {
idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
}
if time.Since(fsvc.Atime) < idlePodReapTime {
continue
}
currentDeploy, err := deploy.kubernetesClient.AppsV1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil {
deploy.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue
}
go func() {
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil {
deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
return
}
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
currentDeploy, err := deploy.kubernetesClient.AppsV1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil {
deploy.logger.Error("error getting function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
return
}
// do nothing if the current replicas is already lower than minScale
if *currentDeploy.Spec.Replicas <= minScale {
continue
}
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
if err != nil {
deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
}
// do nothing if the current replicas is already lower than minScale
if *currentDeploy.Spec.Replicas <= minScale {
return
}
err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
if err != nil {
deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
}
}()
}
}
}
pkg/executor/executortype/poolmgr/gp.go
-2
View File
@@ -57,7 +57,6 @@ type (
namespace string // namespace to keep our resources
functionNamespace string // fallback namespace for fission functions
podReadyTimeout time.Duration // timeout for generic pods to become ready
idlePodReapTime time.Duration // pods unused for idlePodReapTime are deleted
fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
useSvc bool // create k8s service for specialized pods
useIstio bool
@@ -113,7 +112,6 @@ func MakeGenericPool(
namespace: namespace,
functionNamespace: functionNamespace,
podReadyTimeout: 5 * time.Minute, // TODO make this an env param?
idlePodReapTime: 3 * time.Minute, // TODO make this configurable
fsCache: fsCache,
poolInstanceId: uniuri.NewLen(8),
fetcherConfig: fetcherConfig,
pkg/executor/executortype/poolmgr/gpm.go
+39 -16
View File
@@ -77,7 +77,7 @@ type (
pkgStore k8sCache.Store
pkgController k8sCache.Controller
idlePodReapTime time.Duration
defaultIdlePodReapTime time.Duration
}
request struct {
requestType
@@ -102,17 +102,17 @@ func MakeGenericPoolManager(
gpmLogger := logger.Named("generic_pool_manager")
gpm := &GenericPoolManager{
logger: gpmLogger,
pools: make(map[string]*GenericPool),
kubernetesClient: kubernetesClient,
namespace: functionNamespace,
fissionClient: fissionClient,
functionEnv: cache.MakeCache(10*time.Second, 0),
fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
instanceId: instanceId,
requestChannel: make(chan *request),
idlePodReapTime: 2 * time.Minute,
fetcherConfig: fetcherConfig,
logger: gpmLogger,
pools: make(map[string]*GenericPool),
kubernetesClient: kubernetesClient,
namespace: functionNamespace,
fissionClient: fissionClient,
functionEnv: cache.MakeCache(10*time.Second, 0),
fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
instanceId: instanceId,
requestChannel: make(chan *request),
defaultIdlePodReapTime: 2 * time.Minute,
fetcherConfig: fetcherConfig,
}
go gpm.service()
@@ -555,13 +555,14 @@ func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
// idleObjectReaper reaps objects after certain idle time
func (gpm *GenericPoolManager) idleObjectReaper() {
pollSleep := time.Duration(gpm.idlePodReapTime)
pollSleep := 5 * time.Second
for {
time.Sleep(pollSleep)
envs, err := gpm.fissionClient.CoreV1().Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
gpm.logger.Error("failed to get environment list", zap.Error(err))
continue
}
envList := make(map[k8sTypes.UID]struct{})
@@ -569,7 +570,18 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
envList[env.ObjectMeta.UID] = struct{}{}
}
funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
fns, err := gpm.fissionClient.CoreV1().Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
gpm.logger.Error("failed to get environment list", zap.Error(err))
continue
}
fnList := make(map[k8sTypes.UID]fv1.Function)
for i, fn := range fns.Items {
fnList[fn.ObjectMeta.UID] = fns.Items[i]
}
funcSvcs, err := gpm.fsCache.ListOld(pollSleep)
if err != nil {
gpm.logger.Error("error reaping idle pods", zap.Error(err))
continue
@@ -594,8 +606,19 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
continue
}
idlePodReapTime := gpm.defaultIdlePodReapTime
if fn, ok := fnList[fsvc.Function.UID]; ok {
if fn.Spec.IdleTimeout != nil {
idlePodReapTime = time.Duration(*fn.Spec.IdleTimeout) * time.Second
}
}
if time.Since(fsvc.Atime) < idlePodReapTime {
continue
}
go func() {
deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
deleted, err := gpm.fsCache.DeleteOld(fsvc, idlePodReapTime)
if err != nil {
gpm.logger.Error("error deleting Kubernetes objects for function service",
zap.Error(err),