Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix poolmanager terminates running function pod periodically (#1435)

Browse Source 
The pool manager keeps terminating function pod periodically even there are
traffic to the function. The root cause is that executor, poolmgr, newdeploy
manage their own functionServiceCache separately. And when router taps a
function, executor updates the access time of the function service entry in its
own cache without notifying executor types to do the update as well. Hence,
the access time of function service entry in poolmanager cache never gets updated.
Due to the access time never gets updated, the idle pod reaper in poolmanager
then thinks the function pod is in idle state and recycle it.

This PR removes the cache in executor itself, and when router tries to tap a function,
executor will call executor type to tap the function and update access time.
This commit is contained in:
Ta-Ching Chen
2019-11-26 01:16:30 +08:00
committed by GitHub
parent 6d2fe08973
commit 51b264e8ca
19 changed files with 371 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/fission-cli/cmd/function/create.go
+9 -11
View File
@@ -36,7 +36,6 @@ import (
"github.com/fission/fission/pkg/fission-cli/console"
flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
"github.com/fission/fission/pkg/fission-cli/util"
"github.com/fission/fission/pkg/types"
)
const (
@@ -363,18 +362,17 @@ func (opts *CreateSubCommand) run(input cli.Input) error {
}
func getInvokeStrategy(input cli.Input, existingInvokeStrategy *fv1.InvokeStrategy) (strategy *fv1.InvokeStrategy, err error) {
var fnExecutor, newFnExecutor fv1.ExecutorType
switch input.String(flagkey.FnExecutorType) {
case "":
fallthrough
case types.ExecutorTypePoolmgr:
newFnExecutor = types.ExecutorTypePoolmgr
case types.ExecutorTypeNewdeploy:
newFnExecutor = types.ExecutorTypeNewdeploy
case string(fv1.ExecutorTypePoolmgr):
newFnExecutor = fv1.ExecutorTypePoolmgr
case string(fv1.ExecutorTypeNewdeploy):
newFnExecutor = fv1.ExecutorTypeNewdeploy
default:
return nil, errors.New("executor type must be one of 'poolmgr' or 'newdeploy', defaults to 'poolmgr'")
return nil, errors.Errorf("executor type must be one of '%v' or '%v'", fv1.ExecutorTypePoolmgr, fv1.ExecutorTypeNewdeploy)
}
if existingInvokeStrategy != nil {
@@ -388,11 +386,11 @@ func getInvokeStrategy(input cli.Input, existingInvokeStrategy *fv1.InvokeStrate
fnExecutor = newFnExecutor
}
if input.IsSet(flagkey.FnSpecializationTimeout) && fnExecutor != types.ExecutorTypeNewdeploy {
if input.IsSet(flagkey.FnSpecializationTimeout) && fnExecutor != fv1.ExecutorTypeNewdeploy {
return nil, errors.Errorf("%v flag is only applicable for newdeploy type of executor", flagkey.FnSpecializationTimeout)
}
if fnExecutor == types.ExecutorTypePoolmgr {
if fnExecutor == fv1.ExecutorTypePoolmgr {
if input.IsSet(flagkey.RuntimeTargetcpu) || input.IsSet(flagkey.ReplicasMinscale) || input.IsSet(flagkey.ReplicasMaxscale) {
return nil, errors.New("to set target CPU or min/max scale for function, please specify \"--executortype newdeploy\"")
}
@@ -403,7 +401,7 @@ func getInvokeStrategy(input cli.Input, existingInvokeStrategy *fv1.InvokeStrate
strategy = &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
ExecutionStrategy: fv1.ExecutionStrategy{
ExecutorType: types.ExecutorTypePoolmgr,
ExecutorType: fv1.ExecutorTypePoolmgr,
},
}
} else {
@@ -413,7 +411,7 @@ func getInvokeStrategy(input cli.Input, existingInvokeStrategy *fv1.InvokeStrate
maxScale := minScale
specializationTimeout := fv1.DefaultSpecializationTimeOut
if existingInvokeStrategy != nil && existingInvokeStrategy.ExecutionStrategy.ExecutorType == types.ExecutorTypeNewdeploy {
if existingInvokeStrategy != nil && existingInvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
minScale = existingInvokeStrategy.ExecutionStrategy.MinScale
maxScale = existingInvokeStrategy.ExecutionStrategy.MaxScale
targetCPU = existingInvokeStrategy.ExecutionStrategy.TargetCPUPercent
pkg/fission-cli/cmd/function/function_test.go
+16 -16
View File
@@ -48,7 +48,7 @@ func TestGetInvokeStrategy(t *testing.T) {
},
{
// case: executor type set to poolmgr
testArgs: map[string]interface{}{flagkey.FnExecutorType: fv1.ExecutorTypePoolmgr},
testArgs: map[string]interface{}{flagkey.FnExecutorType: string(fv1.ExecutorTypePoolmgr)},
existingInvokeStrategy: nil,
expectedResult: &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
@@ -60,7 +60,7 @@ func TestGetInvokeStrategy(t *testing.T) {
},
{
// case: executor type set to newdeploy
testArgs: map[string]interface{}{flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy},
testArgs: map[string]interface{}{flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy)},
existingInvokeStrategy: nil,
expectedResult: &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
@@ -76,7 +76,7 @@ func TestGetInvokeStrategy(t *testing.T) {
},
{
// case: executor type change from poolmgr to newdeploy
testArgs: map[string]interface{}{flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy},
testArgs: map[string]interface{}{flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy)},
existingInvokeStrategy: &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
ExecutionStrategy: fv1.ExecutionStrategy{
@@ -97,7 +97,7 @@ func TestGetInvokeStrategy(t *testing.T) {
},
{
// case: executor type change from newdeploy to poolmgr
testArgs: map[string]interface{}{flagkey.FnExecutorType: fv1.ExecutorTypePoolmgr},
testArgs: map[string]interface{}{flagkey.FnExecutorType: string(fv1.ExecutorTypePoolmgr)},
existingInvokeStrategy: &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
ExecutionStrategy: fv1.ExecutionStrategy{
@@ -119,7 +119,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: minscale < maxscale
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMinscale: 2,
flagkey.ReplicasMaxscale: 3,
},
@@ -139,7 +139,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: minscale > maxscale
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMinscale: 5,
flagkey.ReplicasMaxscale: 3,
},
@@ -150,7 +150,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: maxscale not specified
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMinscale: 5,
},
existingInvokeStrategy: nil,
@@ -160,7 +160,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: minscale not specified
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMaxscale: 3,
},
existingInvokeStrategy: nil,
@@ -179,7 +179,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: maxscale set to 0
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMaxscale: 0,
},
existingInvokeStrategy: nil,
@@ -189,7 +189,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: maxscale set to 9 when existing is 5
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.ReplicasMaxscale: 9,
},
existingInvokeStrategy: &fv1.InvokeStrategy{
@@ -217,7 +217,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: change nothing for existing strategy
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
},
existingInvokeStrategy: &fv1.InvokeStrategy{
StrategyType: fv1.StrategyTypeExecution,
@@ -244,7 +244,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: set target cpu percentage
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.RuntimeTargetcpu: 50,
},
existingInvokeStrategy: nil,
@@ -263,7 +263,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: change target cpu percentage
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.RuntimeTargetcpu: 20,
},
existingInvokeStrategy: &fv1.InvokeStrategy{
@@ -291,7 +291,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: change specializationtimeout
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.FnSpecializationTimeout: 200,
},
existingInvokeStrategy: &fv1.InvokeStrategy{
@@ -318,7 +318,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: specializationtimeout should not work for poolmgr
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypePoolmgr,
flagkey.FnExecutorType: string(fv1.ExecutorTypePoolmgr),
flagkey.FnSpecializationTimeout: 10,
},
existingInvokeStrategy: nil,
@@ -328,7 +328,7 @@ func TestGetInvokeStrategy(t *testing.T) {
{
// case: specializationtimeout should not be less than 120
testArgs: map[string]interface{}{
flagkey.FnExecutorType: fv1.ExecutorTypeNewdeploy,
flagkey.FnExecutorType: string(fv1.ExecutorTypeNewdeploy),
flagkey.FnSpecializationTimeout: 90,
},
existingInvokeStrategy: nil,
pkg/fission-cli/cmd/function/update.go
+1 -2
View File
@@ -30,7 +30,6 @@ import (
"github.com/fission/fission/pkg/fission-cli/console"
flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
"github.com/fission/fission/pkg/fission-cli/util"
"github.com/fission/fission/pkg/types"
)
type UpdateSubCommand struct {
@@ -171,7 +170,7 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error {
function.Spec.InvokeStrategy = *strategy
if input.IsSet(flagkey.FnSpecializationTimeout) {
if strategy.ExecutionStrategy.ExecutorType != types.ExecutorTypeNewdeploy {
if strategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
return errors.Errorf("--%v flag is only applicable for newdeploy type of executor", flagkey.FnSpecializationTimeout)
}
pkg/fission-cli/flag/flag.go
+1 -2
View File
@@ -24,7 +24,6 @@ import (
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
"github.com/fission/fission/pkg/types"
)
type (
@@ -94,7 +93,7 @@ var (
FnBuildCmd = Flag{Type: String, Name: flagkey.FnBuildCmd, Usage: "Package build command for builder to run with"}
FnSecret = Flag{Type: StringSlice, Name: flagkey.FnSecret, Usage: "Function access to secret, should be present in the same namespace as the function. You can provide multiple secrets using multiple --secrets flags. In the case of fn update the the secrets will be replaced by the provided list of secrets."}
FnCfgMap = Flag{Type: StringSlice, Name: flagkey.FnCfgMap, Usage: "Function access to configmap, should be present in the same namespace as the function. You can provide multiple configmaps using multiple --configmap flags. In case of fn update the configmaps will be replaced by the provided list of configmaps."}
FnExecutorType = Flag{Type: String, Name: flagkey.FnExecutorType, Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy'", DefaultValue: types.ExecutorTypePoolmgr}
FnExecutorType = Flag{Type: String, Name: flagkey.FnExecutorType, Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy'", DefaultValue: string(fv1.ExecutorTypePoolmgr)}
FnExecutionTimeout = Flag{Type: Int, Name: flagkey.FnExecutionTimeout, Aliases: []string{"ft"}, Usage: "Maximum time for a request to wait for the response from the function", DefaultValue: 60}
FnLogPod = Flag{Type: String, Name: flagkey.FnLogPod, Usage: "Function pod name (use the latest pod name if unspecified)"}
FnLogFollow = Flag{Type: Bool, Name: flagkey.FnLogFollow, Short: "f", Usage: "Specify if the logs should be streamed"}