Fix executor tries to create same name deployment (#1082)

The root cause of the issue was introduced by PR https://github.com/fission/fission/pull/1009/files .
To be short, even the CRD of environment was delete, it still takes time for executor (poolmgr) to destroy env pool. In our cases, the previous test creates an env and delete it when test finished, then the next one creates the same name env, but failed to create pool due to the deploy name conflict. So the executor selects the pod from the first created env pool. Then, executor starts to delete the env pool, and makes the pod state became Termination state. To fix this problem, a unique name of deployment will be returned after this PR to prevent the name conflict.
This commit is contained in:
Ta-Ching Chen
2019-01-30 22:14:14 +08:00
committed by GitHub
parent 1806259637
commit 8b0a201f69
8 changed files with 187 additions and 244 deletions
+2 -5
View File
@@ -362,12 +362,9 @@ func (gp *GenericPool) specializePod(pod *apiv1.Pod, metadata *metav1.ObjectMeta
return nil
}
// getPoolName returns a unique name of an environment
func (gp *GenericPool) getPoolName() string {
// Use executor type as delimiter between function name and namespace to prevent deployment name conflict.
// For example:
// 1. fn-name: a-b fn-namespace: c => a-b-poolmgr-c
// 2. fn-name: a fn-namespace: b-c => a-poolmgr-b-c
return strings.ToLower(fmt.Sprintf("%v-poolmgr-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace))
return strings.ToLower(fmt.Sprintf("poolmgr-%v-%v-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace, uniuri.NewLen(8)))
}
// A pool is a deployment of generic containers for an env. This
+52 -3
View File
@@ -30,6 +30,7 @@ import (
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission"
"github.com/fission/fission/cache"
"github.com/fission/fission/crd"
"github.com/fission/fission/executor/fscache"
"github.com/fission/fission/executor/reaper"
@@ -49,6 +50,7 @@ type (
namespace string
fissionClient *crd.FissionClient
functionEnv *cache.Cache
fsCache *fscache.FunctionServiceCache
instanceId string
requestChannel chan *request
@@ -77,7 +79,6 @@ func MakeGenericPoolManager(
fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset,
functionNamespace string,
fsCache *fscache.FunctionServiceCache,
instanceId string) *GenericPoolManager {
gpm := &GenericPoolManager{
@@ -85,7 +86,8 @@ func MakeGenericPoolManager(
kubernetesClient: kubernetesClient,
namespace: functionNamespace,
fissionClient: fissionClient,
fsCache: fsCache,
functionEnv: cache.MakeCache(10*time.Second, 0),
fsCache: fscache.MakeFunctionServiceCache(),
instanceId: instanceId,
requestChannel: make(chan *request),
idlePodReapTime: 2 * time.Minute,
@@ -157,7 +159,7 @@ func (gpm *GenericPoolManager) service() {
if !ok || poolsize == 0 {
// Env no longer exists or pool size changed to zero
log.Printf("Destroying generic pool for environment [%v]", key)
log.Printf("Destroying generic pool for environment %v", pool.env.Metadata)
delete(gpm.pools, key)
// and delete the pool asynchronously.
@@ -187,6 +189,53 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) {
}
}
func (gpm *GenericPoolManager) GetFuncSvc(metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
// from Func -> get Env
log.Printf("[%v] getting environment for function", metadata.Name)
env, err := gpm.getFunctionEnv(metadata)
if err != nil {
return nil, err
}
pool, err := gpm.GetPool(env)
if err != nil {
return nil, err
}
// from GenericPool -> get one function container
// (this also adds to the cache)
log.Printf("[%v] getting function service from pool", metadata.Name)
return pool.GetFuncSvc(metadata)
}
func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) {
var env *crd.Environment
// Cached ?
result, err := gpm.functionEnv.Get(crd.CacheKey(m))
if err == nil {
env = result.(*crd.Environment)
return env, nil
}
// Cache miss -- get func from controller
f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
if err != nil {
return nil, err
}
// Get env from metadata
log.Printf("[%v] getting env", m)
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
if err != nil {
return nil, err
}
// cache for future lookups
gpm.functionEnv.Set(crd.CacheKey(m), env)
return env, nil
}
func (gpm *GenericPoolManager) eagerPoolCreator() {
pollSleep := time.Duration(2 * time.Second)
for {