Fix executor tries to create same name deployment (#1082)
The root cause of the issue was introduced by PR https://github.com/fission/fission/pull/1009/files . To be short, even the CRD of environment was delete, it still takes time for executor (poolmgr) to destroy env pool. In our cases, the previous test creates an env and delete it when test finished, then the next one creates the same name env, but failed to create pool due to the deploy name conflict. So the executor selects the pod from the first created env pool. Then, executor starts to delete the env pool, and makes the pod state became Termination state. To fix this problem, a unique name of deployment will be returned after this PR to prevent the name conflict.
This commit is contained in:
@@ -362,12 +362,9 @@ func (gp *GenericPool) specializePod(pod *apiv1.Pod, metadata *metav1.ObjectMeta
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPoolName returns a unique name of an environment
|
||||
func (gp *GenericPool) getPoolName() string {
|
||||
// Use executor type as delimiter between function name and namespace to prevent deployment name conflict.
|
||||
// For example:
|
||||
// 1. fn-name: a-b fn-namespace: c => a-b-poolmgr-c
|
||||
// 2. fn-name: a fn-namespace: b-c => a-poolmgr-b-c
|
||||
return strings.ToLower(fmt.Sprintf("%v-poolmgr-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace))
|
||||
return strings.ToLower(fmt.Sprintf("poolmgr-%v-%v-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace, uniuri.NewLen(8)))
|
||||
}
|
||||
|
||||
// A pool is a deployment of generic containers for an env. This
|
||||
|
||||
+52
-3
@@ -30,6 +30,7 @@ import (
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
"github.com/fission/fission"
|
||||
"github.com/fission/fission/cache"
|
||||
"github.com/fission/fission/crd"
|
||||
"github.com/fission/fission/executor/fscache"
|
||||
"github.com/fission/fission/executor/reaper"
|
||||
@@ -49,6 +50,7 @@ type (
|
||||
namespace string
|
||||
|
||||
fissionClient *crd.FissionClient
|
||||
functionEnv *cache.Cache
|
||||
fsCache *fscache.FunctionServiceCache
|
||||
instanceId string
|
||||
requestChannel chan *request
|
||||
@@ -77,7 +79,6 @@ func MakeGenericPoolManager(
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
functionNamespace string,
|
||||
fsCache *fscache.FunctionServiceCache,
|
||||
instanceId string) *GenericPoolManager {
|
||||
|
||||
gpm := &GenericPoolManager{
|
||||
@@ -85,7 +86,8 @@ func MakeGenericPoolManager(
|
||||
kubernetesClient: kubernetesClient,
|
||||
namespace: functionNamespace,
|
||||
fissionClient: fissionClient,
|
||||
fsCache: fsCache,
|
||||
functionEnv: cache.MakeCache(10*time.Second, 0),
|
||||
fsCache: fscache.MakeFunctionServiceCache(),
|
||||
instanceId: instanceId,
|
||||
requestChannel: make(chan *request),
|
||||
idlePodReapTime: 2 * time.Minute,
|
||||
@@ -157,7 +159,7 @@ func (gpm *GenericPoolManager) service() {
|
||||
if !ok || poolsize == 0 {
|
||||
// Env no longer exists or pool size changed to zero
|
||||
|
||||
log.Printf("Destroying generic pool for environment [%v]", key)
|
||||
log.Printf("Destroying generic pool for environment %v", pool.env.Metadata)
|
||||
delete(gpm.pools, key)
|
||||
|
||||
// and delete the pool asynchronously.
|
||||
@@ -187,6 +189,53 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) {
|
||||
}
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) GetFuncSvc(metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
// from Func -> get Env
|
||||
log.Printf("[%v] getting environment for function", metadata.Name)
|
||||
env, err := gpm.getFunctionEnv(metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pool, err := gpm.GetPool(env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// from GenericPool -> get one function container
|
||||
// (this also adds to the cache)
|
||||
log.Printf("[%v] getting function service from pool", metadata.Name)
|
||||
return pool.GetFuncSvc(metadata)
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) {
|
||||
var env *crd.Environment
|
||||
|
||||
// Cached ?
|
||||
result, err := gpm.functionEnv.Get(crd.CacheKey(m))
|
||||
if err == nil {
|
||||
env = result.(*crd.Environment)
|
||||
return env, nil
|
||||
}
|
||||
|
||||
// Cache miss -- get func from controller
|
||||
f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get env from metadata
|
||||
log.Printf("[%v] getting env", m)
|
||||
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// cache for future lookups
|
||||
gpm.functionEnv.Set(crd.CacheKey(m), env)
|
||||
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) eagerPoolCreator() {
|
||||
pollSleep := time.Duration(2 * time.Second)
|
||||
for {
|
||||
|
||||
Reference in New Issue
Block a user