Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Poolmanager wait for function specialization timeout when specializing a pod (#1392)

Browse Source
This commit is contained in:
Ta-Ching Chen
2019-11-10 00:01:19 +08:00
committed by GitHub
parent d3d8ff6c1b
commit c7d0c09f01
7 changed files with 98 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/poolmgr/gp.go
+21 -28
View File
@@ -298,7 +298,7 @@ func (gp *GenericPool) getFetcherUrl(podIP string) string {
// specializePod chooses a pod, copies the required user-defined function to that pod
// (via fetcher), and calls the function-run container to load it, resulting in a
// specialized pod.
func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metadata *metav1.ObjectMeta) error {
func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, fn *fv1.Function) error {
// for fetcher we don't need to create a service, just talk to the pod directly
podIP := pod.Status.PodIP
if len(podIP) == 0 {
@@ -306,28 +306,21 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
}
// specialize pod with service
if gp.useIstio {
svc := utils.GetFunctionIstioServiceName(metadata.Name, metadata.Namespace)
svc := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
podIP = fmt.Sprintf("%v.%v", svc, gp.namespace)
}
// tell fetcher to get the function.
fetcherUrl := gp.getFetcherUrl(podIP)
gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
fn, err := gp.fissionClient.
Functions(metadata.Namespace).
Get(metadata.Name)
if err != nil {
return err
}
gp.logger.Info("calling fetcher to copy function", zap.String("function", fn.Metadata.Name), zap.String("url", fetcherUrl))
specializeReq := gp.fetcherConfig.NewSpecializeRequest(fn, gp.env)
gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
gp.logger.Info("specializing pod", zap.String("function", fn.Metadata.Name))
// Fetcher will download user function to share volume of pod, and
// invoke environment specialize api for pod specialization.
err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
err := fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
if err != nil {
return err
}
@@ -508,9 +501,9 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.
return svc, err
}
func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
newLabels := gp.labelsForFunction(m)
func (gp *GenericPool) getFuncSvc(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
gp.logger.Info("choosing pod from pool", zap.Any("function", fn.Metadata))
newLabels := gp.labelsForFunction(&fn.Metadata)
if gp.useIstio {
// Istio only allows accessing pod through k8s service, and requests come to
@@ -536,8 +529,8 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
// and make sure that there is only one pod behind the service
sel := map[string]string{
"functionName": m.Name,
"functionUid": string(m.UID),
"functionName": fn.Metadata.Name,
"functionUid": string(fn.Metadata.UID),
}
podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
LabelSelector: labels.Set(sel).AsSelector().String(),
@@ -558,22 +551,21 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
return nil, err
}
err = gp.specializePod(ctx, pod, m)
err = gp.specializePod(ctx, pod, fn)
if err != nil {
gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, err
}
gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.Any("function", fn.Metadata))
var svcHost string
if gp.useSvc && !gp.useIstio {
svcName := fmt.Sprintf("svc-%v", m.Name)
if len(m.UID) > 0 {
svcName = fmt.Sprintf("%s-%v", svcName, m.UID)
svcName := fmt.Sprintf("svc-%v", fn.Metadata.Name)
if len(fn.Metadata.UID) > 0 {
svcName = fmt.Sprintf("%s-%v", svcName, fn.Metadata.UID)
}
labels := gp.labelsForFunction(m)
svc, err := gp.createSvc(svcName, labels)
svc, err := gp.createSvc(svcName, newLabels)
if err != nil {
gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, err
@@ -587,7 +579,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
// namespace-qualified hostname
svcHost = fmt.Sprintf("%v.%v", svcName, gp.namespace)
} else if gp.useIstio {
svc := utils.GetFunctionIstioServiceName(m.Name, m.Namespace)
svc := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
} else {
svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
@@ -596,8 +588,8 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
gp.logger.Info("specialized pod",
zap.String("pod", pod.ObjectMeta.Name),
zap.String("podNamespace", pod.ObjectMeta.Namespace),
zap.String("function", m.Name),
zap.String("functionNamespace", m.Namespace),
zap.String("function", fn.Metadata.Name),
zap.String("functionNamespace", fn.Metadata.Namespace),
zap.String("specialization_host", svcHost))
kubeObjRefs := []apiv1.ObjectReference{
@@ -611,9 +603,10 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
},
}
m := fn.Metadata // only cache necessary part
fsvc := &fscache.FuncSvc{
Name: pod.ObjectMeta.Name,
Function: m,
Function: &m,
Environment: gp.env,
Address: svcHost,
KubernetesObjects: kubeObjRefs,
pkg/executor/poolmgr/gpm.go
+21 -24
View File
@@ -139,7 +139,7 @@ func (gpm *GenericPoolManager) RefreshFuncPods(logger *zap.Logger, f fv1.Functio
return err
}
gp, err := gpm.GetPool(env)
gp, err := gpm.getPool(env)
if err != nil {
return err
}
@@ -228,7 +228,7 @@ func (gpm *GenericPoolManager) service() {
}
}
func (gpm *GenericPoolManager) GetPool(env *fv1.Environment) (*GenericPool, error) {
func (gpm *GenericPoolManager) getPool(env *fv1.Environment) (*GenericPool, error) {
c := make(chan *response)
gpm.requestChannel <- &request{
requestType: GET_POOL,
@@ -239,61 +239,58 @@ func (gpm *GenericPoolManager) GetPool(env *fv1.Environment) (*GenericPool, erro
return resp.pool, resp.error
}
func (gpm *GenericPoolManager) CleanupPools(envs []fv1.Environment) {
func (gpm *GenericPoolManager) cleanupPools(envs []fv1.Environment) {
gpm.requestChannel <- &request{
requestType: CLEANUP_POOLS,
envList: envs,
}
}
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
// from Func -> get Env
gpm.logger.Debug("getting environment for function", zap.String("function", metadata.Name))
env, err := gpm.getFunctionEnv(metadata)
gpm.logger.Debug("getting environment for function", zap.String("function", fn.Metadata.Name))
env, err := gpm.getFunctionEnv(fn)
if err != nil {
return nil, err
}
pool, err := gpm.GetPool(env)
pool, err := gpm.getPool(env)
if err != nil {
return nil, err
}
// from GenericPool -> get one function container
// (this also adds to the cache)
gpm.logger.Debug("getting function service from pool", zap.String("function", metadata.Name))
return pool.GetFuncSvc(ctx, metadata)
gpm.logger.Debug("getting function service from pool", zap.String("function", fn.Metadata.Name))
return pool.getFuncSvc(ctx, fn)
}
func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*fv1.Environment, error) {
func (gpm *GenericPoolManager) getFunctionEnv(fn *fv1.Function) (*fv1.Environment, error) {
var env *fv1.Environment
// Cached ?
result, err := gpm.functionEnv.Get(crd.CacheKey(m))
// TODO: the cache should be able to search by <env name, fn namespace> instead of function metadata.
result, err := gpm.functionEnv.Get(crd.CacheKey(&fn.Metadata))
if err == nil {
env = result.(*fv1.Environment)
return env, nil
}
// Cache miss -- get func from controller
f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
if err != nil {
return nil, err
}
// Get env from metadata
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
// Get env from controller
env, err = gpm.fissionClient.Environments(fn.Spec.Environment.Namespace).Get(fn.Spec.Environment.Name)
if err != nil {
return nil, err
}
// cache for future lookups
gpm.functionEnv.Set(crd.CacheKey(m), env)
m := fn.Metadata
gpm.functionEnv.Set(crd.CacheKey(&m), env)
return env, nil
}
func (gpm *GenericPoolManager) eagerPoolCreator() {
pollSleep := time.Duration(2 * time.Second)
pollSleep := 2 * time.Second
for {
// get list of envs from controller
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
@@ -303,7 +300,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
time.Sleep(5 * time.Second)
continue
}
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
gpm.logger.Error("failed to get environment list", zap.Error(err))
}
// Create pools for all envs. TODO: we should make this a bit less eager, only
@@ -314,7 +311,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
env := envs.Items[i]
// Create pool only if poolsize greater than zero
if gpm.getEnvPoolsize(&env) > 0 {
_, err := gpm.GetPool(&envs.Items[i])
_, err := gpm.getPool(&envs.Items[i])
if err != nil {
gpm.logger.Error("eager-create pool failed", zap.Error(err))
}
@@ -322,7 +319,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
}
// Clean up pools whose env was deleted
gpm.CleanupPools(envs.Items)
gpm.cleanupPools(envs.Items)
time.Sleep(pollSleep)
}
}