Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve executor bootstrap speed (#1446)

Browse Source
This commit is contained in:
Ta-Ching Chen
2019-11-29 14:18:23 +08:00
committed by GitHub
parent 3067ecdee3
commit 47aaa85108
9 changed files with 134 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/reaper/reaper.go
+37 -51
View File
@@ -20,7 +20,7 @@ import (
"strings"
"time"
"github.com/fission/fission/pkg/utils"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,6 +29,7 @@ import (
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/crd"
"github.com/fission/fission/pkg/types"
"github.com/fission/fission/pkg/utils"
)
var (
@@ -38,45 +39,39 @@ var (
// CleanupOldExecutorObjects cleans up resources created by old executor instances
func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) {
err := cleanup(logger, kubernetesClient, instanceId)
if err != nil {
// TODO retry reaper; logged and ignored for now
logger.Error("Failed to cleanup old executor objects", zap.Error(err))
}
}
errs := &multierror.Error{}
func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
// Pods might still be running user functions, so we give them
err := cleanupHpa(logger, kubernetesClient, instanceId)
if err != nil {
errs = multierror.Append(errs, err)
}
err = cleanupDeployments(logger, kubernetesClient, instanceId)
if err != nil {
errs = multierror.Append(errs, err)
}
// Pods might be running user functions, so we give them
// a few minutes before terminating them. This time is the
// maximum function runtime, plus the time a router might
// still route to an old instance, i.e. router cache expiry
// time.
time.Sleep(6 * time.Minute)
err := cleanupServices(logger, client, instanceId)
err = cleanupServices(logger, kubernetesClient, instanceId)
if err != nil {
return err
errs = multierror.Append(errs, err)
}
err = cleanupHpa(logger, client, instanceId)
err = cleanupPods(logger, kubernetesClient, instanceId)
if err != nil {
return err
errs = multierror.Append(errs, err)
}
// Deployments are used for idle pools and can be cleaned up
// immediately. (We should "adopt" these instead of creating
// a new pool.)
err = cleanupDeployments(logger, client, instanceId)
if err != nil {
return err
if errs.ErrorOrNil() != nil {
// TODO retry reaper; logged and ignored for now
logger.Error("Failed to cleanup old executor objects", zap.Error(err))
}
err = cleanupPods(logger, client, instanceId)
if err != nil {
return err
}
return nil
}
// CleanupKubeObject deletes given kubernetes object
@@ -119,20 +114,11 @@ func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instan
}
for _, dep := range deploymentList.Items {
id, ok := dep.ObjectMeta.Annotations[types.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId {
logger.Debug("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.AppsV1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
if err != nil {
logger.Error("error cleaning up deployment",
zap.Error(err),
zap.String("deployment_name", dep.ObjectMeta.Name),
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
}
// ignore err
if !ok {
// Backward compatibility with older label name
id, ok = dep.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
}
// Backward compatibility with older label name
pid, pok := dep.ObjectMeta.Annotations[types.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId {
if ok && id != instanceId {
logger.Debug("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.AppsV1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
if err != nil {
@@ -154,6 +140,10 @@ func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId st
}
for _, pod := range podList.Items {
id, ok := pod.ObjectMeta.Annotations[types.EXECUTOR_INSTANCEID_LABEL]
if !ok {
// Backward compatibility with older label name
id, ok = pod.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
}
if ok && id != instanceId {
logger.Debug("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
@@ -165,18 +155,6 @@ func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId st
}
// ignore err
}
// Backward compatibility with older label name
pid, pok := pod.ObjectMeta.Annotations[types.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId {
logger.Debug("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
if err != nil {
logger.Error("error cleaning up pod",
zap.Error(err),
zap.String("pod_name", pod.ObjectMeta.Name),
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
}
}
}
return nil
}
@@ -188,6 +166,10 @@ func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceI
}
for _, svc := range svcList.Items {
id, ok := svc.ObjectMeta.Annotations[types.EXECUTOR_INSTANCEID_LABEL]
if !ok {
// Backward compatibility with older label name
id, ok = svc.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
}
if ok && id != instanceId {
logger.Debug("cleaning up service", zap.String("service", svc.ObjectMeta.Name))
err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil)
@@ -211,6 +193,10 @@ func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId str
for _, hpa := range hpaList.Items {
id, ok := hpa.ObjectMeta.Annotations[types.EXECUTOR_INSTANCEID_LABEL]
if !ok {
// Backward compatibility with older label name
id, ok = hpa.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
}
if ok && id != instanceId {
logger.Debug("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name))
err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil)