@@ -19,7 +19,7 @@ package poolmgr
|
||||
import (
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -72,11 +72,15 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
|
||||
// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
|
||||
// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
|
||||
// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
|
||||
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
|
||||
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
log.Printf("Error : %v creating %s RoleBinding", err, fission.SecretConfigMapGetterRB)
|
||||
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
|
||||
} else {
|
||||
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in func's ns for func : %s/%s", fission.FissionFetcherSA, envNs, fn.Metadata.Namespace, fn.Metadata.Name)
|
||||
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
|
||||
zap.String("service_account", fission.FissionFetcherSA),
|
||||
zap.String("service_account_namepsace", envNs),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
}
|
||||
|
||||
if istioEnabled {
|
||||
@@ -126,7 +130,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
|
||||
// create function istio service if it does not exist
|
||||
_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
|
||||
if err != nil && !kerrors.IsAlreadyExists(err) {
|
||||
log.Printf("Error creating function istio service: %v", err)
|
||||
gpm.logger.Error("error creating istio service for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_name", svcName),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.Any("selectors", sel))
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -149,7 +157,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
|
||||
// delete function istio service
|
||||
err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
log.Printf("Error deleting function istio service: %v", err)
|
||||
gpm.logger.Error("error deleting istio service for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_name", svcName),
|
||||
zap.String("function_name", fn.Metadata.Name))
|
||||
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -178,13 +190,18 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
|
||||
envNs = newFunc.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB,
|
||||
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB,
|
||||
newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole,
|
||||
fission.FissionFetcherSA, envNs)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error : %v creating GetSecretConfigMapRoleBinding", err)
|
||||
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
|
||||
} else {
|
||||
log.Printf("Set up rolebinding for fetcher SA in func's env ns : %s, in func's ns : %s, for func : %s", newFunc.Spec.Environment.Namespace, newFunc.Metadata.Namespace, newFunc.Metadata.Name)
|
||||
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
|
||||
zap.String("service_account", fission.FissionFetcherSA),
|
||||
zap.String("service_account_namepsace", envNs),
|
||||
zap.String("function_name", newFunc.Metadata.Name),
|
||||
zap.String("function_namespace", newFunc.Metadata.Namespace))
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
+37
-30
@@ -19,7 +19,6 @@ package poolmgr
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
@@ -30,6 +29,7 @@ import (
|
||||
"github.com/dchest/uniuri"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -46,6 +46,7 @@ import (
|
||||
|
||||
type (
|
||||
GenericPool struct {
|
||||
logger *zap.Logger
|
||||
env *crd.Environment
|
||||
replicas int32 // num idle pods
|
||||
deployment *v1beta1.Deployment // kubernetes deployment
|
||||
@@ -83,6 +84,7 @@ type (
|
||||
)
|
||||
|
||||
func MakeGenericPool(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
env *crd.Environment,
|
||||
@@ -94,7 +96,9 @@ func MakeGenericPool(
|
||||
enableIstio bool,
|
||||
collectorEndpoint string) (*GenericPool, error) {
|
||||
|
||||
log.Printf("Creating pool for environment %v", env.Metadata)
|
||||
gpLogger := logger.Named("generic_pool")
|
||||
|
||||
gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
|
||||
|
||||
fetcherImage := os.Getenv("FETCHER_IMAGE")
|
||||
if len(fetcherImage) == 0 {
|
||||
@@ -104,6 +108,7 @@ func MakeGenericPool(
|
||||
// TODO: in general we need to provide the user a way to configure pools. Initial
|
||||
// replicas, autoscaling params, various timeouts, etc.
|
||||
gp := &GenericPool{
|
||||
logger: gpLogger,
|
||||
env: env,
|
||||
replicas: initialReplicas, // TODO make this an env param instead?
|
||||
requestChannel: make(chan *choosePodRequest),
|
||||
@@ -128,13 +133,12 @@ func MakeGenericPool(
|
||||
gp.runtimeImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
|
||||
gp.fetcherImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("FETCHER_IMAGE_PULL_POLICY"))
|
||||
|
||||
log.Printf("fetcher image: %v, pull policy: %v", gp.fetcherImage, gp.fetcherImagePullPolicy)
|
||||
gpLogger.Info("fetcher", zap.String("image", gp.fetcherImage), zap.Any("pull_policy", gp.fetcherImagePullPolicy))
|
||||
|
||||
// create fetcher SA in this ns, if not already created
|
||||
_, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace)
|
||||
if err != nil {
|
||||
log.Printf("Error : %v creating fetcher SA in ns : %s", err, gp.namespace)
|
||||
return nil, err
|
||||
return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
|
||||
}
|
||||
|
||||
// Labels for generic deployment/RS/pods.
|
||||
@@ -145,7 +149,7 @@ func MakeGenericPool(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("[%v] Deployment created", env.Metadata)
|
||||
gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
|
||||
|
||||
go gp.choosePodService()
|
||||
|
||||
@@ -195,7 +199,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
|
||||
for {
|
||||
// Retries took too long, error out.
|
||||
if time.Since(startTime) > gp.podReadyTimeout {
|
||||
log.Printf("[%v] Erroring out, timed out", newLabels)
|
||||
gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
|
||||
return nil, errors.New("timeout: waited too long to get a ready pod")
|
||||
}
|
||||
|
||||
@@ -220,7 +224,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
|
||||
// add it to the list of ready pods
|
||||
readyPods = append(readyPods, &pod)
|
||||
}
|
||||
log.Printf("[%v] found %v ready pods of %v total", newLabels, len(readyPods), len(podList.Items))
|
||||
gp.logger.Info("found ready pods",
|
||||
zap.Any("labels", newLabels),
|
||||
zap.Int("ready_count", len(readyPods)),
|
||||
zap.Int("total", len(podList.Items)))
|
||||
|
||||
// If there are no ready pods, wait and retry.
|
||||
if len(readyPods) == 0 {
|
||||
@@ -241,14 +248,14 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
|
||||
// modified, this should fail; in that case just
|
||||
// retry.
|
||||
chosenPod.ObjectMeta.Labels = newLabels
|
||||
log.Printf("relabeling pod: [%v]", chosenPod.ObjectMeta.Name)
|
||||
gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
|
||||
_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
|
||||
if err != nil {
|
||||
log.Printf("failed to relabel pod [%v]: %v", chosenPod.ObjectMeta.Name, err)
|
||||
gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
|
||||
continue
|
||||
}
|
||||
}
|
||||
log.Printf("Chosen pod: %v (in %v)", chosenPod.ObjectMeta.Name, time.Since(startTime))
|
||||
gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
|
||||
return chosenPod, nil
|
||||
}
|
||||
}
|
||||
@@ -267,7 +274,7 @@ func (gp *GenericPool) scheduleDeletePod(name string) {
|
||||
// The sleep allows debugging or collecting logs from the pod before it's
|
||||
// cleaned up. (We need a better solutions for both those things; log
|
||||
// aggregation and storage will help.)
|
||||
log.Printf("Error in pod '%v', scheduling cleanup", name)
|
||||
gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
|
||||
// Ignore sleep here if istio feature is enabled, function pod
|
||||
// will be deleted after 6 mins (terminationGracePeriodSeconds).
|
||||
if !gp.useIstio {
|
||||
@@ -319,7 +326,7 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
|
||||
|
||||
// tell fetcher to get the function.
|
||||
fetcherUrl := gp.getSpecializeUrl(podIP)
|
||||
log.Printf("[%v] calling fetcher to copy function with fetcher url: %v", metadata.Name, fetcherUrl)
|
||||
gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
|
||||
|
||||
fn, err := gp.fissionClient.
|
||||
Functions(metadata.Namespace).
|
||||
@@ -356,9 +363,9 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
|
||||
},
|
||||
}
|
||||
|
||||
log.Printf("[%v] specializing pod", metadata.Name)
|
||||
gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
|
||||
|
||||
err = fetcherClient.MakeClient(fetcherUrl).Specialize(ctx, &specializeReq)
|
||||
err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -550,7 +557,7 @@ func (gp *GenericPool) createPool() error {
|
||||
|
||||
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
|
||||
if err != nil {
|
||||
log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err)
|
||||
gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
|
||||
return err
|
||||
}
|
||||
gp.deployment = depl
|
||||
@@ -564,11 +571,9 @@ func (gp *GenericPool) waitForReadyPod() error {
|
||||
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
|
||||
gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf(
|
||||
"Error waiting for ready pod of deployment %v in namespace %v",
|
||||
gp.deployment.ObjectMeta.Name, gp.namespace))
|
||||
log.Print(err)
|
||||
return err
|
||||
e := "error waiting for ready pod for deployment"
|
||||
gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
|
||||
return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
|
||||
}
|
||||
|
||||
gp.deployment = depl
|
||||
@@ -582,7 +587,7 @@ func (gp *GenericPool) waitForReadyPod() error {
|
||||
gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Error getting pod list after timeout waiting for ready pod: %v", err)
|
||||
gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
|
||||
}
|
||||
|
||||
// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
|
||||
@@ -593,9 +598,8 @@ func (gp *GenericPool) waitForReadyPod() error {
|
||||
multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
|
||||
}
|
||||
}
|
||||
return errors.Errorf(
|
||||
"Timeout: waited too long for pod of deployment %v in namespace %v to be ready, error: %v",
|
||||
gp.deployment.ObjectMeta.Name, gp.namespace, multierr)
|
||||
return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
|
||||
gp.deployment.ObjectMeta.Name, gp.namespace)
|
||||
}
|
||||
time.Sleep(1000 * time.Millisecond)
|
||||
}
|
||||
@@ -623,7 +627,7 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.
|
||||
}
|
||||
|
||||
func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
log.Printf("[%v] Choosing pod from pool", m.Name)
|
||||
gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
|
||||
newLabels := gp.labelsForFunction(m)
|
||||
|
||||
if gp.useIstio {
|
||||
@@ -677,7 +681,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
|
||||
gp.scheduleDeletePod(pod.ObjectMeta.Name)
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("Specialized pod: %v", pod.ObjectMeta.Name)
|
||||
gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
|
||||
|
||||
var svcHost string
|
||||
if gp.useSvc && !gp.useIstio {
|
||||
@@ -694,7 +698,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
|
||||
}
|
||||
if svc.ObjectMeta.Name != svcName {
|
||||
gp.scheduleDeletePod(pod.ObjectMeta.Name)
|
||||
return nil, errors.New(fmt.Sprintf("sanity check failed for svc %v", svc.ObjectMeta.Name))
|
||||
return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
|
||||
}
|
||||
|
||||
// the fission router isn't in the same namespace, so return a
|
||||
@@ -704,7 +708,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
|
||||
svc := fission.GetFunctionIstioServiceName(m.Name, m.Namespace)
|
||||
svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
|
||||
} else {
|
||||
log.Printf("Using pod IP for specialized pod")
|
||||
gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
|
||||
svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
|
||||
}
|
||||
|
||||
@@ -746,7 +750,10 @@ func (gp *GenericPool) destroy() error {
|
||||
err := gp.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
|
||||
if err != nil {
|
||||
log.Printf("Error destroying deployment: %v", err)
|
||||
gp.logger.Error("error destroying deployment",
|
||||
zap.Error(err),
|
||||
zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
|
||||
zap.String("deployment_namespace", gp.namespace))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
+27
-18
@@ -18,12 +18,12 @@ package poolmgr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -45,6 +45,8 @@ const (
|
||||
|
||||
type (
|
||||
GenericPoolManager struct {
|
||||
logger *zap.Logger
|
||||
|
||||
pools map[string]*GenericPool
|
||||
kubernetesClient *kubernetes.Clientset
|
||||
namespace string
|
||||
@@ -78,18 +80,22 @@ type (
|
||||
)
|
||||
|
||||
func MakeGenericPoolManager(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
functionNamespace string,
|
||||
instanceId string) *GenericPoolManager {
|
||||
|
||||
gpmLogger := logger.Named("generic_pool_manager")
|
||||
|
||||
gpm := &GenericPoolManager{
|
||||
logger: gpmLogger,
|
||||
pools: make(map[string]*GenericPool),
|
||||
kubernetesClient: kubernetesClient,
|
||||
namespace: functionNamespace,
|
||||
fissionClient: fissionClient,
|
||||
functionEnv: cache.MakeCache(10*time.Second, 0),
|
||||
fsCache: fscache.MakeFunctionServiceCache(),
|
||||
fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
|
||||
instanceId: instanceId,
|
||||
requestChannel: make(chan *request),
|
||||
idlePodReapTime: 2 * time.Minute,
|
||||
@@ -101,7 +107,7 @@ func MakeGenericPoolManager(
|
||||
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
|
||||
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
|
||||
if err != nil {
|
||||
log.Println("Failed to parse ENABLE_ISTIO")
|
||||
gpmLogger.Info("failed to parse ENABLE_ISTIO")
|
||||
}
|
||||
gpm.enableIstio = istio
|
||||
}
|
||||
@@ -142,7 +148,7 @@ func (gpm *GenericPoolManager) service() {
|
||||
ns = req.env.Metadata.Namespace
|
||||
}
|
||||
|
||||
pool, err = MakeGenericPool(
|
||||
pool, err = MakeGenericPool(gpm.logger,
|
||||
gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
|
||||
ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio,
|
||||
gpm.collectorEndpoint)
|
||||
@@ -163,7 +169,7 @@ func (gpm *GenericPoolManager) service() {
|
||||
if !ok || poolsize == 0 {
|
||||
// Env no longer exists or pool size changed to zero
|
||||
|
||||
log.Printf("Destroying generic pool for environment %v", pool.env.Metadata)
|
||||
gpm.logger.Info("eestroying generic pool", zap.Any("environment", pool.env.Metadata))
|
||||
delete(gpm.pools, key)
|
||||
|
||||
// and delete the pool asynchronously.
|
||||
@@ -195,7 +201,7 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) {
|
||||
|
||||
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
// from Func -> get Env
|
||||
log.Printf("[%v] getting environment for function", metadata.Name)
|
||||
gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
|
||||
env, err := gpm.getFunctionEnv(metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -207,7 +213,7 @@ func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.
|
||||
}
|
||||
// from GenericPool -> get one function container
|
||||
// (this also adds to the cache)
|
||||
log.Printf("[%v] getting function service from pool", metadata.Name)
|
||||
gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
|
||||
return pool.GetFuncSvc(ctx, metadata)
|
||||
}
|
||||
|
||||
@@ -228,7 +234,7 @@ func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Enviro
|
||||
}
|
||||
|
||||
// Get env from metadata
|
||||
log.Printf("[%v] getting env", m)
|
||||
gpm.logger.Info("getting env", zap.Any("function", m))
|
||||
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -247,11 +253,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
|
||||
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
if fission.IsNetworkError(err) {
|
||||
log.Printf("Encountered network error, retrying: %v", err)
|
||||
gpm.logger.Error("encountered network error, retrying", zap.Error(err))
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
log.Fatalf("Failed to get environment list: %v", err)
|
||||
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
// Create pools for all envs. TODO: we should make this a bit less eager, only
|
||||
@@ -264,7 +270,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
|
||||
if gpm.getEnvPoolsize(&env) > 0 {
|
||||
_, err := gpm.GetPool(&envs.Items[i])
|
||||
if err != nil {
|
||||
log.Printf("eager-create pool failed: %v", err)
|
||||
gpm.logger.Error("eager-create pool failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -292,7 +298,7 @@ func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
|
||||
if obj.Kind == "pod" {
|
||||
pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
|
||||
if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && fission.IsReadyPod(pod) {
|
||||
log.Printf("Valid pod address : %s", fsvc.Address)
|
||||
gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -309,7 +315,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
|
||||
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get environment list: %v", err)
|
||||
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
envList := make(map[types.UID]struct{})
|
||||
@@ -319,7 +325,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
|
||||
funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
|
||||
if err != nil {
|
||||
log.Printf("Error reaping idle pods: %v", err)
|
||||
gpm.logger.Error("error reaping idle pods", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -331,8 +337,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
// For function with the environment that no longer exists, executor
|
||||
// cleanups the idle pod as usual and prints log to notify user.
|
||||
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
|
||||
log.Printf("Environment %v for function %v no longer exists",
|
||||
fsvc.Environment.Metadata.Name, fsvc.Name)
|
||||
gpm.logger.Info("function environment no longer exists",
|
||||
zap.String("environment", fsvc.Environment.Metadata.Name),
|
||||
zap.String("function", fsvc.Name))
|
||||
}
|
||||
|
||||
if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fission.AllowedFunctionsPerContainerInfinite {
|
||||
@@ -341,7 +348,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
|
||||
deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
|
||||
if err != nil {
|
||||
log.Printf("Error deleting Kubernetes objects for fsvc '%v': %v", fsvc, err)
|
||||
gpm.logger.Error("error deleting Kubernetes objects for function service",
|
||||
zap.Error(err),
|
||||
zap.Any("service", fsvc))
|
||||
}
|
||||
|
||||
if !deleted {
|
||||
@@ -349,7 +358,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
}
|
||||
|
||||
for _, kubeobj := range fsvc.KubernetesObjects {
|
||||
reaper.CleanupKubeObject(gpm.kubernetesClient, &kubeobj)
|
||||
reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ package poolmgr
|
||||
import (
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -39,7 +39,9 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
|
||||
k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
pkg := obj.(*crd.Package)
|
||||
log.Printf("List watch for package reported a new package addition: %s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)
|
||||
gpm.logger.Info("list watch for package reported a new package addition",
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namepsace", pkg.Metadata.Namespace))
|
||||
|
||||
// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
|
||||
envNs := fissionfnNamespace
|
||||
@@ -49,13 +51,21 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
|
||||
|
||||
// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
|
||||
// every function's package to be loaded into its env. without that, there's no point to move forward.
|
||||
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
|
||||
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
log.Printf("Error creating %s for package: %s.%s, err: %v", fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace, err)
|
||||
gpm.logger.Error("error creating rolebinding for package",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", fission.PackageGetterRB),
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namespace", pkg.Metadata.Namespace))
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, pkg.Metadata.Namespace, pkg.Metadata.Name)
|
||||
gpm.logger.Info("successfully set up rolebinding for fetcher service account",
|
||||
zap.String("service_account", fission.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", envNs),
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namespace", pkg.Metadata.Namespace))
|
||||
},
|
||||
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
@@ -75,14 +85,23 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
|
||||
envNs = newPkg.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB,
|
||||
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB,
|
||||
newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole,
|
||||
fission.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
log.Printf("Error : %v updating %s for package: %s.%s", err, fission.PackageGetterRB, newPkg.Metadata.Name, newPkg.Metadata.Namespace)
|
||||
gpm.logger.Error("error updating rolebinding for package",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", fission.PackageGetterRB),
|
||||
zap.String("package_name", newPkg.Metadata.Name),
|
||||
zap.String("package_namespace", newPkg.Metadata.Namespace))
|
||||
return
|
||||
}
|
||||
log.Printf("Updated rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, newPkg.Metadata.Namespace, newPkg.Metadata.Name)
|
||||
|
||||
gpm.logger.Info("successfully updated rolebinding for fetcher service account",
|
||||
zap.String("service_account", fission.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", envNs),
|
||||
zap.String("package_name", newPkg.Metadata.Name),
|
||||
zap.String("package_namespace", newPkg.Metadata.Namespace))
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user