From a34840b0a70232c7ea83f5bc20890d1066425bc2 Mon Sep 17 00:00:00 2001 From: soharab-ic <156293296+soharab-ic@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:25:12 +0530 Subject: [PATCH] Set OwnerReferences to K8s resources created by Fission (#2964) * Poolmanager deployment is created based on environment. Set environment as owner to poolmanager deployment. Signed-off-by: Md Soharab Ansari * Set OwnerReferences to K8s resources created by fission resources. ``` Set OwnerReferences to deployment, service and HPA created by newdeploy function. Set OwnerReferences to builderManager deployment and service created by environment. Set OwnerReferences to deployment, service and HPA created by container function. ``` Signed-off-by: Md Soharab Ansari * Use ControllerRef Signed-off-by: Md Soharab Ansari --------- Signed-off-by: Md Soharab Ansari --- pkg/buildermgr/envwatcher.go | 15 +++++++++++++++ .../executortype/container/containermgr.go | 2 +- .../executortype/container/deployment.go | 9 +++++++++ pkg/executor/executortype/container/svc.go | 9 +++++++++ .../executortype/newdeploy/newdeploy.go | 19 ++++++++++++++++++- .../executortype/newdeploy/newdeploymgr.go | 4 ++-- .../executortype/poolmgr/gp_deployment.go | 8 ++++++++ pkg/executor/util/hpa/hpa.go | 11 ++++++++++- pkg/executor/util/hpa/hpa_test.go | 10 +++++++++- 9 files changed, 81 insertions(+), 6 deletions(-) diff --git a/pkg/buildermgr/envwatcher.go b/pkg/buildermgr/envwatcher.go index 89dcf1f0..d50c6f7d 100644 --- a/pkg/buildermgr/envwatcher.go +++ b/pkg/buildermgr/envwatcher.go @@ -28,6 +28,7 @@ import ( apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -329,6 +330,13 @@ func (envw *environmentWatcher) createBuilderService(ctx context.Context, env *f Namespace: ns, Name: name, Labels: sel, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, }, Spec: apiv1.ServiceSpec{ Selector: sel, @@ -440,6 +448,13 @@ func (envw *environmentWatcher) createBuilderDeployment(ctx context.Context, env Namespace: ns, Name: name, Labels: sel, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, diff --git a/pkg/executor/executortype/container/containermgr.go b/pkg/executor/executortype/container/containermgr.go index 391c0b1b..7bcf205c 100644 --- a/pkg/executor/executortype/container/containermgr.go +++ b/pkg/executor/executortype/container/containermgr.go @@ -421,7 +421,7 @@ func (caaf *Container) fnCreate(ctx context.Context, fn *fv1.Function) (*fscache return nil, fmt.Errorf("error creating deployment %s: %w", objName, err) } - hpa, err := caaf.hpaops.CreateOrGetHpa(ctx, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) + hpa, err := caaf.hpaops.CreateOrGetHpa(ctx, fn, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) if err != nil { caaf.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName)) go cleanupFunc(ns, objName) diff --git a/pkg/executor/executortype/container/deployment.go b/pkg/executor/executortype/container/deployment.go index 6afaa6ba..61b51965 100644 --- a/pkg/executor/executortype/container/deployment.go +++ b/pkg/executor/executortype/container/deployment.go @@ -27,6 +27,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -86,6 +87,7 @@ func (cn *Container) createOrGetDeployment(ctx context.Context, fn *fv1.Function if existingDepl.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != cn.instanceID { existingDepl.Annotations = deployment.Annotations existingDepl.Labels = deployment.Labels + existingDepl.OwnerReferences = deployment.OwnerReferences existingDepl.Spec.Template.Spec.Containers = deployment.Spec.Template.Spec.Containers existingDepl.Spec.Template.Spec.ServiceAccountName = deployment.Spec.Template.Spec.ServiceAccountName existingDepl.Spec.Template.Spec.TerminationGracePeriodSeconds = deployment.Spec.Template.Spec.TerminationGracePeriodSeconds @@ -268,6 +270,13 @@ func (cn *Container) getDeploymentSpec(ctx context.Context, fn *fv1.Function, ta Name: deployName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, diff --git a/pkg/executor/executortype/container/svc.go b/pkg/executor/executortype/container/svc.go index c2b8622a..7f7aaab0 100644 --- a/pkg/executor/executortype/container/svc.go +++ b/pkg/executor/executortype/container/svc.go @@ -24,6 +24,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -54,6 +55,13 @@ func (cn *Container) createOrGetSvc(ctx context.Context, fn *fv1.Function, deplo Name: svcName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: apiv1.ServiceSpec{ Ports: []apiv1.ServicePort{ @@ -74,6 +82,7 @@ func (cn *Container) createOrGetSvc(ctx context.Context, fn *fv1.Function, deplo if existingSvc.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != cn.instanceID { existingSvc.Annotations = service.Annotations existingSvc.Labels = service.Labels + existingSvc.OwnerReferences = service.OwnerReferences existingSvc.Spec.Ports = service.Spec.Ports existingSvc.Spec.Selector = service.Spec.Selector existingSvc.Spec.Type = service.Spec.Type diff --git a/pkg/executor/executortype/newdeploy/newdeploy.go b/pkg/executor/executortype/newdeploy/newdeploy.go index 18d8e340..68e35d62 100644 --- a/pkg/executor/executortype/newdeploy/newdeploy.go +++ b/pkg/executor/executortype/newdeploy/newdeploy.go @@ -29,6 +29,7 @@ import ( k8s_err "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -61,6 +62,7 @@ func (deploy *NewDeploy) createOrGetDeployment(ctx context.Context, fn *fv1.Func if existingDepl.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != deploy.instanceID { existingDepl.Annotations = deployment.Annotations existingDepl.Labels = deployment.Labels + existingDepl.OwnerReferences = deployment.OwnerReferences existingDepl.Spec.Template.Spec.Containers = deployment.Spec.Template.Spec.Containers existingDepl.Spec.Template.Spec.ServiceAccountName = deployment.Spec.Template.Spec.ServiceAccountName existingDepl.Spec.Template.Spec.TerminationGracePeriodSeconds = deployment.Spec.Template.Spec.TerminationGracePeriodSeconds @@ -249,6 +251,13 @@ func (deploy *NewDeploy) getDeploymentSpec(ctx context.Context, fn *fv1.Function Name: deployName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, @@ -323,13 +332,20 @@ func (deploy *NewDeploy) getResources(env *fv1.Environment, fn *fv1.Function) ap return resources } -func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, deployLabels map[string]string, deployAnnotations map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) { +func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, fn *fv1.Function, deployLabels map[string]string, deployAnnotations map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) { logger := otelUtils.LoggerWithTraceID(ctx, deploy.logger) service := &apiv1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: svcName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: apiv1.ServiceSpec{ Ports: []apiv1.ServicePort{ @@ -351,6 +367,7 @@ func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, deployLabels map[st if existingSvc.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != deploy.instanceID { existingSvc.Annotations = service.Annotations existingSvc.Labels = service.Labels + existingSvc.OwnerReferences = service.OwnerReferences existingSvc.Spec.Ports = service.Spec.Ports existingSvc.Spec.Selector = service.Spec.Selector existingSvc.Spec.Type = service.Spec.Type diff --git a/pkg/executor/executortype/newdeploy/newdeploymgr.go b/pkg/executor/executortype/newdeploy/newdeploymgr.go index ac983546..c81b9de2 100644 --- a/pkg/executor/executortype/newdeploy/newdeploymgr.go +++ b/pkg/executor/executortype/newdeploy/newdeploymgr.go @@ -451,7 +451,7 @@ func (deploy *NewDeploy) fnCreate(ctx context.Context, fn *fv1.Function) (*fscac // Since newdeploy waits for pods of deployment to be ready, // change the order of kubeObject creation (create service first, // then deployment) to take advantage of waiting time. - svc, err := deploy.createOrGetSvc(ctx, deployLabels, deployAnnotations, objName, ns) + svc, err := deploy.createOrGetSvc(ctx, fn, deployLabels, deployAnnotations, objName, ns) if err != nil { deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName)) go cleanupFunc(context.Background(), ns, objName) @@ -466,7 +466,7 @@ func (deploy *NewDeploy) fnCreate(ctx context.Context, fn *fv1.Function) (*fscac return nil, fmt.Errorf("error creating deployment %s: %w", objName, err) } - hpa, err := deploy.hpaops.CreateOrGetHpa(ctx, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) + hpa, err := deploy.hpaops.CreateOrGetHpa(ctx, fn, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) if err != nil { deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName)) go cleanupFunc(context.Background(), ns, objName) diff --git a/pkg/executor/executortype/poolmgr/gp_deployment.go b/pkg/executor/executortype/poolmgr/gp_deployment.go index 4d2f4155..0e7b598a 100644 --- a/pkg/executor/executortype/poolmgr/gp_deployment.go +++ b/pkg/executor/executortype/poolmgr/gp_deployment.go @@ -26,6 +26,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8sErrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/executor/util" @@ -62,6 +63,13 @@ func (gp *GenericPool) genDeploymentMeta(env *fv1.Environment) metav1.ObjectMeta Name: getPoolName(env), Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, } } diff --git a/pkg/executor/util/hpa/hpa.go b/pkg/executor/util/hpa/hpa.go index aa770a4f..1ab24d3d 100644 --- a/pkg/executor/util/hpa/hpa.go +++ b/pkg/executor/util/hpa/hpa.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -72,7 +73,7 @@ func getScaleTargetRef(deployment *appsv1.Deployment) asv2.CrossVersionObjectRef } } -func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, execStrategy *fv1.ExecutionStrategy, +func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, fn *fv1.Function, hpaName string, execStrategy *fv1.ExecutionStrategy, depl *appsv1.Deployment, deployLabels map[string]string, deployAnnotations map[string]string) (*asv2.HorizontalPodAutoscaler, error) { if depl == nil { @@ -103,6 +104,13 @@ func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, Name: hpaName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: asv2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: getScaleTargetRef(depl), @@ -119,6 +127,7 @@ func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, if existingHpa.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != hpaops.instanceID { existingHpa.Annotations = hpa.Annotations existingHpa.Labels = hpa.Labels + existingHpa.OwnerReferences = hpa.OwnerReferences existingHpa.Spec = hpa.Spec existingHpa, err = hpaops.kubernetesClient.AutoscalingV2().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Update(ctx, existingHpa, metav1.UpdateOptions{}) if err != nil { diff --git a/pkg/executor/util/hpa/hpa_test.go b/pkg/executor/util/hpa/hpa_test.go index 30106056..a4b17155 100644 --- a/pkg/executor/util/hpa/hpa_test.go +++ b/pkg/executor/util/hpa/hpa_test.go @@ -25,6 +25,7 @@ import ( asv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes/fake" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -67,7 +68,14 @@ func TestHpaOps(t *testing.T) { "test-annotation": "test-annotation-value", } // Test CreateHPA - hpa, err := hpaops.CreateOrGetHpa(ctx, "test-hpa", + hpa, err := hpaops.CreateOrGetHpa(ctx, + &fv1.Function{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-fn", + UID: uuid.NewUUID(), + }, + }, + "test-hpa", &fv1.ExecutionStrategy{ ExecutorType: fv1.ExecutorTypeNewdeploy, MinScale: 1,