From 5edf34adba8cdbcac77f4e5c7a478fc0cec93117 Mon Sep 17 00:00:00 2001 From: soharab-ic <156293296+soharab-ic@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:25:12 +0530 Subject: [PATCH] Set OwnerReferences to K8s resources created by Fission (#2964) * Poolmanager deployment is created based on environment. Set environment as owner to poolmanager deployment. Signed-off-by: Md Soharab Ansari * Set OwnerReferences to K8s resources created by fission resources. ``` Set OwnerReferences to deployment, service and HPA created by newdeploy function. Set OwnerReferences to builderManager deployment and service created by environment. Set OwnerReferences to deployment, service and HPA created by container function. ``` Signed-off-by: Md Soharab Ansari * Use ControllerRef Signed-off-by: Md Soharab Ansari --------- Signed-off-by: Md Soharab Ansari --- pkg/buildermgr/envwatcher.go | 15 +++++++++++++++ .../executortype/container/containermgr.go | 2 +- .../executortype/container/deployment.go | 9 +++++++++ pkg/executor/executortype/container/svc.go | 9 +++++++++ .../executortype/newdeploy/newdeploy.go | 19 ++++++++++++++++++- .../executortype/newdeploy/newdeploymgr.go | 4 ++-- .../executortype/poolmgr/gp_deployment.go | 8 ++++++++ pkg/executor/util/hpa/hpa.go | 11 ++++++++++- pkg/executor/util/hpa/hpa_test.go | 10 +++++++++- 9 files changed, 81 insertions(+), 6 deletions(-) diff --git a/pkg/buildermgr/envwatcher.go b/pkg/buildermgr/envwatcher.go index 89dcf1f06..d50c6f7dd 100644 --- a/pkg/buildermgr/envwatcher.go +++ b/pkg/buildermgr/envwatcher.go @@ -28,6 +28,7 @@ import ( apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -329,6 +330,13 @@ func (envw *environmentWatcher) createBuilderService(ctx context.Context, env *f Namespace: ns, Name: name, Labels: sel, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, }, Spec: apiv1.ServiceSpec{ Selector: sel, @@ -440,6 +448,13 @@ func (envw *environmentWatcher) createBuilderDeployment(ctx context.Context, env Namespace: ns, Name: name, Labels: sel, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, diff --git a/pkg/executor/executortype/container/containermgr.go b/pkg/executor/executortype/container/containermgr.go index 391c0b1bf..7bcf205c3 100644 --- a/pkg/executor/executortype/container/containermgr.go +++ b/pkg/executor/executortype/container/containermgr.go @@ -421,7 +421,7 @@ func (caaf *Container) fnCreate(ctx context.Context, fn *fv1.Function) (*fscache return nil, fmt.Errorf("error creating deployment %s: %w", objName, err) } - hpa, err := caaf.hpaops.CreateOrGetHpa(ctx, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) + hpa, err := caaf.hpaops.CreateOrGetHpa(ctx, fn, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) if err != nil { caaf.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName)) go cleanupFunc(ns, objName) diff --git a/pkg/executor/executortype/container/deployment.go b/pkg/executor/executortype/container/deployment.go index 6afaa6ba5..61b51965c 100644 --- a/pkg/executor/executortype/container/deployment.go +++ b/pkg/executor/executortype/container/deployment.go @@ -27,6 +27,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -86,6 +87,7 @@ func (cn *Container) createOrGetDeployment(ctx context.Context, fn *fv1.Function if existingDepl.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != cn.instanceID { existingDepl.Annotations = deployment.Annotations existingDepl.Labels = deployment.Labels + existingDepl.OwnerReferences = deployment.OwnerReferences existingDepl.Spec.Template.Spec.Containers = deployment.Spec.Template.Spec.Containers existingDepl.Spec.Template.Spec.ServiceAccountName = deployment.Spec.Template.Spec.ServiceAccountName existingDepl.Spec.Template.Spec.TerminationGracePeriodSeconds = deployment.Spec.Template.Spec.TerminationGracePeriodSeconds @@ -268,6 +270,13 @@ func (cn *Container) getDeploymentSpec(ctx context.Context, fn *fv1.Function, ta Name: deployName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, diff --git a/pkg/executor/executortype/container/svc.go b/pkg/executor/executortype/container/svc.go index c2b8622a4..7f7aaab0d 100644 --- a/pkg/executor/executortype/container/svc.go +++ b/pkg/executor/executortype/container/svc.go @@ -24,6 +24,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -54,6 +55,13 @@ func (cn *Container) createOrGetSvc(ctx context.Context, fn *fv1.Function, deplo Name: svcName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: apiv1.ServiceSpec{ Ports: []apiv1.ServicePort{ @@ -74,6 +82,7 @@ func (cn *Container) createOrGetSvc(ctx context.Context, fn *fv1.Function, deplo if existingSvc.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != cn.instanceID { existingSvc.Annotations = service.Annotations existingSvc.Labels = service.Labels + existingSvc.OwnerReferences = service.OwnerReferences existingSvc.Spec.Ports = service.Spec.Ports existingSvc.Spec.Selector = service.Spec.Selector existingSvc.Spec.Type = service.Spec.Type diff --git a/pkg/executor/executortype/newdeploy/newdeploy.go b/pkg/executor/executortype/newdeploy/newdeploy.go index 18d8e340d..68e35d62e 100644 --- a/pkg/executor/executortype/newdeploy/newdeploy.go +++ b/pkg/executor/executortype/newdeploy/newdeploy.go @@ -29,6 +29,7 @@ import ( k8s_err "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" @@ -61,6 +62,7 @@ func (deploy *NewDeploy) createOrGetDeployment(ctx context.Context, fn *fv1.Func if existingDepl.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != deploy.instanceID { existingDepl.Annotations = deployment.Annotations existingDepl.Labels = deployment.Labels + existingDepl.OwnerReferences = deployment.OwnerReferences existingDepl.Spec.Template.Spec.Containers = deployment.Spec.Template.Spec.Containers existingDepl.Spec.Template.Spec.ServiceAccountName = deployment.Spec.Template.Spec.ServiceAccountName existingDepl.Spec.Template.Spec.TerminationGracePeriodSeconds = deployment.Spec.Template.Spec.TerminationGracePeriodSeconds @@ -249,6 +251,13 @@ func (deploy *NewDeploy) getDeploymentSpec(ctx context.Context, fn *fv1.Function Name: deployName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, @@ -323,13 +332,20 @@ func (deploy *NewDeploy) getResources(env *fv1.Environment, fn *fv1.Function) ap return resources } -func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, deployLabels map[string]string, deployAnnotations map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) { +func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, fn *fv1.Function, deployLabels map[string]string, deployAnnotations map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) { logger := otelUtils.LoggerWithTraceID(ctx, deploy.logger) service := &apiv1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: svcName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: apiv1.ServiceSpec{ Ports: []apiv1.ServicePort{ @@ -351,6 +367,7 @@ func (deploy *NewDeploy) createOrGetSvc(ctx context.Context, deployLabels map[st if existingSvc.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != deploy.instanceID { existingSvc.Annotations = service.Annotations existingSvc.Labels = service.Labels + existingSvc.OwnerReferences = service.OwnerReferences existingSvc.Spec.Ports = service.Spec.Ports existingSvc.Spec.Selector = service.Spec.Selector existingSvc.Spec.Type = service.Spec.Type diff --git a/pkg/executor/executortype/newdeploy/newdeploymgr.go b/pkg/executor/executortype/newdeploy/newdeploymgr.go index ac9835465..c81b9de29 100644 --- a/pkg/executor/executortype/newdeploy/newdeploymgr.go +++ b/pkg/executor/executortype/newdeploy/newdeploymgr.go @@ -451,7 +451,7 @@ func (deploy *NewDeploy) fnCreate(ctx context.Context, fn *fv1.Function) (*fscac // Since newdeploy waits for pods of deployment to be ready, // change the order of kubeObject creation (create service first, // then deployment) to take advantage of waiting time. - svc, err := deploy.createOrGetSvc(ctx, deployLabels, deployAnnotations, objName, ns) + svc, err := deploy.createOrGetSvc(ctx, fn, deployLabels, deployAnnotations, objName, ns) if err != nil { deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName)) go cleanupFunc(context.Background(), ns, objName) @@ -466,7 +466,7 @@ func (deploy *NewDeploy) fnCreate(ctx context.Context, fn *fv1.Function) (*fscac return nil, fmt.Errorf("error creating deployment %s: %w", objName, err) } - hpa, err := deploy.hpaops.CreateOrGetHpa(ctx, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) + hpa, err := deploy.hpaops.CreateOrGetHpa(ctx, fn, objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl, deployLabels, deployAnnotations) if err != nil { deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName)) go cleanupFunc(context.Background(), ns, objName) diff --git a/pkg/executor/executortype/poolmgr/gp_deployment.go b/pkg/executor/executortype/poolmgr/gp_deployment.go index 4d2f41557..0e7b598a7 100644 --- a/pkg/executor/executortype/poolmgr/gp_deployment.go +++ b/pkg/executor/executortype/poolmgr/gp_deployment.go @@ -26,6 +26,7 @@ import ( apiv1 "k8s.io/api/core/v1" k8sErrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/executor/util" @@ -62,6 +63,13 @@ func (gp *GenericPool) genDeploymentMeta(env *fv1.Environment) metav1.ObjectMeta Name: getPoolName(env), Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(env, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Environment", + }), + }, } } diff --git a/pkg/executor/util/hpa/hpa.go b/pkg/executor/util/hpa/hpa.go index aa770a4f5..1ab24d3db 100644 --- a/pkg/executor/util/hpa/hpa.go +++ b/pkg/executor/util/hpa/hpa.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_err "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -72,7 +73,7 @@ func getScaleTargetRef(deployment *appsv1.Deployment) asv2.CrossVersionObjectRef } } -func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, execStrategy *fv1.ExecutionStrategy, +func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, fn *fv1.Function, hpaName string, execStrategy *fv1.ExecutionStrategy, depl *appsv1.Deployment, deployLabels map[string]string, deployAnnotations map[string]string) (*asv2.HorizontalPodAutoscaler, error) { if depl == nil { @@ -103,6 +104,13 @@ func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, Name: hpaName, Labels: deployLabels, Annotations: deployAnnotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(fn, schema.GroupVersionKind{ + Group: "fission.io", + Version: "v1", + Kind: "Function", + }), + }, }, Spec: asv2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: getScaleTargetRef(depl), @@ -119,6 +127,7 @@ func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, if existingHpa.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != hpaops.instanceID { existingHpa.Annotations = hpa.Annotations existingHpa.Labels = hpa.Labels + existingHpa.OwnerReferences = hpa.OwnerReferences existingHpa.Spec = hpa.Spec existingHpa, err = hpaops.kubernetesClient.AutoscalingV2().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Update(ctx, existingHpa, metav1.UpdateOptions{}) if err != nil { diff --git a/pkg/executor/util/hpa/hpa_test.go b/pkg/executor/util/hpa/hpa_test.go index 301060560..a4b171557 100644 --- a/pkg/executor/util/hpa/hpa_test.go +++ b/pkg/executor/util/hpa/hpa_test.go @@ -25,6 +25,7 @@ import ( asv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes/fake" fv1 "github.com/fission/fission/pkg/apis/core/v1" @@ -67,7 +68,14 @@ func TestHpaOps(t *testing.T) { "test-annotation": "test-annotation-value", } // Test CreateHPA - hpa, err := hpaops.CreateOrGetHpa(ctx, "test-hpa", + hpa, err := hpaops.CreateOrGetHpa(ctx, + &fv1.Function{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-fn", + UID: uuid.NewUUID(), + }, + }, + "test-hpa", &fv1.ExecutionStrategy{ ExecutorType: fv1.ExecutorTypeNewdeploy, MinScale: 1,