Migrate HPA v1 to v2beta2 (#2421)

* Migrate HPA v1 to v2beta2 HPA v2beta2 is defined and supported from 1.19+ onwards. Also HPA v2 is stable from 1.23 onwards. As we support 1.19+ onwards using HPA v2beta2. This change is base for custom metrics support we want to add later by modifying Function spec. * Add unit tests for hpa operations * Use constants instead of strings Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
2022-05-02 13:30:21 +05:30
parent e72641c0f3
commit ed4bd2573b
11 changed files with 353 additions and 242 deletions
@@ -0,0 +1,153 @@
+/*
+Copyright 2022 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package hpa
+
+import (
+	"context"
+	"errors"
+
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	asv2beta2 "k8s.io/api/autoscaling/v2beta2"
+	corev1 "k8s.io/api/core/v1"
+	k8s_err "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+	otelUtils "github.com/fission/fission/pkg/utils/otel"
+)
+
+// Deployment Constants
+const (
+	DeploymentKind    = "Deployment"
+	DeploymentVersion = "apps/v1"
+)
+
+type HpaOperations struct {
+	logger           *zap.Logger
+	kubernetesClient kubernetes.Interface
+	instanceID       string
+}
+
+func NewHpaOperations(logger *zap.Logger, kubernetesClient kubernetes.Interface, instanceID string) *HpaOperations {
+	return &HpaOperations{
+		logger:           logger,
+		kubernetesClient: kubernetesClient,
+		instanceID:       instanceID,
+	}
+}
+
+func ConvertTargetCPUToCustomMetric(targetCPUVal int32) asv2beta2.MetricSpec {
+	return asv2beta2.MetricSpec{
+		Type: asv2beta2.ResourceMetricSourceType,
+		Resource: &asv2beta2.ResourceMetricSource{
+			Name: corev1.ResourceCPU,
+			Target: asv2beta2.MetricTarget{
+				Type:               asv2beta2.UtilizationMetricType,
+				AverageUtilization: &targetCPUVal,
+			},
+		},
+	}
+}
+
+func getScaleTargetRef(deployment *appsv1.Deployment) asv2beta2.CrossVersionObjectReference {
+	return asv2beta2.CrossVersionObjectReference{
+		APIVersion: DeploymentVersion,
+		Kind:       DeploymentKind,
+		Name:       deployment.ObjectMeta.Name,
+	}
+}
+
+func (hpaops *HpaOperations) CreateOrGetHpa(ctx context.Context, hpaName string, execStrategy *fv1.ExecutionStrategy,
+	depl *appsv1.Deployment, deployLabels map[string]string, deployAnnotations map[string]string) (*asv2beta2.HorizontalPodAutoscaler, error) {
+
+	if depl == nil {
+		return nil, errors.New("failed to create HPA, found empty deployment")
+	}
+	logger := otelUtils.LoggerWithTraceID(ctx, hpaops.logger)
+
+	minRepl := int32(execStrategy.MinScale)
+	if minRepl == 0 {
+		minRepl = 1
+	}
+	maxRepl := int32(execStrategy.MaxScale)
+	if maxRepl == 0 {
+		maxRepl = minRepl
+	}
+	targetCPU := int32(execStrategy.TargetCPUPercent)
+	var hpaMetrics []asv2beta2.MetricSpec
+	if targetCPU > 0 {
+		hpaMetrics = append(hpaMetrics, ConvertTargetCPUToCustomMetric(targetCPU))
+	}
+
+	hpa := &asv2beta2.HorizontalPodAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        hpaName,
+			Labels:      deployLabels,
+			Annotations: deployAnnotations,
+		},
+		Spec: asv2beta2.HorizontalPodAutoscalerSpec{
+			ScaleTargetRef: getScaleTargetRef(depl),
+			MinReplicas:    &minRepl,
+			MaxReplicas:    maxRepl,
+			Metrics:        hpaMetrics,
+		},
+	}
+
+	existingHpa, err := hpaops.GetHpa(ctx, depl.ObjectMeta.Namespace, hpaName)
+	if err == nil {
+		// to adopt orphan service
+		if existingHpa.Annotations[fv1.EXECUTOR_INSTANCEID_LABEL] != hpaops.instanceID {
+			existingHpa.Annotations = hpa.Annotations
+			existingHpa.Labels = hpa.Labels
+			existingHpa.Spec = hpa.Spec
+			existingHpa, err = hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Update(ctx, existingHpa, metav1.UpdateOptions{})
+			if err != nil {
+				logger.Warn("error adopting HPA", zap.Error(err),
+					zap.String("HPA", hpaName), zap.String("ns", depl.ObjectMeta.Namespace))
+				return nil, err
+			}
+		}
+		return existingHpa, err
+	} else if k8s_err.IsNotFound(err) {
+		cHpa, err := hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Create(ctx, hpa, metav1.CreateOptions{})
+		if err != nil {
+			if k8s_err.IsAlreadyExists(err) {
+				cHpa, err = hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Get(ctx, hpaName, metav1.GetOptions{})
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+		otelUtils.SpanTrackEvent(ctx, "hpaCreated", otelUtils.GetAttributesForHPA(cHpa)...)
+		return cHpa, nil
+	}
+	return nil, err
+}
+
+func (hpaops *HpaOperations) GetHpa(ctx context.Context, ns, name string) (*asv2beta2.HorizontalPodAutoscaler, error) {
+	return hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(ns).Get(ctx, name, metav1.GetOptions{})
+}
+
+func (hpaops *HpaOperations) UpdateHpa(ctx context.Context, hpa *asv2beta2.HorizontalPodAutoscaler) error {
+	_, err := hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Update(ctx, hpa, metav1.UpdateOptions{})
+	return err
+}
+
+func (hpaops *HpaOperations) DeleteHpa(ctx context.Context, ns string, name string) error {
+	return hpaops.kubernetesClient.AutoscalingV2beta2().HorizontalPodAutoscalers(ns).Delete(ctx, name, metav1.DeleteOptions{})
+}
@@ -0,0 +1,147 @@
+/*
+Copyright 2022 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package hpa
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/dchest/uniuri"
+	appsv1 "k8s.io/api/apps/v1"
+	asv2beta2 "k8s.io/api/autoscaling/v2beta2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+	"github.com/fission/fission/pkg/utils/loggerfactory"
+)
+
+func TestConvertTargetCPUToCustomMetric(t *testing.T) {
+	metricSpec := ConvertTargetCPUToCustomMetric(50)
+	if metricSpec.Type != asv2beta2.ResourceMetricSourceType {
+		t.Errorf("Expected metric type to be Resource, got %v", metricSpec.Type)
+	}
+	if metricSpec.Resource.Name != corev1.ResourceCPU {
+		t.Errorf("Expected metric name to be cpu, got %v", metricSpec.Resource.Name)
+	}
+	if metricSpec.Resource.Target.Type != asv2beta2.UtilizationMetricType {
+		t.Errorf("Expected metric target type to be Utilization, got %v", metricSpec.Resource.Target.Type)
+	}
+	if metricSpec.Resource.Target.AverageUtilization == nil {
+		t.Errorf("Expected metric target average utilization to be set, got nil")
+	}
+}
+
+func TestHpaOps(t *testing.T) {
+	logger := loggerfactory.GetLogger()
+	kubernetesClient := fake.NewSimpleClientset()
+	instanceID := strings.ToLower(uniuri.NewLen(8))
+	ns := "test-namespace"
+	hpaops := NewHpaOperations(logger, kubernetesClient, instanceID)
+	if hpaops.instanceID != instanceID {
+		t.Errorf("Expected instanceID to be %v, got %v", instanceID, hpaops.instanceID)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	deployLabels := map[string]string{
+		"test-label": "test-label-value",
+	}
+	deployAnnotations := map[string]string{
+		"test-annotation": "test-annotation-value",
+	}
+	// Test CreateHPA
+	hpa, err := hpaops.CreateOrGetHpa(ctx, "test-hpa",
+		&fv1.ExecutionStrategy{
+			ExecutorType:          fv1.ExecutorTypeNewdeploy,
+			MinScale:              1,
+			MaxScale:              5,
+			TargetCPUPercent:      50,
+			SpecializationTimeout: 300,
+		},
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "test-deployment",
+				Namespace: ns,
+			},
+		},
+		deployLabels,
+		deployAnnotations)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if *hpa.Spec.MinReplicas != 1 {
+		t.Errorf("Expected min replicas to be 1, got %v", hpa.Spec.MinReplicas)
+	}
+	if hpa.Spec.MaxReplicas != 5 {
+		t.Errorf("Expected max replicas to be 5, got %v", hpa.Spec.MaxReplicas)
+	}
+	if hpa.Spec.Metrics[0].Type != asv2beta2.ResourceMetricSourceType {
+		t.Errorf("Expected metric type to be Resource, got %v", hpa.Spec.Metrics[0].Type)
+	}
+	if hpa.Spec.Metrics[0].Resource.Name != corev1.ResourceCPU {
+		t.Errorf("Expected metric name to be cpu, got %v", hpa.Spec.Metrics[0].Resource.Name)
+	}
+	if hpa.Spec.Metrics[0].Resource.Target.Type != asv2beta2.UtilizationMetricType {
+		t.Errorf("Expected metric target type to be Utilization, got %v", hpa.Spec.Metrics[0].Resource.Target.Type)
+	}
+	if hpa.Spec.Metrics[0].Resource.Target.AverageUtilization == nil {
+		t.Errorf("Expected metric target average utilization to be set, got nil")
+	}
+	if *hpa.Spec.Metrics[0].Resource.Target.AverageUtilization != 50 {
+		t.Errorf("Expected metric target average utilization to be 50, got %v", *hpa.Spec.Metrics[0].Resource.Target.AverageUtilization)
+	}
+	if hpa.ObjectMeta.Labels["test-label"] != "test-label-value" {
+		t.Errorf("Expected label to be set, got %v", hpa.ObjectMeta.Labels["test-label"])
+	}
+	if hpa.ObjectMeta.Annotations["test-annotation"] != "test-annotation-value" {
+		t.Errorf("Expected annotation to be set, got %v", hpa.ObjectMeta.Annotations["test-annotation"])
+	}
+
+	hpa, err = hpaops.GetHpa(ctx, ns, "test-hpa")
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	hpa.Spec.MaxReplicas = 10
+
+	// Test UpdateHPA
+	err = hpaops.UpdateHpa(ctx, hpa)
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	hpa, err = hpaops.GetHpa(ctx, ns, "test-hpa")
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+	if hpa.Spec.MaxReplicas != 10 {
+		t.Errorf("Expected max replicas to be 10, got %v", hpa.Spec.MaxReplicas)
+	}
+
+	// Test DeleteHPA
+	err = hpaops.DeleteHpa(ctx, ns, "test-hpa")
+	if err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	_, err = hpaops.GetHpa(ctx, ns, "test-hpa")
+	if err == nil {
+		t.Errorf("Expected error, got nil")
+	}
+}