Add kubernetes QPS and Burst limit configuration (#2899)

Added configurable kubernetes client limits burst and QPS
Default QPS 200 and burst 500. Configurable via helm chart values.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

charts/fission-all/templates/_helpers.tpl

@@ -103,6 +103,13 @@ This template generates the image name for the deployment depending on the value
 {{- end }}
 {{- end }}
 
+{{- define "kube_client.envs" }}
+- name: KUBE_CLIENT_QPS
+  value: "{{ .Values.kubernetesClientQPS }}"
+- name: KUBE_CLIENT_BURST
+  value: "{{ .Values.kubernetesClientBurst }}"
+{{- end}}
+
 {{/*
 Define the svc's name
 */}}

charts/fission-all/templates/buildermgr/deployment.yaml

@@ -56,6 +56,7 @@ spec:
         - name: HELM_RELEASE_NAME
           value: {{ .Release.Name | quote }}
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         {{- if .Values.builderPodSpec.enabled }}
         volumeMounts:

charts/fission-all/templates/canary-config/deployment.yaml

@@ -37,7 +37,8 @@ spec:
           value: {{ .Values.debugEnv | quote }}
         - name: PPROF_ENABLED
           value: {{ .Values.pprof.enabled | quote }}
-        {{- include "fission-resource-namespace.envs" . | indent 8 }} 
+        {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:

charts/fission-all/templates/executor/deployment.yaml

@@ -78,6 +78,7 @@ spec:
           value: {{ .Values.executor.serviceAccountCheck.interval | quote }}
         {{- end}}  
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         - name: HELM_RELEASE_NAME
           value: {{ .Release.Name | quote }}
         {{- include "opentelemtry.envs" . | indent 8 }}

charts/fission-all/templates/kubewatcher/deployment.yaml

@@ -29,7 +29,8 @@ spec:
           value: {{ .Values.debugEnv | quote }}
         - name: PPROF_ENABLED
           value: {{ .Values.pprof.enabled | quote }}
-        {{- include "fission-resource-namespace.envs" . | indent 8 }}  
+        {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         resources:
           {{- toYaml .Values.kubewatcher.resources | nindent 10 }}

charts/fission-all/templates/mqt-keda/deployment.yaml

@@ -47,6 +47,7 @@ spec:
         - name: REDIS_IMAGE
           value: "{{ .Values.mqt_keda.connector_images.redis.image }}:{{ .Values.mqt_keda.connector_images.redis.tag }}"
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         resources:
           {{- toYaml .Values.mqt_keda.resources | nindent 10 }}

charts/fission-all/templates/pre-upgrade-checks/pre-upgrade-job.yaml

@@ -37,6 +37,7 @@ spec:
         command: [ "/pre-upgrade-checks" ]
         env:
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- if .Values.terminationMessagePath }}
         terminationMessagePath: {{ .Values.terminationMessagePath }}
         {{- end }}

charts/fission-all/templates/router/deployment.yaml

@@ -83,7 +83,8 @@ spec:
           value: {{ .Values.pprof.enabled | quote }}
         - name: DISPLAY_ACCESS_LOG
           value: {{ .Values.router.displayAccessLog | default false | quote }}
-        {{- include "fission-resource-namespace.envs" . | indent 8 }}  
+        {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         resources:
           {{- toYaml .Values.router.resources | nindent 10 }}

charts/fission-all/templates/storagesvc/deployment.yaml

@@ -61,6 +61,7 @@ spec:
           value: {{ .Values.persistence.s3.region }}
         {{- end }}
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         resources:
           {{- toYaml .Values.storagesvc.resources | nindent 10 }}

charts/fission-all/templates/timer/deployment.yaml

@@ -30,6 +30,7 @@ spec:
         - name: PPROF_ENABLED
           value: {{ .Values.pprof.enabled | quote }}
         {{- include "fission-resource-namespace.envs" . | indent 8 }}
+        {{- include "kube_client.envs" . | indent 8 }}
         {{- include "opentelemtry.envs" . | indent 8 }}
         resources:
           {{- toYaml .Values.timer.resources | nindent 10 }}

charts/fission-all/values.yaml

@@ -92,6 +92,13 @@ createNamespace: true
 ##
 enableIstio: false
 
+## Kubernetes client QPS and Burst settings
+##
+## kubernetesClientQPS represents the maximum queries per second to the kubernetes api server from client instances of fission components.
+kubernetesClientQPS: 200
+## kubernetesClientBurst represents the maximum burst queries to the kubernetes api server from client instances of fission components.
+kubernetesClientBurst: 500
+
 ## fetcher is a light weight component that helps in running functions.
 ## fetcher helps in fetching function source code/build and uploading it when function is invoked.
 ##

pkg/crd/client.go

@@ -35,6 +35,11 @@ import (
 	"github.com/fission/fission/pkg/utils"
 )
 
+const (
+	EnvKubeClientQps   = "KUBE_CLIENT_QPS"
+	EnvKubeClientBurst = "KUBE_CLIENT_BURST"
+)
+
 type (
 	ClientGeneratorInterface interface {
 		GetRestConfig() (*rest.Config, error)
@@ -60,6 +65,20 @@ func (cg *ClientGenerator) getRestConfig() (*rest.Config, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	qps, _ := utils.GetUIntValueFromEnv(EnvKubeClientQps)
+	burst, _ := utils.GetIntValueFromEnv(EnvKubeClientBurst)
+
+	// Set QPS and Burst to higher values to avoid throttling
+	if qps == 0 {
+		qps = 200
+	}
+	if burst == 0 {
+		burst = 500
+	}
+	cg.restConfig.QPS = float32(qps)
+	cg.restConfig.Burst = burst
+
 	return cg.restConfig, nil
 }
 

pkg/utils/utils.go

@@ -232,6 +232,18 @@ func GetUIntValueFromEnv(envVar string) (uint, error) {
 	return uint(value), nil
 }
 
+func GetIntValueFromEnv(envVar string) (int, error) {
+	s, err := GetStringValueFromEnv(envVar)
+	if err != nil {
+		return 0, err
+	}
+	value, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, err
+	}
+	return value, nil
+}
+
 func FindFreePort() (int, error) {
 	listener, err := net.Listen("tcp", ":0")
 	if err != nil {

pkg/utils/utils_test.go

@@ -162,3 +162,48 @@ func TestGetUIntValueFromEnv(t *testing.T) {
 		})
 	}
 }
+
+func TestGetIntValueFromEnv(t *testing.T) {
+	varName := "TEST_VAR"
+	tests := []struct {
+		name    string
+		value   string
+		want    int
+		wantErr bool
+	}{
+		{
+			name:    "empty string case",
+			value:   "",
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name:    "string case",
+			value:   "test string",
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name:    "not int case",
+			value:   "-100",
+			want:    -100,
+			wantErr: false,
+		},
+		{
+			name:    "int case",
+			value:   "7",
+			want:    7,
+			wantErr: false,
+		}}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			os.Setenv(varName, tt.value)
+			got, err := GetIntValueFromEnv(varName)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetIntValueFromEnv() error = %v, wantErr %v, got %d", err, tt.wantErr, got)
+				return
+			}
+		})
+	}
+}