Run canary config server separate from controller (#2617)

* add canary config server
* remove the canary config code from the controller
* remove port exposure for canary config

charts/fission-all/templates/_fission-component-roles.tpl

@@ -170,3 +170,16 @@ rules:
   - patch
   - delete
 {{- end }}
+{{- define "canaryconfig-rules" }}
+rules:
+- apiGroups:
+  - fission.io
+  resources:
+  - canaryconfigs
+  - httptriggers
+  verbs:
+  - list
+  - watch
+  - get
+  - update
+{{- end }}

charts/fission-all/templates/_fission-role-generator.tpl

@@ -41,6 +41,10 @@ metadata:
 {{- if eq "timer" .component }}
 {{- include "timer-rules" . }}
 {{- end }}
+{{- if eq "canaryconfig" .component }}
+{{- include "canaryconfig-rules" . }}
+{{- end }}
+
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

charts/fission-all/templates/canary-config/clusterrole.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.canaryDeployment.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-canaryconfig
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+{{- end }}

charts/fission-all/templates/canary-config/clusterrolebinding.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.canaryDeployment.enabled }}
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ .Release.Name }}-canaryconfig
+subjects:
+  - kind: ServiceAccount
+    name: fission-canaryconfig
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Release.Name }}-canaryconfig
+  apiGroup: rbac.authorization.k8s.io
+{{- end -}}

charts/fission-all/templates/canary-config/deployment.yaml

@@ -0,0 +1,81 @@
+{{- if .Values.canaryDeployment.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: canaryconfig
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: canaryconfig
+    application: fission-canaryconfig
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      svc: canaryconfig
+      application: fission-canaryconfig
+  template:
+    metadata:
+      labels:
+        svc: canaryconfig
+        application: fission-canaryconfig
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "8080"
+    spec:
+      {{- if .Values.canaryDeployment.securityContext.enabled }}
+      securityContext: {{- omit .Values.canaryDeployment.securityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      containers:
+      - name: canaryconfig
+        image: {{ include "fission-bundleImage" . | quote }}
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command: ["/fission-bundle"]
+        args: ["--canaryConfig"]
+        env:
+        - name: DEBUG_ENV
+          value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
+        {{- include "fission-resource-namespace.envs" . | indent 8 }} 
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        {{- include "opentelemtry.envs" . | indent 8 }}
+        resources:
+          {{- toYaml .Values.canaryDeployment.resources | nindent 10 }}
+        {{- if .Values.terminationMessagePath }}
+        terminationMessagePath: {{ .Values.terminationMessagePath }}
+        {{- end }}
+        {{- if .Values.terminationMessagePolicy }}
+        terminationMessagePolicy: {{ .Values.terminationMessagePolicy }}
+        {{- end }}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/config/config.yaml
+          subPath: config.yaml
+        ports:
+          - containerPort: 8080
+            name: metrics
+        {{- if .Values.pprof.enabled }}
+          - containerPort: 6060
+            name: pprof
+        {{- end }}
+  
+      serviceAccountName: fission-canaryconfig
+      volumes:
+      - name: config-volume
+        configMap:
+          name: feature-config
+{{- if .Values.priorityClassName }}
+      priorityClassName: {{ .Values.priorityClassName }}
+{{- end }}
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets: 
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- if .Values.extraCoreComponentPodConfig }}
+{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
+{{- end }}
+{{- end -}}

charts/fission-all/templates/canary-config/role-fission-cr.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.canaryDeployment.enabled }}
+{{- include "fission-role-generator" (merge (dict "namespace" .Values.defaultNamespace "component" "canaryconfig") .) }}
+
+{{- if not .Values.singleDefaultNamespace }}
+{{- range $namespace := $.Values.additionalFissionNamespaces }}
+{{ include "fission-role-generator" (merge (dict "namespace" $namespace "component" "canaryconfig") $) }}
+{{- end }}
+{{- end }}
+{{- end -}}

charts/fission-all/templates/canary-config/serviceaccount.yaml

@@ -0,0 +1,7 @@
+{{- if .Values.canaryDeployment.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fission-canaryconfig
+  namespace: {{ .Release.Namespace }}
+{{- end -}}

charts/fission-all/templates/canary-config/servicemonitor.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.canaryDeployment.enabled }}
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: canaryconfig-monitor
+  {{- if .Values.serviceMonitor.namespace }}
+  namespace: {{ .Values.serviceMonitor.namespace }}
+  {{- end }}
+  {{- with .Values.serviceMonitor.additionalServiceMonitorLabels }}
+  labels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      svc: canaryconfig
+  endpoints:
+  - targetPort: 8080
+{{- end -}}
+{{- end -}}

charts/fission-all/values.yaml

@@ -701,11 +701,34 @@ prometheus:
   ##
   serviceEndpoint: ""
 
-## set this flag to true if you need canary deployment feature
+
-##
 canaryDeployment:
+## set this flag to true if you need canary deployment feature
   enabled: false
 
+  ## Pod resources as:
+  ##  resources:
+  ##    limits:
+  ##      cpu: <tbd>
+  ##      memory: <tbd>
+  ##    requests:
+  ##      cpu: <tbd>
+  ##      memory: <tbd>
+  ##
+  resources: {}
+
+  ## Security Context
+  ## It holds pod-level and container level security configuration.
+  ## This is an experimental section, please verify before enabling in production.
+  ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-1
+  securityContext:
+    enabled: false
+    ## Mark it false, if you want to stop the non root user validation
+    runAsNonRoot: true
+    fsGroup: 10001
+    runAsUser: 10001
+    runAsGroup: 10001
+
 ## Enable authentication for fission function invocation via Fission router
 ##
 authentication:

cmd/fission-bundle/main.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/fission/fission/cmd/fission-bundle/mqtrigger"
 	"github.com/fission/fission/pkg/buildermgr"
+	"github.com/fission/fission/pkg/canaryconfigmgr"
 	"github.com/fission/fission/pkg/controller"
 	"github.com/fission/fission/pkg/executor"
 	"github.com/fission/fission/pkg/info"
@@ -53,6 +54,10 @@ func runController(ctx context.Context, logger *zap.Logger, port int) {
 	controller.Start(ctx, logger, port, false)
 }
 
+func runCanaryConfigServer(ctx context.Context, logger *zap.Logger) error {
+	return canaryconfigmgr.StartCanaryServer(ctx, logger, false)
+}
+
 func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string) {
 	router.Start(ctx, logger, port, executorUrl)
 }
@@ -173,6 +178,7 @@ Use it to start one or more of the fission servers:
 
 Usage:
   fission-bundle --controllerPort=<port>
+  fission-bundle --canaryConfig
   fission-bundle --routerPort=<port> [--executorUrl=<url>]
   fission-bundle --executorPort=<port> [--namespace=<namespace>] [--fission-namespace=<namespace>]
   fission-bundle --kubewatcher [--routerUrl=<url>]
@@ -186,6 +192,7 @@ Usage:
   fission-bundle --version
 Options:
   --controllerPort=<port>         Port that the controller should listen on.
+  --canaryConfig		  		  Start canary config server.
   --webhookPort=<port>             Port that the webhook should listen on.
   --routerPort=<port>             Port that the router should listen on.
   --executorPort=<port>           Port that the executor should listen on.
@@ -246,6 +253,14 @@ Options:
 		return
 	}
 
+	if arguments["--canaryConfig"] == true {
+		err := runCanaryConfigServer(ctx, logger)
+		if err != nil {
+			logger.Error("canary config server exited with error: ", zap.Error(err))
+			return
+		}
+	}
+
 	if arguments["--routerPort"] != nil {
 		port := getPort(logger, arguments["--routerPort"])
 		runRouter(ctx, logger, port, executorUrl)

pkg/canaryconfigmgr/canaryConfigMgr.go

@@ -32,6 +32,7 @@ import (
 	k8sCache "k8s.io/client-go/tools/cache"
 
 	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+	"github.com/fission/fission/pkg/crd"
 	"github.com/fission/fission/pkg/generated/clientset/versioned"
 	"github.com/fission/fission/pkg/utils"
 )
@@ -550,3 +551,18 @@ func getEnvValue(envVar string) string {
 	envVarSplit := strings.Split(envVar, "=")
 	return envVarSplit[1]
 }
+
+func StartCanaryServer(ctx context.Context, logger *zap.Logger, unitTestFlag bool) error {
+	cLogger := logger.Named("CanaryServer")
+
+	fc, kc, _, _, err := crd.MakeFissionClient()
+	if err != nil {
+		cLogger.Fatal("failed to connect to k8s API", zap.Error(err))
+	}
+
+	err = ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
+	if err != nil {
+		cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err))
+	}
+	return err
+}

pkg/canaryconfigmgr/config.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"k8s.io/client-go/kubernetes"
+
+	config "github.com/fission/fission/pkg/featureconfig"
+	"github.com/fission/fission/pkg/generated/clientset/versioned"
+)
+
+// ConfigureFeatures gets the feature config and configures the features that are enabled
+func ConfigureFeatures(ctx context.Context, logger *zap.Logger, unitTestMode bool, fissionClient versioned.Interface, kubeClient kubernetes.Interface) error {
+	// set feature enabled to false if unitTestMode
+	if unitTestMode {
+		return nil
+	}
+
+	// get the featureConfig from config map mounted onto the file system
+	featureConfig, err := config.GetFeatureConfig()
+	if err != nil {
+		logger.Error("error getting feature config", zap.Error(err))
+		return err
+	}
+
+	// configure respective features
+	// in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here
+	canaryCfgMgr, err := MakeCanaryConfigMgr(ctx, logger, fissionClient, kubeClient, featureConfig.CanaryConfig.PrometheusSvc)
+	if err != nil {
+		return errors.Wrap(err, "failed to start canary config manager")
+	}
+	canaryCfgMgr.Run(ctx)
+
+	return err
+}

pkg/controller/api.go

@@ -67,7 +67,7 @@ type (
 	}
 )
 
-func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) {
+func MakeAPI(logger *zap.Logger) (*API, error) {
 	api, err := makeCRDBackedAPI(logger)
 
 	u := os.Getenv("STORAGE_SERVICE_URL")
@@ -98,8 +98,6 @@ func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error)
 		api.functionNamespace = "fission-function"
 	}
 
-	api.featureStatus = featureStatus
-
 	return api, err
 }
 

pkg/controller/controller.go

@@ -27,7 +27,7 @@ import (
 func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool) {
 	cLogger := logger.Named("controller")
 
-	fc, kc, apiExtClient, _, err := crd.MakeFissionClient()
+	fc, _, apiExtClient, _, err := crd.MakeFissionClient()
 	if err != nil {
 		cLogger.Fatal("failed to connect to k8s API", zap.Error(err))
 	}
@@ -42,12 +42,7 @@ func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool)
 		cLogger.Fatal("error waiting for CRDs", zap.Error(err))
 	}
 
-	featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
+	api, err := MakeAPI(cLogger)
-	if err != nil {
-		cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err))
-	}
-
-	api, err := MakeAPI(cLogger, featureStatus)
 	if err != nil {
 		cLogger.Fatal("failed to start controller", zap.Error(err))
 	}