diff --git a/charts/fission-all/templates/_fission-component-roles.tpl b/charts/fission-all/templates/_fission-component-roles.tpl index bacb9fa7..a1ff202a 100644 --- a/charts/fission-all/templates/_fission-component-roles.tpl +++ b/charts/fission-all/templates/_fission-component-roles.tpl @@ -169,4 +169,17 @@ rules: - update - patch - delete +{{- end }} +{{- define "canaryconfig-rules" }} +rules: +- apiGroups: + - fission.io + resources: + - canaryconfigs + - httptriggers + verbs: + - list + - watch + - get + - update {{- end }} \ No newline at end of file diff --git a/charts/fission-all/templates/_fission-role-generator.tpl b/charts/fission-all/templates/_fission-role-generator.tpl index 12664b24..aee04232 100644 --- a/charts/fission-all/templates/_fission-role-generator.tpl +++ b/charts/fission-all/templates/_fission-role-generator.tpl @@ -41,6 +41,10 @@ metadata: {{- if eq "timer" .component }} {{- include "timer-rules" . }} {{- end }} +{{- if eq "canaryconfig" .component }} +{{- include "canaryconfig-rules" . }} +{{- end }} + --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/fission-all/templates/canary-config/clusterrole.yaml b/charts/fission-all/templates/canary-config/clusterrole.yaml new file mode 100644 index 00000000..f9a3eb0b --- /dev/null +++ b/charts/fission-all/templates/canary-config/clusterrole.yaml @@ -0,0 +1,43 @@ +{{- if .Values.canaryDeployment.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-canaryconfig +rules: +- apiGroups: + - "" + resources: + - services + verbs: + - list +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - create +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +{{- end }} \ No newline at end of file diff --git a/charts/fission-all/templates/canary-config/clusterrolebinding.yaml b/charts/fission-all/templates/canary-config/clusterrolebinding.yaml new file mode 100644 index 00000000..cf486f57 --- /dev/null +++ b/charts/fission-all/templates/canary-config/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +{{- if .Values.canaryDeployment.enabled }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-canaryconfig +subjects: + - kind: ServiceAccount + name: fission-canaryconfig + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}-canaryconfig + apiGroup: rbac.authorization.k8s.io +{{- end -}} \ No newline at end of file diff --git a/charts/fission-all/templates/canary-config/deployment.yaml b/charts/fission-all/templates/canary-config/deployment.yaml new file mode 100644 index 00000000..8dc7e2e8 --- /dev/null +++ b/charts/fission-all/templates/canary-config/deployment.yaml @@ -0,0 +1,81 @@ +{{- if .Values.canaryDeployment.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: canaryconfig + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + svc: canaryconfig + application: fission-canaryconfig +spec: + replicas: 1 + selector: + matchLabels: + svc: canaryconfig + application: fission-canaryconfig + template: + metadata: + labels: + svc: canaryconfig + application: fission-canaryconfig + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "8080" + spec: + {{- if .Values.canaryDeployment.securityContext.enabled }} + securityContext: {{- omit .Values.canaryDeployment.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + containers: + - name: canaryconfig + image: {{ include "fission-bundleImage" . | quote }} + imagePullPolicy: {{ .Values.pullPolicy }} + command: ["/fission-bundle"] + args: ["--canaryConfig"] + env: + - name: DEBUG_ENV + value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} + {{- include "fission-resource-namespace.envs" . | indent 8 }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- include "opentelemtry.envs" . | indent 8 }} + resources: + {{- toYaml .Values.canaryDeployment.resources | nindent 10 }} + {{- if .Values.terminationMessagePath }} + terminationMessagePath: {{ .Values.terminationMessagePath }} + {{- end }} + {{- if .Values.terminationMessagePolicy }} + terminationMessagePolicy: {{ .Values.terminationMessagePolicy }} + {{- end }} + volumeMounts: + - name: config-volume + mountPath: /etc/config/config.yaml + subPath: config.yaml + ports: + - containerPort: 8080 + name: metrics + {{- if .Values.pprof.enabled }} + - containerPort: 6060 + name: pprof + {{- end }} + + serviceAccountName: fission-canaryconfig + volumes: + - name: config-volume + configMap: + name: feature-config +{{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} +{{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- if .Values.extraCoreComponentPodConfig }} +{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/fission-all/templates/canary-config/role-fission-cr.yaml b/charts/fission-all/templates/canary-config/role-fission-cr.yaml new file mode 100644 index 00000000..e1a23288 --- /dev/null +++ b/charts/fission-all/templates/canary-config/role-fission-cr.yaml @@ -0,0 +1,9 @@ +{{- if .Values.canaryDeployment.enabled }} +{{- include "fission-role-generator" (merge (dict "namespace" .Values.defaultNamespace "component" "canaryconfig") .) }} + +{{- if not .Values.singleDefaultNamespace }} +{{- range $namespace := $.Values.additionalFissionNamespaces }} +{{ include "fission-role-generator" (merge (dict "namespace" $namespace "component" "canaryconfig") $) }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/fission-all/templates/canary-config/serviceaccount.yaml b/charts/fission-all/templates/canary-config/serviceaccount.yaml new file mode 100644 index 00000000..c99ac28b --- /dev/null +++ b/charts/fission-all/templates/canary-config/serviceaccount.yaml @@ -0,0 +1,7 @@ +{{- if .Values.canaryDeployment.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fission-canaryconfig + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/fission-all/templates/canary-config/servicemonitor.yaml b/charts/fission-all/templates/canary-config/servicemonitor.yaml new file mode 100644 index 00000000..fe68900e --- /dev/null +++ b/charts/fission-all/templates/canary-config/servicemonitor.yaml @@ -0,0 +1,24 @@ +{{- if .Values.canaryDeployment.enabled }} +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: canaryconfig-monitor + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} + {{- end }} + {{- with .Values.serviceMonitor.additionalServiceMonitorLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + svc: canaryconfig + endpoints: + - targetPort: 8080 +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/fission-all/values.yaml b/charts/fission-all/values.yaml index 7c7aa823..0064e820 100644 --- a/charts/fission-all/values.yaml +++ b/charts/fission-all/values.yaml @@ -359,7 +359,7 @@ controller: runAsNonRoot: true fsGroup: 10001 runAsUser: 10001 - runAsGroup: 10001 + runAsGroup: 10001 ## webhook is the component that validates API calls. ## It contains validation and mutation for functions, triggers, environments, Kubernetes event watches, etc. @@ -701,11 +701,34 @@ prometheus: ## serviceEndpoint: "" -## set this flag to true if you need canary deployment feature -## + canaryDeployment: +## set this flag to true if you need canary deployment feature enabled: false + ## Pod resources as: + ## resources: + ## limits: + ## cpu: + ## memory: + ## requests: + ## cpu: + ## memory: + ## + resources: {} + + ## Security Context + ## It holds pod-level and container level security configuration. + ## This is an experimental section, please verify before enabling in production. + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-1 + securityContext: + enabled: false + ## Mark it false, if you want to stop the non root user validation + runAsNonRoot: true + fsGroup: 10001 + runAsUser: 10001 + runAsGroup: 10001 + ## Enable authentication for fission function invocation via Fission router ## authentication: diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 736ada1a..53491918 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -29,6 +29,7 @@ import ( "github.com/fission/fission/cmd/fission-bundle/mqtrigger" "github.com/fission/fission/pkg/buildermgr" + "github.com/fission/fission/pkg/canaryconfigmgr" "github.com/fission/fission/pkg/controller" "github.com/fission/fission/pkg/executor" "github.com/fission/fission/pkg/info" @@ -53,6 +54,10 @@ func runController(ctx context.Context, logger *zap.Logger, port int) { controller.Start(ctx, logger, port, false) } +func runCanaryConfigServer(ctx context.Context, logger *zap.Logger) error { + return canaryconfigmgr.StartCanaryServer(ctx, logger, false) +} + func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string) { router.Start(ctx, logger, port, executorUrl) } @@ -173,6 +178,7 @@ Use it to start one or more of the fission servers: Usage: fission-bundle --controllerPort= + fission-bundle --canaryConfig fission-bundle --routerPort= [--executorUrl=] fission-bundle --executorPort= [--namespace=] [--fission-namespace=] fission-bundle --kubewatcher [--routerUrl=] @@ -186,6 +192,7 @@ Usage: fission-bundle --version Options: --controllerPort= Port that the controller should listen on. + --canaryConfig Start canary config server. --webhookPort= Port that the webhook should listen on. --routerPort= Port that the router should listen on. --executorPort= Port that the executor should listen on. @@ -246,6 +253,14 @@ Options: return } + if arguments["--canaryConfig"] == true { + err := runCanaryConfigServer(ctx, logger) + if err != nil { + logger.Error("canary config server exited with error: ", zap.Error(err)) + return + } + } + if arguments["--routerPort"] != nil { port := getPort(logger, arguments["--routerPort"]) runRouter(ctx, logger, port, executorUrl) diff --git a/pkg/canaryconfigmgr/canaryConfigMgr.go b/pkg/canaryconfigmgr/canaryConfigMgr.go index d13d9546..a98adb9b 100644 --- a/pkg/canaryconfigmgr/canaryConfigMgr.go +++ b/pkg/canaryconfigmgr/canaryConfigMgr.go @@ -32,6 +32,7 @@ import ( k8sCache "k8s.io/client-go/tools/cache" fv1 "github.com/fission/fission/pkg/apis/core/v1" + "github.com/fission/fission/pkg/crd" "github.com/fission/fission/pkg/generated/clientset/versioned" "github.com/fission/fission/pkg/utils" ) @@ -550,3 +551,18 @@ func getEnvValue(envVar string) string { envVarSplit := strings.Split(envVar, "=") return envVarSplit[1] } + +func StartCanaryServer(ctx context.Context, logger *zap.Logger, unitTestFlag bool) error { + cLogger := logger.Named("CanaryServer") + + fc, kc, _, _, err := crd.MakeFissionClient() + if err != nil { + cLogger.Fatal("failed to connect to k8s API", zap.Error(err)) + } + + err = ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) + if err != nil { + cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err)) + } + return err +} diff --git a/pkg/canaryconfigmgr/config.go b/pkg/canaryconfigmgr/config.go new file mode 100644 index 00000000..2f20dea6 --- /dev/null +++ b/pkg/canaryconfigmgr/config.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package canaryconfigmgr + +import ( + "context" + + "github.com/pkg/errors" + "go.uber.org/zap" + "k8s.io/client-go/kubernetes" + + config "github.com/fission/fission/pkg/featureconfig" + "github.com/fission/fission/pkg/generated/clientset/versioned" +) + +// ConfigureFeatures gets the feature config and configures the features that are enabled +func ConfigureFeatures(ctx context.Context, logger *zap.Logger, unitTestMode bool, fissionClient versioned.Interface, kubeClient kubernetes.Interface) error { + // set feature enabled to false if unitTestMode + if unitTestMode { + return nil + } + + // get the featureConfig from config map mounted onto the file system + featureConfig, err := config.GetFeatureConfig() + if err != nil { + logger.Error("error getting feature config", zap.Error(err)) + return err + } + + // configure respective features + // in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here + canaryCfgMgr, err := MakeCanaryConfigMgr(ctx, logger, fissionClient, kubeClient, featureConfig.CanaryConfig.PrometheusSvc) + if err != nil { + return errors.Wrap(err, "failed to start canary config manager") + } + canaryCfgMgr.Run(ctx) + + return err +} diff --git a/pkg/controller/api.go b/pkg/controller/api.go index d99a2088..d799f280 100644 --- a/pkg/controller/api.go +++ b/pkg/controller/api.go @@ -67,7 +67,7 @@ type ( } ) -func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) { +func MakeAPI(logger *zap.Logger) (*API, error) { api, err := makeCRDBackedAPI(logger) u := os.Getenv("STORAGE_SERVICE_URL") @@ -98,8 +98,6 @@ func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) api.functionNamespace = "fission-function" } - api.featureStatus = featureStatus - return api, err } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index d34ddb1d..0f54cec9 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -27,7 +27,7 @@ import ( func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool) { cLogger := logger.Named("controller") - fc, kc, apiExtClient, _, err := crd.MakeFissionClient() + fc, _, apiExtClient, _, err := crd.MakeFissionClient() if err != nil { cLogger.Fatal("failed to connect to k8s API", zap.Error(err)) } @@ -42,12 +42,7 @@ func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool) cLogger.Fatal("error waiting for CRDs", zap.Error(err)) } - featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) - if err != nil { - cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err)) - } - - api, err := MakeAPI(cLogger, featureStatus) + api, err := MakeAPI(cLogger) if err != nil { cLogger.Fatal("failed to start controller", zap.Error(err)) }