From bfd005a2959f62add114e75b1e0df227af6cbc16 Mon Sep 17 00:00:00 2001 From: Ta-Ching Chen Date: Mon, 7 Oct 2019 18:09:36 +0800 Subject: [PATCH] Allow to deploy router as DaemonSet (#1342) ReplicaSet generated by Deployment distributes pods to nodes based on nodes resource usage, which means in some cases the newly created pods may be assigned to only a few nodes. The requests will go to the same node if there multiple pods on it and may increase the node resource usage and overall request latency. To solve the problem mentioned above, this PR allows users to deploy the router as DaemonSet so that we can distribute requests across all nodes for better workload distribution and lower latency. --- charts/README.md | 3 +- charts/fission-all/templates/deployment.yaml | 83 ----------------- charts/fission-all/templates/router.yaml | 88 +++++++++++++++++++ charts/fission-all/values.yaml | 1 + charts/fission-core/templates/deployment.yaml | 83 ----------------- charts/fission-core/templates/router.yaml | 88 +++++++++++++++++++ charts/fission-core/values.yaml | 1 + 7 files changed, 180 insertions(+), 167 deletions(-) create mode 100644 charts/fission-all/templates/router.yaml create mode 100644 charts/fission-core/templates/router.yaml diff --git a/charts/README.md b/charts/README.md index 08d1e0b0..e39847e0 100644 --- a/charts/README.md +++ b/charts/README.md @@ -5,7 +5,7 @@ ## Prerequisites -- Kubernetes 1.6 or later +- Kubernetes 1.9 or later ## Helm charts @@ -65,6 +65,7 @@ Parameter | Description | Default `prometheusDeploy` | Set to true if prometheus needs to be deployed along with fission | `true` in `fission-all`, `false` in `fission-core` `canaryDeployment.enabled` | Set to true if you need canary deployment feature | `true` in `fission-all`, `false` in `fission-core` `extraCoreComponentPodConfig` | Extend the container specs for the core fission pods. Can be used to add things like affinty/tolerations/nodeSelectors/etc. | None +`router.deployAsDaemonSet` | Deploy router as DaemonSet instead of Deployment | `false` `router.svcAddressMaxRetries` | Max retries times for router to retry on a certain service URL returns from cache/executor | `5` `router.svcAddressUpdateTimeout` | The length of update lock expiry time for router to get a service URL returns from executor | `30` `router.svcAnnotations` | Annotations for router service | None diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index a30decc6..a057ca62 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -177,89 +177,6 @@ spec: {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} {{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: router - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - svc: router - application: fission-router -spec: - replicas: 1 - selector: - matchLabels: - svc: router - application: fission-router - template: - metadata: - labels: - svc: router - application: fission-router - annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" - prometheus.io/port: "8080" - spec: - containers: - - name: router - image: {{ include "fission-bundleImage" . | quote }} - imagePullPolicy: {{ .Values.pullPolicy }} - command: ["/fission-bundle"] - args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}", "--collectorEndpoint", "{{ .Values.traceCollectorEndpoint }}"] - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROUTER_ROUND_TRIP_TIMEOUT - value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} - - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT - value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} - - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME - value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} - - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE - value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} - - name: ROUTER_ROUND_TRIP_MAX_RETRIES - value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} - - name: ROUTER_SVC_ADDRESS_MAX_RETRIES - value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} - - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT - value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} - - name: DEBUG_ENV - value: {{ .Values.debugEnv | quote }} - - name: TRACING_SAMPLING_RATE - value: {{ .Values.traceSamplingRate | default "0.5" | quote }} -{{ if .Values.analytics }} - - name: ANALYTICS_URL - value: "https://g.fission.sh/metrics" -{{ else if .Values.analyticsNonHelmInstall }} - - name: ANALYTICS_URL - value: "https://g.fission.sh/metrics" -{{ end }} - readinessProbe: - httpGet: - path: "/router-healthz" - port: 8888 - initialDelaySeconds: 1 - periodSeconds: 1 - failureThreshold: 30 - livenessProbe: - httpGet: - path: "/router-healthz" - port: 8888 - initialDelaySeconds: 35 - periodSeconds: 5 - ports: - - containerPort: 8080 - name: metrics - - containerPort: 8888 - name: http - serviceAccount: fission-svc -{{- if .Values.extraCoreComponentPodConfig }} -{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} -{{- end }} --- apiVersion: v1 diff --git a/charts/fission-all/templates/router.yaml b/charts/fission-all/templates/router.yaml new file mode 100644 index 00000000..79ecaafd --- /dev/null +++ b/charts/fission-all/templates/router.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +{{- if not .Values.router.deployAsDaemonSet }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: router + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + svc: router + application: fission-router +spec: +{{- if not .Values.router.deployAsDaemonSet }} + replicas: 1 +{{- end }} + selector: + matchLabels: + application: fission-router + svc: router + template: + metadata: + labels: + application: fission-router + svc: router + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "8080" + spec: + containers: + - name: router + image: {{ include "fission-bundleImage" . | quote }} + imagePullPolicy: {{ .Values.pullPolicy }} + command: ["/fission-bundle"] + args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}", "--collectorEndpoint", "{{ .Values.traceCollectorEndpoint }}"] + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROUTER_ROUND_TRIP_TIMEOUT + value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} + - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT + value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} + - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME + value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} + - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE + value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} + - name: ROUTER_ROUND_TRIP_MAX_RETRIES + value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} + - name: ROUTER_SVC_ADDRESS_MAX_RETRIES + value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} + - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT + value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} + - name: DEBUG_ENV + value: {{ .Values.debugEnv | quote }} + - name: TRACING_SAMPLING_RATE + value: {{ .Values.traceSamplingRate | default "0.5" | quote }} +{{- if .Values.analytics }} + - name: ANALYTICS_URL + value: "https://g.fission.sh/metrics" +{{- else if .Values.analyticsNonHelmInstall }} + - name: ANALYTICS_URL + value: "https://g.fission.sh/metrics" +{{- end }} + readinessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 35 + periodSeconds: 5 + ports: + - containerPort: 8080 + name: metrics + - containerPort: 8888 + name: http + serviceAccount: fission-svc +{{- if .Values.extraCoreComponentPodConfig }} +{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} +{{- end }} diff --git a/charts/fission-all/values.yaml b/charts/fission-all/values.yaml index a0ad2306..2ecc4665 100644 --- a/charts/fission-all/values.yaml +++ b/charts/fission-all/values.yaml @@ -60,6 +60,7 @@ logger: ## Router config router: + deployAsDaemonSet: false svcAddressMaxRetries: 5 svcAddressUpdateTimeout: 30s ## Add annotations for router diff --git a/charts/fission-core/templates/deployment.yaml b/charts/fission-core/templates/deployment.yaml index 0ed8ed37..9985d1e5 100644 --- a/charts/fission-core/templates/deployment.yaml +++ b/charts/fission-core/templates/deployment.yaml @@ -177,89 +177,6 @@ spec: {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} {{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: router - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - svc: router - application: fission-router -spec: - replicas: 1 - selector: - matchLabels: - application: fission-router - svc: router - template: - metadata: - labels: - application: fission-router - svc: router - annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" - prometheus.io/port: "8080" - spec: - containers: - - name: router - image: {{ include "fission-bundleImage" . | quote }} - imagePullPolicy: {{ .Values.pullPolicy }} - command: ["/fission-bundle"] - args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}", "--collectorEndpoint", "{{ .Values.traceCollectorEndpoint }}"] - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROUTER_ROUND_TRIP_TIMEOUT - value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} - - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT - value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} - - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME - value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} - - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE - value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} - - name: ROUTER_ROUND_TRIP_MAX_RETRIES - value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} - - name: ROUTER_SVC_ADDRESS_MAX_RETRIES - value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} - - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT - value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} - - name: DEBUG_ENV - value: {{ .Values.debugEnv | quote }} - - name: TRACING_SAMPLING_RATE - value: {{ .Values.traceSamplingRate | default "0.5" | quote }} -{{ if .Values.analytics }} - - name: ANALYTICS_URL - value: "https://g.fission.sh/metrics" -{{ else if .Values.analyticsNonHelmInstall }} - - name: ANALYTICS_URL - value: "https://g.fission.sh/metrics" -{{ end }} - readinessProbe: - httpGet: - path: "/router-healthz" - port: 8888 - initialDelaySeconds: 1 - periodSeconds: 1 - failureThreshold: 30 - livenessProbe: - httpGet: - path: "/router-healthz" - port: 8888 - initialDelaySeconds: 35 - periodSeconds: 5 - ports: - - containerPort: 8080 - name: metrics - - containerPort: 8888 - name: http - serviceAccount: fission-svc -{{- if .Values.extraCoreComponentPodConfig }} -{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} -{{- end }} --- apiVersion: v1 diff --git a/charts/fission-core/templates/router.yaml b/charts/fission-core/templates/router.yaml new file mode 100644 index 00000000..79ecaafd --- /dev/null +++ b/charts/fission-core/templates/router.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +{{- if not .Values.router.deployAsDaemonSet }} +kind: Deployment +{{- else }} +kind: DaemonSet +{{- end }} +metadata: + name: router + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + svc: router + application: fission-router +spec: +{{- if not .Values.router.deployAsDaemonSet }} + replicas: 1 +{{- end }} + selector: + matchLabels: + application: fission-router + svc: router + template: + metadata: + labels: + application: fission-router + svc: router + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "8080" + spec: + containers: + - name: router + image: {{ include "fission-bundleImage" . | quote }} + imagePullPolicy: {{ .Values.pullPolicy }} + command: ["/fission-bundle"] + args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}", "--collectorEndpoint", "{{ .Values.traceCollectorEndpoint }}"] + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROUTER_ROUND_TRIP_TIMEOUT + value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} + - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT + value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} + - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME + value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} + - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE + value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} + - name: ROUTER_ROUND_TRIP_MAX_RETRIES + value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} + - name: ROUTER_SVC_ADDRESS_MAX_RETRIES + value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} + - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT + value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} + - name: DEBUG_ENV + value: {{ .Values.debugEnv | quote }} + - name: TRACING_SAMPLING_RATE + value: {{ .Values.traceSamplingRate | default "0.5" | quote }} +{{- if .Values.analytics }} + - name: ANALYTICS_URL + value: "https://g.fission.sh/metrics" +{{- else if .Values.analyticsNonHelmInstall }} + - name: ANALYTICS_URL + value: "https://g.fission.sh/metrics" +{{- end }} + readinessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 1 + periodSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: "/router-healthz" + port: 8888 + initialDelaySeconds: 35 + periodSeconds: 5 + ports: + - containerPort: 8080 + name: metrics + - containerPort: 8888 + name: http + serviceAccount: fission-svc +{{- if .Values.extraCoreComponentPodConfig }} +{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} +{{- end }} diff --git a/charts/fission-core/values.yaml b/charts/fission-core/values.yaml index c9ed3902..6c1e0331 100644 --- a/charts/fission-core/values.yaml +++ b/charts/fission-core/values.yaml @@ -47,6 +47,7 @@ enableIstio: false ## Router config router: + deployAsDaemonSet: false svcAddressMaxRetries: 5 svcAddressUpdateTimeout: 30s ## Add annotations for router