From 4a0bd1aa2182219ba5962d232c6447f6a9768478 Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Thu, 16 Sep 2021 17:48:26 +0530 Subject: [PATCH] fix: Use standard env variables for opentelemtry (#2194) Opentelemetry defines standard variables which are supported by multiple observability platforms. [1] https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md [2] https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md By defining support for standard env vars, Fission open telemetry support can be leveraged with different platforms such as NewRelic, SigNoz, DataDog etc. Signed-off-by: Sanket Sudake --- charts/fission-all/templates/_helpers.tpl | 20 ++++ charts/fission-all/templates/deployment.yaml | 94 ++++--------------- charts/fission-all/templates/router.yaml | 66 ++++++------- charts/fission-all/values.yaml | 14 ++- charts/fission-core/templates/_helpers.tpl | 20 ++++ charts/fission-core/templates/deployment.yaml | 84 +++++------------ charts/fission-core/templates/router.yaml | 62 ++++++------ charts/fission-core/values.yaml | 14 ++- cmd/fetcher/app/server.go | 7 +- cmd/fission-bundle/main.go | 6 +- go.mod | 1 + pkg/fetcher/config/config.go | 12 +-- pkg/utils/otel/provider.go | 92 ++++++++++++++---- pkg/utils/tracing/tracing.go | 6 +- skaffold.yaml | 5 +- 15 files changed, 253 insertions(+), 250 deletions(-) diff --git a/charts/fission-all/templates/_helpers.tpl b/charts/fission-all/templates/_helpers.tpl index 91d4951f..fbb8c13a 100644 --- a/charts/fission-all/templates/_helpers.tpl +++ b/charts/fission-all/templates/_helpers.tpl @@ -51,3 +51,23 @@ This template generates the image name for the deployment depending on the value {{- end }} {{- end }} {{- end -}} + +{{- define "opentelemtry.envs" }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.openTelemetry.otlpCollectorEndpoint }}" +- name: OTEL_EXPORTER_OTLP_INSECURE + value: "{{ .Values.openTelemetry.otlpInsecure }}" +{{- if .Values.openTelemetry.otlpHeaders }} +- name: OTEL_EXPORTER_OTLP_HEADERS + value: "{{ .Values.openTelemetry.otlpHeaders }}" +{{- end }} +{{- end }} + +{{- define "opentracing.envs" }} +- name: OPENTRACING_ENABLED + value: {{ .Values.openTracing.enabled | default false | quote }} +- name: TRACE_JAEGER_COLLECTOR_ENDPOINT + value: "{{ .Values.openTracing.collectorEndpoint }}" +- name: TRACING_SAMPLING_RATE + value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index f13da361..4f7a1553 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -335,14 +335,6 @@ spec: env: - name: FISSION_FUNCTION_NAMESPACE value: "{{ .Values.functionNamespace }}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED @@ -351,6 +343,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} readinessProbe: httpGet: path: "/healthz" @@ -429,14 +423,6 @@ spec: value: {{ .Values.executor.podReadyTimeout | default false | quote }} - name: ENABLE_ISTIO value: "{{ .Values.enableIstio }}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: FETCHER_MINCPU value: {{ .Values.fetcher.resource.cpu.requests | quote }} - name: FETCHER_MINMEM @@ -449,6 +435,8 @@ spec: value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} readinessProbe: httpGet: path: "/healthz" @@ -513,14 +501,6 @@ spec: value: "{{ .Values.pullPolicy }}" - name: ENABLE_ISTIO value: "{{ .Values.enableIstio }}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: FETCHER_MINCPU value: {{ .Values.fetcher.resource.cpu.requests | quote }} - name: FETCHER_MINMEM @@ -533,6 +513,8 @@ spec: value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -563,18 +545,12 @@ spec: command: ["/fission-bundle"] args: ["--kubewatcher", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -711,8 +687,8 @@ spec: value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -846,18 +822,12 @@ spec: {{- else }} value: nats://{{ .Values.nats.hostaddress }} {{- end }} - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -903,18 +873,12 @@ spec: value: "{{.Values.kafka.brokers}}" - name: MESSAGE_QUEUE_KAFKA_VERSION value: "{{.Values.kafka.version}}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} # TLS authentication is TLS with authentication (2 way) # More info: https://docs.confluent.io/current/kafka/authentication_ssl.html#ssl-overview {{- if .Values.kafka.authentication.tls.enabled }} @@ -999,14 +963,6 @@ spec: command: ["/fission-bundle"] args: ["--mqt", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: MESSAGE_QUEUE_TYPE value: azure-storage-queue - name: AZURE_STORAGE_ACCOUNT_NAME @@ -1020,6 +976,8 @@ spec: value: {{ .Values.debugEnv | quote }} - name: PPROF_ENABLED value: {{ .Values.pprof.enabled | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -1057,14 +1015,6 @@ spec: args: ["--storageServicePort", "8000", "--storageType", "local"] {{- end }} env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: PRUNE_INTERVAL value: "{{.Values.pruneInterval}}" - name: DEBUG_ENV @@ -1085,6 +1035,8 @@ spec: - name: STORAGE_S3_REGION value: {{ .Values.persistence.s3.region }} {{- end }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} {{- if ne (.Values.persistence.storageType | default "local") "s3" }} volumeMounts: - name: fission-storage @@ -1154,14 +1106,6 @@ spec: command: ["/fission-bundle"] args: ["--mqt_keda", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: CONNECTOR_IMAGE_PULL_POLICY @@ -1180,6 +1124,8 @@ spec: value: "{{ .Values.mqt_keda.connector_images.gcp_pub_sub.image }}:{{ .Values.mqt_keda.connector_images.gcp_pub_sub.tag }}" - name: REDIS_IMAGE value: "{{ .Values.mqt_keda.connector_images.redis.image }}:{{ .Values.mqt_keda.connector_images.redis.tag }}" + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} diff --git a/charts/fission-all/templates/router.yaml b/charts/fission-all/templates/router.yaml index 60e852b4..03480847 100644 --- a/charts/fission-all/templates/router.yaml +++ b/charts/fission-all/templates/router.yaml @@ -35,42 +35,36 @@ spec: command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROUTER_ROUND_TRIP_TIMEOUT - value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} - - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT - value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} - - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME - value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} - - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE - value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} - - name: ROUTER_ROUND_TRIP_MAX_RETRIES - value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} - - name: ROUTER_SVC_ADDRESS_MAX_RETRIES - value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} - - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT - value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} - - name: ROUTER_UNTAP_SERVICE_TIMEOUT - value: {{ .Values.router.unTapServiceTimeout | default "3600s" | quote }} - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.router.traceSamplingRate | default "0.5" | quote }} - - name: USE_ENCODED_PATH - value: {{ .Values.router.useEncodedPath | default false | quote }} - - name: DEBUG_ENV - value: {{ .Values.debugEnv | quote }} - - name: PPROF_ENABLED - value: {{ .Values.pprof.enabled | quote }} - - name: DISPLAY_ACCESS_LOG - value: {{ .Values.router.displayAccessLog | default false | quote }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROUTER_ROUND_TRIP_TIMEOUT + value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} + - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT + value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} + - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME + value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} + - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE + value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} + - name: ROUTER_ROUND_TRIP_MAX_RETRIES + value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} + - name: ROUTER_SVC_ADDRESS_MAX_RETRIES + value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} + - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT + value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} + - name: ROUTER_UNTAP_SERVICE_TIMEOUT + value: {{ .Values.router.unTapServiceTimeout | default "3600s" | quote }} + - name: USE_ENCODED_PATH + value: {{ .Values.router.useEncodedPath | default false | quote }} + - name: DEBUG_ENV + value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} + - name: DISPLAY_ACCESS_LOG + value: {{ .Values.router.displayAccessLog | default false | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} resources: {{- toYaml .Values.router.resources | indent 10 }} readinessProbe: diff --git a/charts/fission-all/values.yaml b/charts/fission-all/values.yaml index bf80d793..5e58e367 100644 --- a/charts/fission-all/values.yaml +++ b/charts/fission-all/values.yaml @@ -352,10 +352,18 @@ openTracing: ## uniformly sample traces with the given probabilistic sampling rate #samplingRate: 0.75 -# Use this flag to set the collector endpoint for OpenTelemetry. # It is an alternate to OpenTracing. -# OpenTracing should be disabled (openTracing.enabled: false) if you are using OpenTelemetry. -# otelCollectorEndpoint: "otel-collector.observability.svc:4317" +openTelemetry: + # Use this flag to set the collector endpoint for OpenTelemetry. + # The variable is endpoint of the collector in the format shown below. + # otlpCollectorEndpoint: "otel-collector.observability.svc:4317" + otlpCollectorEndpoint: "" + # Set this flag to false if you are using secure endpoint for the collector. + otlpInsecure: true + # Key-value pairs to be used as headers associated with gRPC or HTTP requests + # to the collector. + # Eg. otlpHeaders: "key1=value1,key2=value2" + otlpHeaders: "" ## Message Queue Trigger Kind, KEDA: enable and configuration mqt_keda: diff --git a/charts/fission-core/templates/_helpers.tpl b/charts/fission-core/templates/_helpers.tpl index 395f8714..b4674d06 100644 --- a/charts/fission-core/templates/_helpers.tpl +++ b/charts/fission-core/templates/_helpers.tpl @@ -42,3 +42,23 @@ This template generates the image name for the deployment depending on the value {{ .Values.image }}:{{ .Values.imageTag }} {{- end }} {{- end -}} + +{{- define "opentelemtry.envs" }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "{{ .Values.openTelemetry.otlpCollectorEndpoint }}" +- name: OTEL_EXPORTER_OTLP_INSECURE + value: "{{ .Values.openTelemetry.otlpInsecure }}" +{{- if .Values.openTelemetry.otlpHeaders }} +- name: OTEL_EXPORTER_OTLP_HEADERS + value: "{{ .Values.openTelemetry.otlpHeaders }}" +{{- end }} +{{- end }} + +{{- define "opentracing.envs" }} +- name: OPENTRACING_ENABLED + value: {{ .Values.openTracing.enabled | default false | quote }} +- name: TRACE_JAEGER_COLLECTOR_ENDPOINT + value: "{{ .Values.openTracing.collectorEndpoint }}" +- name: TRACING_SAMPLING_RATE + value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} +{{- end }} diff --git a/charts/fission-core/templates/deployment.yaml b/charts/fission-core/templates/deployment.yaml index c71aae45..badac004 100644 --- a/charts/fission-core/templates/deployment.yaml +++ b/charts/fission-core/templates/deployment.yaml @@ -140,20 +140,14 @@ spec: command: ["/fission-bundle"] args: ["--controllerPort", "8888"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - - name: FISSION_FUNCTION_NAMESPACE - value: "{{ .Values.functionNamespace }}" - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + - name: FISSION_FUNCTION_NAMESPACE + value: "{{ .Values.functionNamespace }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} readinessProbe: httpGet: path: "/healthz" @@ -219,14 +213,6 @@ spec: value: "{{ .Values.pullPolicy }}" - name: FETCHER_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: ADOPT_EXISTING_RESOURCES value: {{ .Values.executor.adoptExistingResources | default false | quote }} - name: POD_READY_TIMEOUT @@ -241,6 +227,8 @@ spec: value: {{ .Values.fetcher.resource.cpu.limits | quote }} - name: FETCHER_MAXMEM value: {{ .Values.fetcher.resource.mem.limits | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} readinessProbe: httpGet: path: "/healthz" @@ -294,15 +282,7 @@ spec: - name: FETCHER_IMAGE_PULL_POLICY value: "{{ .Values.pullPolicy }}" - name: BUILDER_IMAGE_PULL_POLICY - value: "{{ .Values.pullPolicy }}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} + value: "{{ .Values.pullPolicy }}" - name: ENABLE_ISTIO value: "{{ .Values.enableIstio }}" - name: FETCHER_MINCPU @@ -313,6 +293,8 @@ spec: value: {{ .Values.fetcher.resource.cpu.limits | quote }} - name: FETCHER_MAXMEM value: {{ .Values.fetcher.resource.mem.limits | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -343,14 +325,8 @@ spec: command: ["/fission-bundle"] args: ["--kubewatcher", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -381,14 +357,8 @@ spec: command: ["/fission-bundle"] args: ["--timer", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -428,14 +398,6 @@ spec: env: - name: PRUNE_INTERVAL value: "{{.Values.pruneInterval}}" - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }} - name: STORAGE_S3_ENDPOINT value: {{ .Values.persistence.s3.endPoint }} @@ -450,6 +412,8 @@ spec: - name: STORAGE_S3_REGION value: {{ .Values.persistence.s3.region }} {{- end }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} {{- if ne (.Values.persistence.storageType | default "local") "s3" }} volumeMounts: - name: fission-storage @@ -502,14 +466,6 @@ spec: command: ["/fission-bundle"] args: ["--mqt_keda", "--routerUrl", "http://router.{{ .Release.Namespace }}"] env: - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.openTracing.samplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} - name: CONNECTOR_IMAGE_PULL_POLICY @@ -528,6 +484,8 @@ spec: value: "{{ .Values.mqt_keda.connector_images.gcp_pub_sub.image }}:{{ .Values.mqt_keda.connector_images.gcp_pub_sub.tag }}" - name: REDIS_IMAGE value: "{{ .Values.mqt_keda.connector_images.redis.image }}:{{ .Values.mqt_keda.connector_images.redis.tag }}" + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} diff --git a/charts/fission-core/templates/router.yaml b/charts/fission-core/templates/router.yaml index 431e7a13..be21eb4d 100644 --- a/charts/fission-core/templates/router.yaml +++ b/charts/fission-core/templates/router.yaml @@ -35,40 +35,34 @@ spec: command: ["/fission-bundle"] args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"] env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ROUTER_ROUND_TRIP_TIMEOUT - value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} - - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT - value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} - - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME - value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} - - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE - value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} - - name: ROUTER_ROUND_TRIP_MAX_RETRIES - value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} - - name: ROUTER_SVC_ADDRESS_MAX_RETRIES - value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} - - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT - value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} - - name: ROUTER_UNTAP_SERVICE_TIMEOUT - value: {{ .Values.router.unTapServiceTimeout | default "3600s" | quote }} - - name: OTEL_COLLECTOR_ENDPOINT - value: "{{ .Values.otelCollectorEndpoint }}" - - name: OPENTRACING_ENABLED - value: {{ .Values.openTracing.enabled | default false | quote }} - - name: TRACE_JAEGER_COLLECTOR_ENDPOINT - value: "{{ .Values.openTracing.collectorEndpoint }}" - - name: TRACING_SAMPLING_RATE - value: {{ .Values.router.traceSamplingRate | default "0.5" | quote }} - - name: USE_ENCODED_PATH - value: {{ .Values.router.useEncodedPath | default false | quote }} - - name: DEBUG_ENV - value: {{ .Values.debugEnv | quote }} - - name: DISPLAY_ACCESS_LOG - value: {{ .Values.router.displayAccessLog | default false | quote }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROUTER_ROUND_TRIP_TIMEOUT + value: {{ .Values.router.roundTrip.timeout | default "50ms" | quote }} + - name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT + value: {{ .Values.router.roundTrip.timeoutExponent | default 2 | quote }} + - name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME + value: {{ .Values.router.roundTrip.keepAliveTime | default "30s" | quote }} + - name: ROUTER_ROUND_TRIP_DISABLE_KEEP_ALIVE + value: {{ .Values.router.roundTrip.disableKeepAlive | default true | quote }} + - name: ROUTER_ROUND_TRIP_MAX_RETRIES + value: {{ .Values.router.roundTrip.maxRetries | default 10 | quote }} + - name: ROUTER_SVC_ADDRESS_MAX_RETRIES + value: {{ .Values.router.svcAddressMaxRetries | default 5 | quote }} + - name: ROUTER_SVC_ADDRESS_UPDATE_TIMEOUT + value: {{ .Values.router.svcAddressUpdateTimeout | default "30s" | quote }} + - name: ROUTER_UNTAP_SERVICE_TIMEOUT + value: {{ .Values.router.unTapServiceTimeout | default "3600s" | quote }} + - name: USE_ENCODED_PATH + value: {{ .Values.router.useEncodedPath | default false | quote }} + - name: DEBUG_ENV + value: {{ .Values.debugEnv | quote }} + - name: DISPLAY_ACCESS_LOG + value: {{ .Values.router.displayAccessLog | default false | quote }} + {{- include "opentracing.envs" . | indent 8 }} + {{- include "opentelemtry.envs" . | indent 8 }} resources: {{- toYaml .Values.router.resources | indent 10 }} readinessProbe: diff --git a/charts/fission-core/values.yaml b/charts/fission-core/values.yaml index 72ad490f..86f523e3 100644 --- a/charts/fission-core/values.yaml +++ b/charts/fission-core/values.yaml @@ -242,10 +242,18 @@ openTracing: ## uniformly sample traces with the given probabilistic sampling rate #samplingRate: 0.75 -# Use this flag to set the collector endpoint for OpenTelemetry. # It is an alternate to OpenTracing. -# OpenTracing should be disabled (openTracing.enabled: false) if you are using OpenTelemetry. -# otelCollectorEndpoint: "otel-collector.observability.svc:4317" +openTelemetry: + # Use this flag to set the collector endpoint for OpenTelemetry. + # The variable is endpoint of the collector in the format shown below. + # otlpCollectorEndpoint: "otel-collector.observability.svc:4317" + otlpCollectorEndpoint: "" + # Set this flag to false if you are using secure endpoint for the collector. + otlpInsecure: true + # Key-value pairs to be used as headers associated with gRPC or HTTP requests + # to the collector. + # Eg. otlpHeaders: "key1=value1,key2=value2" + otlpHeaders: "" ## Message Queue Trigger Kind, KEDA: enable and configuration mqt_keda: diff --git a/cmd/fetcher/app/server.go b/cmd/fetcher/app/server.go index fa609311..d2927f79 100644 --- a/cmd/fetcher/app/server.go +++ b/cmd/fetcher/app/server.go @@ -63,6 +63,7 @@ func Run(logger *zap.Logger) { } } + ctx := context.Background() openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { go func() { @@ -71,17 +72,17 @@ func Run(logger *zap.Logger) { } }() } else { - shutdown, err := otelUtils.InitProvider(logger, "Fission-Fetcher") + shutdown, err := otelUtils.InitProvider(ctx, logger, "Fission-Fetcher") if err != nil { logger.Fatal("error initializing provider for OTLP", zap.Error(err)) } if shutdown != nil { - defer shutdown() + defer shutdown(ctx) } } tracer := otel.Tracer("fetcher") - ctx, span := tracer.Start(context.Background(), "fetcher/Run") + ctx, span := tracer.Start(ctx, "fetcher/Run") defer span.End() f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir) diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 63cc1629..818d942a 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "fmt" "log" @@ -225,6 +226,7 @@ Options: logger.Fatal("Could not parse command line arguments", zap.Error(err)) } + ctx := context.Background() openTracingEnabled := tracing.TracingEnabled(logger) if openTracingEnabled { err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments)) @@ -232,12 +234,12 @@ Options: logger.Fatal("Could not register trace exporter", zap.Error(err), zap.Any("argument", arguments)) } } else { - shutdown, err := otel.InitProvider(logger, getServiceName(arguments)) + shutdown, err := otel.InitProvider(ctx, logger, getServiceName(arguments)) if err != nil { logger.Fatal("error initializing provider for OTLP", zap.Error(err), zap.Any("argument", arguments)) } if shutdown != nil { - defer shutdown() + defer shutdown(ctx) } } diff --git a/go.mod b/go.mod index 394d11ff..491e5846 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( go.opencensus.io v0.23.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0 go.opentelemetry.io/otel v1.0.0-RC2 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.0-RC2 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.0-RC2 go.opentelemetry.io/otel/sdk v1.0.0-RC2 go.opentelemetry.io/otel/trace v1.0.0-RC2 diff --git a/pkg/fetcher/config/config.go b/pkg/fetcher/config/config.go index 1667e660..50270d38 100644 --- a/pkg/fetcher/config/config.go +++ b/pkg/fetcher/config/config.go @@ -18,6 +18,7 @@ import ( fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/fetcher" "github.com/fission/fission/pkg/utils" + "github.com/fission/fission/pkg/utils/otel" ) type Config struct { @@ -283,16 +284,7 @@ func (cfg *Config) addFetcherToPodSpecWithCommand(podSpec *apiv1.PodSpec, mainCo }, }, }, - Env: []apiv1.EnvVar{ - { - Name: "OPENTRACING_ENABLED", - Value: os.Getenv("OPENTRACING_ENABLED"), - }, - { - Name: "OTEL_COLLECTOR_ENDPOINT", - Value: os.Getenv("OTEL_COLLECTOR_ENDPOINT"), - }, - }, + Env: otel.OtelEnvForContainer(), } // Pod is removed from endpoints list for service when it's diff --git a/pkg/utils/otel/provider.go b/pkg/utils/otel/provider.go index dc6ef230..5867366f 100644 --- a/pkg/utils/otel/provider.go +++ b/pkg/utils/otel/provider.go @@ -3,8 +3,11 @@ package otel import ( "context" "os" + "strconv" + "strings" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" @@ -12,31 +15,58 @@ import ( semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + apiv1 "k8s.io/api/core/v1" ) -func getSpanProcessor(ctx context.Context, logger *zap.Logger) (*sdktrace.SpanProcessor, error) { - collectorEndpoint := os.Getenv("OTEL_COLLECTOR_ENDPOINT") - if collectorEndpoint == "" { - logger.Info("skipping trace exporter registration") +const ( + OtelEnvPrefix = "OTEL_" + OtelEndpointEnvVar = "OTEL_EXPORTER_OTLP_ENDPOINT" + OtelInsecureEnvVar = "OTEL_EXPORTER_OTLP_INSECURE" +) + +type OtelConfig struct { + endpoint string + insecure bool +} + +func parseOtelConfig() OtelConfig { + config := OtelConfig{} + config.endpoint = os.Getenv(OtelEndpointEnvVar) + insecure, err := strconv.ParseBool(os.Getenv(OtelInsecureEnvVar)) + if err != nil { + insecure = false + } + config.insecure = insecure + return config +} + +func getTraceExporter(ctx context.Context, logger *zap.Logger) (*otlptrace.Exporter, error) { + otelConfig := parseOtelConfig() + if otelConfig.endpoint == "" { + logger.Info("OTEL_EXPORTER_OTLP_ENDPOINT not set, skipping Opentelemtry tracing") return nil, nil } - traceExporter, err := otlptracegrpc.New(ctx, - otlptracegrpc.WithInsecure(), - otlptracegrpc.WithEndpoint(collectorEndpoint), + grpcOpts := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpoint(otelConfig.endpoint), otlptracegrpc.WithDialOption(grpc.WithBlock()), - ) + } + if otelConfig.insecure { + grpcOpts = append(grpcOpts, otlptracegrpc.WithInsecure()) + } else { + grpcOpts = append(grpcOpts, otlptracegrpc.WithTLSCredentials(credentials.NewClientTLSFromCert(nil, ""))) + } + + exporter, err := otlptracegrpc.New(ctx, grpcOpts...) if err != nil { return nil, err } - - bsp := sdktrace.NewBatchSpanProcessor(traceExporter) - return &bsp, nil + return exporter, nil } // Initializes an OTLP exporter, and configures the corresponding trace and metric providers. -func InitProvider(logger *zap.Logger, serviceName string) (func(), error) { - ctx := context.Background() +func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (func(context.Context), error) { res, err := resource.New(ctx, resource.WithAttributes( semconv.ServiceNameKey.String(serviceName), @@ -46,26 +76,50 @@ func InitProvider(logger *zap.Logger, serviceName string) (func(), error) { return nil, err } tracerProvider := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), sdktrace.WithResource(res), ) - bsp, err := getSpanProcessor(ctx, logger) + traceExporter, err := getTraceExporter(ctx, logger) if err != nil { return nil, err } - if bsp != nil { - tracerProvider.RegisterSpanProcessor(*bsp) + + if traceExporter != nil { + bsp := sdktrace.NewBatchSpanProcessor(traceExporter) + tracerProvider.RegisterSpanProcessor(bsp) } otel.SetTracerProvider(tracerProvider) - otel.SetTextMapPropagator(propagation.TraceContext{}) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, propagation.Baggage{})) // Shutdown will flush any remaining spans and shut down the exporter. - return func() { + return func(ctx context.Context) { err := tracerProvider.Shutdown(ctx) if err != nil { logger.Fatal("error shutting down trace provider", zap.Error(err)) } + if traceExporter != nil { + if err = traceExporter.Shutdown(ctx); err != nil { + logger.Fatal("error shutting down trace exporter", zap.Error(err)) + } + } }, nil } + +// OtelEnvForContainer returns a list of environment variables +// for the container, which start with prefix OTEL_ +func OtelEnvForContainer() []apiv1.EnvVar { + otelEnvs := []apiv1.EnvVar{} + for _, e := range os.Environ() { + if strings.HasPrefix(e, OtelEnvPrefix) { + pair := strings.SplitN(e, "=", 2) + otelEnvs = append(otelEnvs, apiv1.EnvVar{ + Name: pair[0], + Value: pair[1], + }) + + } + } + return otelEnvs +} diff --git a/pkg/utils/tracing/tracing.go b/pkg/utils/tracing/tracing.go index 47683578..7bd2eb97 100644 --- a/pkg/utils/tracing/tracing.go +++ b/pkg/utils/tracing/tracing.go @@ -11,7 +11,11 @@ import ( ) func TracingEnabled(logger *zap.Logger) bool { - openTracingEnabled, err := strconv.ParseBool(os.Getenv("OPENTRACING_ENABLED")) + tracingEnabled := os.Getenv("TRACING_ENABLED") + if len(tracingEnabled) == 0 { + return false + } + openTracingEnabled, err := strconv.ParseBool(tracingEnabled) if err != nil { if logger != nil { logger.Error("Error parsing OpenTracing enabled flag", zap.Error(err)) diff --git a/skaffold.yaml b/skaffold.yaml index 0dc2a92e..800a04ff 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -52,7 +52,8 @@ deploy: repository: index.docker.io routerServiceType: LoadBalancer openTracing.enabled: false - otelCollectorEndpoint: "" + openTelemetry.otlpCollectorEndpoint: "" + openTelemetry.otlpInsecure: true wait: true flags: install: @@ -113,5 +114,5 @@ profiles: path: /deploy/helm/releases/0/setValues/pprof.enabled value: true - op: replace - path: /deploy/helm/releases/0/setValues/otelCollectorEndpoint + path: /deploy/helm/releases/0/setValues/openTelemetry.otlpCollectorEndpoint value: "otel-collector.opentelemetry-operator-system.svc:4317"