From f3e1f9df90310597d8e47fb8822226a65c275702 Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Fri, 16 Jul 2021 13:34:03 +0530 Subject: [PATCH] Add go pprof support across fission components (#2112) Signed-off-by: Sanket Sudake --- Documentation/wip/profiling.md | 33 +++++++++ charts/fission-all/templates/deployment.yaml | 40 +++++++++-- charts/fission-all/templates/router.yaml | 6 ++ charts/fission-all/values.yaml | 4 ++ cmd/builder/main.go | 3 + cmd/fetcher/main.go | 3 + cmd/fission-bundle/main.go | 2 + pkg/utils/profile/profile.go | 74 ++++++++++++++++++++ skaffold.yaml | 4 ++ 9 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 Documentation/wip/profiling.md create mode 100644 pkg/utils/profile/profile.go diff --git a/Documentation/wip/profiling.md b/Documentation/wip/profiling.md new file mode 100644 index 00000000..4cf4c3f5 --- /dev/null +++ b/Documentation/wip/profiling.md @@ -0,0 +1,33 @@ +# Profiling Fission with Pprof + +Fission uses [net/pprof](https://pkg.go.dev/net/http/pprof) for profiling the code across Fission components. +It would be helpful in identifying performance bottlenecks. + +To enable profiling, just set `pprof.enabled` to `true` while installing Fission helm chart. + +## Pprof data of component pod + +Do port forwarding to port 6060 of the pod, + +```sh +kubectl port-forward pod/executor-668dfd7c89-2b2ff 6060:6060 +``` + +Run different commands to get or analyze pprof data, + +```sh +go tool pprof http://localhost:6060/debug/pprof/flamegraph + +go tool pprof http://localhost:6060/debug/pprof/profile\?seconds\=60 +``` + +You can also analyze with binary to get correct references of source, + +```sh +# Download binary from pod +kubectl cp fission/executor-668dfd7c89-2b2ff:/fission-bundle fission-bundle + +go tool pprof -http ":49816" fission-bundle http://localhost:49513/debug/pprof +``` + +You can also download pprof data and visualize/analyze with different compatible tools. diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index 8e20dbfa..27d3f82b 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -340,6 +340,8 @@ spec: value: {{ .Values.traceSamplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} - name: POD_NAMESPACE valueFrom: fieldRef: @@ -364,6 +366,10 @@ spec: ports: - containerPort: 8888 name: http + {{- if .Values.pprof.enabled }} + - containerPort: 6060 + name: pprof + {{- end }} serviceAccountName: fission-svc volumes: - name: config-volume @@ -432,6 +438,8 @@ spec: value: {{ .Values.fetcher.resource.mem.limits | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} readinessProbe: httpGet: path: "/healthz" @@ -450,6 +458,10 @@ spec: name: metrics - containerPort: 8888 name: http + {{- if .Values.pprof.enabled }} + - containerPort: 6060 + name: pprof + {{- end }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -506,6 +518,8 @@ spec: value: {{ .Values.fetcher.resource.mem.limits | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -542,6 +556,8 @@ spec: value: {{ .Values.traceSamplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -615,7 +631,7 @@ metadata: kubernetes.io/cluster-service: 'true' kubernetes.io/name: heapster spec: - type: ClusterIP + type: ClusterIP ports: - port: 80 targetPort: 8082 @@ -644,7 +660,7 @@ spec: - name: heapster image: gcr.io/google_containers/heapster-amd64:v1.5.0 imagePullPolicy: {{ .Values.pullPolicy }} - command: + command: - /heapster - --source=kubernetes:https://kubernetes.default serviceAccountName: {{ .Release.Namespace }}/fission-svc @@ -676,6 +692,8 @@ spec: env: - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -815,6 +833,8 @@ spec: value: {{ .Values.traceSamplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -866,6 +886,8 @@ spec: value: {{ .Values.traceSamplingRate | default "0.5" | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} # TLS authentication is TLS with authentication (2 way) # More info: https://docs.confluent.io/current/kafka/authentication_ssl.html#ssl-overview {{- if .Values.kafka.authentication.tls.enabled }} @@ -878,7 +900,7 @@ spec: volumeMounts: - name: kafka-secrets mountPath: /etc/fission/secrets - {{- end }} + {{- end }} serviceAccountName: fission-svc {{- if .Values.kafka.authentication.tls.enabled }} volumes: @@ -891,7 +913,7 @@ spec: {{- if .Values.kafka.authentication.tls.enabled }} apiVersion: v1 kind: Secret -metadata: +metadata: name: mqtrigger-kafka-secrets labels: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" @@ -953,7 +975,7 @@ spec: - name: TRACE_JAEGER_COLLECTOR_ENDPOINT value: "{{ .Values.traceCollectorEndpoint }}" - name: TRACING_SAMPLING_RATE - value: {{ .Values.traceSamplingRate | default "0.5" | quote }} + value: {{ .Values.traceSamplingRate | default "0.5" | quote }} - name: MESSAGE_QUEUE_TYPE value: azure-storage-queue - name: AZURE_STORAGE_ACCOUNT_NAME @@ -965,6 +987,8 @@ spec: key: key - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} serviceAccountName: fission-svc {{- if .Values.extraCoreComponentPodConfig }} {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}} @@ -1010,6 +1034,8 @@ spec: value: "{{.Values.pruneInterval}}" - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }} - name: STORAGE_S3_ENDPOINT value: {{ .Values.persistence.s3.endPoint }} @@ -1045,6 +1071,10 @@ spec: ports: - containerPort: 8000 name: http + {{- if .Values.pprof.enabled }} + - containerPort: 6060 + name: pprof + {{- end }} serviceAccountName: fission-svc {{- if and (.Values.persistence.enabled) (ne (.Values.persistence.storageType | default "local") "s3") }} volumes: diff --git a/charts/fission-all/templates/router.yaml b/charts/fission-all/templates/router.yaml index 7d11dcfa..b7fde4b2 100644 --- a/charts/fission-all/templates/router.yaml +++ b/charts/fission-all/templates/router.yaml @@ -63,6 +63,8 @@ spec: value: {{ .Values.router.useEncodedPath | default false | quote }} - name: DEBUG_ENV value: {{ .Values.debugEnv | quote }} + - name: PPROF_ENABLED + value: {{ .Values.pprof.enabled | quote }} - name: DISPLAY_ACCESS_LOG value: {{ .Values.router.displayAccessLog | default false | quote }} resources: @@ -85,6 +87,10 @@ spec: name: metrics - containerPort: 8888 name: http + {{- if .Values.pprof.enabled }} + - containerPort: 6060 + name: pprof + {{- end }} serviceAccountName: fission-svc {{- if .Values.router.extraCoreComponentPodConfig }} {{ toYaml .Values.router.extraCoreComponentPodConfig | indent 6 -}} diff --git a/charts/fission-all/values.yaml b/charts/fission-all/values.yaml index fdee524f..3dfdb584 100644 --- a/charts/fission-all/values.yaml +++ b/charts/fission-all/values.yaml @@ -366,3 +366,7 @@ mqt_keda: gcp_pub_sub: image: fission/keda-gcp-pubsub-http-connector tag: v0.2 + +## Enable Pprof based profiling +pprof: + enabled: false diff --git a/cmd/builder/main.go b/cmd/builder/main.go index 865de011..1946c235 100644 --- a/cmd/builder/main.go +++ b/cmd/builder/main.go @@ -24,10 +24,13 @@ import ( "go.uber.org/zap/zapcore" "github.com/fission/fission/cmd/builder/app" + "github.com/fission/fission/pkg/utils/profile" ) // Usage: builder func main() { + profile.ProfileIfEnabled() + config := zap.NewProductionConfig() config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder logger, err := config.Build() diff --git a/cmd/fetcher/main.go b/cmd/fetcher/main.go index fbc16e64..3fa71e5f 100644 --- a/cmd/fetcher/main.go +++ b/cmd/fetcher/main.go @@ -23,10 +23,13 @@ import ( "go.uber.org/zap/zapcore" "github.com/fission/fission/cmd/fetcher/app" + "github.com/fission/fission/pkg/utils/profile" ) // Usage: fetcher func main() { + profile.ProfileIfEnabled() + config := zap.NewProductionConfig() config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder logger, err := config.Build() diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index b0363e7f..62c60d26 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -40,6 +40,7 @@ import ( "github.com/fission/fission/pkg/router" "github.com/fission/fission/pkg/storagesvc" "github.com/fission/fission/pkg/timer" + "github.com/fission/fission/pkg/utils/profile" ) func runController(logger *zap.Logger, port int) { @@ -237,6 +238,7 @@ Options: --builderMgr Start builder manager. --version Print version information ` + profile.ProfileIfEnabled() var logger *zap.Logger var config zap.Config diff --git a/pkg/utils/profile/profile.go b/pkg/utils/profile/profile.go new file mode 100644 index 00000000..22bfcd29 --- /dev/null +++ b/pkg/utils/profile/profile.go @@ -0,0 +1,74 @@ +/* +Copyright 2021 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// To use profile in a go program over http +// import this package and call ProfileIfEnabled() +// in your main function. +// Please set the environment variable PPROF_ENABLE=true to enable/disable it runtime. +// To customize host and port you can set PPROF_HOST and PPROF_PORT environment variables. +// $ PPROF_ENABLE=true PPROF_HOST=localhost PPROF_PORT=6060 go run myprogram.go + +package profile + +import ( + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func getPprofAddr() string { + pprofHost := os.Getenv("PPROF_HOST") + if pprofHost == "" { + pprofHost = "localhost" + } + pprofPort := os.Getenv("PPROF_PORT") + if pprofPort == "" { + pprofPort = "6060" + } + return fmt.Sprintf("%s:%s", pprofHost, pprofPort) +} + +func ProfileIfEnabled() { + enablePprof := os.Getenv("PPROF_ENABLED") + if enablePprof != "true" { + return + } + + pprofMux := http.DefaultServeMux + http.DefaultServeMux = http.NewServeMux() + + config := zap.NewProductionConfig() + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logger, err := config.Build() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + + addr := getPprofAddr() + logger.Info("Running pprof server", zap.String("addr", addr)) + go func() { + err = http.ListenAndServe(addr, pprofMux) + if err != nil { + logger.Fatal("pprof http server failed", zap.Error(err)) + } + }() +} diff --git a/skaffold.yaml b/skaffold.yaml index 924b1ac7..54510e61 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -56,6 +56,7 @@ deploy: prometheus.enabled: false debugEnv: false analytics: false + pprof.enabled: false wait: true recreatePods: false packaged: null @@ -83,6 +84,9 @@ profiles: - op: replace path: /deploy/helm/releases/0/setValues/debugEnv value: true + - op: replace + path: /deploy/helm/releases/0/setValues/pprof.enabled + value: true - name: kind-ci patches: - op: replace