Add go pprof support across fission components (#2112)

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

Documentation/wip/profiling.md

@@ -0,0 +1,33 @@
+# Profiling Fission with Pprof
+
+Fission uses [net/pprof](https://pkg.go.dev/net/http/pprof) for profiling the code across Fission components.
+It would be helpful in identifying performance bottlenecks.
+
+To enable profiling, just set `pprof.enabled` to `true` while installing Fission helm chart.
+
+## Pprof data of component pod
+
+Do port forwarding to port 6060 of the pod,
+
+```sh
+kubectl port-forward pod/executor-668dfd7c89-2b2ff 6060:6060
+```
+
+Run different commands to get or analyze pprof data,
+
+```sh
+go tool pprof http://localhost:6060/debug/pprof/flamegraph
+
+go tool pprof http://localhost:6060/debug/pprof/profile\?seconds\=60
+```
+
+You can also analyze with binary to get correct references of source,
+
+```sh
+# Download binary from pod
+kubectl cp fission/executor-668dfd7c89-2b2ff:/fission-bundle fission-bundle
+
+go tool pprof -http ":49816" fission-bundle http://localhost:49513/debug/pprof
+```
+
+You can also download pprof data and visualize/analyze with different compatible tools.

charts/fission-all/templates/deployment.yaml

@@ -340,6 +340,8 @@ spec:
           value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
@@ -364,6 +366,10 @@ spec:
         ports:
           - containerPort: 8888
             name: http
+        {{- if .Values.pprof.enabled }}
+          - containerPort: 6060
+            name: pprof
+        {{- end }}
       serviceAccountName: fission-svc
       volumes:
       - name: config-volume
@@ -432,6 +438,8 @@ spec:
           value: {{ .Values.fetcher.resource.mem.limits | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
         readinessProbe:
           httpGet:
             path: "/healthz"
@@ -450,6 +458,10 @@ spec:
           name: metrics
         - containerPort: 8888
           name: http
+        {{- if .Values.pprof.enabled }}
+        - containerPort: 6060
+          name: pprof
+        {{- end }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -506,6 +518,8 @@ spec:
           value: {{ .Values.fetcher.resource.mem.limits | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -542,6 +556,8 @@ spec:
           value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -615,7 +631,7 @@ metadata:
     kubernetes.io/cluster-service: 'true'
     kubernetes.io/name: heapster
 spec:
-  type: ClusterIP 
+  type: ClusterIP
   ports:
   - port: 80
     targetPort: 8082
@@ -644,7 +660,7 @@ spec:
       - name: heapster
         image: gcr.io/google_containers/heapster-amd64:v1.5.0
         imagePullPolicy: {{ .Values.pullPolicy }}
-        command: 
+        command:
         - /heapster
         - --source=kubernetes:https://kubernetes.default
       serviceAccountName: {{ .Release.Namespace }}/fission-svc
@@ -676,6 +692,8 @@ spec:
         env:
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -815,6 +833,8 @@ spec:
           value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -866,6 +886,8 @@ spec:
           value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
         # TLS authentication is TLS with authentication (2 way)
         # More info: https://docs.confluent.io/current/kafka/authentication_ssl.html#ssl-overview
         {{- if .Values.kafka.authentication.tls.enabled }}
@@ -878,7 +900,7 @@ spec:
         volumeMounts:
         - name: kafka-secrets
           mountPath: /etc/fission/secrets
-        {{- end }}        
+        {{- end }}
       serviceAccountName: fission-svc
       {{- if .Values.kafka.authentication.tls.enabled }}
       volumes:
@@ -891,7 +913,7 @@ spec:
 {{- if .Values.kafka.authentication.tls.enabled }}
 apiVersion: v1
 kind: Secret
-metadata: 
+metadata:
   name: mqtrigger-kafka-secrets
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
@@ -953,7 +975,7 @@ spec:
         - name: TRACE_JAEGER_COLLECTOR_ENDPOINT
           value: "{{ .Values.traceCollectorEndpoint }}"
         - name: TRACING_SAMPLING_RATE
-          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}        
+          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
         - name: MESSAGE_QUEUE_TYPE
           value: azure-storage-queue
         - name: AZURE_STORAGE_ACCOUNT_NAME
@@ -965,6 +987,8 @@ spec:
               key: key
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
       serviceAccountName: fission-svc
 {{- if .Values.extraCoreComponentPodConfig }}
 {{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -1010,6 +1034,8 @@ spec:
           value: "{{.Values.pruneInterval}}"
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        - name: PPROF_ENABLED
+          value: {{ .Values.pprof.enabled | quote }}
         {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
         - name: STORAGE_S3_ENDPOINT
           value: {{ .Values.persistence.s3.endPoint }}
@@ -1045,6 +1071,10 @@ spec:
         ports:
           - containerPort: 8000
             name: http
+          {{- if .Values.pprof.enabled }}
+          - containerPort: 6060
+            name: pprof
+          {{- end }}
       serviceAccountName: fission-svc
       {{- if and (.Values.persistence.enabled) (ne (.Values.persistence.storageType | default "local") "s3") }}
       volumes:

charts/fission-all/templates/router.yaml

@@ -63,6 +63,8 @@ spec:
             value: {{ .Values.router.useEncodedPath | default false | quote }}
           - name: DEBUG_ENV
             value: {{ .Values.debugEnv | quote }}
+          - name: PPROF_ENABLED
+            value: {{ .Values.pprof.enabled | quote }}
           - name: DISPLAY_ACCESS_LOG
             value: {{ .Values.router.displayAccessLog | default false | quote }}
         resources:
@@ -85,6 +87,10 @@ spec:
           name: metrics
         - containerPort: 8888
           name: http
+        {{- if .Values.pprof.enabled }}
+        - containerPort: 6060
+          name: pprof
+        {{- end }}
       serviceAccountName: fission-svc
 {{- if .Values.router.extraCoreComponentPodConfig }}
 {{ toYaml .Values.router.extraCoreComponentPodConfig | indent 6 -}}

charts/fission-all/values.yaml

@@ -366,3 +366,7 @@ mqt_keda:
     gcp_pub_sub:
       image: fission/keda-gcp-pubsub-http-connector
       tag: v0.2
+
+## Enable Pprof based profiling
+pprof:
+  enabled: false

cmd/builder/main.go

@@ -24,10 +24,13 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/fission/fission/cmd/builder/app"
+	"github.com/fission/fission/pkg/utils/profile"
 )
 
 // Usage: builder <shared volume path>
 func main() {
+	profile.ProfileIfEnabled()
+
 	config := zap.NewProductionConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	logger, err := config.Build()

cmd/fetcher/main.go

@@ -23,10 +23,13 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/fission/fission/cmd/fetcher/app"
+	"github.com/fission/fission/pkg/utils/profile"
 )
 
 // Usage: fetcher <shared volume path>
 func main() {
+	profile.ProfileIfEnabled()
+
 	config := zap.NewProductionConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	logger, err := config.Build()

cmd/fission-bundle/main.go

@@ -40,6 +40,7 @@ import (
 	"github.com/fission/fission/pkg/router"
 	"github.com/fission/fission/pkg/storagesvc"
 	"github.com/fission/fission/pkg/timer"
+	"github.com/fission/fission/pkg/utils/profile"
 )
 
 func runController(logger *zap.Logger, port int) {
@@ -237,6 +238,7 @@ Options:
   --builderMgr                    Start builder manager.
   --version                       Print version information
 `
+	profile.ProfileIfEnabled()
 
 	var logger *zap.Logger
 	var config zap.Config

pkg/utils/profile/profile.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2021 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// To use profile in a go program over http
+// import this package and call ProfileIfEnabled()
+// in your main function.
+// Please set the environment variable PPROF_ENABLE=true to enable/disable it runtime.
+// To customize host and port you can set PPROF_HOST and PPROF_PORT environment variables.
+//	$ PPROF_ENABLE=true PPROF_HOST=localhost PPROF_PORT=6060 go run myprogram.go
+
+package profile
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+func getPprofAddr() string {
+	pprofHost := os.Getenv("PPROF_HOST")
+	if pprofHost == "" {
+		pprofHost = "localhost"
+	}
+	pprofPort := os.Getenv("PPROF_PORT")
+	if pprofPort == "" {
+		pprofPort = "6060"
+	}
+	return fmt.Sprintf("%s:%s", pprofHost, pprofPort)
+}
+
+func ProfileIfEnabled() {
+	enablePprof := os.Getenv("PPROF_ENABLED")
+	if enablePprof != "true" {
+		return
+	}
+
+	pprofMux := http.DefaultServeMux
+	http.DefaultServeMux = http.NewServeMux()
+
+	config := zap.NewProductionConfig()
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+	logger, err := config.Build()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer logger.Sync()
+
+	addr := getPprofAddr()
+	logger.Info("Running pprof server", zap.String("addr", addr))
+	go func() {
+		err = http.ListenAndServe(addr, pprofMux)
+		if err != nil {
+			logger.Fatal("pprof http server failed", zap.Error(err))
+		}
+	}()
+}

skaffold.yaml

@@ -56,6 +56,7 @@ deploy:
           prometheus.enabled: false
           debugEnv: false
           analytics: false
+          pprof.enabled: false
         wait: true
         recreatePods: false
         packaged: null
@@ -83,6 +84,9 @@ profiles:
       - op: replace
         path: /deploy/helm/releases/0/setValues/debugEnv
         value: true
+      - op: replace
+        path: /deploy/helm/releases/0/setValues/pprof.enabled
+        value: true
   - name: kind-ci
     patches:
     - op: replace