Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add go pprof support across fission components (#2112)

Browse Source 
Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
2021-07-16 13:34:03 +05:30
committed by GitHub
parent c85b9e8cbc
commit f3e1f9df90
9 changed files with 164 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/wip/profiling.md
+33
View File
@@ -0,0 +1,33 @@
# Profiling Fission with Pprof
Fission uses [net/pprof](https://pkg.go.dev/net/http/pprof) for profiling the code across Fission components.
It would be helpful in identifying performance bottlenecks.
To enable profiling, just set `pprof.enabled` to `true` while installing Fission helm chart.
## Pprof data of component pod
Do port forwarding to port 6060 of the pod,
```sh
kubectl port-forward pod/executor-668dfd7c89-2b2ff 6060:6060
```
Run different commands to get or analyze pprof data,
```sh
go tool pprof http://localhost:6060/debug/pprof/flamegraph
go tool pprof http://localhost:6060/debug/pprof/profile\?seconds\=60
```
You can also analyze with binary to get correct references of source,
```sh
# Download binary from pod
kubectl cp fission/executor-668dfd7c89-2b2ff:/fission-bundle fission-bundle
go tool pprof -http ":49816" fission-bundle http://localhost:49513/debug/pprof
```
You can also download pprof data and visualize/analyze with different compatible tools.
charts/fission-all/templates/deployment.yaml
+30
View File
@@ -340,6 +340,8 @@ spec:
value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
- name: POD_NAMESPACE
valueFrom:
fieldRef:
@@ -364,6 +366,10 @@ spec:
ports:
- containerPort: 8888
name: http
{{- if .Values.pprof.enabled }}
- containerPort: 6060
name: pprof
{{- end }}
serviceAccountName: fission-svc
volumes:
- name: config-volume
@@ -432,6 +438,8 @@ spec:
value: {{ .Values.fetcher.resource.mem.limits | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
readinessProbe:
httpGet:
path: "/healthz"
@@ -450,6 +458,10 @@ spec:
name: metrics
- containerPort: 8888
name: http
{{- if .Values.pprof.enabled }}
- containerPort: 6060
name: pprof
{{- end }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -506,6 +518,8 @@ spec:
value: {{ .Values.fetcher.resource.mem.limits | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -542,6 +556,8 @@ spec:
value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -676,6 +692,8 @@ spec:
env:
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -815,6 +833,8 @@ spec:
value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -866,6 +886,8 @@ spec:
value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
# TLS authentication is TLS with authentication (2 way)
# More info: https://docs.confluent.io/current/kafka/authentication_ssl.html#ssl-overview
{{- if .Values.kafka.authentication.tls.enabled }}
@@ -965,6 +987,8 @@ spec:
key: key
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
serviceAccountName: fission-svc
{{- if .Values.extraCoreComponentPodConfig }}
{{ toYaml .Values.extraCoreComponentPodConfig | indent 6 -}}
@@ -1010,6 +1034,8 @@ spec:
value: "{{.Values.pruneInterval}}"
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
{{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
- name: STORAGE_S3_ENDPOINT
value: {{ .Values.persistence.s3.endPoint }}
@@ -1045,6 +1071,10 @@ spec:
ports:
- containerPort: 8000
name: http
{{- if .Values.pprof.enabled }}
- containerPort: 6060
name: pprof
{{- end }}
serviceAccountName: fission-svc
{{- if and (.Values.persistence.enabled) (ne (.Values.persistence.storageType | default "local") "s3") }}
volumes:
charts/fission-all/templates/router.yaml
+6
View File
@@ -63,6 +63,8 @@ spec:
value: {{ .Values.router.useEncodedPath | default false | quote }}
- name: DEBUG_ENV
value: {{ .Values.debugEnv | quote }}
- name: PPROF_ENABLED
value: {{ .Values.pprof.enabled | quote }}
- name: DISPLAY_ACCESS_LOG
value: {{ .Values.router.displayAccessLog | default false | quote }}
resources:
@@ -85,6 +87,10 @@ spec:
name: metrics
- containerPort: 8888
name: http
{{- if .Values.pprof.enabled }}
- containerPort: 6060
name: pprof
{{- end }}
serviceAccountName: fission-svc
{{- if .Values.router.extraCoreComponentPodConfig }}
{{ toYaml .Values.router.extraCoreComponentPodConfig | indent 6 -}}
charts/fission-all/values.yaml
+4
View File
@@ -366,3 +366,7 @@ mqt_keda:
gcp_pub_sub:
image: fission/keda-gcp-pubsub-http-connector
tag: v0.2
## Enable Pprof based profiling
pprof:
enabled: false
cmd/builder/main.go
+3
View File
@@ -24,10 +24,13 @@ import (
"go.uber.org/zap/zapcore"
"github.com/fission/fission/cmd/builder/app"
"github.com/fission/fission/pkg/utils/profile"
)
// Usage: builder <shared volume path>
func main() {
profile.ProfileIfEnabled()
config := zap.NewProductionConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
cmd/fetcher/main.go
+3
View File
@@ -23,10 +23,13 @@ import (
"go.uber.org/zap/zapcore"
"github.com/fission/fission/cmd/fetcher/app"
"github.com/fission/fission/pkg/utils/profile"
)
// Usage: fetcher <shared volume path>
func main() {
profile.ProfileIfEnabled()
config := zap.NewProductionConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
cmd/fission-bundle/main.go
+2
View File
@@ -40,6 +40,7 @@ import (
"github.com/fission/fission/pkg/router"
"github.com/fission/fission/pkg/storagesvc"
"github.com/fission/fission/pkg/timer"
"github.com/fission/fission/pkg/utils/profile"
)
func runController(logger *zap.Logger, port int) {
@@ -237,6 +238,7 @@ Options:
--builderMgr Start builder manager.
--version Print version information
`
profile.ProfileIfEnabled()
var logger *zap.Logger
var config zap.Config
pkg/utils/profile/profile.go
+74
View File
@@ -0,0 +1,74 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// To use profile in a go program over http
// import this package and call ProfileIfEnabled()
// in your main function.
// Please set the environment variable PPROF_ENABLE=true to enable/disable it runtime.
// To customize host and port you can set PPROF_HOST and PPROF_PORT environment variables.
// $ PPROF_ENABLE=true PPROF_HOST=localhost PPROF_PORT=6060 go run myprogram.go
package profile
import (
"fmt"
"log"
"net/http"
_ "net/http/pprof"
"os"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func getPprofAddr() string {
pprofHost := os.Getenv("PPROF_HOST")
if pprofHost == "" {
pprofHost = "localhost"
}
pprofPort := os.Getenv("PPROF_PORT")
if pprofPort == "" {
pprofPort = "6060"
}
return fmt.Sprintf("%s:%s", pprofHost, pprofPort)
}
func ProfileIfEnabled() {
enablePprof := os.Getenv("PPROF_ENABLED")
if enablePprof != "true" {
return
}
pprofMux := http.DefaultServeMux
http.DefaultServeMux = http.NewServeMux()
config := zap.NewProductionConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
addr := getPprofAddr()
logger.Info("Running pprof server", zap.String("addr", addr))
go func() {
err = http.ListenAndServe(addr, pprofMux)
if err != nil {
logger.Fatal("pprof http server failed", zap.Error(err))
}
}()
}
skaffold.yaml
+4
View File
@@ -56,6 +56,7 @@ deploy:
prometheus.enabled: false
debugEnv: false
analytics: false
pprof.enabled: false
wait: true
recreatePods: false
packaged: null
@@ -83,6 +84,9 @@ profiles:
- op: replace
path: /deploy/helm/releases/0/setValues/debugEnv
value: true
- op: replace
path: /deploy/helm/releases/0/setValues/pprof.enabled
value: true
- name: kind-ci
patches:
- op: replace