Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prometheus metrics improvements (#2398)

Browse Source 
- Enabled metrics in storagesvc, buildermgr and controller.
- Added a middleware in storagesvc, router, executor and controller to monitor total number of http requests, each request's duration and number of requests that are currently being served. These requests can be filtered on their path, method or statuscode.
- Removed functionCallDuration and functionCallResponseSize metrics from router.
- Removed funcAliveSummary, funcIsAlive, funcReapTime and idleTime metrics.
- Replaced function calls for collecting metrics to direct metric calls.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Ankit Chawla
2022-04-13 21:49:46 +05:30
committed by GitHub
co-authored by Sanket Sudake
parent 231de707dc
commit b638a6d047
26 changed files with 359 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/fscache/functionServiceCache.go
+2 -6
View File
@@ -34,6 +34,7 @@ import (
"github.com/fission/fission/pkg/cache"
"github.com/fission/fission/pkg/crd"
ferror "github.com/fission/fission/pkg/error"
"github.com/fission/fission/pkg/executor/metrics"
"github.com/fission/fission/pkg/poolcache"
)
@@ -227,8 +228,6 @@ func (fsc *FunctionServiceCache) AddFunc(ctx context.Context, fsvc FuncSvc) {
now := time.Now()
fsvc.Ctime = now
fsvc.Atime = now
fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), true)
}
// SetCPUUtilizaton updates/sets CPUutilization in the pool cache
@@ -284,7 +283,6 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
return nil, err
}
fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), true)
return nil, nil
}
@@ -345,9 +343,7 @@ func (fsc *FunctionServiceCache) DeleteEntry(fsvc *FuncSvc) {
)
}
fsc.observeFuncRunningTime(fsvc.Function.Name, string(fsvc.Function.UID), fsvc.Atime.Sub(fsvc.Ctime).Seconds())
fsc.observeFuncAliveTime(fsvc.Function.Name, string(fsvc.Function.UID), time.Since(fsvc.Ctime).Seconds())
fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), false)
metrics.FuncRunningSummary.WithLabelValues(fsvc.Function.Name, fsvc.Function.Namespace).Observe(fsvc.Atime.Sub(fsvc.Ctime).Seconds())
}
// DeleteFunctionSvc deletes a function service at key composed of [function][address].
pkg/executor/fscache/metrics.go
-100
View File
@@ -1,100 +0,0 @@
package fscache
import (
"github.com/prometheus/client_golang/prometheus"
)
var (
// function_name: the function's name
// function_uid: the function's version id
// function_address: the address of the pod from which the function was called
functionLabels = []string{"function_name", "function_uid"}
functionPodLabels = []string{"function_name", "function_address"}
coldStarts = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_function_cold_starts_total",
Help: "How many cold starts are made by function_name, function_uid.",
},
functionLabels,
)
funcRunningSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_running_seconds",
Help: "The running time (last access - create) in seconds of the function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
functionLabels,
)
funcAliveSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_alive_seconds",
Help: "The alive time in seconds of the function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
functionLabels,
)
funcIsAlive = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "fission_function_is_alive",
Help: "A binary value indicating is the function_name, function_uid alive",
},
functionLabels,
)
funcReapTime = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_pod_reaptime_seconds",
Help: "Amount of seconds to reap a pod",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
functionPodLabels,
)
idleTime = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_idle_pod_time",
Help: "Number of seconds it took for Reaper to detect the pod was idle",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
functionPodLabels,
)
)
func init() {
// Register the function calls counter with Prometheus's default registry.
prometheus.MustRegister(coldStarts)
prometheus.MustRegister(funcRunningSummary)
prometheus.MustRegister(funcAliveSummary)
prometheus.MustRegister(funcIsAlive)
prometheus.MustRegister(funcReapTime)
prometheus.MustRegister(idleTime)
}
// IncreaseColdStarts increments the counter by 1.
func (fsc *FunctionServiceCache) IncreaseColdStarts(funcname, funcuid string) {
coldStarts.WithLabelValues(funcname, funcuid).Inc()
}
func (fsc *FunctionServiceCache) observeFuncRunningTime(funcname, funcuid string, running float64) {
funcRunningSummary.WithLabelValues(funcname, funcuid).Observe(running)
}
func (fsc *FunctionServiceCache) observeFuncAliveTime(funcname, funcuid string, alive float64) {
funcAliveSummary.WithLabelValues(funcname, funcuid).Observe(alive)
}
func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive bool) {
count := 0
if isAlive {
count = 1
}
funcIsAlive.WithLabelValues(funcname, funcuid).Set(float64(count))
}
// ReapTime is the amount of time taken to reap a pod
func (fsc *FunctionServiceCache) ReapTime(funcName, funcAddress string, time float64) {
funcReapTime.WithLabelValues(funcName, funcAddress).Observe(time)
}
// IdleTime is the amount of time it took Reaper to find out the pod was idle
func (fsc *FunctionServiceCache) IdleTime(funcName, funcAddress string, time float64) {
idleTime.WithLabelValues(funcName, funcAddress).Observe(time)
}