Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prometheus metrics improvements (#2398)

Browse Source 
- Enabled metrics in storagesvc, buildermgr and controller.
- Added a middleware in storagesvc, router, executor and controller to monitor total number of http requests, each request's duration and number of requests that are currently being served. These requests can be filtered on their path, method or statuscode.
- Removed functionCallDuration and functionCallResponseSize metrics from router.
- Removed funcAliveSummary, funcIsAlive, funcReapTime and idleTime metrics.
- Replaced function calls for collecting metrics to direct metric calls.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Ankit Chawla
2022-04-13 21:49:46 +05:30
committed by GitHub
co-authored by Sanket Sudake
parent 231de707dc
commit b638a6d047
26 changed files with 359 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/router/functionHandler.go
+16 -16
View File
@@ -786,30 +786,30 @@ func (fh functionHandler) getProxyErrorHandler(start time.Time, rrt *RetryingRou
func (fh functionHandler) collectFunctionMetric(start time.Time, rrt *RetryingRoundTripper, req *http.Request, resp *http.Response) {
duration := time.Since(start)
var path string
// Metrics stuff
funcMetricLabels := &functionLabels{
namespace: fh.function.ObjectMeta.Namespace,
name: fh.function.ObjectMeta.Name,
}
httpMetricLabels := &httpLabels{
method: req.Method,
}
if fh.httpTrigger != nil {
httpMetricLabels.host = fh.httpTrigger.Spec.Host
if fh.httpTrigger.Spec.Prefix != nil && *fh.httpTrigger.Spec.Prefix != "" {
httpMetricLabels.path = *fh.httpTrigger.Spec.Prefix
path = *fh.httpTrigger.Spec.Prefix
} else {
httpMetricLabels.path = fh.httpTrigger.Spec.RelativeURL
path = fh.httpTrigger.Spec.RelativeURL
}
}
// Track metrics
httpMetricLabels.code = resp.StatusCode
funcMetricLabels.cached = rrt.urlFromCache
functionCalls.WithLabelValues(fh.function.ObjectMeta.Namespace,
fh.function.ObjectMeta.Name, path, req.Method,
fmt.Sprint(resp.StatusCode)).Inc()
functionCallCompleted(funcMetricLabels, httpMetricLabels,
duration, duration, resp.ContentLength)
if resp.StatusCode >= 400 {
functionCallErrors.WithLabelValues(fh.function.ObjectMeta.Namespace,
fh.function.ObjectMeta.Name, path, req.Method,
fmt.Sprint(resp.StatusCode)).Inc()
}
functionCallOverhead.WithLabelValues(fh.function.ObjectMeta.Namespace,
fh.function.ObjectMeta.Name, path, req.Method,
fmt.Sprint(resp.StatusCode)).
Observe(float64(duration.Nanoseconds()) / 1e9)
// tapService before invoking roundTrip for the serviceUrl
if rrt.urlFromCache {
pkg/router/httpTriggers.go
+2
View File
@@ -41,6 +41,7 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/metrics"
"github.com/fission/fission/pkg/utils/otel"
"github.com/fission/fission/pkg/utils/tracing"
)
@@ -217,6 +218,7 @@ func authLoginHandler(w http.ResponseWriter, r *http.Request) {
func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) *mux.Router {
muxRouter := mux.NewRouter()
muxRouter.Use(metrics.HTTPMetricMiddleware())
openTracingEnabled := tracing.TracingEnabled(ts.logger)
pkg/router/metrics.go
+5 -109
View File
@@ -1,80 +1,35 @@
package router
import (
"fmt"
"sync/atomic"
"time"
"github.com/prometheus/client_golang/prometheus"
)
var globalFunctionCallCount uint64
type (
// functionLabels is the set of metrics labels that relate to
// functions.
//
// cached indicates whether or not the function call hit the
// cache in this service.
//
// namespace and name are the metadata of the function.
functionLabels struct {
cached bool
namespace string
name string
}
// httpLabels is the set of metrics labels that relate to HTTP
// requests.
//
// host is the host that the HTTP request was made to
// path is the relative URL of the request
// method is the HTTP method ("GET", "POST", ...)
// code is the HTTP status code
httpLabels struct {
host string
path string
method string
code int
}
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
metricAddr = ":8080"
// function + http labels as strings
labelsStrings = []string{"cached", "function_namespace", "function_name", "host", "path", "method", "code"}
labelsStrings = []string{"function_namespace", "function_name", "path", "method", "code"}
// Function http calls count
// cached: true | false, is this function service address cached locally
// function_namespace: function namespace
// function_name: function name
// code: http status code
// path: the client call the function on which http path
// method: the function's http method
functionCalls = prometheus.NewCounterVec(
functionCalls = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_function_calls_total",
Help: "Count of Fission function calls",
},
labelsStrings,
)
functionCallErrors = prometheus.NewCounterVec(
functionCallErrors = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_function_errors_total",
Help: "Count of Fission function errors",
},
labelsStrings,
)
functionCallDuration = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_duration_seconds",
Help: "Runtime duration of the Fission function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labelsStrings,
)
functionCallOverhead = prometheus.NewSummaryVec(
functionCallOverhead = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_overhead_seconds",
Help: "The function call delay caused by fission.",
@@ -82,63 +37,4 @@ var (
},
labelsStrings,
)
functionCallResponseSize = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_response_size_bytes",
Help: "The response size of the http call to target function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labelsStrings,
)
)
func init() {
prometheus.MustRegister(functionCalls)
prometheus.MustRegister(functionCallErrors)
prometheus.MustRegister(functionCallDuration)
prometheus.MustRegister(functionCallOverhead)
prometheus.MustRegister(functionCallResponseSize)
}
func labelsToStrings(f *functionLabels, h *httpLabels) []string {
var cached string
if f.cached {
cached = "true"
} else {
cached = "false"
}
return []string{
cached,
f.namespace,
f.name,
h.host,
h.path,
h.method,
fmt.Sprint(h.code),
}
}
func functionCallCompleted(f *functionLabels, h *httpLabels, overhead, duration time.Duration, respSize int64) {
atomic.AddUint64(&globalFunctionCallCount, 1)
l := labelsToStrings(f, h)
// overhead: time from request ingress into router up to proxing into function pod
functionCallOverhead.WithLabelValues(l...).Observe(float64(overhead.Nanoseconds()) / 1e9)
// total function call counter
functionCalls.WithLabelValues(l...).Inc()
// error counter
if h.code >= 400 {
functionCallErrors.WithLabelValues(l...).Inc()
}
// duration summary
functionCallDuration.WithLabelValues(l...).Observe(float64(duration.Nanoseconds()) / 1e9)
// Response size. -1 means the size unknown, in which case we don't report it.
if respSize != -1 {
functionCallResponseSize.WithLabelValues(l...).Observe(float64(respSize))
}
}
pkg/router/mutablemux_test.go
+4
View File
@@ -25,6 +25,8 @@ import (
"github.com/gorilla/mux"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"github.com/fission/fission/pkg/utils/metrics"
)
func OldHandler(responseWriter http.ResponseWriter, request *http.Request) {
@@ -73,6 +75,7 @@ func TestMutableMux(t *testing.T) {
// make a simple mutable router
log.Print("Create mutable router")
muxRouter := mux.NewRouter()
muxRouter.Use(metrics.HTTPMetricMiddleware())
muxRouter.HandleFunc("/", OldHandler)
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
@@ -99,6 +102,7 @@ func TestMutableMux(t *testing.T) {
// change the muxer
log.Print("Change mux router")
newMuxRouter := mux.NewRouter()
muxRouter.Use(metrics.HTTPMetricMiddleware())
newMuxRouter.HandleFunc("/", NewHandler)
mr.updateRouter(newMuxRouter)
pkg/router/router.go
+11 -16
View File
@@ -50,7 +50,6 @@ import (
"time"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/trace"
"go.opentelemetry.io/otel"
@@ -59,6 +58,7 @@ import (
"github.com/fission/fission/pkg/crd"
executorClient "github.com/fission/fission/pkg/executor/client"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils/metrics"
otelUtils "github.com/fission/fission/pkg/utils/otel"
)
@@ -68,13 +68,15 @@ import (
func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet) *mutableRouter {
var mr *mutableRouter
mux := mux.NewRouter()
mux.Use(metrics.HTTPMetricMiddleware())
// see issue https://github.com/fission/fission/issues/1317
useEncodedPath, _ := strconv.ParseBool(os.Getenv("USE_ENCODED_PATH"))
if useEncodedPath {
mr = newMutableRouter(logger, mux.NewRouter().UseEncodedPath())
mr = newMutableRouter(logger, mux.UseEncodedPath())
} else {
mr = newMutableRouter(logger, mux.NewRouter())
mr = newMutableRouter(logger, mux)
}
httpTriggerSet.subscribeRouter(ctx, mr)
@@ -86,9 +88,9 @@ func serve(ctx context.Context, logger *zap.Logger, port int, tracingSamplingRat
mr := router(ctx, logger, httpTriggerSet)
url := fmt.Sprintf(":%v", port)
var err error
var handler http.Handler
if openTracingEnabled {
err = http.ListenAndServe(url, &ochttp.Handler{
handler = &ochttp.Handler{
Handler: mr,
GetStartOptions: func(r *http.Request) trace.StartOptions {
// do not trace router healthz endpoint
@@ -108,23 +110,16 @@ func serve(ctx context.Context, logger *zap.Logger, port int, tracingSamplingRat
Sampler: trace.ProbabilitySampler(tracingSamplingRate),
}
},
})
}
} else {
err = http.ListenAndServe(url, otelUtils.GetHandlerWithOTEL(mr, "fission-router", otelUtils.UrlsToIgnore("/router-healthz")))
handler = otelUtils.GetHandlerWithOTEL(mr, "fission-router", otelUtils.UrlsToIgnore("/router-healthz"))
}
err := http.ListenAndServe(url, handler)
if err != nil {
logger.Error("HTTP server error", zap.Error(err))
}
}
func serveMetric(logger *zap.Logger) {
// Expose the registered metrics via HTTP.
http.Handle("/metrics", promhttp.Handler())
err := http.ListenAndServe(metricAddr, nil)
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
}
// Start starts a router
func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
fmap := makeFunctionServiceMap(logger, time.Minute)
@@ -253,7 +248,7 @@ func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string
svcAddrRetryCount: svcAddrRetryCount,
}, isDebugEnv, unTapServiceTimeout, throttler.MakeThrottler(svcAddrUpdateTimeout))
go serveMetric(logger)
go metrics.ServeMetrics(ctx, logger)
logger.Info("starting router", zap.Int("port", port))