From ded13229c0e1fa7d9e721438edef14168021a04c Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Wed, 2 Mar 2022 10:10:58 +0530 Subject: [PATCH] Refactoring prometheus metrics and labels (#2375) This change mainly fixes few things around router and executor exposed metrices. 1. We are trying to follow standard in metric names. 2. Lables such as namespace are colliding with kube-prometheus standards so they are getting relabled to exported_namespace. Added function prefix to resolve this. Signed-off-by: Sanket Sudake --- pkg/canaryconfigmgr/prometheusClient.go | 19 +++++++++---- pkg/executor/fscache/metrics.go | 37 +++++++++++++------------ pkg/router/metrics.go | 6 ++-- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/pkg/canaryconfigmgr/prometheusClient.go b/pkg/canaryconfigmgr/prometheusClient.go index 818f4e62..4fa56e0d 100644 --- a/pkg/canaryconfigmgr/prometheusClient.go +++ b/pkg/canaryconfigmgr/prometheusClient.go @@ -17,6 +17,7 @@ limitations under the License. package canaryconfigmgr import ( + "context" "fmt" "time" @@ -25,7 +26,6 @@ import ( prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "go.uber.org/zap" - "golang.org/x/net/context" ) type PrometheusApiClient struct { @@ -81,15 +81,20 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path stri return failurePercentForFunc, nil } +func (PrometheusApiClient *PrometheusApiClient) getFunctionQueryLabels(functionName, functionNamespace, path, method string) string { + return fmt.Sprintf("function_name=\"%s\",function_namespace=\"%s\",path=\"%s\",method=\"%s\"", functionName, functionNamespace, path, method) +} + func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, method string, funcName string, funcNs string, window string) (float64, error) { - queryString := fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"}[%v]", path, method, funcName, funcNs, window) + queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method) + queryString := fmt.Sprintf("fission_function_calls_total{%s}[%v]", queryLabels, window) reqs, err := promApiClient.executeQuery(queryString) if err != nil { return 0, errors.Wrapf(err, "error executing query: %s", queryString) } - queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window) + queryString = fmt.Sprintf("fission_function_calls_total{%s} offset %v", queryLabels, window) reqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { @@ -107,14 +112,15 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, } func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(funcName string, funcNs string, path string, method string, window string) (float64, error) { - queryString := fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"}[%v]", funcName, funcNs, path, method, window) + queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method) + queryString := fmt.Sprintf("fission_function_errors_total{%s}[%v]", queryLabels, window) failedRequests, err := promApiClient.executeQuery(queryString) if err != nil { return 0, errors.Wrapf(err, "error executing query: %s", queryString) } - queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window) + queryString = fmt.Sprintf("fission_function_errors_total{%s} offset %v", queryLabels, window) failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { @@ -132,6 +138,9 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f } func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) { + // TODO: Change to debug level once we have a better understanding of what is happening + promApiClient.logger.Info("prometheus executing query", zap.String("query", queryString)) + val, warn, err := promApiClient.client.Query(context.Background(), queryString, time.Now()) if err != nil { return 0, errors.Wrapf(err, "error querying prometheus") diff --git a/pkg/executor/fscache/metrics.go b/pkg/executor/fscache/metrics.go index b3e58bc5..93b0abe5 100644 --- a/pkg/executor/fscache/metrics.go +++ b/pkg/executor/fscache/metrics.go @@ -5,53 +5,56 @@ import ( ) var ( - // funcname: the function's name - // funcuid: the function's version id - coldStarts = prometheus.NewCounterVec( + // function_name: the function's name + // function_uid: the function's version id + // function_address: the address of the pod from which the function was called + functionLabels = []string{"function_name", "function_uid"} + functionPodLabels = []string{"function_name", "function_address"} + coldStarts = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "fission_cold_starts_total", - Help: "How many cold starts are made by funcname, funcuid.", + Name: "fission_function_cold_starts_total", + Help: "How many cold starts are made by function_name, function_uid.", }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcRunningSummary = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_func_running_seconds_summary", + Name: "fission_function_running_seconds", Help: "The running time (last access - create) in seconds of the function.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcAliveSummary = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_func_alive_seconds_summary", + Name: "fission_function_alive_seconds", Help: "The alive time in seconds of the function.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcIsAlive = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "fission_func_is_alive", - Help: "A binary value indicating is the funcname, funcuid alive", + Name: "fission_function_is_alive", + Help: "A binary value indicating is the function_name, function_uid alive", }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcReapTime = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_pod_reaptime_seconds", + Name: "fission_function_pod_reaptime_seconds", Help: "Amount of seconds to reap a pod", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcaddress"}, + functionPodLabels, ) idleTime = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_idle_pod_time", + Name: "fission_function_idle_pod_time", Help: "Number of seconds it took for Reaper to detect the pod was idle", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcaddress"}, + functionPodLabels, ) ) diff --git a/pkg/router/metrics.go b/pkg/router/metrics.go index dd0aedd3..99092b6a 100644 --- a/pkg/router/metrics.go +++ b/pkg/router/metrics.go @@ -43,12 +43,12 @@ var ( metricAddr = ":8080" // function + http labels as strings - labelsStrings = []string{"cached", "namespace", "name", "host", "path", "method", "code"} + labelsStrings = []string{"cached", "function_namespace", "function_name", "host", "path", "method", "code"} // Function http calls count // cached: true | false, is this function service address cached locally - // namespace: function namespace - // name: function name + // function_namespace: function namespace + // function_name: function name // code: http status code // path: the client call the function on which http path // method: the function's http method