Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring prometheus metrics and labels (#2375)

Browse Source 
This change mainly fixes few things around router and executor
exposed metrices.
1. We are trying to follow standard in metric names.
2. Lables such as namespace are colliding with kube-prometheus standards
so they are getting relabled to exported_namespace. Added function prefix
to resolve this.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
2022-03-02 10:10:58 +05:30
committed by GitHub
parent 10f64a8d91
commit ded13229c0
3 changed files with 37 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/canaryconfigmgr/prometheusClient.go
+14 -5
View File
@@ -17,6 +17,7 @@ limitations under the License.
package canaryconfigmgr
import (
"context"
"fmt"
"time"
@@ -25,7 +26,6 @@ import (
prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
"go.uber.org/zap"
"golang.org/x/net/context"
)
type PrometheusApiClient struct {
@@ -81,15 +81,20 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path stri
return failurePercentForFunc, nil
}
func (PrometheusApiClient *PrometheusApiClient) getFunctionQueryLabels(functionName, functionNamespace, path, method string) string {
return fmt.Sprintf("function_name=\"%s\",function_namespace=\"%s\",path=\"%s\",method=\"%s\"", functionName, functionNamespace, path, method)
}
func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, method string, funcName string, funcNs string, window string) (float64, error) {
queryString := fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"}[%v]", path, method, funcName, funcNs, window)
queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method)
queryString := fmt.Sprintf("fission_function_calls_total{%s}[%v]", queryLabels, window)
reqs, err := promApiClient.executeQuery(queryString)
if err != nil {
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window)
queryString = fmt.Sprintf("fission_function_calls_total{%s} offset %v", queryLabels, window)
reqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil {
@@ -107,14 +112,15 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string,
}
func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(funcName string, funcNs string, path string, method string, window string) (float64, error) {
queryString := fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"}[%v]", funcName, funcNs, path, method, window)
queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method)
queryString := fmt.Sprintf("fission_function_errors_total{%s}[%v]", queryLabels, window)
failedRequests, err := promApiClient.executeQuery(queryString)
if err != nil {
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window)
queryString = fmt.Sprintf("fission_function_errors_total{%s} offset %v", queryLabels, window)
failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil {
@@ -132,6 +138,9 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f
}
func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) {
// TODO: Change to debug level once we have a better understanding of what is happening
promApiClient.logger.Info("prometheus executing query", zap.String("query", queryString))
val, warn, err := promApiClient.client.Query(context.Background(), queryString, time.Now())
if err != nil {
return 0, errors.Wrapf(err, "error querying prometheus")
pkg/executor/fscache/metrics.go
+20 -17
View File
@@ -5,53 +5,56 @@ import (
)
var (
// funcname: the function's name
// funcuid: the function's version id
coldStarts = prometheus.NewCounterVec(
// function_name: the function's name
// function_uid: the function's version id
// function_address: the address of the pod from which the function was called
functionLabels = []string{"function_name", "function_uid"}
functionPodLabels = []string{"function_name", "function_address"}
coldStarts = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_cold_starts_total",
Help: "How many cold starts are made by funcname, funcuid.",
Name: "fission_function_cold_starts_total",
Help: "How many cold starts are made by function_name, function_uid.",
},
[]string{"funcname", "funcuid"},
functionLabels,
)
funcRunningSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_func_running_seconds_summary",
Name: "fission_function_running_seconds",
Help: "The running time (last access - create) in seconds of the function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"funcname", "funcuid"},
functionLabels,
)
funcAliveSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_func_alive_seconds_summary",
Name: "fission_function_alive_seconds",
Help: "The alive time in seconds of the function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"funcname", "funcuid"},
functionLabels,
)
funcIsAlive = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "fission_func_is_alive",
Help: "A binary value indicating is the funcname, funcuid alive",
Name: "fission_function_is_alive",
Help: "A binary value indicating is the function_name, function_uid alive",
},
[]string{"funcname", "funcuid"},
functionLabels,
)
funcReapTime = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_pod_reaptime_seconds",
Name: "fission_function_pod_reaptime_seconds",
Help: "Amount of seconds to reap a pod",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"funcname", "funcaddress"},
functionPodLabels,
)
idleTime = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_idle_pod_time",
Name: "fission_function_idle_pod_time",
Help: "Number of seconds it took for Reaper to detect the pod was idle",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"funcname", "funcaddress"},
functionPodLabels,
)
)
pkg/router/metrics.go
+3 -3
View File
@@ -43,12 +43,12 @@ var (
metricAddr = ":8080"
// function + http labels as strings
labelsStrings = []string{"cached", "namespace", "name", "host", "path", "method", "code"}
labelsStrings = []string{"cached", "function_namespace", "function_name", "host", "path", "method", "code"}
// Function http calls count
// cached: true | false, is this function service address cached locally
// namespace: function namespace
// name: function name
// function_namespace: function namespace
// function_name: function name
// code: http status code
// path: the client call the function on which http path
// method: the function's http method