diff --git a/pkg/canaryconfigmgr/prometheusClient.go b/pkg/canaryconfigmgr/prometheusClient.go index 818f4e62..4fa56e0d 100644 --- a/pkg/canaryconfigmgr/prometheusClient.go +++ b/pkg/canaryconfigmgr/prometheusClient.go @@ -17,6 +17,7 @@ limitations under the License. package canaryconfigmgr import ( + "context" "fmt" "time" @@ -25,7 +26,6 @@ import ( prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "go.uber.org/zap" - "golang.org/x/net/context" ) type PrometheusApiClient struct { @@ -81,15 +81,20 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path stri return failurePercentForFunc, nil } +func (PrometheusApiClient *PrometheusApiClient) getFunctionQueryLabels(functionName, functionNamespace, path, method string) string { + return fmt.Sprintf("function_name=\"%s\",function_namespace=\"%s\",path=\"%s\",method=\"%s\"", functionName, functionNamespace, path, method) +} + func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, method string, funcName string, funcNs string, window string) (float64, error) { - queryString := fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"}[%v]", path, method, funcName, funcNs, window) + queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method) + queryString := fmt.Sprintf("fission_function_calls_total{%s}[%v]", queryLabels, window) reqs, err := promApiClient.executeQuery(queryString) if err != nil { return 0, errors.Wrapf(err, "error executing query: %s", queryString) } - queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window) + queryString = fmt.Sprintf("fission_function_calls_total{%s} offset %v", queryLabels, window) reqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { @@ -107,14 +112,15 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, } func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(funcName string, funcNs string, path string, method string, window string) (float64, error) { - queryString := fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"}[%v]", funcName, funcNs, path, method, window) + queryLabels := promApiClient.getFunctionQueryLabels(funcName, funcNs, path, method) + queryString := fmt.Sprintf("fission_function_errors_total{%s}[%v]", queryLabels, window) failedRequests, err := promApiClient.executeQuery(queryString) if err != nil { return 0, errors.Wrapf(err, "error executing query: %s", queryString) } - queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window) + queryString = fmt.Sprintf("fission_function_errors_total{%s} offset %v", queryLabels, window) failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { @@ -132,6 +138,9 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f } func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) { + // TODO: Change to debug level once we have a better understanding of what is happening + promApiClient.logger.Info("prometheus executing query", zap.String("query", queryString)) + val, warn, err := promApiClient.client.Query(context.Background(), queryString, time.Now()) if err != nil { return 0, errors.Wrapf(err, "error querying prometheus") diff --git a/pkg/executor/fscache/metrics.go b/pkg/executor/fscache/metrics.go index b3e58bc5..93b0abe5 100644 --- a/pkg/executor/fscache/metrics.go +++ b/pkg/executor/fscache/metrics.go @@ -5,53 +5,56 @@ import ( ) var ( - // funcname: the function's name - // funcuid: the function's version id - coldStarts = prometheus.NewCounterVec( + // function_name: the function's name + // function_uid: the function's version id + // function_address: the address of the pod from which the function was called + functionLabels = []string{"function_name", "function_uid"} + functionPodLabels = []string{"function_name", "function_address"} + coldStarts = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "fission_cold_starts_total", - Help: "How many cold starts are made by funcname, funcuid.", + Name: "fission_function_cold_starts_total", + Help: "How many cold starts are made by function_name, function_uid.", }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcRunningSummary = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_func_running_seconds_summary", + Name: "fission_function_running_seconds", Help: "The running time (last access - create) in seconds of the function.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcAliveSummary = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_func_alive_seconds_summary", + Name: "fission_function_alive_seconds", Help: "The alive time in seconds of the function.", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcIsAlive = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "fission_func_is_alive", - Help: "A binary value indicating is the funcname, funcuid alive", + Name: "fission_function_is_alive", + Help: "A binary value indicating is the function_name, function_uid alive", }, - []string{"funcname", "funcuid"}, + functionLabels, ) funcReapTime = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_pod_reaptime_seconds", + Name: "fission_function_pod_reaptime_seconds", Help: "Amount of seconds to reap a pod", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcaddress"}, + functionPodLabels, ) idleTime = prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: "fission_idle_pod_time", + Name: "fission_function_idle_pod_time", Help: "Number of seconds it took for Reaper to detect the pod was idle", Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, - []string{"funcname", "funcaddress"}, + functionPodLabels, ) ) diff --git a/pkg/router/metrics.go b/pkg/router/metrics.go index dd0aedd3..99092b6a 100644 --- a/pkg/router/metrics.go +++ b/pkg/router/metrics.go @@ -43,12 +43,12 @@ var ( metricAddr = ":8080" // function + http labels as strings - labelsStrings = []string{"cached", "namespace", "name", "host", "path", "method", "code"} + labelsStrings = []string{"cached", "function_namespace", "function_name", "host", "path", "method", "code"} // Function http calls count // cached: true | false, is this function service address cached locally - // namespace: function namespace - // name: function name + // function_namespace: function namespace + // function_name: function name // code: http status code // path: the client call the function on which http path // method: the function's http method