Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fission metrics integration (#677)

Browse Source 
Add prometheus metrics collection endpoints to router and executor. Add prometheus annotations to router and executor pods.
This commit is contained in:
Soam Vasani
2018-05-21 13:57:18 -07:00
committed by GitHub
parent d9b5b9b8a8
commit d846aed612
11 changed files with 322 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/functionHandler.go
+30 -6
View File
@@ -29,13 +29,15 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/crd"
executorClient "github.com/fission/fission/executor/client"
)
type functionHandler struct {
fmap *functionServiceMap
executor *executorClient.Client
function *metav1.ObjectMeta
fmap *functionServiceMap
executor *executorClient.Client
function *metav1.ObjectMeta
httpTrigger *crd.HTTPTrigger
}
// A layer on top of http.DefaultTransport, with retries.
@@ -75,6 +77,20 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
var needExecutor, serviceUrlFromExecutor bool
var serviceUrl *url.URL
// Metrics stuff
startTime := time.Now()
funcMetricLabels := &functionLabels{
namespace: roundTripper.funcHandler.function.Namespace,
name: roundTripper.funcHandler.function.Name,
}
httpMetricLabels := &httpLabels{
method: req.Method,
}
if roundTripper.funcHandler.httpTrigger != nil {
httpMetricLabels.host = roundTripper.funcHandler.httpTrigger.Spec.Host
httpMetricLabels.path = roundTripper.funcHandler.httpTrigger.Spec.RelativeURL
}
// set the timeout for transport context
timeout := roundTripper.initialTimeout
transport := http.DefaultTransport.(*http.Transport)
@@ -135,13 +151,23 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
KeepAlive: 30 * time.Second,
}).DialContext
overhead := time.Since(startTime)
// forward the request to the function service
resp, err = transport.RoundTrip(req)
if err == nil {
// Track metrics
httpMetricLabels.code = resp.StatusCode
funcMetricLabels.cached = !serviceUrlFromExecutor
functionCallCompleted(funcMetricLabels, httpMetricLabels,
overhead, time.Since(startTime), resp.ContentLength)
// if transport.RoundTrip succeeds and it was a cached entry, then tapService
if !serviceUrlFromExecutor {
go roundTripper.funcHandler.tapService(serviceUrl)
}
// return response back to user
return resp, nil
}
@@ -190,11 +216,9 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
request.Header.Add(fmt.Sprintf("X-Fission-Params-%v", k), v)
}
// System Params
// system params
MetadataToHeaders(HEADERS_FISSION_FUNCTION_PREFIX, fh.function, request)
// TODO: As an optimization we may want to cache proxies too -- this might get us
// connection reuse and possibly better performance
director := func(req *http.Request) {
if _, ok := req.Header["User-Agent"]; !ok {
// explicitly disable User-Agent so it's not set to default value
router/httpTriggers.go
+4 -3
View File
@@ -117,9 +117,10 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
}
fh := &functionHandler{
fmap: ts.functionServiceMap,
function: rr.functionMetadata,
executor: ts.executor,
fmap: ts.functionServiceMap,
function: rr.functionMetadata,
executor: ts.executor,
httpTrigger: &trigger,
}
ht := muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
router/metrics.go
+139
View File
@@ -0,0 +1,139 @@
package router
import (
"fmt"
"time"
"github.com/prometheus/client_golang/prometheus"
)
type (
// functionLabels is the set of metrics labels that relate to
// functions.
//
// cached indicates whether or not the function call hit the
// cache in this service.
//
// namespace and name are the metadata of the function.
functionLabels struct {
cached bool
namespace string
name string
}
// httpLabels is the set of metrics labels that relate to HTTP
// requests.
//
// host is the host that the HTTP request was made to
// path is the relative URL of the request
// method is the HTTP method ("GET", "POST", ...)
// code is the HTTP status code
httpLabels struct {
host string
path string
method string
code int
}
)
var (
metricAddr = ":8080"
// function + http labels as strings
labelsStrings = []string{"cached", "namespace", "name", "host", "path", "method", "code"}
// Function http calls count
// cached: true | false, is this function service address cached locally
// namespace: function namespace
// name: function name
// code: http status code
// path: the client call the function on which http path
// method: the function's http method
functionCalls = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_function_calls_total",
Help: "Count of Fission function calls",
},
labelsStrings,
)
functionCallErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "fission_function_errors_total",
Help: "Count of Fission function errors",
},
labelsStrings,
)
functionCallDuration = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_duration_seconds",
Help: "Runtime duration of the Fission function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labelsStrings,
)
functionCallOverhead = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_overhead_seconds",
Help: "The function call delay caused by fission.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labelsStrings,
)
functionCallResponseSize = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "fission_function_response_size_bytes",
Help: "The response size of the http call to target function.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labelsStrings,
)
)
func init() {
prometheus.MustRegister(functionCalls)
prometheus.MustRegister(functionCallErrors)
prometheus.MustRegister(functionCallDuration)
prometheus.MustRegister(functionCallOverhead)
prometheus.MustRegister(functionCallResponseSize)
}
func labelsToStrings(f *functionLabels, h *httpLabels) []string {
var cached string
if f.cached {
cached = "true"
} else {
cached = "false"
}
return []string{
cached,
f.namespace,
f.name,
h.host,
h.path,
h.method,
fmt.Sprint(h.code),
}
}
func functionCallCompleted(f *functionLabels, h *httpLabels, overhead, duration time.Duration, respSize int64) {
l := labelsToStrings(f, h)
// overhead: time from request ingress into router upto proxing into function pod
functionCallOverhead.WithLabelValues(l...).Observe(float64(overhead.Nanoseconds()) / 1e9)
// total function call counter
functionCalls.WithLabelValues(l...).Inc()
// error counter
if h.code >= 400 {
functionCallErrors.WithLabelValues(l...).Inc()
}
// duration summary
functionCallDuration.WithLabelValues(l...).Observe(float64(duration.Nanoseconds()) / 1e9)
// Response size. -1 means the size unknown, in which case we don't report it.
if respSize != -1 {
functionCallResponseSize.WithLabelValues(l...).Observe(float64(respSize))
}
}
router/router.go
+9
View File
@@ -47,6 +47,7 @@ import (
"time"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/fission/fission"
"github.com/fission/fission/crd"
@@ -71,6 +72,12 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
http.ListenAndServe(url, mr)
}
func serveMetric() {
// Expose the registered metrics via HTTP.
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricAddr, nil))
}
func Start(port int, executorUrl string) {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
@@ -93,6 +100,8 @@ func Start(port int, executorUrl string) {
triggers, _, fnStore := makeHTTPTriggerSet(fmap, fissionClient, executor, restClient)
resolver := makeFunctionReferenceResolver(fnStore)
go serveMetric()
log.Printf("Starting router at port %v\n", port)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()