Fission metrics integration (#677)

Add prometheus metrics collection endpoints to router and executor. Add prometheus annotations to router and executor pods.

executor/api.go

@@ -89,6 +89,7 @@ func (executor *Executor) getServiceForFunction(m *metav1.ObjectMeta) (string, e
 	if resp.err != nil {
 		return "", resp.err
 	}
+	executor.fsCache.IncreaseColdStarts(m.Name, string(m.UID))
 	return resp.funcSvc.Address, resp.err
 }
 

executor/executor.go

@@ -18,12 +18,14 @@ package executor
 
 import (
 	"log"
+	"net/http"
 	"runtime/debug"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/dchest/uniuri"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
@@ -202,8 +204,15 @@ func dumpStackTrace() {
 	debug.PrintStack()
 }
 
-// StartExecutor Starts executor and the executor components such as Poolmgr,
-// deploymgr and potential future executor types
+func serveMetric() {
+	// Expose the registered metrics via HTTP.
+	metricAddr := ":8080"
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(metricAddr, nil))
+}
+
+// StartExecutor Starts executor and the backend components that executor uses such as Poolmgr,
+// deploymgr and potential future backends
 func StartExecutor(fissionNamespace string, functionNamespace string, port int) error {
 	// setup a signal handler for SIGTERM
 	fission.SetupStackTraceHandler()
@@ -238,6 +247,7 @@ func StartExecutor(fissionNamespace string, functionNamespace string, port int)
 	api := MakeExecutor(gpm, ndm, fissionClient, fsCache)
 
 	go api.Serve(port)
+	go serveMetric()
 
 	return nil
 }

executor/fscache/functionServiceCache.go

@@ -216,6 +216,7 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
 		return nil, err
 	}
 
+	fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), true)
 	return nil, nil
 }
 
@@ -249,6 +250,10 @@ func (fsc *FunctionServiceCache) DeleteEntry(fsvc *FuncSvc) {
 	fsc.byFunction.Delete(crd.CacheKey(fsvc.Function))
 	fsc.byAddress.Delete(fsvc.Address)
 	fsc.byFunctionUID.Delete(fsvc.Function.UID)
+
+	fsc.observeFuncRunningTime(fsvc.Function.Name, string(fsvc.Function.UID), fsvc.Atime.Sub(fsvc.Ctime).Seconds())
+	fsc.observeFuncAliveTime(fsvc.Function.Name, string(fsvc.Function.UID), time.Now().Sub(fsvc.Ctime).Seconds())
+	fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), false)
 }
 
 func (fsc *FunctionServiceCache) DeleteOld(fsvc *FuncSvc, minAge time.Duration) (bool, error) {

executor/fscache/metrics.go

@@ -0,0 +1,70 @@
+package fscache
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	metricAddr = ":8080"
+
+	// funcname: the function's name
+	// funcuid: the function's version id
+	coldStarts = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "fission_cold_starts_total",
+			Help: "How many cold starts are made by funcname, funcuid.",
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcRunningSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_func_running_seconds_summary",
+			Help:       "The running time (last access - create) in seconds of the function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcAliveSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_func_alive_seconds_summary",
+			Help:       "The alive time in seconds of the function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcIsAlive = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "fission_func_is_alive",
+			Help: "A binary value indicating is the funcname, funcuid alive",
+		},
+		[]string{"funcname", "funcuid"},
+	)
+)
+
+func init() {
+	// Register the function calls counter with Prometheus's default registry.
+	prometheus.MustRegister(coldStarts)
+	prometheus.MustRegister(funcRunningSummary)
+	prometheus.MustRegister(funcAliveSummary)
+	prometheus.MustRegister(funcIsAlive)
+}
+
+func (fsc *FunctionServiceCache) IncreaseColdStarts(funcname, funcuid string) {
+	coldStarts.WithLabelValues(funcname, funcuid).Inc()
+}
+
+func (fsc *FunctionServiceCache) observeFuncRunningTime(funcname, funcuid string, running float64) {
+	funcRunningSummary.WithLabelValues(funcname, funcuid).Observe(running)
+}
+
+func (fsc *FunctionServiceCache) observeFuncAliveTime(funcname, funcuid string, alive float64) {
+	funcAliveSummary.WithLabelValues(funcname, funcuid).Observe(alive)
+}
+
+func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive bool) {
+	count := 0
+	if isAlive {
+		count = 1
+	}
+	funcIsAlive.WithLabelValues(funcname, funcuid).Set(float64(count))
+}