Add metrics to Reaper (#2019)

* Add metrics to Reaper

Signed-off-by: Harsh Thakur <harshthakur9030@gmail.com>

* Change duration to seconds and fix labels

Signed-off-by: Harsh Thakur <harshthakur9030@gmail.com>

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -845,7 +845,10 @@ func (deploy *NewDeploy) idleObjectReaper() {
 				continue
 			}
 
+			deploy.fsCache.IdleTime(fsvc.Name, fsvc.Address, float64(time.Since(fsvc.Atime)-idlePodReapTime))
+
 			go func() {
+				startTime := time.Now()
 				deployObj := getDeploymentObj(fsvc.KubernetesObjects)
 				if deployObj == nil {
 					deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
@@ -870,6 +873,7 @@ func (deploy *NewDeploy) idleObjectReaper() {
 				if err != nil {
 					deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
 				}
+				deploy.fsCache.ReapTime(fsvc.Function.Name, fsvc.Address, time.Since(startTime).Seconds())
 			}()
 		}
 	}

pkg/executor/executortype/poolmgr/gpm.go

@@ -669,8 +669,11 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
 			if time.Since(fsvc.Atime) < idlePodReapTime {
 				continue
 			}
+			idleTime := (time.Since(fsvc.Atime) - idlePodReapTime).Seconds()
+			gpm.fsCache.IdleTime(fsvc.Name, fsvc.Address, idleTime)
 
 			go func() {
+				startTime := time.Now()
 				deleted, err := gpm.fsCache.DeleteOldPoolCache(fsvc, idlePodReapTime)
 				if err != nil {
 					gpm.logger.Error("error deleting Kubernetes objects for function service",
@@ -687,6 +690,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
 						)
 						reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &fsvc.KubernetesObjects[i])
 						time.Sleep(50 * time.Millisecond)
+						gpm.fsCache.ReapTime(fsvc.Function.Name, fsvc.Address, time.Since(startTime).Seconds())
 					}
 				}
 			}()

pkg/executor/fscache/metrics.go

@@ -37,6 +37,22 @@ var (
 		},
 		[]string{"funcname", "funcuid"},
 	)
+	funcReapTime = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_pod_reaptime_seconds",
+			Help:       "Amount of seconds to reap a pod",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcaddress"},
+	)
+	idleTime = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_idle_pod_time",
+			Help:       "Number of seconds it took for Reaper to detect the pod was idle",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcaddress"},
+	)
 )
 
 func init() {
@@ -45,6 +61,8 @@ func init() {
 	prometheus.MustRegister(funcRunningSummary)
 	prometheus.MustRegister(funcAliveSummary)
 	prometheus.MustRegister(funcIsAlive)
+	prometheus.MustRegister(funcReapTime)
+	prometheus.MustRegister(idleTime)
 }
 
 // IncreaseColdStarts increments the counter by 1.
@@ -67,3 +85,13 @@ func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive
 	}
 	funcIsAlive.WithLabelValues(funcname, funcuid).Set(float64(count))
 }
+
+// ReapTime is the amount of time taken to reap a pod
+func (fsc *FunctionServiceCache) ReapTime(funcName, funcAddress string, time float64) {
+	funcReapTime.WithLabelValues(funcName, funcAddress).Observe(float64(time))
+}
+
+// IdleTime is the amount of time it took Reaper to find out the pod was idle
+func (fsc *FunctionServiceCache) IdleTime(funcName, funcAddress string, time float64) {
+	idleTime.WithLabelValues(funcName, funcAddress).Observe(time)
+}