Prometheus metrics improvements (#2398)

- Enabled metrics in storagesvc, buildermgr and controller.
- Added a middleware in storagesvc, router, executor and controller to monitor total number of http requests, each request's duration and number of requests that are currently being served. These requests can be filtered on their path, method or statuscode.
- Removed functionCallDuration and functionCallResponseSize metrics from router.
- Removed funcAliveSummary, funcIsAlive, funcReapTime and idleTime metrics.
- Replaced function calls for collecting metrics to direct metric calls.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/executor.go

@@ -19,7 +19,6 @@ package executor
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -28,7 +27,6 @@ import (
 
 	"github.com/dchest/uniuri"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
 	k8sInformers "k8s.io/client-go/informers"
 	k8sCache "k8s.io/client-go/tools/cache"
@@ -46,6 +44,7 @@ import (
 	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
 	genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/metrics"
 	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
 
@@ -250,15 +249,6 @@ func (executor *Executor) getFunctionServiceFromCache(ctx context.Context, fn *f
 	return e.GetFuncSvcFromCache(ctx, fn)
 }
 
-func serveMetric(logger *zap.Logger) {
-	// Expose the registered metrics via HTTP.
-	metricAddr := ":8080"
-	http.Handle("/metrics", promhttp.Handler())
-	err := http.ListenAndServe(metricAddr, nil)
-
-	logger.Fatal("done listening on metrics endpoint", zap.Error(err))
-}
-
 // StartExecutor Starts executor and the executor components such as Poolmgr,
 // deploymgr and potential future executor types
 func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
@@ -379,8 +369,8 @@ func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace st
 		return err
 	}
 	go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
+	go metrics.ServeMetrics(ctx, logger)
 	go api.Serve(port, openTracingEnabled)
-	go serveMetric(logger)
 
 	return nil
 }