Use client generator to generate all k8s clients and add respective client-go metrics (#2668)

* Define client generator to generate all k8s clients
* Increase QPS and burst values
* Capture client-go metrics
* Support for controller runtime metrics

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/utils/metrics/http_metrics.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/fission/fission/pkg/router/util"
 )
@@ -33,14 +32,14 @@ type ResponseWriterWrapper struct {
 }
 
 var (
-	httpRequestsTotal = promauto.NewCounterVec(
+	httpRequestsTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "http_requests_total",
 			Help: "Number of requests by path, method and status code.",
 		},
 		[]string{"path", "method", "code"},
 	)
-	httpRequestDuration = promauto.NewSummaryVec(
+	httpRequestDuration = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Name:       "http_requests_duration_seconds",
 			Help:       "Time taken to serve the request by path and method.",
@@ -48,7 +47,7 @@ var (
 		},
 		[]string{"path", "method"},
 	)
-	httpRequestInFlight = promauto.NewGaugeVec(
+	httpRequestInFlight = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "http_requests_in_flight",
 			Help: "Number of requests currently being served by path and method.",
@@ -57,6 +56,12 @@ var (
 	)
 )
 
+func init() {
+	Registry.MustRegister(httpRequestsTotal)
+	Registry.MustRegister(httpRequestDuration)
+	Registry.MustRegister(httpRequestInFlight)
+}
+
 func (rw *ResponseWriterWrapper) WriteHeader(statuscode int) {
 	rw.statusCode = statuscode
 	rw.ResponseWriter.WriteHeader(statuscode)

pkg/utils/metrics/registry.go

@@ -0,0 +1,9 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	Registry = prometheus.NewRegistry()
+)

pkg/utils/metrics/server.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/fission/fission/pkg/utils/httpserver"
 )
@@ -32,7 +33,17 @@ func ServeMetrics(ctx context.Context, logger *zap.Logger) {
 	if metricsAddr == "" {
 		metricsAddr = "8080"
 	}
+	err := metrics.Registry.Register(Registry)
+	if err != nil {
+		logger.Error("failed to register metrics", zap.Error(err))
+	}
 	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle("/metrics", promhttp.HandlerFor(
+		metrics.Registry,
+		promhttp.HandlerOpts{
+			// Opt into OpenMetrics to support exemplars.
+			EnableOpenMetrics: true,
+		},
+	))
 	httpserver.StartServer(ctx, logger, "metrics", metricsAddr, mux)
 }