Retrieve pod metrics only if metrics server is running and Go lint fixes (#2094)

* Retrieve pod metrics only if metrics server is running

Currently we query pod metrics every 30 sec which floods executor logs,
added check which confirms if metrics server is running then only we start
querying pod metrics for identifying CPU utilization.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Fixed couple of typos and misspells with Go CI

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Remove unnecessary conversions with Go CI

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/executortype/newdeploy/newdeploy.go

@@ -48,7 +48,7 @@ const (
 func (deploy *NewDeploy) createOrGetDeployment(fn *fv1.Function, env *fv1.Environment,
 	deployName string, deployLabels map[string]string, deployAnnotations map[string]string, deployNamespace string) (*appsv1.Deployment, error) {
 
-	specializationTimeout := int(fn.Spec.InvokeStrategy.ExecutionStrategy.SpecializationTimeout)
+	specializationTimeout := fn.Spec.InvokeStrategy.ExecutionStrategy.SpecializationTimeout
 	minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
 
 	// Always scale to at least one pod when createOrGetDeployment

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -207,7 +207,7 @@ func (deploy *NewDeploy) getServiceInfo(obj apiv1.ObjectReference) (*apiv1.Servi
 
 	if err != nil || !exists {
 		deploy.logger.Debug(
-			"Falling back to getting service info from k8s API -- this may cause performace issues for your function.",
+			"Falling back to getting service info from k8s API -- this may cause performance issues for your function.",
 			zap.Bool("exists", exists),
 			zap.Error(err),
 		)
@@ -224,7 +224,7 @@ func (deploy *NewDeploy) getDeploymentInfo(obj apiv1.ObjectReference) (*appsv1.D
 
 	if err != nil || !exists {
 		deploy.logger.Debug(
-			"Falling back to getting deployment info from k8s API -- this may cause performace issues for your function.",
+			"Falling back to getting deployment info from k8s API -- this may cause performance issues for your function.",
 			zap.Bool("exists", exists),
 			zap.Error(err),
 		)

pkg/executor/executortype/poolmgr/gp.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/dchest/uniuri"
-	"github.com/fission/fission/pkg/utils"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
@@ -50,6 +49,7 @@ import (
 	"github.com/fission/fission/pkg/executor/util"
 	fetcherClient "github.com/fission/fission/pkg/fetcher/client"
 	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/utils"
 )
 
 type (
@@ -171,32 +171,58 @@ func (gp *GenericPool) getDeployAnnotations() map[string]string {
 	}
 }
 
+func (gp *GenericPool) checkMetricsApi() bool {
+	apiGroups, err := gp.metricsClient.DiscoveryClient.ServerGroups()
+	if err != nil {
+		gp.logger.Error("faied to discover API groups", zap.Error(err))
+		return false
+	}
+	return utils.SupportedMetricsAPIVersionAvailable(apiGroups)
+}
+
 func (gp *GenericPool) updateCPUUtilizationSvc() {
-	for {
+	var metricsApiAvailabe bool
+	checkDuration := 30
+
+	if !gp.checkMetricsApi() {
+		checkDuration = 180
+		gp.logger.Error("Metrics API not available")
+	}
+
+	serviceFunc := func() {
 		podMetricsList, err := gp.metricsClient.MetricsV1beta1().PodMetricses(gp.namespace).List(context.TODO(), metav1.ListOptions{
 			LabelSelector: "managed=false",
 		})
-
 		if err != nil {
 			gp.logger.Error("failed to fetch pod metrics list", zap.Error(err))
-		} else {
-			gp.logger.Debug("pods found", zap.Any("length", len(podMetricsList.Items)))
-			for _, val := range podMetricsList.Items {
-				p, _ := resource.ParseQuantity("0m")
-				for _, container := range val.Containers {
-					p.Add(container.Usage["cpu"])
-				}
-				if value, ok := gp.podFSVCMap.Load(val.ObjectMeta.Name); ok {
-					if valArray, ok1 := value.([]interface{}); ok1 {
-						function, address := valArray[0], valArray[1]
-						gp.fsCache.SetCPUUtilizaton(function.(string), address.(string), p)
-						gp.logger.Info(fmt.Sprintf("updated function %s, address %s, cpuUsage %+v", function.(string), address.(string), p))
-					}
+			return
+		}
+		gp.logger.Debug("pods found", zap.Any("length", len(podMetricsList.Items)))
+		for _, val := range podMetricsList.Items {
+			p, _ := resource.ParseQuantity("0m")
+			for _, container := range val.Containers {
+				p.Add(container.Usage["cpu"])
+			}
+			if value, ok := gp.podFSVCMap.Load(val.ObjectMeta.Name); ok {
+				if valArray, ok1 := value.([]interface{}); ok1 {
+					function, address := valArray[0], valArray[1]
+					gp.fsCache.SetCPUUtilizaton(function.(string), address.(string), p)
+					gp.logger.Info(fmt.Sprintf("updated function %s, address %s, cpuUsage %+v", function.(string), address.(string), p))
 				}
 			}
 		}
+	}
 
-		time.Sleep(30 * time.Second)
+	for {
+		if metricsApiAvailabe {
+			serviceFunc()
+		} else {
+			if gp.checkMetricsApi() {
+				metricsApiAvailabe = true
+				checkDuration = 30
+			}
+		}
+		time.Sleep(time.Duration(checkDuration) * time.Second)
 	}
 }
 

pkg/executor/executortype/poolmgr/gpm.go

@@ -211,7 +211,7 @@ func (gpm *GenericPoolManager) getPodInfo(obj apiv1.ObjectReference) (*apiv1.Pod
 	}
 
 	if err != nil || !exists {
-		gpm.logger.Debug("Falling back to getting pod info from k8s API -- this may cause performace issues for your function.")
+		gpm.logger.Debug("Falling back to getting pod info from k8s API -- this may cause performance issues for your function.")
 		pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(context.TODO(), obj.Name, metav1.GetOptions{})
 		return pod, err
 	}

pkg/executor/executortype/poolmgr/packagewatcher.go

@@ -19,7 +19,6 @@ package poolmgr
 import (
 	"time"
 
-	"github.com/fission/fission/pkg/utils"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -28,6 +27,7 @@ import (
 
 	fv1 "github.com/fission/fission/pkg/apis/core/v1"
 	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/utils"
 )
 
 // TODO : It may make sense to make each of add, update, delete funcs run as separate go routines.

pkg/executor/fscache/metrics.go

@@ -88,7 +88,7 @@ func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive
 
 // ReapTime is the amount of time taken to reap a pod
 func (fsc *FunctionServiceCache) ReapTime(funcName, funcAddress string, time float64) {
-	funcReapTime.WithLabelValues(funcName, funcAddress).Observe(float64(time))
+	funcReapTime.WithLabelValues(funcName, funcAddress).Observe(time)
 }
 
 // IdleTime is the amount of time it took Reaper to find out the pod was idle