Improvements from scale testing (#1812)

Poolmanager when tested at high load had some issues and this PR fixes one set of them which were found so far. 

Co-authored-by: Vishal <vishal-biyani@users.noreply.github.com>

pkg/executor/api.go

@@ -211,18 +211,8 @@ func (executor *Executor) unTapService(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "Failed to parse request", http.StatusBadRequest)
 		return
 	}
-
-	fn, err := executor.fissionClient.CoreV1().Functions(tapSvcReq.FnMetadata.Namespace).Get(tapSvcReq.FnMetadata.Name, metav1.GetOptions{})
-	if err != nil {
-		if k8serrors.IsNotFound(err) {
-			http.Error(w, "Failed to find function", http.StatusNotFound)
-		} else {
-			http.Error(w, "Failed to get function", http.StatusInternalServerError)
-		}
-		return
-	}
-
-	t := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+	key := fmt.Sprintf("%v_%v", tapSvcReq.FnMetadata.UID, tapSvcReq.FnMetadata.ResourceVersion)
+	t := tapSvcReq.FnExecutorType
 	if t != fv1.ExecutorTypePoolmgr {
 		msg := fmt.Sprintf("Unknown executor type '%v'", t)
 		http.Error(w, msg, http.StatusBadRequest)
@@ -231,7 +221,7 @@ func (executor *Executor) unTapService(w http.ResponseWriter, r *http.Request) {
 
 	et := executor.executorTypes[t]
 
-	et.UnTapService(fn, tapSvcReq.ServiceUrl)
+	et.UnTapService(key, tapSvcReq.ServiceUrl)
 
 	w.WriteHeader(http.StatusOK)
 }

pkg/executor/executortype/executortype.go

@@ -46,7 +46,7 @@ type ExecutorType interface {
 	TapService(serviceUrl string) error
 
 	// UnTapService updates the isActive to false
-	UnTapService(fn *fv1.Function, svcHost string)
+	UnTapService(key string, svcHost string)
 
 	// IsValid returns true if a function service is valid. Different executor types
 	// use distinct ways to examine the function service.

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -153,7 +153,7 @@ func (deploy *NewDeploy) DeleteFuncSvcFromCache(fsvc *fscache.FuncSvc) {
 	deploy.fsCache.DeleteEntry(fsvc)
 }
 
-func (deploy *NewDeploy) UnTapService(fn *fv1.Function, svcHost string) {
+func (deploy *NewDeploy) UnTapService(key string, svcHost string) {
 	// Not Implemented for NewDeployment. Will be used when support of concurrent specialization of same function is added.
 }
 

pkg/executor/executortype/poolmgr/gp.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math/rand"
 	"net"
 	"os"
 	"strings"
@@ -96,6 +95,16 @@ func MakeGenericPool(
 
 	gpLogger := logger.Named("generic_pool")
 
+	podReadyTimeoutStr := os.Getenv("POD_READY_TIMEOUT")
+	podReadyTimeout, err := time.ParseDuration(podReadyTimeoutStr)
+	if err != nil {
+		podReadyTimeout = 300 * time.Second
+		gpLogger.Error("failed to parse pod ready timeout duration from 'POD_READY_TIMEOUT' - set to the default value",
+			zap.Error(err),
+			zap.String("value", podReadyTimeoutStr),
+			zap.Duration("default", podReadyTimeout))
+	}
+
 	gpLogger.Info("creating pool", zap.Any("environment", env.ObjectMeta))
 
 	ctx, stopCh := context.WithCancel(context.Background())
@@ -111,7 +120,7 @@ func MakeGenericPool(
 		kubernetesClient:  kubernetesClient,
 		namespace:         namespace,
 		functionNamespace: functionNamespace,
-		podReadyTimeout:   5 * time.Minute, // TODO make this an env param?
+		podReadyTimeout:   podReadyTimeout,
 		fsCache:           fsCache,
 		poolInstanceId:    uniuri.NewLen(8),
 		fetcherConfig:     fetcherConfig,
@@ -124,7 +133,7 @@ func MakeGenericPool(
 	gp.runtimeImagePullPolicy = utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
 
 	// create fetcher SA in this ns, if not already created
-	err := fetcherConfig.SetupServiceAccount(gp.kubernetesClient, gp.namespace, nil)
+	err = fetcherConfig.SetupServiceAccount(gp.kubernetesClient, gp.namespace, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
 	}
@@ -202,6 +211,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
 		// Get pods; filter the ones that are ready
 		podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(
 			metav1.ListOptions{
+				FieldSelector: "status.phase=Running",
 				LabelSelector: labels.Set(
 					gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
 			})
@@ -219,6 +229,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
 
 			// add it to the list of ready pods
 			readyPods = append(readyPods, &pod)
+			break
 		}
 		gp.logger.Info("found ready pods",
 			zap.Any("labels", newLabels),
@@ -237,7 +248,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
 		// Pick a ready pod.  For now just choose randomly;
 		// ideally we'd care about which node it's running on,
 		// and make a good scheduling decision.
-		chosenPod := readyPods[rand.Intn(len(readyPods))]
+		chosenPod := readyPods[0]
 
 		if gp.env.Spec.AllowedFunctionsPerContainer != fv1.AllowedFunctionsPerContainerInfinite {
 			// Relabel.  If the pod already got picked and

pkg/executor/executortype/poolmgr/gpm.go

@@ -173,8 +173,8 @@ func (gpm *GenericPoolManager) DeleteFuncSvcFromCache(fsvc *fscache.FuncSvc) {
 	gpm.fsCache.DeleteFunctionSvc(fsvc)
 }
 
-func (gpm *GenericPoolManager) UnTapService(fn *fv1.Function, svcHost string) {
-	gpm.fsCache.MarkAvailable(fn, svcHost)
+func (gpm *GenericPoolManager) UnTapService(key string, svcHost string) {
+	gpm.fsCache.MarkAvailable(key, svcHost)
 }
 
 func (gpm *GenericPoolManager) GetTotalAvailable(fn *fv1.Function) int {

pkg/executor/fscache/functionServiceCache.go

@@ -218,8 +218,8 @@ func (fsc *FunctionServiceCache) GetTotalAvailable(m *metav1.ObjectMeta) int {
 	return fsc.connFunctionCache.GetTotalAvailable(crd.CacheKey(m))
 }
 
-func (fsc *FunctionServiceCache) MarkAvailable(fn *fv1.Function, svcHost string) {
-	fsc.connFunctionCache.MarkAvailable(crd.CacheKey(&fn.ObjectMeta), svcHost)
+func (fsc *FunctionServiceCache) MarkAvailable(key string, svcHost string) {
+	fsc.connFunctionCache.MarkAvailable(key, svcHost)
 }
 
 func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {

pkg/executor/fscache/functionServiceCache_test.go

@@ -189,7 +189,8 @@ func TestFunctionServiceNewCache(t *testing.T) {
 		logger.Panic(fmt.Sprintln("active instances not matched expected 1, found ", active))
 	}
 
-	fsc.MarkAvailable(fn, fsvc.Address)
+	key := fmt.Sprintf("%v_%v", fn.ObjectMeta.UID, fn.ObjectMeta.ResourceVersion)
+	fsc.MarkAvailable(key, fsvc.Address)
 
 	if fsc.GetTotalAvailable(fsvc.Function) != 0 {
 		log.Panicln("active instances not matched")