Improve poolmanager concurrency handling with virtual capacity (#2737)

* add functionality to wait for specialization by keeping track of incoming requests
* format executor package
* fix required capacity to specialise new pod condition
* move handling concurrency logic into pool cache from executor
* remove unused methods and structs
* implement queue in to store the svc wait
* create a queue struct and its methods to handle concurrent inputs
* use newly created queue to store waiting for svc requests
* add waiting requests in queue and use them when a svc is ready
* set function to request in queue if the context is still alive
* remove concurrency approach to set svc for waiting requests
* update the active requests whenever requests from pool are assigned a svc
* add doc to define why the conditions exist
* remove unwanted params in strcut and clean up code
* set error while getting svc value if sum of specialization in progress and specialized is only more than concurrency limit
* remove duplicate functions and unnecessary values in struct
* close svc channel on set value and create constants for default concurrency and rpp
* get next value in queue in case context is timed out for fetched value
* remove specializationInProgress counter from pool cache
* return in case the queue is empty wihle setting func to svc
* test getSvcVaue and setSvcValue in poolcache
* add unit tests for GetConcurrent and GetRequestsPerPod methods
* reorder imports
* add fuzzy testing for getSVCValue and setSVCValue in poolcache
* restructure go mod file and update pool cache test cases
* Add tests and bug fixes
* refactor code and add test cases
* add svcWaiting check while setting svc value

---------

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/api.go

@@ -63,19 +63,10 @@ func (executor *Executor) getServiceForFunctionAPI(w http.ResponseWriter, r *htt
 		zap.String("function_name", fn.ObjectMeta.Name),
 		zap.String("function_namespace", fn.ObjectMeta.Namespace))
 	if t == fv1.ExecutorTypePoolmgr && !fn.Spec.OnceOnly {
-		concurrency := fn.Spec.Concurrency
-		if concurrency == 0 {
-			concurrency = 500
-		}
-		requestsPerpod := fn.Spec.RequestsPerPod
-		if requestsPerpod == 0 {
-			requestsPerpod = 1
-		}
-		fsvc, active, err := et.GetFuncSvcFromPoolCache(ctx, fn, requestsPerpod)
+		fsvc, err := et.GetFuncSvcFromCache(ctx, fn)
 		// check if its a cache hit (check if there is already specialized function pod that can serve another request)
 		if err == nil {
 			// if a pod is already serving request then it already exists else validated
-			logger.Debug("from cache", zap.Int("active", active))
 			if et.IsValid(ctx, fsvc) {
 				// Cached, return svc address
 				logger.Debug("served from cache", zap.String("name", fsvc.Name), zap.String("address", fsvc.Address))
@@ -87,15 +78,19 @@ func (executor *Executor) getServiceForFunctionAPI(w http.ResponseWriter, r *htt
 				zap.String("function_namespace", fn.ObjectMeta.Namespace),
 				zap.String("address", fsvc.Address))
 			et.DeleteFuncSvcFromCache(ctx, fsvc)
-			active--
+		} else {
+			code, msg := ferror.GetHTTPError(err)
+			if code == http.StatusNotFound {
+				logger.Debug("cache miss", zap.String("function_name", fn.ObjectMeta.Name))
+			} else {
+				logger.Error("error getting service for function",
+					zap.Error(err),
+					zap.String("function_name", fn.ObjectMeta.Name))
+				http.Error(w, msg, code)
+				return
+			}
 		}
 
-		if active >= concurrency {
-			errMsg := fmt.Sprintf("max concurrency reached for %v. All %v instance are active", fn.ObjectMeta.Name, concurrency)
-			logger.Error("error occurred", zap.String("error", errMsg))
-			http.Error(w, html.EscapeString(errMsg), http.StatusTooManyRequests)
-			return
-		}
 	} else if t == fv1.ExecutorTypeNewdeploy || t == fv1.ExecutorTypeContainer {
 		fsvc, err := et.GetFuncSvcFromCache(ctx, fn)
 		if err == nil {