Improve poolmanager concurrency handling with virtual capacity (#2737)

* add functionality to wait for specialization by keeping track of incoming requests
* format executor package
* fix required capacity to specialise new pod condition
* move handling concurrency logic into pool cache from executor
* remove unused methods and structs
* implement queue in to store the svc wait
* create a queue struct and its methods to handle concurrent inputs
* use newly created queue to store waiting for svc requests
* add waiting requests in queue and use them when a svc is ready
* set function to request in queue if the context is still alive
* remove concurrency approach to set svc for waiting requests
* update the active requests whenever requests from pool are assigned a svc
* add doc to define why the conditions exist
* remove unwanted params in strcut and clean up code
* set error while getting svc value if sum of specialization in progress and specialized is only more than concurrency limit
* remove duplicate functions and unnecessary values in struct
* close svc channel on set value and create constants for default concurrency and rpp
* get next value in queue in case context is timed out for fetched value
* remove specializationInProgress counter from pool cache
* return in case the queue is empty wihle setting func to svc
* test getSvcVaue and setSvcValue in poolcache
* add unit tests for GetConcurrent and GetRequestsPerPod methods
* reorder imports
* add fuzzy testing for getSVCValue and setSVCValue in poolcache
* restructure go mod file and update pool cache test cases
* Add tests and bug fixes
* refactor code and add test cases
* add svcWaiting check while setting svc value

---------

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/fscache/queue_test.go

@@ -0,0 +1,115 @@
+package fscache
+
+import (
+	"sync"
+	"testing"
+)
+
+func TestNewQueue(t *testing.T) {
+	q := NewQueue()
+	if q == nil {
+		t.Error("NewQueue returned nil")
+	}
+}
+
+func TestQueuePushWithSingleRequest(t *testing.T) {
+	q := NewQueue()
+	item := &svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        nil,
+	}
+	q.Push(item)
+	if q.Len() != 1 {
+		t.Errorf("Expected queue length to be 1, got %d", q.Len())
+	}
+}
+
+func TestQueuePopWithSingleRequest(t *testing.T) {
+	q := NewQueue()
+	item := &svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        nil,
+	}
+	q.Push(item)
+	popped := q.Pop()
+	if popped == nil {
+		t.Error("Expected Pop to return a non-nil value")
+	}
+	if popped != item {
+		t.Error("Expected Pop to return the same element that was pushed")
+	}
+	if q.Len() != 0 {
+		t.Errorf("Expected queue length to be 0, got %d", q.Len())
+	}
+}
+
+func TestQueuePushWithConcurrentRequest(t *testing.T) {
+	q := NewQueue()
+	noOfRequests := 20
+	var wg sync.WaitGroup
+	wg.Add(noOfRequests)
+	for i := 0; i < noOfRequests; i++ {
+		go func() {
+			defer wg.Done()
+			item := &svcWait{
+				svcChannel: make(chan *FuncSvc),
+				ctx:        nil,
+			}
+			q.Push(item)
+		}()
+	}
+
+	wg.Wait()
+
+	if q.Len() != noOfRequests {
+		t.Errorf("Expected queue length to be 20, got %d", q.Len())
+	}
+}
+
+func TestQueuePopWithConcurrentRequest(t *testing.T) {
+	q := NewQueue()
+	noOfPush := 20
+	noOfPop := 15
+
+	var wg sync.WaitGroup
+	wg.Add(noOfPush + noOfPop)
+
+	for i := 0; i < noOfPush; i++ {
+		go func() {
+			defer wg.Done()
+			item := &svcWait{
+				svcChannel: make(chan *FuncSvc),
+				ctx:        nil,
+			}
+			q.Push(item)
+		}()
+	}
+
+	for i := 0; i < noOfPop; i++ {
+		go func() {
+			defer wg.Done()
+			q.Pop()
+		}()
+	}
+
+	wg.Wait()
+
+	if q.Len() != 5 {
+		t.Errorf("Expected queue length to be 5, got %d", q.Len())
+	}
+}
+
+func TestQueueLen(t *testing.T) {
+	q := NewQueue()
+	if q.Len() != 0 {
+		t.Errorf("Expected queue length to be 0, got %d", q.Len())
+	}
+	item := &svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        nil,
+	}
+	q.Push(item)
+	if q.Len() != 1 {
+		t.Errorf("Expected queue length to be 1, got %d", q.Len())
+	}
+}