Ensure handling for specialization failure in pool manager (#2788)

* Add fixes for failure in specialization
* reduce specialization in progress and remove expired requests from queue when specialization is timed out
* rename markSpecializationFailure and remove logger from the queue
* refactor clean up code in api.go and add test case for queue

Details:

- Cleanup svc waiting for the counter in the pool manager if specialization fails
- Cleanup active requests counter in pool manager if client exists the demand for function service while we have allocated function service
- Consider specialization timeout if pod ready timeout > specialization timeout in waiting for ready pod. We also consider if the request to choosePod is cancelled.
- We ensure if we have requests waiting for service requests but if there is no pod in the specialization we clean up those.
---------

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Pranoy Kundu <pranoy1998k@gmail.com>

pkg/executor/fscache/functionServiceCache.go

@@ -244,6 +244,10 @@ func (fsc *FunctionServiceCache) MarkAvailable(key string, svcHost string) {
 	fsc.connFunctionCache.MarkAvailable(key, svcHost)
 }
 
+func (fsc *FunctionServiceCache) MarkSpecializationFailure(key string) {
+	fsc.connFunctionCache.MarkSpecializationFailure(key)
+}
+
 // Add adds a function service to cache if it does not exist already.
 func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
 	existing, err := fsc.byFunction.Set(crd.CacheKey(fsvc.Function), &fsvc)

pkg/executor/fscache/poolcache.go

@@ -36,6 +36,7 @@ const (
 	markAvailable
 	deleteValue
 	setCPUUtilization
+	markSpecializationFailure
 )
 
 type (
@@ -232,6 +233,14 @@ func (c *PoolCache) service() {
 					}
 				}
 			}
+		case markSpecializationFailure:
+			if c.cache[req.function].svcWaiting > c.cache[req.function].queue.Len() {
+				c.cache[req.function].svcWaiting--
+				if c.cache[req.function].svcWaiting == c.cache[req.function].queue.Len() {
+					expiredRequests := c.cache[req.function].queue.Expired()
+					c.cache[req.function].svcWaiting = c.cache[req.function].svcWaiting - expiredRequests
+				}
+			}
 		case deleteValue:
 			delete(c.cache[req.function].svcs, req.address)
 			req.responseChannel <- resp
@@ -328,3 +337,12 @@ func (c *PoolCache) DeleteValue(ctx context.Context, function, address string) e
 	resp := <-respChannel
 	return resp.error
 }
+
+// ReduceSpecializationInProgress reduces the svcWaiting count
+func (c *PoolCache) MarkSpecializationFailure(function string) {
+	c.requestChannel <- &request{
+		requestType:     markSpecializationFailure,
+		function:        function,
+		responseChannel: make(chan *response),
+	}
+}

pkg/executor/fscache/queue.go

@@ -38,6 +38,31 @@ func (q *Queue) Pop() *svcWait {
 	return svcWait
 }
 
+func (q *Queue) Expired() int {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	expired := 0
+	svcExpired := []*list.Element{}
+	for item := q.items.Front(); item != nil; item = item.Next() {
+		svcWait, ok := item.Value.(*svcWait)
+		if !ok {
+			continue
+		}
+		if svcWait.ctx.Err() != nil {
+			close(svcWait.svcChannel)
+			svcExpired = append(svcExpired, item)
+			expired = expired + 1
+		}
+	}
+
+	for _, item := range svcExpired {
+		q.items.Remove(item)
+	}
+
+	return expired
+}
+
 func (q *Queue) Len() int {
 	q.mutex.Lock()
 	defer q.mutex.Unlock()

pkg/executor/fscache/queue_test.go

@@ -1,6 +1,7 @@
 package fscache
 
 import (
+	"context"
 	"sync"
 	"testing"
 )
@@ -113,3 +114,78 @@ func TestQueueLen(t *testing.T) {
 		t.Errorf("Expected queue length to be 1, got %d", q.Len())
 	}
 }
+
+func TestExpiredWhenAllItemsExpired(t *testing.T) {
+	q := NewQueue()
+	if q.Expired() != 0 {
+		t.Errorf("Expected Expired to return 0, got %d", q.Expired())
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	item := &svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        ctx,
+	}
+	q.Push(item)
+	if q.Len() != 1 {
+		t.Errorf("Expected queue length to be 1, got %d", q.Len())
+	}
+	cancel()
+	if q.Expired() != 1 {
+		t.Errorf("Expected Expired to return 1, got %d", q.Expired())
+	}
+	if q.Len() != 0 {
+		t.Errorf("Expected queue length to be 0, got %d", q.Len())
+	}
+}
+
+func TestExpiredWhenFewItemsExpired(t *testing.T) {
+	q := NewQueue()
+	if q.Expired() != 0 {
+		t.Errorf("Expected Expired to return 0, got %d", q.Expired())
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+
+	q.Push(&svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        ctx,
+	})
+	q.Push(&svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        context.Background(),
+	})
+	if q.Len() != 2 {
+		t.Errorf("Expected queue length to be 1, got %d", q.Len())
+	}
+	cancel()
+	if q.Expired() != 1 {
+		t.Errorf("Expected Expired to return 1, got %d", q.Expired())
+	}
+	if q.Len() != 1 {
+		t.Errorf("Expected queue length to be 0, got %d", q.Len())
+	}
+}
+
+func TestExpiredWhenNoItemsExpired(t *testing.T) {
+	q := NewQueue()
+	if q.Expired() != 0 {
+		t.Errorf("Expected Expired to return 0, got %d", q.Expired())
+	}
+
+	q.Push(&svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        context.Background(),
+	})
+	q.Push(&svcWait{
+		svcChannel: make(chan *FuncSvc),
+		ctx:        context.Background(),
+	})
+	if q.Len() != 2 {
+		t.Errorf("Expected queue length to be 1, got %d", q.Len())
+	}
+	if q.Expired() != 0 {
+		t.Errorf("Expected Expired to return 1, got %d", q.Expired())
+	}
+	if q.Len() != 2 {
+		t.Errorf("Expected queue length to be 0, got %d", q.Len())
+	}
+}