Executor: Dump function service cache for pool manager functions (#2789)

* dump function service cache for executor
* fix lint issue
* code refactor and lint fixes

---------

Signed-off-by: Shubham Bansal <shubhambansaliimtgn@gmail.com>
Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/fscache/functionServiceCache.go

@@ -35,6 +35,7 @@ import (
 	"github.com/fission/fission/pkg/crd"
 	ferror "github.com/fission/fission/pkg/error"
 	"github.com/fission/fission/pkg/executor/metrics"
+	"github.com/fission/fission/pkg/executor/util"
 )
 
 type fscRequestType int
@@ -170,6 +171,27 @@ func (fsc *FunctionServiceCache) service() {
 	}
 }
 
+// DumpDebugInfo => dump function service cache data to temporary directory of executor pod.
+func (fsc *FunctionServiceCache) DumpDebugInfo(ctx context.Context) error {
+	fsc.logger.Info("dumping function service")
+
+	file, err := util.CreateDumpFile(fsc.logger)
+	if err != nil {
+		fsc.logger.Error("error while creating file/dir", zap.String("error", err.Error()))
+		return err
+	}
+	defer file.Close()
+
+	err = fsc.connFunctionCache.LogFnSvcGroup(ctx, file)
+	if err != nil {
+		fsc.logger.Error("error while logging function service group", zap.String("error", err.Error()))
+		return err
+	}
+
+	fsc.logger.Info("dumped function service")
+	return nil
+}
+
 // GetByFunction gets a function service from cache using function key.
 func (fsc *FunctionServiceCache) GetByFunction(m *metav1.ObjectMeta) (*FuncSvc, error) {
 	key := crd.CacheKey(m)

pkg/executor/fscache/poolcache.go

@@ -17,8 +17,10 @@ limitations under the License.
 package fscache
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -37,6 +39,7 @@ const (
 	deleteValue
 	setCPUUtilization
 	markSpecializationFailure
+	logFuncSvc
 )
 
 type (
@@ -66,6 +69,7 @@ type (
 		ctx             context.Context
 		function        string
 		address         string
+		dumpWriter      io.Writer
 		value           *FuncSvc
 		requestsPerPod  int
 		cpuUsage        resource.Quantity
@@ -244,6 +248,48 @@ func (c *PoolCache) service() {
 		case deleteValue:
 			delete(c.cache[req.function].svcs, req.address)
 			req.responseChannel <- resp
+		case logFuncSvc:
+			datawriter := bufio.NewWriter(req.dumpWriter)
+
+			writefnSvcGrp := func(svcGrp *funcSvcGroup) error {
+				_, err := datawriter.WriteString(fmt.Sprintf("svc_waiting:%d\tqueue_len:%d", svcGrp.svcWaiting, svcGrp.queue.Len()))
+				if err != nil {
+					return err
+				}
+
+				if len(svcGrp.svcs) == 0 {
+					_, err := datawriter.WriteString("\n")
+					if err != nil {
+						return err
+					}
+				}
+
+				for addr, fnSvc := range svcGrp.svcs {
+					_, err := datawriter.WriteString(fmt.Sprintf("\tfunction_name:%s\tfn_svc_address:%s\tactive_req:%d\tcurrent_cpu_usage:%v\tcpu_limit:%v\n",
+						fnSvc.val.Function.Name, addr, fnSvc.activeRequests, fnSvc.currentCPUUsage, fnSvc.cpuLimit))
+					if err != nil {
+						return err
+					}
+				}
+				return nil
+			}
+
+			for _, fnSvcGrp := range c.cache {
+				err := writefnSvcGrp(fnSvcGrp)
+				if err != nil {
+					resp.error = err
+					break
+				}
+			}
+			err := datawriter.Flush()
+			if err != nil {
+				if resp.error == nil {
+					resp.error = err
+				} else {
+					resp.error = fmt.Errorf("%v, %v", resp.error, err)
+				}
+			}
+			req.responseChannel <- resp
 		default:
 			resp.error = ferror.MakeError(ferror.ErrorInvalidArgument,
 				fmt.Sprintf("invalid request type: %v", req.requestType))
@@ -346,3 +392,14 @@ func (c *PoolCache) MarkSpecializationFailure(function string) {
 		responseChannel: make(chan *response),
 	}
 }
+
+func (c *PoolCache) LogFnSvcGroup(ctx context.Context, file io.Writer) error {
+	respChannel := make(chan *response)
+	c.requestChannel <- &request{
+		requestType:     logFuncSvc,
+		dumpWriter:      file,
+		responseChannel: respChannel,
+	}
+	resp := <-respChannel
+	return resp.error
+}