added manger to keep track of go routines in the services (#2869)

- added manager to wait for all go routines to end before exit
- code refactor
- renamed Manafer to Interface and GoRoutineManager to GroupManager
- replaced some go routine calls with manager Add func
- added unit tests for manager

pkg/buildermgr/buildermgr.go

@@ -29,10 +29,11 @@ import (
 	"github.com/fission/fission/pkg/executor/util"
 	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/manager"
 )
 
 // Start the buildermgr service.
-func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, storageSvcUrl string) error {
+func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, mgr manager.Interface, storageSvcUrl string) error {
 	bmLogger := logger.Named("builder_manager")
 
 	fissionClient, err := clientGen.GetFissionClient()
@@ -69,7 +70,7 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
 		kubernetesClient, storageSvcUrl,
 		utils.GetK8sInformersForNamespaces(kubernetesClient, time.Minute*30, fv1.Pods),
 		utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.PackagesResource))
-	err = pkgWatcher.Run(ctx)
+	err = pkgWatcher.Run(ctx, mgr)
 	if err != nil {
 		return err
 	}

pkg/buildermgr/pkgwatcher.go

@@ -32,6 +32,7 @@ import (
 	"github.com/fission/fission/pkg/cache"
 	"github.com/fission/fission/pkg/generated/clientset/versioned"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 )
 
@@ -308,8 +309,12 @@ func (pkgw *packageWatcher) packageInformerHandler(ctx context.Context) k8sCache
 	}
 }
 
-func (pkgw *packageWatcher) Run(ctx context.Context) error {
-	go metrics.ServeMetrics(ctx, "buildermgr", pkgw.logger)
+func (pkgw *packageWatcher) Run(ctx context.Context, mgr manager.Interface) error {
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		metrics.ServeMetrics(ctx, "buildermgr", pkgw.logger, mgr)
+	})
+
 	for _, podInformer := range pkgw.podInformer {
 		go podInformer.Run(ctx.Done())
 	}

pkg/executor/api.go

@@ -34,6 +34,7 @@ import (
 	"github.com/fission/fission/pkg/executor/client"
 	"github.com/fission/fission/pkg/executor/fscache"
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
@@ -286,8 +287,7 @@ func (executor *Executor) GetHandler() http.Handler {
 }
 
 // Serve starts an HTTP server.
-func (executor *Executor) Serve(ctx context.Context, port int) {
+func (executor *Executor) Serve(ctx context.Context, mgr manager.Interface, port int) {
 	handler := otelUtils.GetHandlerWithOTEL(executor.GetHandler(), "fission-executor", otelUtils.UrlsToIgnore("/healthz"))
-	httpserver.StartServer(ctx, executor.logger, "executor", fmt.Sprintf("%d", port), handler)
-
+	httpserver.StartServer(ctx, executor.logger, mgr, "executor", fmt.Sprintf("%d", port), handler)
 }

pkg/executor/executor.go

@@ -43,6 +43,7 @@ import (
 	"github.com/fission/fission/pkg/generated/clientset/versioned"
 	genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
@@ -251,7 +252,7 @@ func (executor *Executor) getFunctionServiceFromCache(ctx context.Context, fn *f
 
 // StartExecutor Starts executor and the executor components such as Poolmgr,
 // deploymgr and potential future executor types
-func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, port int) error {
+func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, mgr manager.Interface, port int) error {
 
 	fissionClient, err := clientGen.GetFissionClient()
 	if err != nil {
@@ -392,8 +393,13 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
 
 	utils.CreateMissingPermissionForSA(ctx, kubernetesClient, logger)
 
-	go metrics.ServeMetrics(ctx, "executor", logger)
-	go api.Serve(ctx, port)
+	mgr.Add(ctx, func(ctx context.Context) {
+		metrics.ServeMetrics(ctx, "executor", logger, mgr)
+	})
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		api.Serve(ctx, mgr, port)
+	})
 
 	return nil
 }

pkg/mqtrigger/mqtmanager.go

@@ -28,6 +28,7 @@ import (
 	"github.com/fission/fission/pkg/generated/clientset/versioned"
 	"github.com/fission/fission/pkg/mqtrigger/messageQueue"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 )
 
@@ -78,7 +79,7 @@ func MakeMessageQueueTriggerManager(logger *zap.Logger,
 	return &mqTriggerMgr
 }
 
-func (mqt *MessageQueueTriggerManager) Run(ctx context.Context) error {
+func (mqt *MessageQueueTriggerManager) Run(ctx context.Context, mgr manager.Interface) error {
 	go mqt.service()
 	for _, informer := range utils.GetInformersForNamespaces(mqt.fissionClient, time.Minute*30, fv1.MessageQueueResource) {
 		_, err := informer.AddEventHandler(mqt.mqtInformerHandlers())
@@ -90,7 +91,9 @@ func (mqt *MessageQueueTriggerManager) Run(ctx context.Context) error {
 			mqt.logger.Fatal("failed to wait for caches to sync")
 		}
 	}
-	go metrics.ServeMetrics(ctx, "mqtrigger", mqt.logger)
+	mgr.Add(ctx, func(ctx context.Context) {
+		metrics.ServeMetrics(ctx, "mqtrigger", mqt.logger, mgr)
+	})
 	return nil
 }
 

pkg/router/auth_test.go

@@ -18,6 +18,7 @@ import (
 	config "github.com/fission/fission/pkg/featureconfig"
 	"github.com/fission/fission/pkg/utils/httpserver"
 	"github.com/fission/fission/pkg/utils/loggerfactory"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 )
 
@@ -59,6 +60,10 @@ func GetRouterWithAuth() *mux.Router {
 }
 
 func TestRouterAuth(t *testing.T) {
+
+	mgr := manager.New()
+	defer mgr.Wait()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	teardown := setup(t)
@@ -66,7 +71,9 @@ func TestRouterAuth(t *testing.T) {
 	logger := loggerfactory.GetLogger()
 	testmux := GetRouterWithAuth()
 
-	go httpserver.StartServer(ctx, logger, "test", "8990", testmux)
+	mgr.Add(ctx, func(ctx context.Context) {
+		httpserver.StartServer(ctx, logger, mgr, "test", "8990", testmux)
+	})
 
 	postBody, _ := json.Marshal(map[string]string{
 		"username": "Foo",

pkg/router/mutablemux_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 )
 
@@ -67,6 +68,9 @@ func spamServer(quit chan bool) {
 }
 
 func TestMutableMux(t *testing.T) {
+	mgr := manager.New()
+	defer mgr.Wait()
+
 	// make a simple mutable router
 	log.Print("Create mutable router")
 	muxRouter := mux.NewRouter()
@@ -80,13 +84,19 @@ func TestMutableMux(t *testing.T) {
 	mr := newMutableRouter(logger, muxRouter)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+
 	// start http server
-	go httpserver.StartServer(ctx, logger, "router", "3333", mr)
+	mgr.Add(ctx, func(ctx context.Context) {
+		httpserver.StartServer(ctx, logger, mgr, "router", "3333", mr)
+	})
 
 	// continuously make requests, panic if any fails
 	time.Sleep(100 * time.Millisecond)
 	q := make(chan bool)
-	go spamServer(q)
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		spamServer(q)
+	})
 
 	time.Sleep(5 * time.Millisecond)
 

pkg/router/router.go

@@ -55,6 +55,7 @@ import (
 	eclient "github.com/fission/fission/pkg/executor/client"
 	"github.com/fission/fission/pkg/throttler"
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
@@ -86,19 +87,21 @@ func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTrigger
 	return mr, nil
 }
 
-func serve(ctx context.Context, logger *zap.Logger, port int,
+func serve(ctx context.Context, logger *zap.Logger, mgr manager.Interface, port int,
 	httpTriggerSet *HTTPTriggerSet, displayAccessLog bool) error {
 	mr, err := router(ctx, logger, httpTriggerSet)
 	if err != nil {
 		return errors.Wrap(err, "error making router")
 	}
 	handler := otelUtils.GetHandlerWithOTEL(mr, "fission-router", otelUtils.UrlsToIgnore("/router-healthz"))
-	go httpserver.StartServer(ctx, logger, "router", fmt.Sprintf("%d", port), handler)
+	mgr.Add(ctx, func(ctx context.Context) {
+		httpserver.StartServer(ctx, logger, mgr, "router", fmt.Sprintf("%d", port), handler)
+	})
 	return nil
 }
 
 // Start starts a router
-func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, port int, executor eclient.ClientInterface) error {
+func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, mgr manager.Interface, port int, executor eclient.ClientInterface) error {
 	fmap := makeFunctionServiceMap(logger, time.Minute)
 
 	fissionClient, err := clientGen.GetFissionClient()
@@ -206,7 +209,10 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
 	if err != nil {
 		return errors.Wrap(err, "error making HTTP trigger set")
 	}
-	go metrics.ServeMetrics(ctx, "router", logger)
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		metrics.ServeMetrics(ctx, "router", logger, mgr)
+	})
 
 	logger.Info("starting router", zap.Int("port", port))
 
@@ -214,5 +220,5 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
 	ctx, span := tracer.Start(ctx, "router/Start")
 	defer span.End()
 
-	return serve(ctx, logger, port, triggers, displayAccessLog)
+	return serve(ctx, logger, mgr, port, triggers, displayAccessLog)
 }

pkg/storagesvc/client/storagesvc_test.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/fission/fission/pkg/crd"
 	"github.com/fission/fission/pkg/storagesvc"
+	"github.com/fission/fission/pkg/utils/manager"
 )
 
 const (
@@ -86,6 +87,9 @@ func TestS3StorageService(t *testing.T) {
 	fmt.Println("Test S3 Storage service")
 	var minioClient *minio.Client
 
+	mgr := manager.New()
+	defer mgr.Wait()
+
 	// Start minio docker container
 	pool, err := dockertest.NewPool("")
 	resource := runMinioDockerContainer(pool)
@@ -139,7 +143,7 @@ func TestS3StorageService(t *testing.T) {
 	storage := storagesvc.NewS3Storage()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	_ = storagesvc.Start(ctx, crd.NewClientGenerator(), logger, storage, port)
+	_ = storagesvc.Start(ctx, crd.NewClientGenerator(), logger, storage, mgr, port)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
@@ -205,6 +209,9 @@ func TestLocalStorageService(t *testing.T) {
 	testID := uniuri.NewLen(8)
 	port := 8082
 
+	mgr := manager.New()
+	defer mgr.Wait()
+
 	config := zap.NewDevelopmentConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	logger, err := config.Build()
@@ -217,7 +224,7 @@ func TestLocalStorageService(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	os.Setenv("METRICS_ADDR", "8083")
-	_ = storagesvc.Start(ctx, crd.NewClientGenerator(), logger, storage, port)
+	_ = storagesvc.Start(ctx, crd.NewClientGenerator(), logger, storage, mgr, port)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))

pkg/storagesvc/storagesvc.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/fission/fission/pkg/crd"
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 	"github.com/fission/fission/pkg/utils/metrics"
 	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
@@ -267,7 +268,7 @@ func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int)
 	}
 }
 
-func (ss *StorageService) Start(ctx context.Context, port int) {
+func (ss *StorageService) Start(ctx context.Context, mgr manager.Interface, port int) {
 	r := mux.NewRouter()
 	r.Use(metrics.HTTPMetricMiddleware)
 	r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
@@ -278,11 +279,11 @@ func (ss *StorageService) Start(ctx context.Context, port int) {
 	r.HandleFunc("/healthz", ss.healthHandler).Methods("GET")
 
 	handler := otelUtils.GetHandlerWithOTEL(r, "fission-storagesvc", otelUtils.UrlsToIgnore("/healthz"))
-	httpserver.StartServer(ctx, ss.logger, "storagesvc", fmt.Sprintf("%d", port), handler)
+	httpserver.StartServer(ctx, ss.logger, mgr, "storagesvc", fmt.Sprintf("%d", port), handler)
 }
 
 // Start runs storage service
-func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, storage Storage, port int) error {
+func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *zap.Logger, storage Storage, mgr manager.Interface, port int) error {
 	enablePruner, err := strconv.ParseBool(os.Getenv("PRUNE_ENABLED"))
 	if err != nil {
 		logger.Warn("PRUNE_ENABLED value not set. Enabling archive pruner by default.", zap.Error(err))
@@ -296,8 +297,13 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
 
 	// create http handlers
 	storageService := MakeStorageService(logger, storageClient, port)
-	go metrics.ServeMetrics(ctx, "storagesvc", logger)
-	go storageService.Start(ctx, port)
+	mgr.Add(ctx, func(ctx context.Context) {
+		metrics.ServeMetrics(ctx, "storagesvc", logger, mgr)
+	})
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		storageService.Start(ctx, mgr, port)
+	})
 
 	// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
 	if enablePruner {

pkg/utils/httpserver/server.go

@@ -6,10 +6,11 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/fission/fission/pkg/utils/manager"
 	"go.uber.org/zap"
 )
 
-func StartServer(ctx context.Context, log *zap.Logger, svc string, port string, handler http.Handler) {
+func StartServer(ctx context.Context, log *zap.Logger, mgr manager.Interface, svc string, port string, handler http.Handler) {
 	if !strings.Contains(port, ":") {
 		port = fmt.Sprintf(":%s", port)
 	}
@@ -19,13 +20,13 @@ func StartServer(ctx context.Context, log *zap.Logger, svc string, port string,
 	}
 	l := log.With(zap.String("service", svc), zap.String("addr", server.Addr))
 	l.Info("starting server")
-	go func() {
+	mgr.Add(ctx, func(ctx context.Context) {
 		if err := server.ListenAndServe(); err != nil {
 			if err != http.ErrServerClosed {
 				l.Error("server error", zap.Error(err))
 			}
 		}
-	}()
+	})
 	<-ctx.Done()
 	l.Info("shutting down server")
 	if err := server.Shutdown(ctx); err != nil {

pkg/utils/httpserver/server_test.go

@@ -12,9 +12,13 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/fission/fission/pkg/utils/loggerfactory"
+	"github.com/fission/fission/pkg/utils/manager"
 )
 
 func TestStartServer(t *testing.T) {
+	mgr := manager.New()
+	defer mgr.Wait()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	logger := loggerfactory.GetLogger()
@@ -26,7 +30,10 @@ func TestStartServer(t *testing.T) {
 			logger.Error("failed to write response", zap.Error(err))
 		}
 	}))
-	go StartServer(ctx, logger, "test", "8999", m)
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		StartServer(ctx, logger, mgr, "test", "8999", m)
+	})
 
 	tests := []struct {
 		Name       string

pkg/utils/manager/manager.go

@@ -0,0 +1,62 @@
+package manager
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+var _ Interface = &GroupManager{}
+
+// Interface keeps track of the go routines in the system and can be used to gracefully shutdown the
+// the system by waiting for completion of go routines added to it.
+type Interface interface {
+	// Add will start a go routine for the given "function" and adds it to the list of go routines
+	// and will also remove the "function" from the list when it completes
+	Add(ctx context.Context, function func(context.Context))
+
+	// Wait blocks the execution of the process until all the go routines in the manager are completed.
+	Wait()
+
+	// WaitWithTimeout blocks the execution of the process until timeout or till all all the go routines in the manager are completed
+	WaitWithTimeout(timeout time.Duration) error
+}
+
+type GroupManager struct {
+	wg sync.WaitGroup
+}
+
+func New() Interface {
+	return &GroupManager{
+		wg: sync.WaitGroup{},
+	}
+}
+func (g *GroupManager) Add(ctx context.Context, f func(context.Context)) {
+	g.wg.Add(1)
+	go func() {
+		defer g.wg.Done()
+		f(ctx)
+	}()
+}
+
+func (g *GroupManager) Wait() {
+	g.wg.Wait()
+}
+
+func (g *GroupManager) WaitWithTimeout(timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	done := make(chan struct{})
+	go func() {
+		g.wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-done:
+		return nil
+	}
+}

pkg/utils/manager/manager_test.go

@@ -0,0 +1,65 @@
+package manager
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddAndWait(t *testing.T) {
+	mgr := New()
+
+	value := 10
+	expectedValue := 11
+
+	mgr.Add(context.Background(), func(ctx context.Context) {
+		time.Sleep(1 * time.Second)
+		value = expectedValue
+	})
+
+	mgr.Wait()
+	require.Equal(t, expectedValue, value, "manager did not wait for go routine to complete")
+}
+
+func TestAddWithContextCancel(t *testing.T) {
+	mgr := New()
+
+	value := 10
+	expectedValue := 11
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		<-ctx.Done()
+		value = 11
+	})
+
+	go cancel()
+	mgr.Wait()
+	require.Equal(t, expectedValue, value, "manager did not wait for go routine to complete when context is cancelled")
+}
+
+func TestAddAndWaitWithTimeout(t *testing.T) {
+	mgr := New()
+
+	value := 10
+
+	mgr.Add(context.Background(), func(ctx context.Context) {
+		time.Sleep(2 * time.Second)
+	})
+
+	err := mgr.WaitWithTimeout(1 * time.Second)
+	require.NotNil(t, err, "manager WaitWithTimeout did not return an error when timeout exceeded")
+
+	expectedValue := 11
+	mgr.Add(context.Background(), func(ctx context.Context) {
+		time.Sleep(100 * time.Millisecond)
+		value = expectedValue
+	})
+
+	err = mgr.WaitWithTimeout(1 * time.Second)
+	require.Nil(t, err, "manager returned error even though all go routins completed successfully before timeout")
+	require.Equal(t, expectedValue, value)
+}

pkg/utils/metrics/server.go

@@ -26,9 +26,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 )
 
-func ServeMetrics(ctx context.Context, parent string, logger *zap.Logger) {
+func ServeMetrics(ctx context.Context, parent string, logger *zap.Logger, mgr manager.Interface) {
 	metricsAddr := os.Getenv("METRICS_ADDR")
 	if metricsAddr == "" {
 		metricsAddr = "8080"
@@ -45,5 +46,5 @@ func ServeMetrics(ctx context.Context, parent string, logger *zap.Logger) {
 			EnableOpenMetrics: true,
 		},
 	))
-	httpserver.StartServer(ctx, logger, parent+"/metrics", metricsAddr, mux)
+	httpserver.StartServer(ctx, logger, mgr, parent+"/metrics", metricsAddr, mux)
 }

pkg/utils/profile/profile.go

@@ -32,9 +32,10 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/fission/fission/pkg/utils/httpserver"
+	"github.com/fission/fission/pkg/utils/manager"
 )
 
-func ProfileIfEnabled(ctx context.Context, logger *zap.Logger) {
+func ProfileIfEnabled(ctx context.Context, logger *zap.Logger, mgr manager.Interface) {
 	enablePprof := os.Getenv("PPROF_ENABLED")
 	if enablePprof != "true" {
 		return
@@ -47,5 +48,7 @@ func ProfileIfEnabled(ctx context.Context, logger *zap.Logger) {
 	pprofMux := http.DefaultServeMux
 	http.DefaultServeMux = http.NewServeMux()
 
-	go httpserver.StartServer(ctx, logger, "pprof", pprofPort, pprofMux)
+	mgr.Add(ctx, func(ctx context.Context) {
+		httpserver.StartServer(ctx, logger, mgr, "pprof", pprofPort, pprofMux)
+	})
 }