Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add informers and internal go routines in manager (#2870)

Browse Source 
* used manager's Add function in more places
* exit when ctx.Done is received in archivePruner go routines
* fix manager tests
* fix data race
* added more gpm function in manager and removed manager from a util function
* closed unused channel and stopped ticker after context is done
* added log statements
* used context.Done inside function instead of stopper channel
This commit is contained in:
Vardhaman Surana
2023-11-10 12:42:21 +05:30
committed by GitHub
parent 2a40b4538c
commit 3fabf64b3c
28 changed files with 195 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/executor.go
+20 -11
View File
@@ -74,7 +74,7 @@ type (
)
// MakeExecutor returns an Executor for given ExecutorType(s).
func MakeExecutor(ctx context.Context, logger *zap.Logger, cms *cms.ConfigSecretController,
func MakeExecutor(ctx context.Context, logger *zap.Logger, mgr manager.Interface, cms *cms.ConfigSecretController,
fissionClient versioned.Interface, types map[fv1.ExecutorType]executortype.ExecutorType,
informers ...k8sCache.SharedIndexInformer) (*Executor, error) {
executor := &Executor{
@@ -88,17 +88,21 @@ func MakeExecutor(ctx context.Context, logger *zap.Logger, cms *cms.ConfigSecret
// Run all informers
for _, informer := range informers {
go informer.Run(ctx.Done())
informer := informer
mgr.Add(ctx, func(ctx context.Context) {
informer.Run(ctx.Done())
})
}
for _, et := range types {
go func(et executortype.ExecutorType) {
et.Run(ctx)
}(et)
et := et
mgr.Add(ctx, func(ctx context.Context) {
et.Run(ctx, mgr)
})
}
go executor.serveCreateFuncServices()
mgr.Add(ctx, func(ctx context.Context) {
executor.serveCreateFuncServices(ctx)
})
return executor, nil
}
@@ -108,9 +112,14 @@ func MakeExecutor(ctx context.Context, logger *zap.Logger, cms *cms.ConfigSecret
// get specialized. In other words, it ensures that when there's an
// ongoing request for a certain function, all other requests wait for
// that request to complete.
func (executor *Executor) serveCreateFuncServices() {
func (executor *Executor) serveCreateFuncServices(ctx context.Context) {
for {
req := <-executor.requestChan
var req *createFuncServiceRequest
select {
case <-ctx.Done():
return
case req = <-executor.requestChan:
}
function := req.function
fnName := k8sCache.MetaObjectToName(function)
fnkeyUR := crd.CacheKeyURFromObject(function)
@@ -384,7 +393,7 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
informerFactory.Start(ctx.Done())
}
api, err := MakeExecutor(ctx, logger, cms, fissionClient, executorTypes,
api, err := MakeExecutor(ctx, logger, mgr, cms, fissionClient, executorTypes,
fissionInformers...,
)
if err != nil {
pkg/executor/executortype/container/containermgr.go
+5 -2
View File
@@ -51,6 +51,7 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
"github.com/fission/fission/pkg/utils/maps"
otelUtils "github.com/fission/fission/pkg/utils/otel"
)
@@ -148,7 +149,7 @@ func MakeContainer(
}
// Run start the function along with an object reaper.
func (caaf *Container) Run(ctx context.Context) {
func (caaf *Container) Run(ctx context.Context, mgr manager.Interface) {
waitSynced := make([]k8sCache.InformerSynced, 0)
for _, deplListerSynced := range caaf.deplListerSynced {
waitSynced = append(waitSynced, deplListerSynced)
@@ -160,7 +161,9 @@ func (caaf *Container) Run(ctx context.Context) {
if ok := k8sCache.WaitForCacheSync(ctx.Done(), waitSynced...); !ok {
caaf.logger.Fatal("failed to wait for caches to sync")
}
go caaf.idleObjectReaper(ctx)
mgr.Add(ctx, func(ctx context.Context) {
caaf.idleObjectReaper(ctx)
})
}
// GetTypeName returns the executor type name.
pkg/executor/executortype/executortype.go
+2 -1
View File
@@ -25,11 +25,12 @@ import (
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/executor/fscache"
"github.com/fission/fission/pkg/utils/manager"
)
type ExecutorType interface {
// Run runs background job.
Run(context.Context)
Run(context.Context, manager.Interface)
// GetTypeName returns the name of executor type
GetTypeName(context.Context) fv1.ExecutorType
pkg/executor/executortype/newdeploy/newdeploymgr.go
+5 -2
View File
@@ -53,6 +53,7 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
"github.com/fission/fission/pkg/utils/maps"
otelUtils "github.com/fission/fission/pkg/utils/otel"
)
@@ -162,7 +163,7 @@ func MakeNewDeploy(
}
// Run start the function and environment controller along with an object reaper.
func (deploy *NewDeploy) Run(ctx context.Context) {
func (deploy *NewDeploy) Run(ctx context.Context, mgr manager.Interface) {
waitSynced := make([]k8sCache.InformerSynced, 0)
for _, deplListerSynced := range deploy.deplListerSynced {
waitSynced = append(waitSynced, deplListerSynced)
@@ -174,7 +175,9 @@ func (deploy *NewDeploy) Run(ctx context.Context) {
if ok := k8sCache.WaitForCacheSync(ctx.Done(), waitSynced...); !ok {
deploy.logger.Fatal("failed to wait for caches to sync")
}
go deploy.idleObjectReaper(ctx)
mgr.Add(ctx, func(ctx context.Context) {
deploy.idleObjectReaper(ctx)
})
}
// GetTypeName returns the executor type name.
pkg/executor/executortype/newdeploy/newdeploymgr_test.go
+6 -1
View File
@@ -21,6 +21,7 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/loggerfactory"
"github.com/fission/fission/pkg/utils/manager"
"github.com/fission/fission/pkg/utils/uuid"
)
@@ -35,6 +36,8 @@ const (
func TestRefreshFuncPods(t *testing.T) {
os.Setenv("DEBUG_ENV", "true")
mgr := manager.New()
defer mgr.Wait()
logger := loggerfactory.GetLogger()
kubernetesClient := fake.NewSimpleClientset()
fissionClient := fClient.NewSimpleClientset()
@@ -70,7 +73,9 @@ func TestRefreshFuncPods(t *testing.T) {
}
ndm.nsResolver = &nsResolver
go ndm.Run(ctx)
mgr.Add(ctx, func(ctx context.Context) {
ndm.Run(ctx, mgr)
})
t.Log("New deploy manager started")
for _, f := range factory {
pkg/executor/executortype/poolmgr/gpm.go
+23 -13
View File
@@ -53,6 +53,7 @@ import (
"github.com/fission/fission/pkg/generated/clientset/versioned"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
otelUtils "github.com/fission/fission/pkg/utils/otel"
)
@@ -169,7 +170,7 @@ func MakeGenericPoolManager(ctx context.Context,
return gpm, nil
}
func (gpm *GenericPoolManager) Run(ctx context.Context) {
func (gpm *GenericPoolManager) Run(ctx context.Context, mgr manager.Interface) {
waitSynced := make([]k8sCache.InformerSynced, 0)
for _, podListerSynced := range gpm.podListerSynced {
waitSynced = append(waitSynced, podListerSynced)
@@ -179,10 +180,25 @@ func (gpm *GenericPoolManager) Run(ctx context.Context) {
}
go gpm.service()
gpm.poolPodC.InjectGpm(gpm)
go gpm.WebsocketStartEventChecker(ctx, gpm.kubernetesClient) //nolint:errcheck
go gpm.NoActiveConnectionEventChecker(ctx, gpm.kubernetesClient) //nolint:errcheck
go gpm.idleObjectReaper(ctx)
go gpm.poolPodC.Run(ctx, ctx.Done())
mgr.Add(ctx, func(ctx context.Context) {
err := gpm.WebsocketStartEventChecker(ctx, gpm.kubernetesClient)
if err != nil {
gpm.logger.Error("error in checking websocket start event from pod: ", zap.Error(err))
}
})
mgr.Add(ctx, func(ctx context.Context) {
err := gpm.NoActiveConnectionEventChecker(ctx, gpm.kubernetesClient) //nolint:errcheck
if err != nil {
gpm.logger.Error("error in checking inactive event from pod: ", zap.Error(err))
}
})
mgr.Add(ctx, func(ctx context.Context) {
gpm.idleObjectReaper(ctx)
})
mgr.Add(ctx, func(ctx context.Context) {
gpm.poolPodC.Run(ctx, ctx.Done(), mgr)
})
}
func (gpm *GenericPoolManager) GetTypeName(ctx context.Context) fv1.ExecutorType {
@@ -702,9 +718,6 @@ func (gpm *GenericPoolManager) doIdleObjectReaper(ctx context.Context) {
// WebsocketStartEventChecker checks if the pod has emitted a websocket connection start event
func (gpm *GenericPoolManager) WebsocketStartEventChecker(ctx context.Context, kubeClient kubernetes.Interface) error {
stopper := make(chan struct{})
defer close(stopper)
var wg wait.Group
for _, informer := range utils.GetInformerEventChecker(ctx, kubeClient, "WsConnectionStarted") {
_, err := informer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
@@ -727,7 +740,7 @@ func (gpm *GenericPoolManager) WebsocketStartEventChecker(ctx context.Context, k
if err != nil {
return err
}
wg.StartWithChannel(stopper, informer.Run)
wg.StartWithChannel(ctx.Done(), informer.Run)
}
wg.Wait()
return nil
@@ -735,9 +748,6 @@ func (gpm *GenericPoolManager) WebsocketStartEventChecker(ctx context.Context, k
// NoActiveConnectionEventChecker checks if the pod has emitted an inactive event
func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(ctx context.Context, kubeClient kubernetes.Interface) error {
stopper := make(chan struct{})
defer close(stopper)
var wg wait.Group
for _, informer := range utils.GetInformerEventChecker(ctx, kubeClient, "NoActiveConnections") {
_, err := informer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
@@ -771,7 +781,7 @@ func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(ctx context.Contex
if err != nil {
return err
}
wg.StartWithChannel(stopper, informer.Run)
wg.StartWithChannel(ctx.Done(), informer.Run)
}
wg.Wait()
return nil
pkg/executor/executortype/poolmgr/poolpodcontroller.go
+11 -4
View File
@@ -41,6 +41,7 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
flisterv1 "github.com/fission/fission/pkg/generated/listers/core/v1"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
)
type (
@@ -246,7 +247,7 @@ func (p *PoolPodController) enqueueEnvDelete(obj interface{}) {
p.envDeleteQueue.Add(env)
}
func (p *PoolPodController) Run(ctx context.Context, stopCh <-chan struct{}) {
func (p *PoolPodController) Run(ctx context.Context, stopCh <-chan struct{}, mgr manager.Interface) {
defer utilruntime.HandleCrash()
defer p.envCreateUpdateQueue.ShutDown()
defer p.envDeleteQueue.ShutDown()
@@ -265,10 +266,16 @@ func (p *PoolPodController) Run(ctx context.Context, stopCh <-chan struct{}) {
p.logger.Fatal("failed to wait for caches to sync")
}
for i := 0; i < 4; i++ {
go wait.Until(p.workerRun(ctx, "envCreateUpdate", p.envCreateUpdateQueueProcessFunc), time.Second, stopCh)
mgr.Add(ctx, func(ctx context.Context) {
wait.Until(p.workerRun(ctx, "envCreateUpdate", p.envCreateUpdateQueueProcessFunc), time.Second, stopCh)
})
}
go wait.Until(p.workerRun(ctx, "envDeleteQueue", p.envDeleteQueueProcessFunc), time.Second, stopCh)
go wait.Until(p.workerRun(ctx, "spCleanupPodQueue", p.spCleanupPodQueueProcessFunc), time.Second, stopCh)
mgr.Add(ctx, func(ctx context.Context) {
wait.Until(p.workerRun(ctx, "envDeleteQueue", p.envDeleteQueueProcessFunc), time.Second, stopCh)
})
mgr.Add(ctx, func(ctx context.Context) {
wait.Until(p.workerRun(ctx, "spCleanupPodQueue", p.spCleanupPodQueueProcessFunc), time.Second, stopCh)
})
p.logger.Info("Started workers for poolPodController")
<-stopCh
p.logger.Info("Shutting down workers for poolPodController")
pkg/executor/executortype/poolmgr/poolpodcontroller_test.go
+4 -1
View File
@@ -34,9 +34,12 @@ import (
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/loggerfactory"
"github.com/fission/fission/pkg/utils/manager"
)
func TestPoolPodControllerPodCleanup(t *testing.T) {
mgr := manager.New()
defer mgr.Wait()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logger := loggerfactory.GetLogger()
@@ -74,7 +77,7 @@ func TestPoolPodControllerPodCleanup(t *testing.T) {
gpm := executor.(*GenericPoolManager)
ppc.InjectGpm(gpm)
go ppc.Run(ctx, ctx.Done())
go ppc.Run(ctx, ctx.Done(), mgr)
for _, f := range factory {
f.Start(ctx.Done())