fix(namespace): executor/router/buildermgr RemoveNamespace + per-NS informer lifecycle
- Add RemoveNamespace(ctx, ns) to executortype.ExecutorType interface - Implement RemoveNamespace in poolmgr, newdeploy, container executor types - Add per-namespace context cancellation (nsCancels map) in all three types so informer factories are stopped when namespace is removed (fixes goroutine leak) - Add PoolPodController.RemoveNamespace to clear envLister/podLister maps - Add deregisterNamespace() in executor multitenant subscriber - Switch executor/router/buildermgr watcher strategy from TrackOnly to DispatchRemove so RemoveFunc is called when fission.io/managed label is removed - Add RemoveFunc to executor/router/buildermgr namespace subscribers - Add RemoveNamespace to environmentWatcher and packageWatcher with per-NS cancel - Add RemoveNamespace to HTTPTriggerSet: cancels informers, removes from maps, calls syncTriggers - Fix ns_watcher_test.go fakeExecutorType to implement new RemoveNamespace method Fixes: - Executor dedup gap: re-added namespace was silently skipped (envLister/deplLister still present) - Goroutine/FD leak: old informer factories ran forever after namespace removal - Router stale routes: HTTPTriggers for removed namespace stayed in routing table
This commit is contained in:
@@ -76,6 +76,8 @@ type (
|
||||
podSpecPatch *apiv1.PodSpec
|
||||
envWatchInformer map[string]k8sCache.SharedIndexInformer
|
||||
enableOwnerReferences bool
|
||||
// nsCancels holds per-namespace context cancel functions.
|
||||
nsCancels map[string]context.CancelFunc
|
||||
}
|
||||
)
|
||||
|
||||
@@ -111,8 +113,8 @@ func makeEnvironmentWatcher(
|
||||
podSpecPatch: podSpecPatch,
|
||||
envWatchInformer: utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.EnvironmentResource),
|
||||
enableOwnerReferences: utils.IsOwnerReferencesEnabled(),
|
||||
nsCancels: make(map[string]context.CancelFunc),
|
||||
}
|
||||
|
||||
err := envWatcher.EnvWatchEventHandlers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -540,6 +542,21 @@ func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr
|
||||
|
||||
envw.envWatchInformer[ns] = envInf
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
|
||||
factory.Start(ctx.Done())
|
||||
|
||||
// Create a per-namespace cancellable context for informer lifecycle.
|
||||
nsCtx, nsCancel := context.WithCancel(ctx)
|
||||
envw.nsCancels[ns] = nsCancel
|
||||
|
||||
factory.Start(nsCtx.Done())
|
||||
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
// RemoveNamespace deregisters a namespace from the environment watcher.
|
||||
func (envw *environmentWatcher) RemoveNamespace(ns string) {
|
||||
if cancel, ok := envw.nsCancels[ns]; ok {
|
||||
cancel()
|
||||
delete(envw.nsCancels, ns)
|
||||
}
|
||||
delete(envw.envWatchInformer, ns)
|
||||
envw.logger.Info("buildermgr.envWatcher.RemoveNamespace: cleaned up", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
@@ -11,10 +11,18 @@ type builderEnvNamespaceAdder interface {
|
||||
AddNamespace(ctx context.Context, ns string, mgr manager.Interface)
|
||||
}
|
||||
|
||||
type builderEnvNamespaceRemover interface {
|
||||
RemoveNamespace(ns string)
|
||||
}
|
||||
|
||||
type builderPkgNamespaceAdder interface {
|
||||
AddNamespace(ctx context.Context, ns string, mgr manager.Interface)
|
||||
}
|
||||
|
||||
type builderPkgNamespaceRemover interface {
|
||||
RemoveNamespace(ns string)
|
||||
}
|
||||
|
||||
func NewNamespaceSubscriber(envw builderEnvNamespaceAdder, pkgw builderPkgNamespaceAdder, mgr manager.Interface) utils.NamespaceSubscriber {
|
||||
return utils.NamespaceSubscriberFuncs{
|
||||
SubscriberName: "buildermgr",
|
||||
@@ -22,6 +30,10 @@ func NewNamespaceSubscriber(envw builderEnvNamespaceAdder, pkgw builderPkgNamesp
|
||||
registerBuilderNamespace(ctx, record.Name, envw, pkgw, mgr)
|
||||
return nil
|
||||
},
|
||||
RemoveFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
|
||||
deregisterBuilderNamespace(record.Name, envw, pkgw)
|
||||
return nil
|
||||
},
|
||||
ResyncFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
|
||||
registerBuilderNamespace(ctx, record.Name, envw, pkgw, mgr)
|
||||
return nil
|
||||
@@ -41,3 +53,16 @@ func registerBuilderNamespace(ctx context.Context, namespace string, envw builde
|
||||
pkgw.AddNamespace(ctx, namespace, mgr)
|
||||
}
|
||||
}
|
||||
|
||||
func deregisterBuilderNamespace(namespace string, envw builderEnvNamespaceAdder, pkgw builderPkgNamespaceAdder) {
|
||||
if namespace == "" {
|
||||
return
|
||||
}
|
||||
utils.DefaultNSResolver().RemoveNamespace(namespace)
|
||||
if r, ok := envw.(builderEnvNamespaceRemover); ok {
|
||||
r.RemoveNamespace(namespace)
|
||||
}
|
||||
if r, ok := pkgw.(builderPkgNamespaceRemover); ok {
|
||||
r.RemoveNamespace(namespace)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,9 @@ func StartNSWatcher(
|
||||
pkgw *packageWatcher,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, utils.NewDefaultManagedNamespaceWatcherConfig("buildermgr.NSWatcher", NewNamespaceSubscriber(envw, pkgw, mgr)))
|
||||
config := utils.NewDefaultManagedNamespaceWatcherConfig("buildermgr.NSWatcher", NewNamespaceSubscriber(envw, pkgw, mgr))
|
||||
config.RemovalStrategy = utils.NamespaceRemovalStrategyDispatchRemove
|
||||
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, config)
|
||||
if err != nil {
|
||||
logger.Error("buildermgr.NSWatcher: BootstrapAndDispatch failed", zap.Error(err))
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ type (
|
||||
pkgInformer map[string]k8sCache.SharedIndexInformer
|
||||
storageSvcUrl string
|
||||
buildCache *cache.Cache[crd.CacheKeyUR, *fv1.Package]
|
||||
// nsCancels holds per-namespace context cancel functions.
|
||||
nsCancels map[string]context.CancelFunc
|
||||
}
|
||||
)
|
||||
|
||||
@@ -64,6 +66,7 @@ func makePackageWatcher(logger *zap.Logger, fissionClient versioned.Interface, k
|
||||
pkgInformer: pkgInformer,
|
||||
storageSvcUrl: storageSvcUrl,
|
||||
buildCache: cache.MakeCache[crd.CacheKeyUR, *fv1.Package](0, 0),
|
||||
nsCancels: make(map[string]context.CancelFunc),
|
||||
}
|
||||
return pkgw
|
||||
}
|
||||
@@ -363,7 +366,23 @@ func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr man
|
||||
ns + "/pkg": pkgInf,
|
||||
ns + "/pod": podInf,
|
||||
})
|
||||
fissionFactory.Start(ctx.Done())
|
||||
podFactory.Start(ctx.Done())
|
||||
|
||||
// Create a per-namespace cancellable context for informer lifecycle.
|
||||
nsCtx, nsCancel := context.WithCancel(ctx)
|
||||
pkgw.nsCancels[ns] = nsCancel
|
||||
|
||||
fissionFactory.Start(nsCtx.Done())
|
||||
podFactory.Start(nsCtx.Done())
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
// RemoveNamespace deregisters a namespace from the package watcher.
|
||||
func (pkgw *packageWatcher) RemoveNamespace(ns string) {
|
||||
if cancel, ok := pkgw.nsCancels[ns]; ok {
|
||||
cancel()
|
||||
delete(pkgw.nsCancels, ns)
|
||||
}
|
||||
delete(pkgw.pkgInformer, ns)
|
||||
delete(pkgw.podInformer, ns)
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.RemoveNamespace: cleaned up", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user