fix(namespace): executor/router/buildermgr RemoveNamespace + per-NS informer lifecycle

- Add RemoveNamespace(ctx, ns) to executortype.ExecutorType interface
- Implement RemoveNamespace in poolmgr, newdeploy, container executor types
- Add per-namespace context cancellation (nsCancels map) in all three types so
  informer factories are stopped when namespace is removed (fixes goroutine leak)
- Add PoolPodController.RemoveNamespace to clear envLister/podLister maps
- Add deregisterNamespace() in executor multitenant subscriber
- Switch executor/router/buildermgr watcher strategy from TrackOnly to DispatchRemove
  so RemoveFunc is called when fission.io/managed label is removed
- Add RemoveFunc to executor/router/buildermgr namespace subscribers
- Add RemoveNamespace to environmentWatcher and packageWatcher with per-NS cancel
- Add RemoveNamespace to HTTPTriggerSet: cancels informers, removes from maps, calls syncTriggers
- Fix ns_watcher_test.go fakeExecutorType to implement new RemoveNamespace method

Fixes:
- Executor dedup gap: re-added namespace was silently skipped (envLister/deplLister still present)
- Goroutine/FD leak: old informer factories ran forever after namespace removal
- Router stale routes: HTTPTriggers for removed namespace stayed in routing table
This commit is contained in:
“Naeel”
2026-05-18 09:04:13 +04:00
parent 3b93c5dc8b
commit 4eedf95f5c
17 changed files with 347 additions and 29 deletions
+19 -2
View File
@@ -76,6 +76,8 @@ type (
podSpecPatch *apiv1.PodSpec
envWatchInformer map[string]k8sCache.SharedIndexInformer
enableOwnerReferences bool
// nsCancels holds per-namespace context cancel functions.
nsCancels map[string]context.CancelFunc
}
)
@@ -111,8 +113,8 @@ func makeEnvironmentWatcher(
podSpecPatch: podSpecPatch,
envWatchInformer: utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.EnvironmentResource),
enableOwnerReferences: utils.IsOwnerReferencesEnabled(),
nsCancels: make(map[string]context.CancelFunc),
}
err := envWatcher.EnvWatchEventHandlers(ctx)
if err != nil {
return nil, err
@@ -540,6 +542,21 @@ func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr
envw.envWatchInformer[ns] = envInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
factory.Start(ctx.Done())
// Create a per-namespace cancellable context for informer lifecycle.
nsCtx, nsCancel := context.WithCancel(ctx)
envw.nsCancels[ns] = nsCancel
factory.Start(nsCtx.Done())
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
}
// RemoveNamespace deregisters a namespace from the environment watcher.
func (envw *environmentWatcher) RemoveNamespace(ns string) {
if cancel, ok := envw.nsCancels[ns]; ok {
cancel()
delete(envw.nsCancels, ns)
}
delete(envw.envWatchInformer, ns)
envw.logger.Info("buildermgr.envWatcher.RemoveNamespace: cleaned up", zap.String("namespace", ns))
}
+25
View File
@@ -11,10 +11,18 @@ type builderEnvNamespaceAdder interface {
AddNamespace(ctx context.Context, ns string, mgr manager.Interface)
}
type builderEnvNamespaceRemover interface {
RemoveNamespace(ns string)
}
type builderPkgNamespaceAdder interface {
AddNamespace(ctx context.Context, ns string, mgr manager.Interface)
}
type builderPkgNamespaceRemover interface {
RemoveNamespace(ns string)
}
func NewNamespaceSubscriber(envw builderEnvNamespaceAdder, pkgw builderPkgNamespaceAdder, mgr manager.Interface) utils.NamespaceSubscriber {
return utils.NamespaceSubscriberFuncs{
SubscriberName: "buildermgr",
@@ -22,6 +30,10 @@ func NewNamespaceSubscriber(envw builderEnvNamespaceAdder, pkgw builderPkgNamesp
registerBuilderNamespace(ctx, record.Name, envw, pkgw, mgr)
return nil
},
RemoveFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
deregisterBuilderNamespace(record.Name, envw, pkgw)
return nil
},
ResyncFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
registerBuilderNamespace(ctx, record.Name, envw, pkgw, mgr)
return nil
@@ -41,3 +53,16 @@ func registerBuilderNamespace(ctx context.Context, namespace string, envw builde
pkgw.AddNamespace(ctx, namespace, mgr)
}
}
func deregisterBuilderNamespace(namespace string, envw builderEnvNamespaceAdder, pkgw builderPkgNamespaceAdder) {
if namespace == "" {
return
}
utils.DefaultNSResolver().RemoveNamespace(namespace)
if r, ok := envw.(builderEnvNamespaceRemover); ok {
r.RemoveNamespace(namespace)
}
if r, ok := pkgw.(builderPkgNamespaceRemover); ok {
r.RemoveNamespace(namespace)
}
}
+3 -1
View File
@@ -25,7 +25,9 @@ func StartNSWatcher(
pkgw *packageWatcher,
mgr manager.Interface,
) {
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, utils.NewDefaultManagedNamespaceWatcherConfig("buildermgr.NSWatcher", NewNamespaceSubscriber(envw, pkgw, mgr)))
config := utils.NewDefaultManagedNamespaceWatcherConfig("buildermgr.NSWatcher", NewNamespaceSubscriber(envw, pkgw, mgr))
config.RemovalStrategy = utils.NamespaceRemovalStrategyDispatchRemove
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, config)
if err != nil {
logger.Error("buildermgr.NSWatcher: BootstrapAndDispatch failed", zap.Error(err))
}
+21 -2
View File
@@ -49,6 +49,8 @@ type (
pkgInformer map[string]k8sCache.SharedIndexInformer
storageSvcUrl string
buildCache *cache.Cache[crd.CacheKeyUR, *fv1.Package]
// nsCancels holds per-namespace context cancel functions.
nsCancels map[string]context.CancelFunc
}
)
@@ -64,6 +66,7 @@ func makePackageWatcher(logger *zap.Logger, fissionClient versioned.Interface, k
pkgInformer: pkgInformer,
storageSvcUrl: storageSvcUrl,
buildCache: cache.MakeCache[crd.CacheKeyUR, *fv1.Package](0, 0),
nsCancels: make(map[string]context.CancelFunc),
}
return pkgw
}
@@ -363,7 +366,23 @@ func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr man
ns + "/pkg": pkgInf,
ns + "/pod": podInf,
})
fissionFactory.Start(ctx.Done())
podFactory.Start(ctx.Done())
// Create a per-namespace cancellable context for informer lifecycle.
nsCtx, nsCancel := context.WithCancel(ctx)
pkgw.nsCancels[ns] = nsCancel
fissionFactory.Start(nsCtx.Done())
podFactory.Start(nsCtx.Done())
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
}
// RemoveNamespace deregisters a namespace from the package watcher.
func (pkgw *packageWatcher) RemoveNamespace(ns string) {
if cancel, ok := pkgw.nsCancels[ns]; ok {
cancel()
delete(pkgw.nsCancels, ns)
}
delete(pkgw.pkgInformer, ns)
delete(pkgw.podInformer, ns)
pkgw.logger.Info("buildermgr.pkgWatcher.RemoveNamespace: cleaned up", zap.String("namespace", ns))
}