fix(namespace): executor/router/buildermgr RemoveNamespace + per-NS informer lifecycle
- Add RemoveNamespace(ctx, ns) to executortype.ExecutorType interface - Implement RemoveNamespace in poolmgr, newdeploy, container executor types - Add per-namespace context cancellation (nsCancels map) in all three types so informer factories are stopped when namespace is removed (fixes goroutine leak) - Add PoolPodController.RemoveNamespace to clear envLister/podLister maps - Add deregisterNamespace() in executor multitenant subscriber - Switch executor/router/buildermgr watcher strategy from TrackOnly to DispatchRemove so RemoveFunc is called when fission.io/managed label is removed - Add RemoveFunc to executor/router/buildermgr namespace subscribers - Add RemoveNamespace to environmentWatcher and packageWatcher with per-NS cancel - Add RemoveNamespace to HTTPTriggerSet: cancels informers, removes from maps, calls syncTriggers - Fix ns_watcher_test.go fakeExecutorType to implement new RemoveNamespace method Fixes: - Executor dedup gap: re-added namespace was silently skipped (envLister/deplLister still present) - Goroutine/FD leak: old informer factories ran forever after namespace removal - Router stale routes: HTTPTriggers for removed namespace stayed in routing table
This commit is contained in:
+43
-12
@@ -50,16 +50,18 @@ type HTTPTriggerSet struct {
|
||||
*functionServiceMap
|
||||
*mutableRouter
|
||||
|
||||
logger *zap.Logger
|
||||
fissionClient versioned.Interface
|
||||
kubeClient kubernetes.Interface
|
||||
executor eclient.ClientInterface
|
||||
resolver *functionReferenceResolver
|
||||
triggers []fv1.HTTPTrigger
|
||||
triggerInformer map[string]k8sCache.SharedIndexInformer
|
||||
functions []fv1.Function
|
||||
funcInformer map[string]k8sCache.SharedIndexInformer
|
||||
informerMu sync.RWMutex
|
||||
logger *zap.Logger
|
||||
fissionClient versioned.Interface
|
||||
kubeClient kubernetes.Interface
|
||||
executor eclient.ClientInterface
|
||||
resolver *functionReferenceResolver
|
||||
triggers []fv1.HTTPTrigger
|
||||
triggerInformer map[string]k8sCache.SharedIndexInformer
|
||||
functions []fv1.Function
|
||||
funcInformer map[string]k8sCache.SharedIndexInformer
|
||||
informerMu sync.RWMutex
|
||||
// nsCancels holds per-namespace context cancel functions for informer lifecycle.
|
||||
nsCancels map[string]context.CancelFunc
|
||||
updateRouterRequestChannel chan struct{}
|
||||
tsRoundTripperParams *tsRoundTripperParams
|
||||
isDebugEnv bool
|
||||
@@ -84,6 +86,7 @@ func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, fissionCli
|
||||
svcAddrUpdateThrottler: actionThrottler,
|
||||
unTapServiceTimeout: unTapServiceTimeout,
|
||||
syncDebouncer: debounce.New(time.Millisecond * 20),
|
||||
nsCancels: make(map[string]context.CancelFunc),
|
||||
}
|
||||
httpTriggerSet.triggerInformer = utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.HttpTriggerResource)
|
||||
httpTriggerSet.funcInformer = utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.FunctionResource)
|
||||
@@ -526,11 +529,39 @@ func (ts *HTTPTriggerSet) AddNamespace(ctx context.Context, ns string, mgr manag
|
||||
ns + "/trigger": triggerInf,
|
||||
ns + "/func": funcInf,
|
||||
})
|
||||
factory.Start(ctx.Done())
|
||||
|
||||
// Create a per-namespace cancellable context for informer lifecycle.
|
||||
nsCtx, nsCancel := context.WithCancel(ctx)
|
||||
ts.nsCancels[ns] = nsCancel
|
||||
|
||||
factory.Start(nsCtx.Done())
|
||||
// Wait for cache to sync before rebuilding the router, so triggers are visible.
|
||||
k8sCache.WaitForCacheSync(ctx.Done(), triggerInf.HasSynced, funcInf.HasSynced)
|
||||
k8sCache.WaitForCacheSync(nsCtx.Done(), triggerInf.HasSynced, funcInf.HasSynced)
|
||||
|
||||
ts.logger.Info("router.AddNamespace: done", zap.String("namespace", ns))
|
||||
ts.syncTriggers()
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveNamespace deregisters a namespace from the router.
|
||||
// Cancels the per-namespace informer context, removes informers from internal maps,
|
||||
// and triggers a syncTriggers so stale routes are removed immediately.
|
||||
func (ts *HTTPTriggerSet) RemoveNamespace(ns string) {
|
||||
if ns == "" {
|
||||
return
|
||||
}
|
||||
ts.logger.Info("router.RemoveNamespace: cleaning up namespace", zap.String("namespace", ns))
|
||||
|
||||
if cancel, ok := ts.nsCancels[ns]; ok {
|
||||
cancel()
|
||||
delete(ts.nsCancels, ns)
|
||||
}
|
||||
|
||||
ts.informerMu.Lock()
|
||||
delete(ts.triggerInformer, ns)
|
||||
delete(ts.funcInformer, ns)
|
||||
ts.informerMu.Unlock()
|
||||
|
||||
ts.syncTriggers()
|
||||
ts.logger.Info("router.RemoveNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
@@ -11,12 +11,22 @@ type routerNamespaceAdder interface {
|
||||
AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error
|
||||
}
|
||||
|
||||
type routerNamespaceRemover interface {
|
||||
RemoveNamespace(ns string)
|
||||
}
|
||||
|
||||
func NewNamespaceSubscriber(ts routerNamespaceAdder, mgr manager.Interface) utils.NamespaceSubscriber {
|
||||
return utils.NamespaceSubscriberFuncs{
|
||||
SubscriberName: "router",
|
||||
AddFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
|
||||
return registerRouterNamespace(ctx, record.Name, ts, mgr)
|
||||
},
|
||||
RemoveFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
|
||||
if r, ok := ts.(routerNamespaceRemover); ok {
|
||||
r.RemoveNamespace(record.Name)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
ResyncFunc: func(ctx context.Context, record utils.NamespaceRecord) error {
|
||||
return registerRouterNamespace(ctx, record.Name, ts, mgr)
|
||||
},
|
||||
|
||||
@@ -25,7 +25,9 @@ func StartNSWatcher(
|
||||
ts *HTTPTriggerSet,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, utils.NewDefaultManagedNamespaceWatcherConfig("router.NSWatcher", NewNamespaceSubscriber(ts, mgr)))
|
||||
config := utils.NewDefaultManagedNamespaceWatcherConfig("router.NSWatcher", NewNamespaceSubscriber(ts, mgr))
|
||||
config.RemovalStrategy = utils.NamespaceRemovalStrategyDispatchRemove
|
||||
_, err := utils.RunManagedNamespaceWatcher(ctx, logger, kubeClient, mgr, config)
|
||||
if err != nil {
|
||||
logger.Error("router.NSWatcher: BootstrapAndDispatch failed", zap.Error(err))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user