fix(namespace): harden lifecycle — RemoveNamespace, parallel dispatch, reconciler

- DefaultNSResolver.RemoveNamespace(): removes NS from global map on label removal
  so Snapshot() and idleObjectReaper stop iterating deleted namespaces.
  Fixes class of dirty-state bugs when NS name is reused by new tenant.

- HandleWatcherNamespaceRemoval: call RemoveNamespace on both TrackOnly and
  DispatchRemove strategies — global resolver cleanup is always required.

- dispatch(): parallel subscriber execution via goroutine per subscriber +
  sync.WaitGroup. Reduces onboarding latency from O(N_subscribers × API_latency)
  to O(max(API_latency)). Safe: MarkPart* are internally mutex-protected.

- inMemoryNamespaceManager.RunReconciler(): 30s ticker scans for
  NamespacePhaseFailed records and retries via DispatchResync. Started
  automatically by RunManagedNamespaceWatcher. Fixes permanent stuck-failed
  state caused by transient k8s API errors.

Analysis source: FORENSIC_ARCHITECTURE_AUDIT.md §Deep Risk Analysis

pkg/utils/namespace.go

@@ -113,6 +113,23 @@ func (nsr *NamespaceResolver) AddNamespace(ns string) bool {
 	return true
 }
 
+// RemoveNamespace removes a namespace from FissionResourceNS.
+// Returns true if the namespace was present and removed, false if it was not found.
+// Thread-safe. Used when a namespace loses the fission.io/managed=true label so that
+// Snapshot() and idleObjectReaper loops no longer iterate over deleted namespaces.
+func (nsr *NamespaceResolver) RemoveNamespace(ns string) bool {
+	nsr.mu.Lock()
+	defer nsr.mu.Unlock()
+	if _, exists := nsr.FissionResourceNS[ns]; !exists {
+		return false
+	}
+	delete(nsr.FissionResourceNS, ns)
+	if nsr.Logger != nil {
+		nsr.Logger.Info("dynamically removed namespace from resolver", zap.String("namespace", ns))
+	}
+	return true
+}
+
 // Snapshot returns a stable copy of the currently registered resource namespaces.
 // The returned slice is detached from the internal mutable map and safe to iterate.
 func (nsr *NamespaceResolver) Snapshot() []string {

pkg/utils/namespace_manager.go

@@ -74,6 +74,9 @@ type NamespaceManager interface {
 	MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
 	MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
 	Remove(name string) bool
+	// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed.
+	// Must be started as a goroutine; exits when ctx is cancelled.
+	RunReconciler(ctx context.Context)
 }
 
 type inMemoryNamespaceManager struct {
@@ -142,6 +145,12 @@ func RunManagedNamespaceWatcher(ctx context.Context, logger *zap.Logger, kubeCli
 	logger = namespaceManagerLogger(logger)
 	manager, handlers, err := PrepareManagedNamespaceWatcher(ctx, logger, config)
 	StartManagedNamespaceWatcher(ctx, logger, config.Component, kubeClient, mgr, handlers)
+	// Start reconciler: retries namespaces stuck in NamespacePhaseFailed every 30s.
+	mgr.Add(ctx, func(ctx context.Context) {
+		logger.Info(config.Component + ": namespace reconciler started")
+		manager.RunReconciler(ctx)
+		logger.Info(config.Component + ": namespace reconciler stopped")
+	})
 	LogNamespaceManagerSummary(logger, config.Component+": started namespace watcher", manager.Summary())
 	return manager, err
 }
@@ -236,6 +245,11 @@ func HandleWatcherNamespaceRemoval(ctx context.Context, logger *zap.Logger, comp
 	if !ok {
 		return
 	}
+	// Always remove from global resolver so Snapshot() and idleObjectReaper
+	// stop iterating this namespace. This is safe: if the same name is re-added
+	// later, AddNamespace will return true and all subscribers will re-register.
+	DefaultNSResolver().RemoveNamespace(record.Name)
+
 	if strategy == NamespaceRemovalStrategyDispatchRemove {
 		if _, _, err := manager.DispatchRemove(ctx, record.Name); err != nil {
 			logger.Error(component+": DispatchRemove failed", zap.String("namespace", record.Name), zap.Error(err))
@@ -509,28 +523,44 @@ func (m *inMemoryNamespaceManager) DispatchResync(ctx context.Context, namespace
 }
 
 func (m *inMemoryNamespaceManager) dispatch(ctx context.Context, namespace string, handler func(NamespaceSubscriber, NamespaceRecord) error) (NamespaceRecord, bool, error) {
-	record, ok := m.Get(namespace)
+	_, ok := m.Get(namespace)
 	if !ok {
 		return NamespaceRecord{}, false, nil
 	}
 
-	var firstErr error
-	for _, subscriber := range m.snapshotSubscriberObjects() {
-		_, _ = m.MarkPartRegistering(namespace, subscriber.Name())
-		currentRecord, _ := m.Get(namespace)
-		err := handler(subscriber, currentRecord)
-		if err != nil {
-			_, _ = m.MarkPartFailed(namespace, subscriber.Name(), err)
-			if firstErr == nil {
-				firstErr = err
-			}
-			continue
-		}
-		_, _ = m.MarkPartActive(namespace, subscriber.Name())
+	subscribers := m.snapshotSubscriberObjects()
+	for _, sub := range subscribers {
+		_, _ = m.MarkPartRegistering(namespace, sub.Name())
 	}
 
-	record, _ = m.Get(namespace)
-	return record, true, firstErr
+	// Run all subscriber handlers in parallel — each handler makes independent k8s API calls.
+	// MarkPart* methods are internally mutex-protected and safe for concurrent calls.
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		errs error
+	)
+	for _, sub := range subscribers {
+		sub := sub
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			currentRecord, _ := m.Get(namespace)
+			err := handler(sub, currentRecord)
+			if err != nil {
+				_, _ = m.MarkPartFailed(namespace, sub.Name(), err)
+				mu.Lock()
+				errs = errors.Join(errs, err)
+				mu.Unlock()
+				return
+			}
+			_, _ = m.MarkPartActive(namespace, sub.Name())
+		}()
+	}
+	wg.Wait()
+
+	record, _ := m.Get(namespace)
+	return record, true, errs
 }
 
 func (m *inMemoryNamespaceManager) MarkPartState(namespace string, part string, state NamespacePartState) (NamespaceRecord, bool) {
@@ -581,6 +611,35 @@ func (m *inMemoryNamespaceManager) Remove(name string) bool {
 	return true
 }
 
+// RunReconciler periodically finds namespaces in NamespacePhaseFailed and retries them
+// via DispatchResync. This ensures transient k8s API errors (e.g. temporary 503) do not
+// permanently strand a namespace. Exits when ctx is cancelled.
+func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context) {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			m.mu.RLock()
+			var failedNS []string
+			for ns, rec := range m.records {
+				if rec.Phase == NamespacePhaseFailed {
+					failedNS = append(failedNS, ns)
+				}
+			}
+			m.mu.RUnlock()
+			for _, ns := range failedNS {
+				if _, _, err := m.DispatchResync(ctx, ns); err != nil {
+					// Still failing — will retry on next tick.
+					_ = err
+				}
+			}
+		}
+	}
+}
+
 func deriveNamespacePhase(record NamespaceRecord) NamespacePhase {
 	if len(record.RegisteredParts) == 0 {
 		return record.Phase