fix(reconciler): health-check Active NS every 60s to restore deleted SA/RoleBindings

RunReconciler now runs two tickers:
- 30s: retry Failed namespaces (existing behavior)
- 60s: DispatchResync on Active namespaces; since registerNamespace is
  idempotent this is a no-op when SA/RoleBindings are intact and
  silently restores them if deleted

Fixes P1-A from integration test 2026-05-18: SA deleted from Active NS
was not being restored because reconciler only processed Failed NS.
This commit is contained in:
“Naeel”
2026-05-18 13:27:50 +04:00
parent 650616b464
commit 7265985309
+32 -7
View File
@@ -74,7 +74,9 @@ type NamespaceManager interface {
MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
Remove(name string) bool
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed.
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed (every 30 s)
// and performs a health-check on Active namespaces (every 60 s) by calling DispatchResync,
// which re-ensures SA/RoleBindings and executor-type registration are intact.
// Must be started as a goroutine; exits when ctx is cancelled.
// logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence.
RunReconciler(ctx context.Context, logger *zap.Logger)
@@ -612,22 +614,27 @@ func (m *inMemoryNamespaceManager) Remove(name string) bool {
return true
}
// RunReconciler periodically finds namespaces in NamespacePhaseFailed and retries them
// via DispatchResync. This ensures transient k8s API errors (e.g. temporary 503 on
// EnsureNamespaceSA or executor-type AddNamespace) do not permanently strand a namespace.
// RunReconciler periodically:
// - retries namespaces stuck in NamespacePhaseFailed via DispatchResync (every 30 s)
// - health-checks Active namespaces by calling DispatchResync (every 60 s); since
// registerNamespace is idempotent, this is a no-op when SA/RoleBindings are intact
// and silently restores them if they were deleted.
//
// logger receives one log line per retry attempt and per outcome.
// Exits when ctx is cancelled.
func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) {
if logger == nil {
logger = zap.NewNop()
}
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
failedTicker := time.NewTicker(30 * time.Second)
defer failedTicker.Stop()
activeTicker := time.NewTicker(60 * time.Second)
defer activeTicker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
case <-failedTicker.C:
m.mu.RLock()
var failedNS []string
for ns, rec := range m.records {
@@ -655,6 +662,24 @@ func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *za
logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns))
}
}
case <-activeTicker.C:
m.mu.RLock()
var activeNS []string
for ns, rec := range m.records {
if rec.Phase == NamespacePhaseActive {
activeNS = append(activeNS, ns)
}
}
m.mu.RUnlock()
for _, ns := range activeNS {
_, _, err := m.DispatchResync(ctx, ns)
if err != nil {
logger.Warn("namespace reconciler: active NS health-check failed, marking failed for retry",
zap.String("namespace", ns),
zap.Error(err),
)
}
}
}
}
}