fix(reconciler): health-check Active NS every 60s to restore deleted SA/RoleBindings

RunReconciler now runs two tickers:
- 30s: retry Failed namespaces (existing behavior)
- 60s: DispatchResync on Active namespaces; since registerNamespace is
  idempotent this is a no-op when SA/RoleBindings are intact and
  silently restores them if deleted

Fixes P1-A from integration test 2026-05-18: SA deleted from Active NS
was not being restored because reconciler only processed Failed NS.
This commit is contained in:
“Naeel”
2026-05-18 13:27:50 +04:00
parent 650616b464
commit 7265985309
+32 -7
View File
@@ -74,7 +74,9 @@ type NamespaceManager interface {
MarkPartActive(namespace string, part string) (NamespaceRecord, bool) MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool) MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
Remove(name string) bool Remove(name string) bool
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed. // RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed (every 30 s)
// and performs a health-check on Active namespaces (every 60 s) by calling DispatchResync,
// which re-ensures SA/RoleBindings and executor-type registration are intact.
// Must be started as a goroutine; exits when ctx is cancelled. // Must be started as a goroutine; exits when ctx is cancelled.
// logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence. // logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence.
RunReconciler(ctx context.Context, logger *zap.Logger) RunReconciler(ctx context.Context, logger *zap.Logger)
@@ -612,22 +614,27 @@ func (m *inMemoryNamespaceManager) Remove(name string) bool {
return true return true
} }
// RunReconciler periodically finds namespaces in NamespacePhaseFailed and retries them // RunReconciler periodically:
// via DispatchResync. This ensures transient k8s API errors (e.g. temporary 503 on // - retries namespaces stuck in NamespacePhaseFailed via DispatchResync (every 30 s)
// EnsureNamespaceSA or executor-type AddNamespace) do not permanently strand a namespace. // - health-checks Active namespaces by calling DispatchResync (every 60 s); since
// registerNamespace is idempotent, this is a no-op when SA/RoleBindings are intact
// and silently restores them if they were deleted.
//
// logger receives one log line per retry attempt and per outcome. // logger receives one log line per retry attempt and per outcome.
// Exits when ctx is cancelled. // Exits when ctx is cancelled.
func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) { func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) {
if logger == nil { if logger == nil {
logger = zap.NewNop() logger = zap.NewNop()
} }
ticker := time.NewTicker(30 * time.Second) failedTicker := time.NewTicker(30 * time.Second)
defer ticker.Stop() defer failedTicker.Stop()
activeTicker := time.NewTicker(60 * time.Second)
defer activeTicker.Stop()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return
case <-ticker.C: case <-failedTicker.C:
m.mu.RLock() m.mu.RLock()
var failedNS []string var failedNS []string
for ns, rec := range m.records { for ns, rec := range m.records {
@@ -655,6 +662,24 @@ func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *za
logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns)) logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns))
} }
} }
case <-activeTicker.C:
m.mu.RLock()
var activeNS []string
for ns, rec := range m.records {
if rec.Phase == NamespacePhaseActive {
activeNS = append(activeNS, ns)
}
}
m.mu.RUnlock()
for _, ns := range activeNS {
_, _, err := m.DispatchResync(ctx, ns)
if err != nil {
logger.Warn("namespace reconciler: active NS health-check failed, marking failed for retry",
zap.String("namespace", ns),
zap.Error(err),
)
}
}
} }
} }
} }