fix(reconciler): health-check Active NS every 60s to restore deleted SA/RoleBindings
RunReconciler now runs two tickers: - 30s: retry Failed namespaces (existing behavior) - 60s: DispatchResync on Active namespaces; since registerNamespace is idempotent this is a no-op when SA/RoleBindings are intact and silently restores them if deleted Fixes P1-A from integration test 2026-05-18: SA deleted from Active NS was not being restored because reconciler only processed Failed NS.
This commit is contained in:
@@ -74,7 +74,9 @@ type NamespaceManager interface {
|
||||
MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
|
||||
MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
|
||||
Remove(name string) bool
|
||||
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed.
|
||||
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed (every 30 s)
|
||||
// and performs a health-check on Active namespaces (every 60 s) by calling DispatchResync,
|
||||
// which re-ensures SA/RoleBindings and executor-type registration are intact.
|
||||
// Must be started as a goroutine; exits when ctx is cancelled.
|
||||
// logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence.
|
||||
RunReconciler(ctx context.Context, logger *zap.Logger)
|
||||
@@ -612,22 +614,27 @@ func (m *inMemoryNamespaceManager) Remove(name string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// RunReconciler periodically finds namespaces in NamespacePhaseFailed and retries them
|
||||
// via DispatchResync. This ensures transient k8s API errors (e.g. temporary 503 on
|
||||
// EnsureNamespaceSA or executor-type AddNamespace) do not permanently strand a namespace.
|
||||
// RunReconciler periodically:
|
||||
// - retries namespaces stuck in NamespacePhaseFailed via DispatchResync (every 30 s)
|
||||
// - health-checks Active namespaces by calling DispatchResync (every 60 s); since
|
||||
// registerNamespace is idempotent, this is a no-op when SA/RoleBindings are intact
|
||||
// and silently restores them if they were deleted.
|
||||
//
|
||||
// logger receives one log line per retry attempt and per outcome.
|
||||
// Exits when ctx is cancelled.
|
||||
func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) {
|
||||
if logger == nil {
|
||||
logger = zap.NewNop()
|
||||
}
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
failedTicker := time.NewTicker(30 * time.Second)
|
||||
defer failedTicker.Stop()
|
||||
activeTicker := time.NewTicker(60 * time.Second)
|
||||
defer activeTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
case <-failedTicker.C:
|
||||
m.mu.RLock()
|
||||
var failedNS []string
|
||||
for ns, rec := range m.records {
|
||||
@@ -655,6 +662,24 @@ func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *za
|
||||
logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns))
|
||||
}
|
||||
}
|
||||
case <-activeTicker.C:
|
||||
m.mu.RLock()
|
||||
var activeNS []string
|
||||
for ns, rec := range m.records {
|
||||
if rec.Phase == NamespacePhaseActive {
|
||||
activeNS = append(activeNS, ns)
|
||||
}
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
for _, ns := range activeNS {
|
||||
_, _, err := m.DispatchResync(ctx, ns)
|
||||
if err != nil {
|
||||
logger.Warn("namespace reconciler: active NS health-check failed, marking failed for retry",
|
||||
zap.String("namespace", ns),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user