fix(reconciler): health-check Active NS every 60s to restore deleted SA/RoleBindings
RunReconciler now runs two tickers: - 30s: retry Failed namespaces (existing behavior) - 60s: DispatchResync on Active namespaces; since registerNamespace is idempotent this is a no-op when SA/RoleBindings are intact and silently restores them if deleted Fixes P1-A from integration test 2026-05-18: SA deleted from Active NS was not being restored because reconciler only processed Failed NS.
This commit is contained in:
@@ -74,7 +74,9 @@ type NamespaceManager interface {
|
|||||||
MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
|
MarkPartActive(namespace string, part string) (NamespaceRecord, bool)
|
||||||
MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
|
MarkPartFailed(namespace string, part string, err error) (NamespaceRecord, bool)
|
||||||
Remove(name string) bool
|
Remove(name string) bool
|
||||||
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed.
|
// RunReconciler periodically retries namespaces stuck in NamespacePhaseFailed (every 30 s)
|
||||||
|
// and performs a health-check on Active namespaces (every 60 s) by calling DispatchResync,
|
||||||
|
// which re-ensures SA/RoleBindings and executor-type registration are intact.
|
||||||
// Must be started as a goroutine; exits when ctx is cancelled.
|
// Must be started as a goroutine; exits when ctx is cancelled.
|
||||||
// logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence.
|
// logger is used to report retry attempts and outcomes; pass zap.NewNop() to silence.
|
||||||
RunReconciler(ctx context.Context, logger *zap.Logger)
|
RunReconciler(ctx context.Context, logger *zap.Logger)
|
||||||
@@ -612,22 +614,27 @@ func (m *inMemoryNamespaceManager) Remove(name string) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunReconciler periodically finds namespaces in NamespacePhaseFailed and retries them
|
// RunReconciler periodically:
|
||||||
// via DispatchResync. This ensures transient k8s API errors (e.g. temporary 503 on
|
// - retries namespaces stuck in NamespacePhaseFailed via DispatchResync (every 30 s)
|
||||||
// EnsureNamespaceSA or executor-type AddNamespace) do not permanently strand a namespace.
|
// - health-checks Active namespaces by calling DispatchResync (every 60 s); since
|
||||||
|
// registerNamespace is idempotent, this is a no-op when SA/RoleBindings are intact
|
||||||
|
// and silently restores them if they were deleted.
|
||||||
|
//
|
||||||
// logger receives one log line per retry attempt and per outcome.
|
// logger receives one log line per retry attempt and per outcome.
|
||||||
// Exits when ctx is cancelled.
|
// Exits when ctx is cancelled.
|
||||||
func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) {
|
func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *zap.Logger) {
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
logger = zap.NewNop()
|
logger = zap.NewNop()
|
||||||
}
|
}
|
||||||
ticker := time.NewTicker(30 * time.Second)
|
failedTicker := time.NewTicker(30 * time.Second)
|
||||||
defer ticker.Stop()
|
defer failedTicker.Stop()
|
||||||
|
activeTicker := time.NewTicker(60 * time.Second)
|
||||||
|
defer activeTicker.Stop()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-failedTicker.C:
|
||||||
m.mu.RLock()
|
m.mu.RLock()
|
||||||
var failedNS []string
|
var failedNS []string
|
||||||
for ns, rec := range m.records {
|
for ns, rec := range m.records {
|
||||||
@@ -655,6 +662,24 @@ func (m *inMemoryNamespaceManager) RunReconciler(ctx context.Context, logger *za
|
|||||||
logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns))
|
logger.Info("namespace reconciler: resync succeeded", zap.String("namespace", ns))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
case <-activeTicker.C:
|
||||||
|
m.mu.RLock()
|
||||||
|
var activeNS []string
|
||||||
|
for ns, rec := range m.records {
|
||||||
|
if rec.Phase == NamespacePhaseActive {
|
||||||
|
activeNS = append(activeNS, ns)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.RUnlock()
|
||||||
|
for _, ns := range activeNS {
|
||||||
|
_, _, err := m.DispatchResync(ctx, ns)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("namespace reconciler: active NS health-check failed, marking failed for retry",
|
||||||
|
zap.String("namespace", ns),
|
||||||
|
zap.Error(err),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user