fix(executor): P2 — pre-register managed NS before adopt/cleanup (closes AdoptExistingResources race)

Problem:
  executor starts → AdoptExistingResources + CleanupOldExecutorObjects run
  against utils.DefaultNSResolver().Snapshot() which returns ONLY static NS
  from FISSION_RESOURCE_NAMESPACES. Managed (labeled) namespaces are
  registered later, asynchronously, by StartNSWatcher.

  Result:
  - Pods from a previous executor in managed NS are never adopted
    (no instanceID patch) → poolmgr creates new pool pods → cold start
    for first request after executor restart.
  - Old executor objects (RS/deployments) in managed NS accumulate
    without being cleaned up (resource leak).

Fix:
  Add multitenant.PreRegisterManagedNamespaces(ctx, logger, kubernetesClient)
  called synchronously in executor.go BEFORE the adopt/cleanup goroutines.

  The function does a single Namespaces.List with label
  fission.io/managed=true and calls DefaultNSResolver().AddNamespace() for
  each result. This is idempotent with the later watcher AddFunc calls.

  Failure is non-fatal: a warning is logged and startup proceeds with
  static NS only (safe degraded mode).

  After this call DefaultNSResolver().Snapshot() includes managed NS, so:
  - AdoptExistingResources patches old pods in managed NS with new instanceID
  - CleanupOldExecutorObjects removes stale objects from managed NS
  - GetReaperNamespace() returns the full tenant NS set

Files:
  pkg/executor/multitenant/ns_watcher.go — PreRegisterManagedNamespaces()
  pkg/executor/executor.go               — call before adopt/cleanup
This commit is contained in:
“Naeel”
2026-05-18 11:42:48 +04:00
parent f5b57173f5
commit 2a7d6101b1
2 changed files with 32 additions and 0 deletions
+5
View File
@@ -347,6 +347,11 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
executorTypes[ndm.GetTypeName(ctx)] = ndm
executorTypes[cnm.GetTypeName(ctx)] = cnm
// Pre-populate DefaultNSResolver with managed (labeled) namespaces so that
// AdoptExistingResources and CleanupOldExecutorObjects cover the full tenant NS set.
// Must run before the adopt/cleanup goroutines below. Non-fatal on error.
multitenant.PreRegisterManagedNamespaces(ctx, logger, kubernetesClient)
adoptExistingResources, _ := strconv.ParseBool(os.Getenv("ADOPT_EXISTING_RESOURCES"))
wg := &sync.WaitGroup{}
+27
View File
@@ -65,6 +65,7 @@ import (
"fmt"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
@@ -172,3 +173,29 @@ func deregisterNamespace(
logger.Info("multitenant.NSWatcher: deregistered namespace", zap.String("namespace", ns))
return joinErr
}
// PreRegisterManagedNamespaces does a one-time synchronous List of all Namespaces
// labeled fission.io/managed=true and adds them to the global DefaultNSResolver.
//
// Called at executor startup BEFORE AdoptExistingResources and CleanupOldExecutorObjects
// so that adopt and cleanup cover managed (dynamic) namespaces, not just static ones
// from FISSION_RESOURCE_NAMESPACES. This closes the P2 race where old executor pods
// in managed NS were never adopted (causing unnecessary cold starts) and never cleaned
// (causing orphaned pod accumulation).
//
// Failure is non-fatal: a warning is logged and the executor proceeds with static NS only.
func PreRegisterManagedNamespaces(ctx context.Context, logger *zap.Logger, client kubernetes.Interface) {
nsList, err := client.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
LabelSelector: utils.ManagedNamespaceLabelSelector(),
})
if err != nil {
logger.Warn("PreRegisterManagedNamespaces: failed to list managed namespaces; adopt/cleanup will use static NS only",
zap.Error(err))
return
}
for i := range nsList.Items {
utils.DefaultNSResolver().AddNamespace(nsList.Items[i].Name)
}
logger.Info("PreRegisterManagedNamespaces: pre-registered managed namespaces",
zap.Int("count", len(nsList.Items)))
}