multi-tenant: EnsureNamespaceSA + ns_watcher SA provisioning (v8)

pkg/executor/executor.go

@@ -37,6 +37,7 @@ import (
 	"github.com/fission/fission/pkg/executor/executortype/newdeploy"
 	"github.com/fission/fission/pkg/executor/executortype/poolmgr"
 	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/multitenant"
 	"github.com/fission/fission/pkg/executor/util"
 	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
 	"github.com/fission/fission/pkg/generated/clientset/versioned"
@@ -399,6 +400,12 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
 
 	utils.CreateMissingPermissionForSA(ctx, kubernetesClient, logger)
 
+	// Start multi-tenant Namespace watcher.
+	// Detects Namespaces labeled fission.io/managed=true and registers them in all
+	// executor types without a pod restart. Backward-compatible with FISSION_RESOURCE_NAMESPACES.
+	// See: pkg/executor/multitenant/ns_watcher.go
+	multitenant.StartNSWatcher(ctx, logger, kubernetesClient, executorTypes, mgr)
+
 	mgr.Add(ctx, func(ctx context.Context) {
 		metrics.ServeMetrics(ctx, "executor", logger, mgr)
 	})

pkg/executor/executortype/container/containermgr.go

@@ -792,3 +792,50 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
 func (caaf *Container) DumpDebugInfo(ctx context.Context) error {
 	return nil
 }
+
+// AddNamespace dynamically registers a new namespace in the container executor without a restart.
+// Sets up deployment and service listers so the executor can manage container functions in the new NS.
+func (caaf *Container) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
+	if ns == "" {
+		return nil
+	}
+	// Use container-specific dedup: check if deplLister is already set up for this NS.
+	// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
+	// shared by all executor types and is now called once in multitenant.registerNamespace.
+	if _, ok := caaf.deplLister[ns]; ok {
+		return nil // already registered
+	}
+
+	caaf.logger.Info("AddNamespace: setting up informers for new namespace (container)", zap.String("namespace", ns))
+
+	finformer := genInformer.NewFilteredSharedInformerFactory(caaf.fissionClient, 30*time.Minute, ns, nil)
+
+	executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeContainer)
+	if err != nil {
+		return fmt.Errorf("AddNamespace %s (container): get executor label: %w", ns, err)
+	}
+	cnmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
+		caaf.kubernetesClient,
+		30*time.Minute,
+		k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
+			opts.LabelSelector = executorLabel.String()
+		}),
+		k8sInformers.WithNamespace(ns),
+	)
+
+	caaf.deplLister[ns] = cnmInformer.Apps().V1().Deployments().Lister()
+	caaf.deplListerSynced[ns] = cnmInformer.Apps().V1().Deployments().Informer().HasSynced
+	caaf.svcLister[ns] = cnmInformer.Core().V1().Services().Lister()
+	caaf.svcListerSynced[ns] = cnmInformer.Core().V1().Services().Informer().HasSynced
+
+	_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(caaf.FuncInformerHandler(ctx))
+	if err != nil {
+		return fmt.Errorf("AddNamespace %s (container): add function handler: %w", ns, err)
+	}
+
+	finformer.Start(ctx.Done())
+	cnmInformer.Start(ctx.Done())
+
+	caaf.logger.Info("AddNamespace: done (container)", zap.String("namespace", ns))
+	return nil
+}

pkg/executor/executortype/executortype.go

@@ -69,4 +69,9 @@ type ExecutorType interface {
 
 	// CleanupOldExecutorObjects cleans up resources created by old executor instances
 	CleanupOldExecutorObjects(context.Context)
+
+	// AddNamespace dynamically registers a new user namespace so the executor
+	// starts watching Fission CRDs and K8s resources in it without a pod restart.
+	// Called when a Namespace with label fission.io/managed=true appears.
+	AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error
 }

pkg/executor/executortype/newdeploy/newdeploymgr.go

@@ -898,3 +898,55 @@ func (deploy *NewDeploy) scaleDeployment(ctx context.Context, deplNS string, dep
 func (deploy *NewDeploy) DumpDebugInfo(ctx context.Context) error {
 	return nil
 }
+
+// AddNamespace dynamically registers a new namespace in the newdeploy executor without a restart.
+// Sets up deployment and service listers so the executor can manage functions in the new NS.
+// Safe to call repeatedly — uses per-executor deplLister for deduplication.
+func (deploy *NewDeploy) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
+	if ns == "" {
+		return nil
+	}
+	// Use newdeploy-specific dedup: check if deplLister is already set up for this NS.
+	// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
+	// shared by all executor types and is now called once in multitenant.registerNamespace.
+	if _, ok := deploy.deplLister[ns]; ok {
+		return nil // already registered
+	}
+
+	deploy.logger.Info("AddNamespace: setting up informers for new namespace (newdeploy)", zap.String("namespace", ns))
+
+	// Fission CRD informer factory for the new NS.
+	finformer := genInformer.NewFilteredSharedInformerFactory(deploy.fissionClient, 30*time.Minute, ns, nil)
+
+	// K8s deployment+service informer factory filtered by newdeploy executor label.
+	executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeNewdeploy)
+	if err != nil {
+		return fmt.Errorf("AddNamespace %s (newdeploy): get executor label: %w", ns, err)
+	}
+	ndmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
+		deploy.kubernetesClient,
+		30*time.Minute,
+		k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
+			opts.LabelSelector = executorLabel.String()
+		}),
+		k8sInformers.WithNamespace(ns),
+	)
+
+	// Register deployment and service listers — same as done at startup in MakeNewDeploy.
+	deploy.deplLister[ns] = ndmInformer.Apps().V1().Deployments().Lister()
+	deploy.deplListerSynced[ns] = ndmInformer.Apps().V1().Deployments().Informer().HasSynced
+	deploy.svcLister[ns] = ndmInformer.Core().V1().Services().Lister()
+	deploy.svcListerSynced[ns] = ndmInformer.Core().V1().Services().Informer().HasSynced
+
+	// Register function event handler so this NS's functions are adopted.
+	_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(deploy.FunctionEventHandlers(ctx))
+	if err != nil {
+		return fmt.Errorf("AddNamespace %s (newdeploy): add function handler: %w", ns, err)
+	}
+
+	finformer.Start(ctx.Done())
+	ndmInformer.Start(ctx.Done())
+
+	deploy.logger.Info("AddNamespace: done (newdeploy)", zap.String("namespace", ns))
+	return nil
+}

pkg/executor/executortype/poolmgr/gpm.go

@@ -784,3 +784,59 @@ func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(ctx context.Contex
 func (gpm *GenericPoolManager) DumpDebugInfo(ctx context.Context) error {
 	return gpm.fsCache.DumpDebugInfo(ctx)
 }
+
+// AddNamespace dynamically registers a new namespace in the poolmgr executor without a restart.
+// Called by watchManagedNamespaces when a Namespace with label fission.io/managed=true appears.
+//
+// What it does:
+//  1. Adds the namespace to the global NamespaceResolver (thread-safe, deduplicates)
+//  2. Creates a Fission CRD informer factory for the NS (watches Environments, Functions, Packages)
+//  3. Creates a K8s pod informer factory filtered by poolmgr executor label
+//  4. Registers the informers with PoolPodController (envLister, podLister, RS watcher)
+//  5. Starts both informer factories
+//
+// If the namespace was already registered, returns nil immediately (no-op).
+func (gpm *GenericPoolManager) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
+	if ns == "" {
+		// Empty NS means "scan all" — for poolmgr this is a no-op; discovery is done by the caller.
+		return nil
+	}
+	// Use poolmgr-specific dedup: check if envLister is already set up for this NS.
+	// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
+	// shared by all executor types and is now called once in multitenant.registerNamespace.
+	if _, ok := gpm.poolPodC.envLister[ns]; ok {
+		return nil
+	}
+
+	gpm.logger.Info("AddNamespace: setting up informers for new namespace", zap.String("namespace", ns))
+
+	// Fission CRD informer factory — watches Environments, Functions, Packages in this NS.
+	finformer := genInformer.NewFilteredSharedInformerFactory(gpm.fissionClient, 30*time.Minute, ns, nil)
+
+	// K8s pod/RS informer factory filtered by poolmgr executor label.
+	// Same label used at startup in GetInformerFactoryByExecutor.
+	executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypePoolmgr)
+	if err != nil {
+		return fmt.Errorf("AddNamespace %s: get executor label: %w", ns, err)
+	}
+	gpmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
+		gpm.kubernetesClient,
+		30*time.Minute,
+		k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
+			opts.LabelSelector = executorLabel.String()
+		}),
+		k8sInformers.WithNamespace(ns),
+	)
+
+	// Register the new informers with PoolPodController.
+	if err := gpm.poolPodC.AddNamespaceInformers(ctx, ns, finformer, gpmInformer); err != nil {
+		return fmt.Errorf("AddNamespace %s: register informers: %w", ns, err)
+	}
+
+	// Start the factories — they will begin syncing immediately.
+	finformer.Start(ctx.Done())
+	gpmInformer.Start(ctx.Done())
+
+	gpm.logger.Info("AddNamespace: informers started for namespace", zap.String("namespace", ns))
+	return nil
+}

pkg/executor/executortype/poolmgr/poolpodcontroller.go

@@ -485,3 +485,46 @@ func (p *PoolPodController) spCleanupPodQueueProcessFunc(ctx context.Context) bo
 	p.spCleanupPodQueue.Forget(key)
 	return false
 }
+
+// AddNamespaceInformers registers informers for a newly-added namespace in PoolPodController.
+// Called from GenericPoolManager.AddNamespace after the informer factories are created.
+//
+// Sets up:
+//   - Environment lister and synced func (for pool creation on env events)
+//   - Pod lister and synced func (for specialized pod tracking)
+//   - ReplicaSet event handler (for RS scale-down pod cleanup)
+func (p *PoolPodController) AddNamespaceInformers(
+	ctx context.Context,
+	ns string,
+	finformer genInformer.SharedInformerFactory,
+	gpmInformer k8sInformers.SharedInformerFactory,
+) error {
+	// Environment informer — triggers pool creation/deletion when envs change in this NS.
+	_, err := finformer.Core().V1().Environments().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
+		AddFunc:    p.enqueueEnvAdd,
+		UpdateFunc: p.enqueueEnvUpdate,
+		DeleteFunc: p.enqueueEnvDelete,
+	})
+	if err != nil {
+		return fmt.Errorf("AddNamespaceInformers %s: add env handler: %w", ns, err)
+	}
+	p.envLister[ns] = finformer.Core().V1().Environments().Lister()
+	p.envListerSynced[ns] = finformer.Core().V1().Environments().Informer().HasSynced
+
+	// Pod lister — used by processRS to find specialized pods in this NS.
+	p.podLister[ns] = gpmInformer.Core().V1().Pods().Lister()
+	p.podListerSynced[ns] = gpmInformer.Core().V1().Pods().Informer().HasSynced
+
+	// ReplicaSet informer — triggers cleanup of specialized pods when RS scales to 0.
+	_, err = gpmInformer.Apps().V1().ReplicaSets().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
+		AddFunc:    p.handleRSAdd,
+		UpdateFunc: p.handleRSUpdate,
+		DeleteFunc: p.handleRSDelete,
+	})
+	if err != nil {
+		return fmt.Errorf("AddNamespaceInformers %s: add RS handler: %w", ns, err)
+	}
+
+	p.logger.Info("AddNamespaceInformers: registered informers for namespace", zap.String("namespace", ns))
+	return nil
+}

pkg/executor/multitenant/ns_watcher.go

@@ -0,0 +1,188 @@
+// Package multitenant provides utilities for running Fission in a multi-tenant
+// Kubernetes environment where user namespaces are created dynamically at runtime.
+//
+// # The Problem
+//
+// In a standard Fission installation, all resource namespaces must be enumerated in the
+// FISSION_RESOURCE_NAMESPACES environment variable before the process starts. Adding a
+// new user namespace requires patching that env var and triggering a rolling restart of
+// every Fission component (executor, router, buildermgr, etc.) — causing ~30 seconds of
+// downtime per new tenant.
+//
+// At scale this becomes a severe operational problem: with hundreds of new tenants being
+// created continuously, the executor is in a permanent rolling restart loop, causing
+// cascading failures for all existing users.
+//
+// # The Solution
+//
+// This package implements hot namespace registration without pod restarts.
+// The mechanism is intentionally simple and decoupled from any specific platform:
+//
+//  1. A platform (a cloud console, a CI/CD system, an operator) creates a Kubernetes
+//     Namespace and sets the label:
+//     fission.io/managed=true
+//
+//  2. StartNSWatcher registers a Kubernetes Namespace Informer that receives an event
+//     the moment a labeled Namespace is created or updated — no polling, no delay.
+//
+//  3. On the AddFunc / UpdateFunc callback NSWatcher calls AddNamespace on every
+//     registered executor type (poolmgr, newdeploy, container). Each type creates
+//     per-NS informer factories, pod listers, and event handlers — live, without restart.
+//
+//  4. NamespaceResolver.AddNamespace deduplicates — calling AddNamespace on an already-
+//     registered namespace is always a safe no-op.
+//
+// # Backward Compatibility
+//
+// The FISSION_RESOURCE_NAMESPACES environment variable continues to work as before.
+// Namespaces listed there are registered at startup and do not require the label.
+// This package adds on top of the existing mechanism — it does not replace it.
+//
+// # Required RBAC
+//
+// The fission-executor ServiceAccount must be granted permission to list and watch
+// Namespaces at the cluster scope. Apply the manifest at:
+//
+//	deploy/multitenant/rbac.yaml
+//
+// # Integration Contract
+//
+// The entire integration contract for external platforms is a single label on a Namespace:
+//
+//	apiVersion: v1
+//	kind: Namespace
+//	metadata:
+//	  name: tenant-abc123
+//	  labels:
+//	    fission.io/managed: "true"
+//
+// No other coupling to Fission internals is required or expected.
+package multitenant
+
+import (
+	"context"
+	"time"
+
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sInformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+	"github.com/fission/fission/pkg/executor/executortype"
+	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/manager"
+)
+
+// ManagedNSLabel is the Kubernetes label key that marks a Namespace as dynamically managed
+// by Fission multi-tenant mode. The expected value is "true".
+//
+// Platforms MUST set this label on every user Namespace they create:
+//
+//	metadata:
+//	  labels:
+//	    fission.io/managed: "true"
+const ManagedNSLabel = "fission.io/managed"
+
+// StartNSWatcher registers a Kubernetes Namespace Informer that reacts immediately
+// when a Namespace with label fission.io/managed=true is created or relabeled.
+//
+// Unlike a polling approach, this uses the standard k8s Watch mechanism — the executor
+// receives the event within milliseconds of the Namespace appearing, with zero wasted
+// API calls between events.
+//
+// The informer is managed via mgr and shuts down cleanly when ctx is cancelled.
+func StartNSWatcher(
+	ctx context.Context,
+	logger *zap.Logger,
+	kubernetesClient kubernetes.Interface,
+	executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
+	mgr manager.Interface,
+) {
+	// Use a label-filtered informer so only Namespaces with our label are delivered.
+	// The resync period of 30m is standard for Fission informers — it re-lists to recover
+	// from any missed events, but normal operation is purely event-driven (no ticking).
+	factory := k8sInformers.NewSharedInformerFactoryWithOptions(
+		kubernetesClient,
+		30*time.Minute,
+		k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
+			opts.LabelSelector = ManagedNSLabel + "=true"
+		}),
+	)
+
+	nsInformer := factory.Core().V1().Namespaces().Informer()
+
+	_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
+		// AddFunc fires when a new Namespace with the label appears.
+		AddFunc: func(obj interface{}) {
+			ns := namespaceName(obj)
+			if ns == "" {
+				return
+			}
+			registerNamespace(ctx, logger, kubernetesClient, ns, executorTypes, mgr)
+		},
+		// UpdateFunc fires when an existing Namespace is updated — covers the case
+		// where the label is added to a pre-existing Namespace.
+		UpdateFunc: func(_, newObj interface{}) {
+			nsObj, ok := newObj.(*corev1.Namespace)
+			if !ok {
+				return
+			}
+			if nsObj.Labels[ManagedNSLabel] != "true" {
+				return // label was removed — nothing to do (executor keeps existing registrations)
+			}
+			registerNamespace(ctx, logger, kubernetesClient, nsObj.Name, executorTypes, mgr)
+		},
+	})
+
+	mgr.Add(ctx, func(ctx context.Context) {
+		logger.Info("multitenant.NSWatcher: started", zap.String("label", ManagedNSLabel+"=true"))
+		factory.Start(ctx.Done())
+		factory.WaitForCacheSync(ctx.Done())
+		logger.Info("multitenant.NSWatcher: cache synced — watching for new namespaces")
+		<-ctx.Done()
+		logger.Info("multitenant.NSWatcher: stopped")
+	})
+}
+
+// registerNamespace calls AddNamespace on every executor type for the given namespace.
+// The global NamespaceResolver is updated here — once, before any executor type is called.
+// Each executor type uses its own internal state for deduplication instead of the
+// global resolver, so all executor types receive the AddNamespace call regardless of
+// iteration order.
+func registerNamespace(
+	ctx context.Context,
+	logger *zap.Logger,
+	kubernetesClient kubernetes.Interface,
+	ns string,
+	executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
+	mgr manager.Interface,
+) {
+	// Update the global resolver once here. Each executor type must NOT call
+	// DefaultNSResolver().AddNamespace() for dedup — they have their own checks.
+	utils.DefaultNSResolver().AddNamespace(ns)
+	// Ensure fission-fetcher SA exists in the new namespace so pool pods can start.
+	utils.EnsureNamespaceSA(ctx, kubernetesClient, logger, ns)
+
+	for _, et := range executorTypes {
+		if err := et.AddNamespace(ctx, ns, mgr); err != nil {
+			logger.Error("multitenant.NSWatcher: AddNamespace failed",
+				zap.String("namespace", ns),
+				zap.Error(err),
+			)
+		}
+	}
+	logger.Info("multitenant.NSWatcher: registered namespace", zap.String("namespace", ns))
+}
+
+// namespaceName extracts the namespace name from an informer event object.
+func namespaceName(obj interface{}) string {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return ""
+	}
+	return accessor.GetName()
+}