// Package multitenant provides utilities for running Fission in a multi-tenant // Kubernetes environment where user namespaces are created dynamically at runtime. // // # The Problem // // In a standard Fission installation, all resource namespaces must be enumerated in the // FISSION_RESOURCE_NAMESPACES environment variable before the process starts. Adding a // new user namespace requires patching that env var and triggering a rolling restart of // every Fission component (executor, router, buildermgr, etc.) — causing ~30 seconds of // downtime per new tenant. // // At scale this becomes a severe operational problem: with hundreds of new tenants being // created continuously, the executor is in a permanent rolling restart loop, causing // cascading failures for all existing users. // // # The Solution // // This package implements hot namespace registration without pod restarts. // The mechanism is intentionally simple and decoupled from any specific platform: // // 1. A platform (a cloud console, a CI/CD system, an operator) creates a Kubernetes // Namespace and sets the label: // fission.io/managed=true // // 2. StartNSWatcher registers a Kubernetes Namespace Informer that receives an event // the moment a labeled Namespace is created or updated — no polling, no delay. // // 3. On the AddFunc / UpdateFunc callback NSWatcher calls AddNamespace on every // registered executor type (poolmgr, newdeploy, container). Each type creates // per-NS informer factories, pod listers, and event handlers — live, without restart. // // 4. NamespaceResolver.AddNamespace deduplicates — calling AddNamespace on an already- // registered namespace is always a safe no-op. // // # Backward Compatibility // // The FISSION_RESOURCE_NAMESPACES environment variable continues to work as before. // Namespaces listed there are registered at startup and do not require the label. // This package adds on top of the existing mechanism — it does not replace it. // // # Required RBAC // // The fission-executor ServiceAccount must be granted permission to list and watch // Namespaces at the cluster scope. Apply the manifest at: // // deploy/multitenant/rbac.yaml // // # Integration Contract // // The entire integration contract for external platforms is a single label on a Namespace: // // apiVersion: v1 // kind: Namespace // metadata: // name: tenant-abc123 // labels: // fission.io/managed: "true" // // No other coupling to Fission internals is required or expected. package multitenant import ( "context" "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sInformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" k8sCache "k8s.io/client-go/tools/cache" fv1 "github.com/fission/fission/pkg/apis/core/v1" "github.com/fission/fission/pkg/executor/executortype" "github.com/fission/fission/pkg/utils" "github.com/fission/fission/pkg/utils/manager" ) // ManagedNSLabel is the Kubernetes label key that marks a Namespace as dynamically managed // by Fission multi-tenant mode. The expected value is "true". // // Platforms MUST set this label on every user Namespace they create: // // metadata: // labels: // fission.io/managed: "true" const ManagedNSLabel = "fission.io/managed" // StartNSWatcher registers a Kubernetes Namespace Informer that reacts immediately // when a Namespace with label fission.io/managed=true is created or relabeled. // // Unlike a polling approach, this uses the standard k8s Watch mechanism — the executor // receives the event within milliseconds of the Namespace appearing, with zero wasted // API calls between events. // // The informer is managed via mgr and shuts down cleanly when ctx is cancelled. func StartNSWatcher( ctx context.Context, logger *zap.Logger, kubernetesClient kubernetes.Interface, executorTypes map[fv1.ExecutorType]executortype.ExecutorType, mgr manager.Interface, ) { // Use a label-filtered informer so only Namespaces with our label are delivered. // The resync period of 30m is standard for Fission informers — it re-lists to recover // from any missed events, but normal operation is purely event-driven (no ticking). factory := k8sInformers.NewSharedInformerFactoryWithOptions( kubernetesClient, 30*time.Minute, k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) { opts.LabelSelector = ManagedNSLabel + "=true" }), ) nsInformer := factory.Core().V1().Namespaces().Informer() _, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{ // AddFunc fires when a new Namespace with the label appears. AddFunc: func(obj interface{}) { ns := namespaceName(obj) if ns == "" { return } registerNamespace(ctx, logger, kubernetesClient, ns, executorTypes, mgr) }, // UpdateFunc fires when an existing Namespace is updated — covers the case // where the label is added to a pre-existing Namespace. UpdateFunc: func(_, newObj interface{}) { nsObj, ok := newObj.(*corev1.Namespace) if !ok { return } if nsObj.Labels[ManagedNSLabel] != "true" { return // label was removed — nothing to do (executor keeps existing registrations) } registerNamespace(ctx, logger, kubernetesClient, nsObj.Name, executorTypes, mgr) }, }) mgr.Add(ctx, func(ctx context.Context) { logger.Info("multitenant.NSWatcher: started", zap.String("label", ManagedNSLabel+"=true")) factory.Start(ctx.Done()) factory.WaitForCacheSync(ctx.Done()) logger.Info("multitenant.NSWatcher: cache synced — watching for new namespaces") <-ctx.Done() logger.Info("multitenant.NSWatcher: stopped") }) } // registerNamespace calls AddNamespace on every executor type for the given namespace. // The global NamespaceResolver is updated here — once, before any executor type is called. // Each executor type uses its own internal state for deduplication instead of the // global resolver, so all executor types receive the AddNamespace call regardless of // iteration order. func registerNamespace( ctx context.Context, logger *zap.Logger, kubernetesClient kubernetes.Interface, ns string, executorTypes map[fv1.ExecutorType]executortype.ExecutorType, mgr manager.Interface, ) { // Update the global resolver once here. Each executor type must NOT call // DefaultNSResolver().AddNamespace() for dedup — they have their own checks. utils.DefaultNSResolver().AddNamespace(ns) // Ensure fission-fetcher SA exists in the new namespace so pool pods can start. utils.EnsureNamespaceSA(ctx, kubernetesClient, logger, ns) for _, et := range executorTypes { if err := et.AddNamespace(ctx, ns, mgr); err != nil { logger.Error("multitenant.NSWatcher: AddNamespace failed", zap.String("namespace", ns), zap.Error(err), ) } } logger.Info("multitenant.NSWatcher: registered namespace", zap.String("namespace", ns)) } // namespaceName extracts the namespace name from an informer event object. func namespaceName(obj interface{}) string { accessor, err := meta.Accessor(obj) if err != nil { return "" } return accessor.GetName() }