multi-tenant: EnsureNamespaceSA + ns_watcher SA provisioning (v8)

This commit is contained in:
Naeel
2026-04-26 07:41:46 +03:00
parent 82e1ff76a5
commit 161de70576
24 changed files with 970 additions and 1 deletions
+3
View File
@@ -0,0 +1,3 @@
FROM cgr.dev/chainguard/static:latest@sha256:a301031ffd4ed67f35ca7fa6cf3dad9937b5fa47d7493955a18d9b4ca5412d1a
COPY fission-bundle /
ENTRYPOINT ["/fission-bundle"]
BIN
View File
Binary file not shown.
+3
View File
@@ -0,0 +1,3 @@
FROM cgr.dev/chainguard/static:latest@sha256:a301031ffd4ed67f35ca7fa6cf3dad9937b5fa47d7493955a18d9b4ca5412d1a
COPY fission-bundle /
ENTRYPOINT ["/fission-bundle"]
Binary file not shown.
+26
View File
@@ -0,0 +1,26 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fission-executor-ns-watcher
labels:
app: fission-executor
rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: fission-executor-ns-watcher
labels:
app: fission-executor
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fission-executor-ns-watcher
subjects:
- kind: ServiceAccount
name: fission-executor
namespace: fission
+42
View File
@@ -0,0 +1,42 @@
# deploy/multitenant/rbac.yaml
#
# RBAC required for the Fission multi-tenant NSWatcher.
#
# The fission-executor ServiceAccount must be allowed to list and watch Namespaces
# at the cluster scope so that NSWatcher can detect newly-labeled Namespaces.
#
# This is a read-only ClusterRole — no write permissions are granted.
# Apply once per cluster after installing Fission:
#
# kubectl apply -f deploy/multitenant/rbac.yaml
#
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fission-executor-ns-watcher
labels:
app.kubernetes.io/name: fission
app.kubernetes.io/component: executor
app.kubernetes.io/part-of: fission-multitenant
rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: fission-executor-ns-watcher
labels:
app.kubernetes.io/name: fission
app.kubernetes.io/component: executor
app.kubernetes.io/part-of: fission-multitenant
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: fission-executor-ns-watcher
subjects:
- kind: ServiceAccount
name: fission-executor
namespace: fission
Executable
BIN
View File
Binary file not shown.
+132
View File
@@ -0,0 +1,132 @@
import re
# ── envwatcher.go ─────────────────────────────────────────────────────────────
with open("/home/naeel/terra/fission-src/pkg/buildermgr/envwatcher.go") as f:
src = f.read()
# добавляем genInformer import если нет
if "genInformer" not in src:
src = src.replace(
'"github.com/fission/fission/pkg/generated/clientset/versioned"',
'"github.com/fission/fission/pkg/generated/clientset/versioned"\n\t'
'genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"',
1
)
# добавляем fmt если нет
if '"fmt"' not in src:
src = src.replace('"context"', '"context"\n\t"fmt"', 1)
addon = r'''
// AddNamespace dynamically registers a new namespace in environmentWatcher.
// Creates a per-NS Environment informer. Safe to call repeatedly — deduplicates via nsResolver.
func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
if !envw.nsResolver.AddNamespace(ns) {
return // already registered
}
envw.logger.Info("buildermgr.envWatcher.AddNamespace: setting up informer", zap.String("namespace", ns))
factory := genInformer.NewFilteredSharedInformerFactory(envw.fissionClient, 30*time.Minute, ns, nil)
envInf := factory.Core().V1().Environments().Informer()
_, err := envInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
envObj := obj.(*fv1.Environment)
envw.AddUpdateBuilder(ctx, envObj)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldEnvObj := oldObj.(*fv1.Environment)
newEnvObj := newObj.(*fv1.Environment)
if oldEnvObj.ResourceVersion == newEnvObj.ResourceVersion {
return
}
envw.AddUpdateBuilder(ctx, newEnvObj)
},
DeleteFunc: func(obj interface{}) {
envObj, ok := obj.(*fv1.Environment)
if !ok {
return
}
envw.deleteBuilder(ctx, envObj)
},
})
if err != nil {
envw.logger.Error("buildermgr.envWatcher.AddNamespace: add handler failed",
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
return
}
envw.envWatchInformer[ns] = envInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
}
'''
with open("/home/naeel/terra/fission-src/pkg/buildermgr/envwatcher.go", "w") as f:
f.write(src + addon)
print("envwatcher.go: done")
# ── pkgwatcher.go ─────────────────────────────────────────────────────────────
with open("/home/naeel/terra/fission-src/pkg/buildermgr/pkgwatcher.go") as f:
src = f.read()
# добавляем genInformer import если нет
if "genInformer" not in src:
src = src.replace(
'"github.com/fission/fission/pkg/generated/clientset/versioned"',
'"github.com/fission/fission/pkg/generated/clientset/versioned"\n\t'
'genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"',
1
)
# добавляем fmt если нет
if '"fmt"' not in src:
src = src.replace('"context"', '"context"\n\t"fmt"', 1)
# Добавляем k8sInformers если нет
if "k8sInformers" not in src:
src = src.replace(
'"k8s.io/client-go/kubernetes"',
'"k8s.io/client-go/kubernetes"\n\tk8sInformers "k8s.io/client-go/informers"',
1
)
addon2 = r'''
// AddNamespace dynamically registers a new namespace in packageWatcher.
// Creates per-NS Package and Pod informers. Safe to call repeatedly.
func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
if !pkgw.nsResolver.AddNamespace(ns) {
return // already registered
}
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: setting up informers", zap.String("namespace", ns))
// Package informer
fissionFactory := genInformer.NewFilteredSharedInformerFactory(pkgw.fissionClient, 30*time.Minute, ns, nil)
pkgInf := fissionFactory.Core().V1().Packages().Informer()
_, err := pkgInf.AddEventHandler(pkgw.packageInformerHandler(ctx))
if err != nil {
pkgw.logger.Error("buildermgr.pkgWatcher.AddNamespace: pkg handler failed",
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
return
}
// Pod informer for build logs
podFactory := k8sInformers.NewSharedInformerFactoryWithOptions(pkgw.k8sClient, 30*time.Minute,
k8sInformers.WithNamespace(ns))
podInf := podFactory.Core().V1().Pods().Informer()
pkgw.pkgInformer[ns] = pkgInf
pkgw.podInformer[ns] = podInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
ns + "/pkg": pkgInf,
ns + "/pod": podInf,
})
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
}
'''
with open("/home/naeel/terra/fission-src/pkg/buildermgr/pkgwatcher.go", "w") as f:
f.write(src + addon2)
print("pkgwatcher.go: done")
+4
View File
@@ -74,5 +74,9 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
if err != nil {
return err
}
// Multi-tenant: watch namespaces labeled fission.io/managed=true
StartNSWatcher(ctx, logger, kubernetesClient, envWatcher, pkgWatcher, mgr)
return nil
}
+45
View File
@@ -39,6 +39,7 @@ import (
"github.com/fission/fission/pkg/executor/util"
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
"github.com/fission/fission/pkg/generated/clientset/versioned"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
)
@@ -498,3 +499,47 @@ func (envw *environmentWatcher) createBuilderDeployment(ctx context.Context, env
return deployment, nil
}
// AddNamespace dynamically registers a new namespace in environmentWatcher.
// Creates a per-NS Environment informer. Safe to call repeatedly — deduplicates via nsResolver.
func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
if !envw.nsResolver.AddNamespace(ns) {
return // already registered
}
envw.logger.Info("buildermgr.envWatcher.AddNamespace: setting up informer", zap.String("namespace", ns))
factory := genInformer.NewFilteredSharedInformerFactory(envw.fissionClient, 30*time.Minute, ns, nil)
envInf := factory.Core().V1().Environments().Informer()
_, err := envInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
envObj := obj.(*fv1.Environment)
envw.AddUpdateBuilder(ctx, envObj)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldEnvObj := oldObj.(*fv1.Environment)
newEnvObj := newObj.(*fv1.Environment)
if oldEnvObj.ResourceVersion == newEnvObj.ResourceVersion {
return
}
envw.AddUpdateBuilder(ctx, newEnvObj)
},
DeleteFunc: func(obj interface{}) {
envObj, ok := obj.(*fv1.Environment)
if !ok {
return
}
envw.DeleteBuilder(ctx, envObj)
},
})
if err != nil {
envw.logger.Error("buildermgr.envWatcher.AddNamespace: add handler failed",
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
return
}
envw.envWatchInformer[ns] = envInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
factory.Start(ctx.Done())
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
}
+80
View File
@@ -0,0 +1,80 @@
// Package buildermgr — NSWatcher for multi-tenant mode.
//
// Listens for Namespaces labeled fission.io/managed=true and calls
// AddNamespace on envWatcher and packageWatcher so they pick up
// Environments and Packages in new tenant namespaces without a restart.
package buildermgr
import (
"context"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sInformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission/pkg/utils/manager"
)
const builderManagedNSLabel = "fission.io/managed"
// StartNSWatcher watches for Namespaces labeled fission.io/managed=true
// and immediately registers per-NS informers in envWatcher and pkgWatcher.
func StartNSWatcher(
ctx context.Context,
logger *zap.Logger,
kubeClient kubernetes.Interface,
envw *environmentWatcher,
pkgw *packageWatcher,
mgr manager.Interface,
) {
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
kubeClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = builderManagedNSLabel + "=true"
}),
)
nsInformer := factory.Core().V1().Namespaces().Informer()
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ns := builderNSName(obj)
if ns == "" {
return
}
envw.AddNamespace(ctx, ns, mgr)
pkgw.AddNamespace(ctx, ns, mgr)
},
UpdateFunc: func(_, newObj interface{}) {
nsObj, ok := newObj.(*corev1.Namespace)
if !ok || nsObj.Labels[builderManagedNSLabel] != "true" {
return
}
envw.AddNamespace(ctx, nsObj.Name, mgr)
pkgw.AddNamespace(ctx, nsObj.Name, mgr)
},
})
mgr.Add(ctx, func(ctx context.Context) {
logger.Info("buildermgr.NSWatcher: started",
zap.String("label", builderManagedNSLabel+"=true"))
factory.Start(ctx.Done())
factory.WaitForCacheSync(ctx.Done())
logger.Info("buildermgr.NSWatcher: cache synced — watching for new namespaces")
<-ctx.Done()
logger.Info("buildermgr.NSWatcher: stopped")
})
}
func builderNSName(obj interface{}) string {
nsObj, ok := obj.(*corev1.Namespace)
if !ok {
return ""
}
return nsObj.Name
}
+38
View File
@@ -25,6 +25,7 @@ import (
apiv1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sInformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
@@ -32,6 +33,7 @@ import (
"github.com/fission/fission/pkg/cache"
"github.com/fission/fission/pkg/crd"
"github.com/fission/fission/pkg/generated/clientset/versioned"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
"github.com/fission/fission/pkg/utils/metrics"
@@ -329,3 +331,39 @@ func setInitialBuildStatus(ctx context.Context, fissionClient versioned.Interfac
// TODO: use UpdateStatus to update status
return fissionClient.CoreV1().Packages(pkg.Namespace).Update(ctx, pkg, metav1.UpdateOptions{})
}
// AddNamespace dynamically registers a new namespace in packageWatcher.
// Creates per-NS Package and Pod informers. Safe to call repeatedly.
func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
if !pkgw.nsResolver.AddNamespace(ns) {
return // already registered
}
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: setting up informers", zap.String("namespace", ns))
// Package informer
fissionFactory := genInformer.NewFilteredSharedInformerFactory(pkgw.fissionClient, 30*time.Minute, ns, nil)
pkgInf := fissionFactory.Core().V1().Packages().Informer()
_, err := pkgInf.AddEventHandler(pkgw.packageInformerHandler(ctx))
if err != nil {
pkgw.logger.Error("buildermgr.pkgWatcher.AddNamespace: pkg handler failed",
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
return
}
// Pod informer for build logs
podFactory := k8sInformers.NewSharedInformerFactoryWithOptions(pkgw.k8sClient, 30*time.Minute,
k8sInformers.WithNamespace(ns))
podInf := podFactory.Core().V1().Pods().Informer()
pkgw.pkgInformer[ns] = pkgInf
pkgw.podInformer[ns] = podInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
ns + "/pkg": pkgInf,
ns + "/pod": podInf,
})
fissionFactory.Start(ctx.Done())
podFactory.Start(ctx.Done())
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
}
+7
View File
@@ -37,6 +37,7 @@ import (
"github.com/fission/fission/pkg/executor/executortype/newdeploy"
"github.com/fission/fission/pkg/executor/executortype/poolmgr"
"github.com/fission/fission/pkg/executor/fscache"
"github.com/fission/fission/pkg/executor/multitenant"
"github.com/fission/fission/pkg/executor/util"
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
"github.com/fission/fission/pkg/generated/clientset/versioned"
@@ -399,6 +400,12 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
utils.CreateMissingPermissionForSA(ctx, kubernetesClient, logger)
// Start multi-tenant Namespace watcher.
// Detects Namespaces labeled fission.io/managed=true and registers them in all
// executor types without a pod restart. Backward-compatible with FISSION_RESOURCE_NAMESPACES.
// See: pkg/executor/multitenant/ns_watcher.go
multitenant.StartNSWatcher(ctx, logger, kubernetesClient, executorTypes, mgr)
mgr.Add(ctx, func(ctx context.Context) {
metrics.ServeMetrics(ctx, "executor", logger, mgr)
})
@@ -792,3 +792,50 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
func (caaf *Container) DumpDebugInfo(ctx context.Context) error {
return nil
}
// AddNamespace dynamically registers a new namespace in the container executor without a restart.
// Sets up deployment and service listers so the executor can manage container functions in the new NS.
func (caaf *Container) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
if ns == "" {
return nil
}
// Use container-specific dedup: check if deplLister is already set up for this NS.
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
// shared by all executor types and is now called once in multitenant.registerNamespace.
if _, ok := caaf.deplLister[ns]; ok {
return nil // already registered
}
caaf.logger.Info("AddNamespace: setting up informers for new namespace (container)", zap.String("namespace", ns))
finformer := genInformer.NewFilteredSharedInformerFactory(caaf.fissionClient, 30*time.Minute, ns, nil)
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeContainer)
if err != nil {
return fmt.Errorf("AddNamespace %s (container): get executor label: %w", ns, err)
}
cnmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
caaf.kubernetesClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = executorLabel.String()
}),
k8sInformers.WithNamespace(ns),
)
caaf.deplLister[ns] = cnmInformer.Apps().V1().Deployments().Lister()
caaf.deplListerSynced[ns] = cnmInformer.Apps().V1().Deployments().Informer().HasSynced
caaf.svcLister[ns] = cnmInformer.Core().V1().Services().Lister()
caaf.svcListerSynced[ns] = cnmInformer.Core().V1().Services().Informer().HasSynced
_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(caaf.FuncInformerHandler(ctx))
if err != nil {
return fmt.Errorf("AddNamespace %s (container): add function handler: %w", ns, err)
}
finformer.Start(ctx.Done())
cnmInformer.Start(ctx.Done())
caaf.logger.Info("AddNamespace: done (container)", zap.String("namespace", ns))
return nil
}
@@ -69,4 +69,9 @@ type ExecutorType interface {
// CleanupOldExecutorObjects cleans up resources created by old executor instances
CleanupOldExecutorObjects(context.Context)
// AddNamespace dynamically registers a new user namespace so the executor
// starts watching Fission CRDs and K8s resources in it without a pod restart.
// Called when a Namespace with label fission.io/managed=true appears.
AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error
}
@@ -898,3 +898,55 @@ func (deploy *NewDeploy) scaleDeployment(ctx context.Context, deplNS string, dep
func (deploy *NewDeploy) DumpDebugInfo(ctx context.Context) error {
return nil
}
// AddNamespace dynamically registers a new namespace in the newdeploy executor without a restart.
// Sets up deployment and service listers so the executor can manage functions in the new NS.
// Safe to call repeatedly — uses per-executor deplLister for deduplication.
func (deploy *NewDeploy) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
if ns == "" {
return nil
}
// Use newdeploy-specific dedup: check if deplLister is already set up for this NS.
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
// shared by all executor types and is now called once in multitenant.registerNamespace.
if _, ok := deploy.deplLister[ns]; ok {
return nil // already registered
}
deploy.logger.Info("AddNamespace: setting up informers for new namespace (newdeploy)", zap.String("namespace", ns))
// Fission CRD informer factory for the new NS.
finformer := genInformer.NewFilteredSharedInformerFactory(deploy.fissionClient, 30*time.Minute, ns, nil)
// K8s deployment+service informer factory filtered by newdeploy executor label.
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeNewdeploy)
if err != nil {
return fmt.Errorf("AddNamespace %s (newdeploy): get executor label: %w", ns, err)
}
ndmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
deploy.kubernetesClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = executorLabel.String()
}),
k8sInformers.WithNamespace(ns),
)
// Register deployment and service listers — same as done at startup in MakeNewDeploy.
deploy.deplLister[ns] = ndmInformer.Apps().V1().Deployments().Lister()
deploy.deplListerSynced[ns] = ndmInformer.Apps().V1().Deployments().Informer().HasSynced
deploy.svcLister[ns] = ndmInformer.Core().V1().Services().Lister()
deploy.svcListerSynced[ns] = ndmInformer.Core().V1().Services().Informer().HasSynced
// Register function event handler so this NS's functions are adopted.
_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(deploy.FunctionEventHandlers(ctx))
if err != nil {
return fmt.Errorf("AddNamespace %s (newdeploy): add function handler: %w", ns, err)
}
finformer.Start(ctx.Done())
ndmInformer.Start(ctx.Done())
deploy.logger.Info("AddNamespace: done (newdeploy)", zap.String("namespace", ns))
return nil
}
+56
View File
@@ -784,3 +784,59 @@ func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(ctx context.Contex
func (gpm *GenericPoolManager) DumpDebugInfo(ctx context.Context) error {
return gpm.fsCache.DumpDebugInfo(ctx)
}
// AddNamespace dynamically registers a new namespace in the poolmgr executor without a restart.
// Called by watchManagedNamespaces when a Namespace with label fission.io/managed=true appears.
//
// What it does:
// 1. Adds the namespace to the global NamespaceResolver (thread-safe, deduplicates)
// 2. Creates a Fission CRD informer factory for the NS (watches Environments, Functions, Packages)
// 3. Creates a K8s pod informer factory filtered by poolmgr executor label
// 4. Registers the informers with PoolPodController (envLister, podLister, RS watcher)
// 5. Starts both informer factories
//
// If the namespace was already registered, returns nil immediately (no-op).
func (gpm *GenericPoolManager) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
if ns == "" {
// Empty NS means "scan all" — for poolmgr this is a no-op; discovery is done by the caller.
return nil
}
// Use poolmgr-specific dedup: check if envLister is already set up for this NS.
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
// shared by all executor types and is now called once in multitenant.registerNamespace.
if _, ok := gpm.poolPodC.envLister[ns]; ok {
return nil
}
gpm.logger.Info("AddNamespace: setting up informers for new namespace", zap.String("namespace", ns))
// Fission CRD informer factory — watches Environments, Functions, Packages in this NS.
finformer := genInformer.NewFilteredSharedInformerFactory(gpm.fissionClient, 30*time.Minute, ns, nil)
// K8s pod/RS informer factory filtered by poolmgr executor label.
// Same label used at startup in GetInformerFactoryByExecutor.
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypePoolmgr)
if err != nil {
return fmt.Errorf("AddNamespace %s: get executor label: %w", ns, err)
}
gpmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
gpm.kubernetesClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = executorLabel.String()
}),
k8sInformers.WithNamespace(ns),
)
// Register the new informers with PoolPodController.
if err := gpm.poolPodC.AddNamespaceInformers(ctx, ns, finformer, gpmInformer); err != nil {
return fmt.Errorf("AddNamespace %s: register informers: %w", ns, err)
}
// Start the factories — they will begin syncing immediately.
finformer.Start(ctx.Done())
gpmInformer.Start(ctx.Done())
gpm.logger.Info("AddNamespace: informers started for namespace", zap.String("namespace", ns))
return nil
}
@@ -485,3 +485,46 @@ func (p *PoolPodController) spCleanupPodQueueProcessFunc(ctx context.Context) bo
p.spCleanupPodQueue.Forget(key)
return false
}
// AddNamespaceInformers registers informers for a newly-added namespace in PoolPodController.
// Called from GenericPoolManager.AddNamespace after the informer factories are created.
//
// Sets up:
// - Environment lister and synced func (for pool creation on env events)
// - Pod lister and synced func (for specialized pod tracking)
// - ReplicaSet event handler (for RS scale-down pod cleanup)
func (p *PoolPodController) AddNamespaceInformers(
ctx context.Context,
ns string,
finformer genInformer.SharedInformerFactory,
gpmInformer k8sInformers.SharedInformerFactory,
) error {
// Environment informer — triggers pool creation/deletion when envs change in this NS.
_, err := finformer.Core().V1().Environments().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: p.enqueueEnvAdd,
UpdateFunc: p.enqueueEnvUpdate,
DeleteFunc: p.enqueueEnvDelete,
})
if err != nil {
return fmt.Errorf("AddNamespaceInformers %s: add env handler: %w", ns, err)
}
p.envLister[ns] = finformer.Core().V1().Environments().Lister()
p.envListerSynced[ns] = finformer.Core().V1().Environments().Informer().HasSynced
// Pod lister — used by processRS to find specialized pods in this NS.
p.podLister[ns] = gpmInformer.Core().V1().Pods().Lister()
p.podListerSynced[ns] = gpmInformer.Core().V1().Pods().Informer().HasSynced
// ReplicaSet informer — triggers cleanup of specialized pods when RS scales to 0.
_, err = gpmInformer.Apps().V1().ReplicaSets().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: p.handleRSAdd,
UpdateFunc: p.handleRSUpdate,
DeleteFunc: p.handleRSDelete,
})
if err != nil {
return fmt.Errorf("AddNamespaceInformers %s: add RS handler: %w", ns, err)
}
p.logger.Info("AddNamespaceInformers: registered informers for namespace", zap.String("namespace", ns))
return nil
}
+188
View File
@@ -0,0 +1,188 @@
// Package multitenant provides utilities for running Fission in a multi-tenant
// Kubernetes environment where user namespaces are created dynamically at runtime.
//
// # The Problem
//
// In a standard Fission installation, all resource namespaces must be enumerated in the
// FISSION_RESOURCE_NAMESPACES environment variable before the process starts. Adding a
// new user namespace requires patching that env var and triggering a rolling restart of
// every Fission component (executor, router, buildermgr, etc.) — causing ~30 seconds of
// downtime per new tenant.
//
// At scale this becomes a severe operational problem: with hundreds of new tenants being
// created continuously, the executor is in a permanent rolling restart loop, causing
// cascading failures for all existing users.
//
// # The Solution
//
// This package implements hot namespace registration without pod restarts.
// The mechanism is intentionally simple and decoupled from any specific platform:
//
// 1. A platform (a cloud console, a CI/CD system, an operator) creates a Kubernetes
// Namespace and sets the label:
// fission.io/managed=true
//
// 2. StartNSWatcher registers a Kubernetes Namespace Informer that receives an event
// the moment a labeled Namespace is created or updated — no polling, no delay.
//
// 3. On the AddFunc / UpdateFunc callback NSWatcher calls AddNamespace on every
// registered executor type (poolmgr, newdeploy, container). Each type creates
// per-NS informer factories, pod listers, and event handlers — live, without restart.
//
// 4. NamespaceResolver.AddNamespace deduplicates — calling AddNamespace on an already-
// registered namespace is always a safe no-op.
//
// # Backward Compatibility
//
// The FISSION_RESOURCE_NAMESPACES environment variable continues to work as before.
// Namespaces listed there are registered at startup and do not require the label.
// This package adds on top of the existing mechanism — it does not replace it.
//
// # Required RBAC
//
// The fission-executor ServiceAccount must be granted permission to list and watch
// Namespaces at the cluster scope. Apply the manifest at:
//
// deploy/multitenant/rbac.yaml
//
// # Integration Contract
//
// The entire integration contract for external platforms is a single label on a Namespace:
//
// apiVersion: v1
// kind: Namespace
// metadata:
// name: tenant-abc123
// labels:
// fission.io/managed: "true"
//
// No other coupling to Fission internals is required or expected.
package multitenant
import (
"context"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sInformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/executor/executortype"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/manager"
)
// ManagedNSLabel is the Kubernetes label key that marks a Namespace as dynamically managed
// by Fission multi-tenant mode. The expected value is "true".
//
// Platforms MUST set this label on every user Namespace they create:
//
// metadata:
// labels:
// fission.io/managed: "true"
const ManagedNSLabel = "fission.io/managed"
// StartNSWatcher registers a Kubernetes Namespace Informer that reacts immediately
// when a Namespace with label fission.io/managed=true is created or relabeled.
//
// Unlike a polling approach, this uses the standard k8s Watch mechanism — the executor
// receives the event within milliseconds of the Namespace appearing, with zero wasted
// API calls between events.
//
// The informer is managed via mgr and shuts down cleanly when ctx is cancelled.
func StartNSWatcher(
ctx context.Context,
logger *zap.Logger,
kubernetesClient kubernetes.Interface,
executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
mgr manager.Interface,
) {
// Use a label-filtered informer so only Namespaces with our label are delivered.
// The resync period of 30m is standard for Fission informers — it re-lists to recover
// from any missed events, but normal operation is purely event-driven (no ticking).
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
kubernetesClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = ManagedNSLabel + "=true"
}),
)
nsInformer := factory.Core().V1().Namespaces().Informer()
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
// AddFunc fires when a new Namespace with the label appears.
AddFunc: func(obj interface{}) {
ns := namespaceName(obj)
if ns == "" {
return
}
registerNamespace(ctx, logger, kubernetesClient, ns, executorTypes, mgr)
},
// UpdateFunc fires when an existing Namespace is updated — covers the case
// where the label is added to a pre-existing Namespace.
UpdateFunc: func(_, newObj interface{}) {
nsObj, ok := newObj.(*corev1.Namespace)
if !ok {
return
}
if nsObj.Labels[ManagedNSLabel] != "true" {
return // label was removed — nothing to do (executor keeps existing registrations)
}
registerNamespace(ctx, logger, kubernetesClient, nsObj.Name, executorTypes, mgr)
},
})
mgr.Add(ctx, func(ctx context.Context) {
logger.Info("multitenant.NSWatcher: started", zap.String("label", ManagedNSLabel+"=true"))
factory.Start(ctx.Done())
factory.WaitForCacheSync(ctx.Done())
logger.Info("multitenant.NSWatcher: cache synced — watching for new namespaces")
<-ctx.Done()
logger.Info("multitenant.NSWatcher: stopped")
})
}
// registerNamespace calls AddNamespace on every executor type for the given namespace.
// The global NamespaceResolver is updated here — once, before any executor type is called.
// Each executor type uses its own internal state for deduplication instead of the
// global resolver, so all executor types receive the AddNamespace call regardless of
// iteration order.
func registerNamespace(
ctx context.Context,
logger *zap.Logger,
kubernetesClient kubernetes.Interface,
ns string,
executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
mgr manager.Interface,
) {
// Update the global resolver once here. Each executor type must NOT call
// DefaultNSResolver().AddNamespace() for dedup — they have their own checks.
utils.DefaultNSResolver().AddNamespace(ns)
// Ensure fission-fetcher SA exists in the new namespace so pool pods can start.
utils.EnsureNamespaceSA(ctx, kubernetesClient, logger, ns)
for _, et := range executorTypes {
if err := et.AddNamespace(ctx, ns, mgr); err != nil {
logger.Error("multitenant.NSWatcher: AddNamespace failed",
zap.String("namespace", ns),
zap.Error(err),
)
}
}
logger.Info("multitenant.NSWatcher: registered namespace", zap.String("namespace", ns))
}
// namespaceName extracts the namespace name from an informer event object.
func namespaceName(obj interface{}) string {
accessor, err := meta.Accessor(obj)
if err != nil {
return ""
}
return accessor.GetName()
}
+83
View File
@@ -18,6 +18,7 @@ package router
import (
"context"
"fmt"
"net/http"
"strings"
"time"
@@ -35,6 +36,7 @@ import (
eclient "github.com/fission/fission/pkg/executor/client"
config "github.com/fission/fission/pkg/featureconfig"
"github.com/fission/fission/pkg/generated/clientset/versioned"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/info"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils"
@@ -426,3 +428,84 @@ func (ts *HTTPTriggerSet) updateRouter(ctx context.Context) {
ts.mutableRouter.updateRouter(router)
}
}
// AddNamespace dynamically registers a new namespace in the router without a restart.
// Creates per-NS informers for HTTPTriggers and Functions, wires up event handlers,
// and triggers a router rebuild. Safe to call repeatedly — deduplicates via NSResolver.
func (ts *HTTPTriggerSet) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
if !utils.DefaultNSResolver().AddNamespace(ns) {
return nil // already registered
}
ts.logger.Info("router.AddNamespace: setting up informers", zap.String("namespace", ns))
factory := genInformer.NewFilteredSharedInformerFactory(ts.fissionClient, 30*time.Minute, ns, nil)
triggerInf := factory.Core().V1().HTTPTriggers().Informer()
funcInf := factory.Core().V1().Functions().Informer()
_, err := triggerInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
trigger := obj.(*fv1.HTTPTrigger)
go createIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
trigger := obj.(*fv1.HTTPTrigger)
go deleteIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldTrigger := oldObj.(*fv1.HTTPTrigger)
newTrigger := newObj.(*fv1.HTTPTrigger)
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
return
}
go updateIngress(context.Background(), ts.logger, oldTrigger, newTrigger, ts.kubeClient)
ts.syncTriggers()
},
})
if err != nil {
return fmt.Errorf("router.AddNamespace %s: trigger handler: %w", ns, err)
}
_, err = funcInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) { ts.syncTriggers() },
DeleteFunc: func(obj interface{}) { ts.syncTriggers() },
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldFn := oldObj.(*fv1.Function)
fn := newObj.(*fv1.Function)
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
return
}
for key, rr := range ts.resolver.copy() {
if key.namespace == fn.ObjectMeta.Namespace &&
rr.functionMap[fn.ObjectMeta.Name] != nil &&
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
ts.logger.Debug("invalidating resolver cache")
_ = ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
break
}
}
ts.syncTriggers()
},
})
if err != nil {
return fmt.Errorf("router.AddNamespace %s: func handler: %w", ns, err)
}
// ts.funcInformer and resolver.funcInformer are the same map reference —
// updating ts.funcInformer also makes the resolver aware of the new namespace.
ts.triggerInformer[ns] = triggerInf
ts.funcInformer[ns] = funcInf
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
ns + "/trigger": triggerInf,
ns + "/func": funcInf,
})
factory.Start(ctx.Done())
// Wait for cache to sync before rebuilding the router, so triggers are visible.
k8sCache.WaitForCacheSync(ctx.Done(), triggerInf.HasSynced, funcInf.HasSynced)
ts.logger.Info("router.AddNamespace: done", zap.String("namespace", ns))
ts.syncTriggers()
return nil
}
+84
View File
@@ -0,0 +1,84 @@
// Package router — NSWatcher for multi-tenant mode.
//
// Listens for Namespaces labeled fission.io/managed=true and calls
// HTTPTriggerSet.AddNamespace so the router picks up HTTPTriggers and
// Functions in tenant namespaces without a restart.
package router
import (
"context"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sInformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission/pkg/utils/manager"
)
const routerManagedNSLabel = "fission.io/managed"
// StartNSWatcher registers a Kubernetes Namespace Informer for the router.
// Whenever a Namespace with label fission.io/managed=true appears (or is relabeled),
// the router immediately subscribes to HTTPTriggers and Functions in that namespace.
func StartNSWatcher(
ctx context.Context,
logger *zap.Logger,
kubeClient kubernetes.Interface,
ts *HTTPTriggerSet,
mgr manager.Interface,
) {
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
kubeClient,
30*time.Minute,
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
opts.LabelSelector = routerManagedNSLabel + "=true"
}),
)
nsInformer := factory.Core().V1().Namespaces().Informer()
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ns := routerNSName(obj)
if ns == "" {
return
}
if err := ts.AddNamespace(ctx, ns, mgr); err != nil {
logger.Error("router.NSWatcher: AddNamespace failed",
zap.String("namespace", ns), zap.Error(err))
}
},
UpdateFunc: func(_, newObj interface{}) {
nsObj, ok := newObj.(*corev1.Namespace)
if !ok || nsObj.Labels[routerManagedNSLabel] != "true" {
return
}
if err := ts.AddNamespace(ctx, nsObj.Name, mgr); err != nil {
logger.Error("router.NSWatcher: AddNamespace failed",
zap.String("namespace", nsObj.Name), zap.Error(err))
}
},
})
mgr.Add(ctx, func(ctx context.Context) {
logger.Info("router.NSWatcher: started",
zap.String("label", routerManagedNSLabel+"=true"))
factory.Start(ctx.Done())
factory.WaitForCacheSync(ctx.Done())
logger.Info("router.NSWatcher: cache synced — watching for new namespaces")
<-ctx.Done()
logger.Info("router.NSWatcher: stopped")
})
}
func routerNSName(obj interface{}) string {
nsObj, ok := obj.(*corev1.Namespace)
if !ok {
return ""
}
return nsObj.Name
}
+4
View File
@@ -213,6 +213,10 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
metrics.ServeMetrics(ctx, "router", logger, mgr)
})
// Multi-tenant: watch namespaces labeled fission.io/managed=true
// and dynamically register HTTPTrigger/Function informers without restart.
StartNSWatcher(ctx, logger, kubeClient, triggers, mgr)
logger.Info("starting router", zap.Int("port", port))
tracer := otel.Tracer("router")
+21 -1
View File
@@ -3,6 +3,7 @@ package utils
import (
"os"
"strings"
"sync"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -19,6 +20,8 @@ const (
type (
NamespaceResolver struct {
mu sync.RWMutex // protects FissionResourceNS
FunctionNamespace string
BuilderNamespace string
DefaultNamespace string
@@ -82,16 +85,33 @@ func WithDefaultNs() option {
}
}
// AddNamespace dynamically adds a namespace to FissionResourceNS without restarting the process.
// Returns true if the namespace was newly added, false if it was already present.
// Thread-safe — multiple goroutines may call this concurrently.
// The label fission.io/managed=true on the Namespace object is the trigger for this call.
func (nsr *NamespaceResolver) AddNamespace(ns string) bool {
nsr.mu.Lock()
defer nsr.mu.Unlock()
if _, exists := nsr.FissionResourceNS[ns]; exists {
return false
}
nsr.FissionResourceNS[ns] = ns
nsr.Logger.Info("dynamically added namespace to resolver", zap.String("namespace", ns))
return true
}
func (nsr *NamespaceResolver) FissionNSWithOptions(option ...option) map[string]string {
var options options
for _, opt := range option {
options = *opt(&options)
}
fissionResourceNS := make(map[string]string)
nsr.mu.RLock()
fissionResourceNS := make(map[string]string, len(nsr.FissionResourceNS))
for k, v := range nsr.FissionResourceNS {
fissionResourceNS[k] = v
}
nsr.mu.RUnlock()
if options.functionNS && nsr.FunctionNamespace != "" {
fissionResourceNS[nsr.FunctionNamespace] = nsr.FunctionNamespace
+7
View File
@@ -308,3 +308,10 @@ func getSAInterval() time.Duration {
SAInterval, _ := GetUIntValueFromEnv(ENV_SA_INTERVAL)
return time.Duration(SAInterval) * time.Minute
}
// EnsureNamespaceSA creates the fission-fetcher ServiceAccount and its Role/RoleBinding
// in the given namespace if they do not already exist. Safe to call repeatedly.
// Used by the multi-tenant NS watcher to provision per-namespace SA on NS registration.
func EnsureNamespaceSA(ctx context.Context, client kubernetes.Interface, logger *zap.Logger, ns string) {
setupSAAndRoleBindings(ctx, client, logger, ns, fetcherCheck)
}