multi-tenant: EnsureNamespaceSA + ns_watcher SA provisioning (v8)
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
FROM cgr.dev/chainguard/static:latest@sha256:a301031ffd4ed67f35ca7fa6cf3dad9937b5fa47d7493955a18d9b4ca5412d1a
|
||||
COPY fission-bundle /
|
||||
ENTRYPOINT ["/fission-bundle"]
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
FROM cgr.dev/chainguard/static:latest@sha256:a301031ffd4ed67f35ca7fa6cf3dad9937b5fa47d7493955a18d9b4ca5412d1a
|
||||
COPY fission-bundle /
|
||||
ENTRYPOINT ["/fission-bundle"]
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,26 @@
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: fission-executor-ns-watcher
|
||||
labels:
|
||||
app: fission-executor
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: fission-executor-ns-watcher
|
||||
labels:
|
||||
app: fission-executor
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: fission-executor-ns-watcher
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: fission-executor
|
||||
namespace: fission
|
||||
@@ -0,0 +1,42 @@
|
||||
# deploy/multitenant/rbac.yaml
|
||||
#
|
||||
# RBAC required for the Fission multi-tenant NSWatcher.
|
||||
#
|
||||
# The fission-executor ServiceAccount must be allowed to list and watch Namespaces
|
||||
# at the cluster scope so that NSWatcher can detect newly-labeled Namespaces.
|
||||
#
|
||||
# This is a read-only ClusterRole — no write permissions are granted.
|
||||
# Apply once per cluster after installing Fission:
|
||||
#
|
||||
# kubectl apply -f deploy/multitenant/rbac.yaml
|
||||
#
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: fission-executor-ns-watcher
|
||||
labels:
|
||||
app.kubernetes.io/name: fission
|
||||
app.kubernetes.io/component: executor
|
||||
app.kubernetes.io/part-of: fission-multitenant
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: fission-executor-ns-watcher
|
||||
labels:
|
||||
app.kubernetes.io/name: fission
|
||||
app.kubernetes.io/component: executor
|
||||
app.kubernetes.io/part-of: fission-multitenant
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: fission-executor-ns-watcher
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: fission-executor
|
||||
namespace: fission
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,132 @@
|
||||
import re
|
||||
|
||||
# ── envwatcher.go ─────────────────────────────────────────────────────────────
|
||||
with open("/home/naeel/terra/fission-src/pkg/buildermgr/envwatcher.go") as f:
|
||||
src = f.read()
|
||||
|
||||
# добавляем genInformer import если нет
|
||||
if "genInformer" not in src:
|
||||
src = src.replace(
|
||||
'"github.com/fission/fission/pkg/generated/clientset/versioned"',
|
||||
'"github.com/fission/fission/pkg/generated/clientset/versioned"\n\t'
|
||||
'genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"',
|
||||
1
|
||||
)
|
||||
|
||||
# добавляем fmt если нет
|
||||
if '"fmt"' not in src:
|
||||
src = src.replace('"context"', '"context"\n\t"fmt"', 1)
|
||||
|
||||
addon = r'''
|
||||
// AddNamespace dynamically registers a new namespace in environmentWatcher.
|
||||
// Creates a per-NS Environment informer. Safe to call repeatedly — deduplicates via nsResolver.
|
||||
func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
|
||||
if !envw.nsResolver.AddNamespace(ns) {
|
||||
return // already registered
|
||||
}
|
||||
envw.logger.Info("buildermgr.envWatcher.AddNamespace: setting up informer", zap.String("namespace", ns))
|
||||
|
||||
factory := genInformer.NewFilteredSharedInformerFactory(envw.fissionClient, 30*time.Minute, ns, nil)
|
||||
envInf := factory.Core().V1().Environments().Informer()
|
||||
|
||||
_, err := envInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
envObj := obj.(*fv1.Environment)
|
||||
envw.AddUpdateBuilder(ctx, envObj)
|
||||
},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldEnvObj := oldObj.(*fv1.Environment)
|
||||
newEnvObj := newObj.(*fv1.Environment)
|
||||
if oldEnvObj.ResourceVersion == newEnvObj.ResourceVersion {
|
||||
return
|
||||
}
|
||||
envw.AddUpdateBuilder(ctx, newEnvObj)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
envObj, ok := obj.(*fv1.Environment)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
envw.deleteBuilder(ctx, envObj)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
envw.logger.Error("buildermgr.envWatcher.AddNamespace: add handler failed",
|
||||
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
|
||||
return
|
||||
}
|
||||
|
||||
envw.envWatchInformer[ns] = envInf
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
|
||||
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
'''
|
||||
|
||||
with open("/home/naeel/terra/fission-src/pkg/buildermgr/envwatcher.go", "w") as f:
|
||||
f.write(src + addon)
|
||||
print("envwatcher.go: done")
|
||||
|
||||
# ── pkgwatcher.go ─────────────────────────────────────────────────────────────
|
||||
with open("/home/naeel/terra/fission-src/pkg/buildermgr/pkgwatcher.go") as f:
|
||||
src = f.read()
|
||||
|
||||
# добавляем genInformer import если нет
|
||||
if "genInformer" not in src:
|
||||
src = src.replace(
|
||||
'"github.com/fission/fission/pkg/generated/clientset/versioned"',
|
||||
'"github.com/fission/fission/pkg/generated/clientset/versioned"\n\t'
|
||||
'genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"',
|
||||
1
|
||||
)
|
||||
|
||||
# добавляем fmt если нет
|
||||
if '"fmt"' not in src:
|
||||
src = src.replace('"context"', '"context"\n\t"fmt"', 1)
|
||||
|
||||
# Добавляем k8sInformers если нет
|
||||
if "k8sInformers" not in src:
|
||||
src = src.replace(
|
||||
'"k8s.io/client-go/kubernetes"',
|
||||
'"k8s.io/client-go/kubernetes"\n\tk8sInformers "k8s.io/client-go/informers"',
|
||||
1
|
||||
)
|
||||
|
||||
addon2 = r'''
|
||||
// AddNamespace dynamically registers a new namespace in packageWatcher.
|
||||
// Creates per-NS Package and Pod informers. Safe to call repeatedly.
|
||||
func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
|
||||
if !pkgw.nsResolver.AddNamespace(ns) {
|
||||
return // already registered
|
||||
}
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: setting up informers", zap.String("namespace", ns))
|
||||
|
||||
// Package informer
|
||||
fissionFactory := genInformer.NewFilteredSharedInformerFactory(pkgw.fissionClient, 30*time.Minute, ns, nil)
|
||||
pkgInf := fissionFactory.Core().V1().Packages().Informer()
|
||||
|
||||
_, err := pkgInf.AddEventHandler(pkgw.packageInformerHandler(ctx))
|
||||
if err != nil {
|
||||
pkgw.logger.Error("buildermgr.pkgWatcher.AddNamespace: pkg handler failed",
|
||||
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
|
||||
return
|
||||
}
|
||||
|
||||
// Pod informer for build logs
|
||||
podFactory := k8sInformers.NewSharedInformerFactoryWithOptions(pkgw.k8sClient, 30*time.Minute,
|
||||
k8sInformers.WithNamespace(ns))
|
||||
podInf := podFactory.Core().V1().Pods().Informer()
|
||||
|
||||
pkgw.pkgInformer[ns] = pkgInf
|
||||
pkgw.podInformer[ns] = podInf
|
||||
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
|
||||
ns + "/pkg": pkgInf,
|
||||
ns + "/pod": podInf,
|
||||
})
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
'''
|
||||
|
||||
with open("/home/naeel/terra/fission-src/pkg/buildermgr/pkgwatcher.go", "w") as f:
|
||||
f.write(src + addon2)
|
||||
print("pkgwatcher.go: done")
|
||||
@@ -74,5 +74,9 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Multi-tenant: watch namespaces labeled fission.io/managed=true
|
||||
StartNSWatcher(ctx, logger, kubernetesClient, envWatcher, pkgWatcher, mgr)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/fission/fission/pkg/executor/util"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/generated/clientset/versioned"
|
||||
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"github.com/fission/fission/pkg/utils/manager"
|
||||
)
|
||||
@@ -498,3 +499,47 @@ func (envw *environmentWatcher) createBuilderDeployment(ctx context.Context, env
|
||||
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in environmentWatcher.
|
||||
// Creates a per-NS Environment informer. Safe to call repeatedly — deduplicates via nsResolver.
|
||||
func (envw *environmentWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
|
||||
if !envw.nsResolver.AddNamespace(ns) {
|
||||
return // already registered
|
||||
}
|
||||
envw.logger.Info("buildermgr.envWatcher.AddNamespace: setting up informer", zap.String("namespace", ns))
|
||||
|
||||
factory := genInformer.NewFilteredSharedInformerFactory(envw.fissionClient, 30*time.Minute, ns, nil)
|
||||
envInf := factory.Core().V1().Environments().Informer()
|
||||
|
||||
_, err := envInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
envObj := obj.(*fv1.Environment)
|
||||
envw.AddUpdateBuilder(ctx, envObj)
|
||||
},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldEnvObj := oldObj.(*fv1.Environment)
|
||||
newEnvObj := newObj.(*fv1.Environment)
|
||||
if oldEnvObj.ResourceVersion == newEnvObj.ResourceVersion {
|
||||
return
|
||||
}
|
||||
envw.AddUpdateBuilder(ctx, newEnvObj)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
envObj, ok := obj.(*fv1.Environment)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
envw.DeleteBuilder(ctx, envObj)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
envw.logger.Error("buildermgr.envWatcher.AddNamespace: add handler failed",
|
||||
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
|
||||
return
|
||||
}
|
||||
|
||||
envw.envWatchInformer[ns] = envInf
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{ns: envInf})
|
||||
factory.Start(ctx.Done())
|
||||
envw.logger.Info("buildermgr.envWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Package buildermgr — NSWatcher for multi-tenant mode.
|
||||
//
|
||||
// Listens for Namespaces labeled fission.io/managed=true and calls
|
||||
// AddNamespace on envWatcher and packageWatcher so they pick up
|
||||
// Environments and Packages in new tenant namespaces without a restart.
|
||||
package buildermgr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8sInformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
"github.com/fission/fission/pkg/utils/manager"
|
||||
)
|
||||
|
||||
const builderManagedNSLabel = "fission.io/managed"
|
||||
|
||||
// StartNSWatcher watches for Namespaces labeled fission.io/managed=true
|
||||
// and immediately registers per-NS informers in envWatcher and pkgWatcher.
|
||||
func StartNSWatcher(
|
||||
ctx context.Context,
|
||||
logger *zap.Logger,
|
||||
kubeClient kubernetes.Interface,
|
||||
envw *environmentWatcher,
|
||||
pkgw *packageWatcher,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
kubeClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = builderManagedNSLabel + "=true"
|
||||
}),
|
||||
)
|
||||
|
||||
nsInformer := factory.Core().V1().Namespaces().Informer()
|
||||
|
||||
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
ns := builderNSName(obj)
|
||||
if ns == "" {
|
||||
return
|
||||
}
|
||||
envw.AddNamespace(ctx, ns, mgr)
|
||||
pkgw.AddNamespace(ctx, ns, mgr)
|
||||
},
|
||||
UpdateFunc: func(_, newObj interface{}) {
|
||||
nsObj, ok := newObj.(*corev1.Namespace)
|
||||
if !ok || nsObj.Labels[builderManagedNSLabel] != "true" {
|
||||
return
|
||||
}
|
||||
envw.AddNamespace(ctx, nsObj.Name, mgr)
|
||||
pkgw.AddNamespace(ctx, nsObj.Name, mgr)
|
||||
},
|
||||
})
|
||||
|
||||
mgr.Add(ctx, func(ctx context.Context) {
|
||||
logger.Info("buildermgr.NSWatcher: started",
|
||||
zap.String("label", builderManagedNSLabel+"=true"))
|
||||
factory.Start(ctx.Done())
|
||||
factory.WaitForCacheSync(ctx.Done())
|
||||
logger.Info("buildermgr.NSWatcher: cache synced — watching for new namespaces")
|
||||
<-ctx.Done()
|
||||
logger.Info("buildermgr.NSWatcher: stopped")
|
||||
})
|
||||
}
|
||||
|
||||
func builderNSName(obj interface{}) string {
|
||||
nsObj, ok := obj.(*corev1.Namespace)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return nsObj.Name
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8sInformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
"github.com/fission/fission/pkg/cache"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/generated/clientset/versioned"
|
||||
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"github.com/fission/fission/pkg/utils/manager"
|
||||
"github.com/fission/fission/pkg/utils/metrics"
|
||||
@@ -329,3 +331,39 @@ func setInitialBuildStatus(ctx context.Context, fissionClient versioned.Interfac
|
||||
// TODO: use UpdateStatus to update status
|
||||
return fissionClient.CoreV1().Packages(pkg.Namespace).Update(ctx, pkg, metav1.UpdateOptions{})
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in packageWatcher.
|
||||
// Creates per-NS Package and Pod informers. Safe to call repeatedly.
|
||||
func (pkgw *packageWatcher) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) {
|
||||
if !pkgw.nsResolver.AddNamespace(ns) {
|
||||
return // already registered
|
||||
}
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: setting up informers", zap.String("namespace", ns))
|
||||
|
||||
// Package informer
|
||||
fissionFactory := genInformer.NewFilteredSharedInformerFactory(pkgw.fissionClient, 30*time.Minute, ns, nil)
|
||||
pkgInf := fissionFactory.Core().V1().Packages().Informer()
|
||||
|
||||
_, err := pkgInf.AddEventHandler(pkgw.packageInformerHandler(ctx))
|
||||
if err != nil {
|
||||
pkgw.logger.Error("buildermgr.pkgWatcher.AddNamespace: pkg handler failed",
|
||||
zap.String("namespace", ns), zap.Error(fmt.Errorf("%w", err)))
|
||||
return
|
||||
}
|
||||
|
||||
// Pod informer for build logs
|
||||
podFactory := k8sInformers.NewSharedInformerFactoryWithOptions(pkgw.k8sClient, 30*time.Minute,
|
||||
k8sInformers.WithNamespace(ns))
|
||||
podInf := podFactory.Core().V1().Pods().Informer()
|
||||
|
||||
pkgw.pkgInformer[ns] = pkgInf
|
||||
pkgw.podInformer[ns] = podInf
|
||||
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
|
||||
ns + "/pkg": pkgInf,
|
||||
ns + "/pod": podInf,
|
||||
})
|
||||
fissionFactory.Start(ctx.Done())
|
||||
podFactory.Start(ctx.Done())
|
||||
pkgw.logger.Info("buildermgr.pkgWatcher.AddNamespace: done", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ import (
|
||||
"github.com/fission/fission/pkg/executor/executortype/newdeploy"
|
||||
"github.com/fission/fission/pkg/executor/executortype/poolmgr"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
"github.com/fission/fission/pkg/executor/multitenant"
|
||||
"github.com/fission/fission/pkg/executor/util"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/generated/clientset/versioned"
|
||||
@@ -399,6 +400,12 @@ func StartExecutor(ctx context.Context, clientGen crd.ClientGeneratorInterface,
|
||||
|
||||
utils.CreateMissingPermissionForSA(ctx, kubernetesClient, logger)
|
||||
|
||||
// Start multi-tenant Namespace watcher.
|
||||
// Detects Namespaces labeled fission.io/managed=true and registers them in all
|
||||
// executor types without a pod restart. Backward-compatible with FISSION_RESOURCE_NAMESPACES.
|
||||
// See: pkg/executor/multitenant/ns_watcher.go
|
||||
multitenant.StartNSWatcher(ctx, logger, kubernetesClient, executorTypes, mgr)
|
||||
|
||||
mgr.Add(ctx, func(ctx context.Context) {
|
||||
metrics.ServeMetrics(ctx, "executor", logger, mgr)
|
||||
})
|
||||
|
||||
@@ -792,3 +792,50 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
|
||||
func (caaf *Container) DumpDebugInfo(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in the container executor without a restart.
|
||||
// Sets up deployment and service listers so the executor can manage container functions in the new NS.
|
||||
func (caaf *Container) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
|
||||
if ns == "" {
|
||||
return nil
|
||||
}
|
||||
// Use container-specific dedup: check if deplLister is already set up for this NS.
|
||||
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
|
||||
// shared by all executor types and is now called once in multitenant.registerNamespace.
|
||||
if _, ok := caaf.deplLister[ns]; ok {
|
||||
return nil // already registered
|
||||
}
|
||||
|
||||
caaf.logger.Info("AddNamespace: setting up informers for new namespace (container)", zap.String("namespace", ns))
|
||||
|
||||
finformer := genInformer.NewFilteredSharedInformerFactory(caaf.fissionClient, 30*time.Minute, ns, nil)
|
||||
|
||||
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeContainer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespace %s (container): get executor label: %w", ns, err)
|
||||
}
|
||||
cnmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
caaf.kubernetesClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = executorLabel.String()
|
||||
}),
|
||||
k8sInformers.WithNamespace(ns),
|
||||
)
|
||||
|
||||
caaf.deplLister[ns] = cnmInformer.Apps().V1().Deployments().Lister()
|
||||
caaf.deplListerSynced[ns] = cnmInformer.Apps().V1().Deployments().Informer().HasSynced
|
||||
caaf.svcLister[ns] = cnmInformer.Core().V1().Services().Lister()
|
||||
caaf.svcListerSynced[ns] = cnmInformer.Core().V1().Services().Informer().HasSynced
|
||||
|
||||
_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(caaf.FuncInformerHandler(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespace %s (container): add function handler: %w", ns, err)
|
||||
}
|
||||
|
||||
finformer.Start(ctx.Done())
|
||||
cnmInformer.Start(ctx.Done())
|
||||
|
||||
caaf.logger.Info("AddNamespace: done (container)", zap.String("namespace", ns))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -69,4 +69,9 @@ type ExecutorType interface {
|
||||
|
||||
// CleanupOldExecutorObjects cleans up resources created by old executor instances
|
||||
CleanupOldExecutorObjects(context.Context)
|
||||
|
||||
// AddNamespace dynamically registers a new user namespace so the executor
|
||||
// starts watching Fission CRDs and K8s resources in it without a pod restart.
|
||||
// Called when a Namespace with label fission.io/managed=true appears.
|
||||
AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error
|
||||
}
|
||||
|
||||
@@ -898,3 +898,55 @@ func (deploy *NewDeploy) scaleDeployment(ctx context.Context, deplNS string, dep
|
||||
func (deploy *NewDeploy) DumpDebugInfo(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in the newdeploy executor without a restart.
|
||||
// Sets up deployment and service listers so the executor can manage functions in the new NS.
|
||||
// Safe to call repeatedly — uses per-executor deplLister for deduplication.
|
||||
func (deploy *NewDeploy) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
|
||||
if ns == "" {
|
||||
return nil
|
||||
}
|
||||
// Use newdeploy-specific dedup: check if deplLister is already set up for this NS.
|
||||
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
|
||||
// shared by all executor types and is now called once in multitenant.registerNamespace.
|
||||
if _, ok := deploy.deplLister[ns]; ok {
|
||||
return nil // already registered
|
||||
}
|
||||
|
||||
deploy.logger.Info("AddNamespace: setting up informers for new namespace (newdeploy)", zap.String("namespace", ns))
|
||||
|
||||
// Fission CRD informer factory for the new NS.
|
||||
finformer := genInformer.NewFilteredSharedInformerFactory(deploy.fissionClient, 30*time.Minute, ns, nil)
|
||||
|
||||
// K8s deployment+service informer factory filtered by newdeploy executor label.
|
||||
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypeNewdeploy)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespace %s (newdeploy): get executor label: %w", ns, err)
|
||||
}
|
||||
ndmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
deploy.kubernetesClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = executorLabel.String()
|
||||
}),
|
||||
k8sInformers.WithNamespace(ns),
|
||||
)
|
||||
|
||||
// Register deployment and service listers — same as done at startup in MakeNewDeploy.
|
||||
deploy.deplLister[ns] = ndmInformer.Apps().V1().Deployments().Lister()
|
||||
deploy.deplListerSynced[ns] = ndmInformer.Apps().V1().Deployments().Informer().HasSynced
|
||||
deploy.svcLister[ns] = ndmInformer.Core().V1().Services().Lister()
|
||||
deploy.svcListerSynced[ns] = ndmInformer.Core().V1().Services().Informer().HasSynced
|
||||
|
||||
// Register function event handler so this NS's functions are adopted.
|
||||
_, err = finformer.Core().V1().Functions().Informer().AddEventHandler(deploy.FunctionEventHandlers(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespace %s (newdeploy): add function handler: %w", ns, err)
|
||||
}
|
||||
|
||||
finformer.Start(ctx.Done())
|
||||
ndmInformer.Start(ctx.Done())
|
||||
|
||||
deploy.logger.Info("AddNamespace: done (newdeploy)", zap.String("namespace", ns))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -784,3 +784,59 @@ func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(ctx context.Contex
|
||||
func (gpm *GenericPoolManager) DumpDebugInfo(ctx context.Context) error {
|
||||
return gpm.fsCache.DumpDebugInfo(ctx)
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in the poolmgr executor without a restart.
|
||||
// Called by watchManagedNamespaces when a Namespace with label fission.io/managed=true appears.
|
||||
//
|
||||
// What it does:
|
||||
// 1. Adds the namespace to the global NamespaceResolver (thread-safe, deduplicates)
|
||||
// 2. Creates a Fission CRD informer factory for the NS (watches Environments, Functions, Packages)
|
||||
// 3. Creates a K8s pod informer factory filtered by poolmgr executor label
|
||||
// 4. Registers the informers with PoolPodController (envLister, podLister, RS watcher)
|
||||
// 5. Starts both informer factories
|
||||
//
|
||||
// If the namespace was already registered, returns nil immediately (no-op).
|
||||
func (gpm *GenericPoolManager) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
|
||||
if ns == "" {
|
||||
// Empty NS means "scan all" — for poolmgr this is a no-op; discovery is done by the caller.
|
||||
return nil
|
||||
}
|
||||
// Use poolmgr-specific dedup: check if envLister is already set up for this NS.
|
||||
// Do NOT use DefaultNSResolver().AddNamespace() — that is a global single-call guard
|
||||
// shared by all executor types and is now called once in multitenant.registerNamespace.
|
||||
if _, ok := gpm.poolPodC.envLister[ns]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
gpm.logger.Info("AddNamespace: setting up informers for new namespace", zap.String("namespace", ns))
|
||||
|
||||
// Fission CRD informer factory — watches Environments, Functions, Packages in this NS.
|
||||
finformer := genInformer.NewFilteredSharedInformerFactory(gpm.fissionClient, 30*time.Minute, ns, nil)
|
||||
|
||||
// K8s pod/RS informer factory filtered by poolmgr executor label.
|
||||
// Same label used at startup in GetInformerFactoryByExecutor.
|
||||
executorLabel, err := utils.GetInformerLabelByExecutor(fv1.ExecutorTypePoolmgr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespace %s: get executor label: %w", ns, err)
|
||||
}
|
||||
gpmInformer := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
gpm.kubernetesClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = executorLabel.String()
|
||||
}),
|
||||
k8sInformers.WithNamespace(ns),
|
||||
)
|
||||
|
||||
// Register the new informers with PoolPodController.
|
||||
if err := gpm.poolPodC.AddNamespaceInformers(ctx, ns, finformer, gpmInformer); err != nil {
|
||||
return fmt.Errorf("AddNamespace %s: register informers: %w", ns, err)
|
||||
}
|
||||
|
||||
// Start the factories — they will begin syncing immediately.
|
||||
finformer.Start(ctx.Done())
|
||||
gpmInformer.Start(ctx.Done())
|
||||
|
||||
gpm.logger.Info("AddNamespace: informers started for namespace", zap.String("namespace", ns))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -485,3 +485,46 @@ func (p *PoolPodController) spCleanupPodQueueProcessFunc(ctx context.Context) bo
|
||||
p.spCleanupPodQueue.Forget(key)
|
||||
return false
|
||||
}
|
||||
|
||||
// AddNamespaceInformers registers informers for a newly-added namespace in PoolPodController.
|
||||
// Called from GenericPoolManager.AddNamespace after the informer factories are created.
|
||||
//
|
||||
// Sets up:
|
||||
// - Environment lister and synced func (for pool creation on env events)
|
||||
// - Pod lister and synced func (for specialized pod tracking)
|
||||
// - ReplicaSet event handler (for RS scale-down pod cleanup)
|
||||
func (p *PoolPodController) AddNamespaceInformers(
|
||||
ctx context.Context,
|
||||
ns string,
|
||||
finformer genInformer.SharedInformerFactory,
|
||||
gpmInformer k8sInformers.SharedInformerFactory,
|
||||
) error {
|
||||
// Environment informer — triggers pool creation/deletion when envs change in this NS.
|
||||
_, err := finformer.Core().V1().Environments().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: p.enqueueEnvAdd,
|
||||
UpdateFunc: p.enqueueEnvUpdate,
|
||||
DeleteFunc: p.enqueueEnvDelete,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespaceInformers %s: add env handler: %w", ns, err)
|
||||
}
|
||||
p.envLister[ns] = finformer.Core().V1().Environments().Lister()
|
||||
p.envListerSynced[ns] = finformer.Core().V1().Environments().Informer().HasSynced
|
||||
|
||||
// Pod lister — used by processRS to find specialized pods in this NS.
|
||||
p.podLister[ns] = gpmInformer.Core().V1().Pods().Lister()
|
||||
p.podListerSynced[ns] = gpmInformer.Core().V1().Pods().Informer().HasSynced
|
||||
|
||||
// ReplicaSet informer — triggers cleanup of specialized pods when RS scales to 0.
|
||||
_, err = gpmInformer.Apps().V1().ReplicaSets().Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: p.handleRSAdd,
|
||||
UpdateFunc: p.handleRSUpdate,
|
||||
DeleteFunc: p.handleRSDelete,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("AddNamespaceInformers %s: add RS handler: %w", ns, err)
|
||||
}
|
||||
|
||||
p.logger.Info("AddNamespaceInformers: registered informers for namespace", zap.String("namespace", ns))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// Package multitenant provides utilities for running Fission in a multi-tenant
|
||||
// Kubernetes environment where user namespaces are created dynamically at runtime.
|
||||
//
|
||||
// # The Problem
|
||||
//
|
||||
// In a standard Fission installation, all resource namespaces must be enumerated in the
|
||||
// FISSION_RESOURCE_NAMESPACES environment variable before the process starts. Adding a
|
||||
// new user namespace requires patching that env var and triggering a rolling restart of
|
||||
// every Fission component (executor, router, buildermgr, etc.) — causing ~30 seconds of
|
||||
// downtime per new tenant.
|
||||
//
|
||||
// At scale this becomes a severe operational problem: with hundreds of new tenants being
|
||||
// created continuously, the executor is in a permanent rolling restart loop, causing
|
||||
// cascading failures for all existing users.
|
||||
//
|
||||
// # The Solution
|
||||
//
|
||||
// This package implements hot namespace registration without pod restarts.
|
||||
// The mechanism is intentionally simple and decoupled from any specific platform:
|
||||
//
|
||||
// 1. A platform (a cloud console, a CI/CD system, an operator) creates a Kubernetes
|
||||
// Namespace and sets the label:
|
||||
// fission.io/managed=true
|
||||
//
|
||||
// 2. StartNSWatcher registers a Kubernetes Namespace Informer that receives an event
|
||||
// the moment a labeled Namespace is created or updated — no polling, no delay.
|
||||
//
|
||||
// 3. On the AddFunc / UpdateFunc callback NSWatcher calls AddNamespace on every
|
||||
// registered executor type (poolmgr, newdeploy, container). Each type creates
|
||||
// per-NS informer factories, pod listers, and event handlers — live, without restart.
|
||||
//
|
||||
// 4. NamespaceResolver.AddNamespace deduplicates — calling AddNamespace on an already-
|
||||
// registered namespace is always a safe no-op.
|
||||
//
|
||||
// # Backward Compatibility
|
||||
//
|
||||
// The FISSION_RESOURCE_NAMESPACES environment variable continues to work as before.
|
||||
// Namespaces listed there are registered at startup and do not require the label.
|
||||
// This package adds on top of the existing mechanism — it does not replace it.
|
||||
//
|
||||
// # Required RBAC
|
||||
//
|
||||
// The fission-executor ServiceAccount must be granted permission to list and watch
|
||||
// Namespaces at the cluster scope. Apply the manifest at:
|
||||
//
|
||||
// deploy/multitenant/rbac.yaml
|
||||
//
|
||||
// # Integration Contract
|
||||
//
|
||||
// The entire integration contract for external platforms is a single label on a Namespace:
|
||||
//
|
||||
// apiVersion: v1
|
||||
// kind: Namespace
|
||||
// metadata:
|
||||
// name: tenant-abc123
|
||||
// labels:
|
||||
// fission.io/managed: "true"
|
||||
//
|
||||
// No other coupling to Fission internals is required or expected.
|
||||
package multitenant
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8sInformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/core/v1"
|
||||
"github.com/fission/fission/pkg/executor/executortype"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"github.com/fission/fission/pkg/utils/manager"
|
||||
)
|
||||
|
||||
// ManagedNSLabel is the Kubernetes label key that marks a Namespace as dynamically managed
|
||||
// by Fission multi-tenant mode. The expected value is "true".
|
||||
//
|
||||
// Platforms MUST set this label on every user Namespace they create:
|
||||
//
|
||||
// metadata:
|
||||
// labels:
|
||||
// fission.io/managed: "true"
|
||||
const ManagedNSLabel = "fission.io/managed"
|
||||
|
||||
// StartNSWatcher registers a Kubernetes Namespace Informer that reacts immediately
|
||||
// when a Namespace with label fission.io/managed=true is created or relabeled.
|
||||
//
|
||||
// Unlike a polling approach, this uses the standard k8s Watch mechanism — the executor
|
||||
// receives the event within milliseconds of the Namespace appearing, with zero wasted
|
||||
// API calls between events.
|
||||
//
|
||||
// The informer is managed via mgr and shuts down cleanly when ctx is cancelled.
|
||||
func StartNSWatcher(
|
||||
ctx context.Context,
|
||||
logger *zap.Logger,
|
||||
kubernetesClient kubernetes.Interface,
|
||||
executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
// Use a label-filtered informer so only Namespaces with our label are delivered.
|
||||
// The resync period of 30m is standard for Fission informers — it re-lists to recover
|
||||
// from any missed events, but normal operation is purely event-driven (no ticking).
|
||||
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
kubernetesClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = ManagedNSLabel + "=true"
|
||||
}),
|
||||
)
|
||||
|
||||
nsInformer := factory.Core().V1().Namespaces().Informer()
|
||||
|
||||
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
// AddFunc fires when a new Namespace with the label appears.
|
||||
AddFunc: func(obj interface{}) {
|
||||
ns := namespaceName(obj)
|
||||
if ns == "" {
|
||||
return
|
||||
}
|
||||
registerNamespace(ctx, logger, kubernetesClient, ns, executorTypes, mgr)
|
||||
},
|
||||
// UpdateFunc fires when an existing Namespace is updated — covers the case
|
||||
// where the label is added to a pre-existing Namespace.
|
||||
UpdateFunc: func(_, newObj interface{}) {
|
||||
nsObj, ok := newObj.(*corev1.Namespace)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if nsObj.Labels[ManagedNSLabel] != "true" {
|
||||
return // label was removed — nothing to do (executor keeps existing registrations)
|
||||
}
|
||||
registerNamespace(ctx, logger, kubernetesClient, nsObj.Name, executorTypes, mgr)
|
||||
},
|
||||
})
|
||||
|
||||
mgr.Add(ctx, func(ctx context.Context) {
|
||||
logger.Info("multitenant.NSWatcher: started", zap.String("label", ManagedNSLabel+"=true"))
|
||||
factory.Start(ctx.Done())
|
||||
factory.WaitForCacheSync(ctx.Done())
|
||||
logger.Info("multitenant.NSWatcher: cache synced — watching for new namespaces")
|
||||
<-ctx.Done()
|
||||
logger.Info("multitenant.NSWatcher: stopped")
|
||||
})
|
||||
}
|
||||
|
||||
// registerNamespace calls AddNamespace on every executor type for the given namespace.
|
||||
// The global NamespaceResolver is updated here — once, before any executor type is called.
|
||||
// Each executor type uses its own internal state for deduplication instead of the
|
||||
// global resolver, so all executor types receive the AddNamespace call regardless of
|
||||
// iteration order.
|
||||
func registerNamespace(
|
||||
ctx context.Context,
|
||||
logger *zap.Logger,
|
||||
kubernetesClient kubernetes.Interface,
|
||||
ns string,
|
||||
executorTypes map[fv1.ExecutorType]executortype.ExecutorType,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
// Update the global resolver once here. Each executor type must NOT call
|
||||
// DefaultNSResolver().AddNamespace() for dedup — they have their own checks.
|
||||
utils.DefaultNSResolver().AddNamespace(ns)
|
||||
// Ensure fission-fetcher SA exists in the new namespace so pool pods can start.
|
||||
utils.EnsureNamespaceSA(ctx, kubernetesClient, logger, ns)
|
||||
|
||||
for _, et := range executorTypes {
|
||||
if err := et.AddNamespace(ctx, ns, mgr); err != nil {
|
||||
logger.Error("multitenant.NSWatcher: AddNamespace failed",
|
||||
zap.String("namespace", ns),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
logger.Info("multitenant.NSWatcher: registered namespace", zap.String("namespace", ns))
|
||||
}
|
||||
|
||||
// namespaceName extracts the namespace name from an informer event object.
|
||||
func namespaceName(obj interface{}) string {
|
||||
accessor, err := meta.Accessor(obj)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return accessor.GetName()
|
||||
}
|
||||
@@ -18,6 +18,7 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -35,6 +36,7 @@ import (
|
||||
eclient "github.com/fission/fission/pkg/executor/client"
|
||||
config "github.com/fission/fission/pkg/featureconfig"
|
||||
"github.com/fission/fission/pkg/generated/clientset/versioned"
|
||||
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
|
||||
"github.com/fission/fission/pkg/info"
|
||||
"github.com/fission/fission/pkg/throttler"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
@@ -426,3 +428,84 @@ func (ts *HTTPTriggerSet) updateRouter(ctx context.Context) {
|
||||
ts.mutableRouter.updateRouter(router)
|
||||
}
|
||||
}
|
||||
|
||||
// AddNamespace dynamically registers a new namespace in the router without a restart.
|
||||
// Creates per-NS informers for HTTPTriggers and Functions, wires up event handlers,
|
||||
// and triggers a router rebuild. Safe to call repeatedly — deduplicates via NSResolver.
|
||||
func (ts *HTTPTriggerSet) AddNamespace(ctx context.Context, ns string, mgr manager.Interface) error {
|
||||
if !utils.DefaultNSResolver().AddNamespace(ns) {
|
||||
return nil // already registered
|
||||
}
|
||||
ts.logger.Info("router.AddNamespace: setting up informers", zap.String("namespace", ns))
|
||||
|
||||
factory := genInformer.NewFilteredSharedInformerFactory(ts.fissionClient, 30*time.Minute, ns, nil)
|
||||
triggerInf := factory.Core().V1().HTTPTriggers().Informer()
|
||||
funcInf := factory.Core().V1().Functions().Informer()
|
||||
|
||||
_, err := triggerInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
trigger := obj.(*fv1.HTTPTrigger)
|
||||
go createIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
||||
ts.syncTriggers()
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
ts.syncTriggers()
|
||||
trigger := obj.(*fv1.HTTPTrigger)
|
||||
go deleteIngress(context.Background(), ts.logger, trigger, ts.kubeClient)
|
||||
},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldTrigger := oldObj.(*fv1.HTTPTrigger)
|
||||
newTrigger := newObj.(*fv1.HTTPTrigger)
|
||||
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
|
||||
return
|
||||
}
|
||||
go updateIngress(context.Background(), ts.logger, oldTrigger, newTrigger, ts.kubeClient)
|
||||
ts.syncTriggers()
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("router.AddNamespace %s: trigger handler: %w", ns, err)
|
||||
}
|
||||
|
||||
_, err = funcInf.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) { ts.syncTriggers() },
|
||||
DeleteFunc: func(obj interface{}) { ts.syncTriggers() },
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldFn := oldObj.(*fv1.Function)
|
||||
fn := newObj.(*fv1.Function)
|
||||
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
|
||||
return
|
||||
}
|
||||
for key, rr := range ts.resolver.copy() {
|
||||
if key.namespace == fn.ObjectMeta.Namespace &&
|
||||
rr.functionMap[fn.ObjectMeta.Name] != nil &&
|
||||
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
|
||||
ts.logger.Debug("invalidating resolver cache")
|
||||
_ = ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
|
||||
break
|
||||
}
|
||||
}
|
||||
ts.syncTriggers()
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("router.AddNamespace %s: func handler: %w", ns, err)
|
||||
}
|
||||
|
||||
// ts.funcInformer and resolver.funcInformer are the same map reference —
|
||||
// updating ts.funcInformer also makes the resolver aware of the new namespace.
|
||||
ts.triggerInformer[ns] = triggerInf
|
||||
ts.funcInformer[ns] = funcInf
|
||||
|
||||
mgr.AddInformers(ctx, map[string]k8sCache.SharedIndexInformer{
|
||||
ns + "/trigger": triggerInf,
|
||||
ns + "/func": funcInf,
|
||||
})
|
||||
factory.Start(ctx.Done())
|
||||
// Wait for cache to sync before rebuilding the router, so triggers are visible.
|
||||
k8sCache.WaitForCacheSync(ctx.Done(), triggerInf.HasSynced, funcInf.HasSynced)
|
||||
|
||||
ts.logger.Info("router.AddNamespace: done", zap.String("namespace", ns))
|
||||
ts.syncTriggers()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Package router — NSWatcher for multi-tenant mode.
|
||||
//
|
||||
// Listens for Namespaces labeled fission.io/managed=true and calls
|
||||
// HTTPTriggerSet.AddNamespace so the router picks up HTTPTriggers and
|
||||
// Functions in tenant namespaces without a restart.
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8sInformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
"github.com/fission/fission/pkg/utils/manager"
|
||||
)
|
||||
|
||||
const routerManagedNSLabel = "fission.io/managed"
|
||||
|
||||
// StartNSWatcher registers a Kubernetes Namespace Informer for the router.
|
||||
// Whenever a Namespace with label fission.io/managed=true appears (or is relabeled),
|
||||
// the router immediately subscribes to HTTPTriggers and Functions in that namespace.
|
||||
func StartNSWatcher(
|
||||
ctx context.Context,
|
||||
logger *zap.Logger,
|
||||
kubeClient kubernetes.Interface,
|
||||
ts *HTTPTriggerSet,
|
||||
mgr manager.Interface,
|
||||
) {
|
||||
factory := k8sInformers.NewSharedInformerFactoryWithOptions(
|
||||
kubeClient,
|
||||
30*time.Minute,
|
||||
k8sInformers.WithTweakListOptions(func(opts *metav1.ListOptions) {
|
||||
opts.LabelSelector = routerManagedNSLabel + "=true"
|
||||
}),
|
||||
)
|
||||
|
||||
nsInformer := factory.Core().V1().Namespaces().Informer()
|
||||
|
||||
_, _ = nsInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
ns := routerNSName(obj)
|
||||
if ns == "" {
|
||||
return
|
||||
}
|
||||
if err := ts.AddNamespace(ctx, ns, mgr); err != nil {
|
||||
logger.Error("router.NSWatcher: AddNamespace failed",
|
||||
zap.String("namespace", ns), zap.Error(err))
|
||||
}
|
||||
},
|
||||
UpdateFunc: func(_, newObj interface{}) {
|
||||
nsObj, ok := newObj.(*corev1.Namespace)
|
||||
if !ok || nsObj.Labels[routerManagedNSLabel] != "true" {
|
||||
return
|
||||
}
|
||||
if err := ts.AddNamespace(ctx, nsObj.Name, mgr); err != nil {
|
||||
logger.Error("router.NSWatcher: AddNamespace failed",
|
||||
zap.String("namespace", nsObj.Name), zap.Error(err))
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
mgr.Add(ctx, func(ctx context.Context) {
|
||||
logger.Info("router.NSWatcher: started",
|
||||
zap.String("label", routerManagedNSLabel+"=true"))
|
||||
factory.Start(ctx.Done())
|
||||
factory.WaitForCacheSync(ctx.Done())
|
||||
logger.Info("router.NSWatcher: cache synced — watching for new namespaces")
|
||||
<-ctx.Done()
|
||||
logger.Info("router.NSWatcher: stopped")
|
||||
})
|
||||
}
|
||||
|
||||
func routerNSName(obj interface{}) string {
|
||||
nsObj, ok := obj.(*corev1.Namespace)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return nsObj.Name
|
||||
}
|
||||
@@ -213,6 +213,10 @@ func Start(ctx context.Context, clientGen crd.ClientGeneratorInterface, logger *
|
||||
metrics.ServeMetrics(ctx, "router", logger, mgr)
|
||||
})
|
||||
|
||||
// Multi-tenant: watch namespaces labeled fission.io/managed=true
|
||||
// and dynamically register HTTPTrigger/Function informers without restart.
|
||||
StartNSWatcher(ctx, logger, kubeClient, triggers, mgr)
|
||||
|
||||
logger.Info("starting router", zap.Int("port", port))
|
||||
|
||||
tracer := otel.Tracer("router")
|
||||
|
||||
+21
-1
@@ -3,6 +3,7 @@ package utils
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -19,6 +20,8 @@ const (
|
||||
|
||||
type (
|
||||
NamespaceResolver struct {
|
||||
mu sync.RWMutex // protects FissionResourceNS
|
||||
|
||||
FunctionNamespace string
|
||||
BuilderNamespace string
|
||||
DefaultNamespace string
|
||||
@@ -82,16 +85,33 @@ func WithDefaultNs() option {
|
||||
}
|
||||
}
|
||||
|
||||
// AddNamespace dynamically adds a namespace to FissionResourceNS without restarting the process.
|
||||
// Returns true if the namespace was newly added, false if it was already present.
|
||||
// Thread-safe — multiple goroutines may call this concurrently.
|
||||
// The label fission.io/managed=true on the Namespace object is the trigger for this call.
|
||||
func (nsr *NamespaceResolver) AddNamespace(ns string) bool {
|
||||
nsr.mu.Lock()
|
||||
defer nsr.mu.Unlock()
|
||||
if _, exists := nsr.FissionResourceNS[ns]; exists {
|
||||
return false
|
||||
}
|
||||
nsr.FissionResourceNS[ns] = ns
|
||||
nsr.Logger.Info("dynamically added namespace to resolver", zap.String("namespace", ns))
|
||||
return true
|
||||
}
|
||||
|
||||
func (nsr *NamespaceResolver) FissionNSWithOptions(option ...option) map[string]string {
|
||||
var options options
|
||||
for _, opt := range option {
|
||||
options = *opt(&options)
|
||||
}
|
||||
|
||||
fissionResourceNS := make(map[string]string)
|
||||
nsr.mu.RLock()
|
||||
fissionResourceNS := make(map[string]string, len(nsr.FissionResourceNS))
|
||||
for k, v := range nsr.FissionResourceNS {
|
||||
fissionResourceNS[k] = v
|
||||
}
|
||||
nsr.mu.RUnlock()
|
||||
|
||||
if options.functionNS && nsr.FunctionNamespace != "" {
|
||||
fissionResourceNS[nsr.FunctionNamespace] = nsr.FunctionNamespace
|
||||
|
||||
@@ -308,3 +308,10 @@ func getSAInterval() time.Duration {
|
||||
SAInterval, _ := GetUIntValueFromEnv(ENV_SA_INTERVAL)
|
||||
return time.Duration(SAInterval) * time.Minute
|
||||
}
|
||||
|
||||
// EnsureNamespaceSA creates the fission-fetcher ServiceAccount and its Role/RoleBinding
|
||||
// in the given namespace if they do not already exist. Safe to call repeatedly.
|
||||
// Used by the multi-tenant NS watcher to provision per-namespace SA on NS registration.
|
||||
func EnsureNamespaceSA(ctx context.Context, client kubernetes.Interface, logger *zap.Logger, ns string) {
|
||||
setupSAAndRoleBindings(ctx, client, logger, ns, fetcherCheck)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user