Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow namespace configuration for different CRD resources in Fission (#2539)

Browse Source 
* Allow multiple namespaces for builder manager
* Enable multiple namespaces for executor informers
* Added missing context
* helm chart support for multiple namespaces
* Directly consume map type from GetInformerForNamespaces fn
* Optimize function resolver by choosing namespace-specific informer
* helm chart support for multiple namespaces
* consider default namespace and move duplicate code to helm template
* Improve documentation for fission namespace values

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: shubham bansal <shubhambansaliimtgn@gmail.com>
This commit is contained in:
Sanket Sudake
2022-10-19 15:48:47 +05:30
committed by GitHub
co-authored by shubham bansal
parent facd14de90
commit 827baea974
27 changed files with 430 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/executor/executor.go
+42 -18
View File
@@ -45,6 +45,7 @@ import (
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
"github.com/fission/fission/pkg/generated/clientset/versioned"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
finformerv1 "github.com/fission/fission/pkg/generated/informers/externalversions/core/v1"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/metrics"
otelUtils "github.com/fission/fission/pkg/utils/otel"
@@ -79,7 +80,7 @@ type (
// MakeExecutor returns an Executor for given ExecutorType(s).
func MakeExecutor(ctx context.Context, logger *zap.Logger, cms *cms.ConfigSecretController,
fissionClient versioned.Interface, types map[fv1.ExecutorType]executortype.ExecutorType,
informers []k8sCache.SharedIndexInformer) (*Executor, error) {
informers ...k8sCache.SharedIndexInformer) (*Executor, error) {
executor := &Executor{
logger: logger.Named("executor"),
cms: cms,
@@ -284,10 +285,24 @@ func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace st
logger.Info("Starting executor", zap.String("instanceID", executorInstanceID))
informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30)
funcInformer := informerFactory.Core().V1().Functions()
pkgInformer := informerFactory.Core().V1().Packages()
envInformer := informerFactory.Core().V1().Environments()
funcInformer := make(map[string]finformerv1.FunctionInformer, 0)
envInformer := make(map[string]finformerv1.EnvironmentInformer, 0)
pkgInformer := make(map[string]finformerv1.PackageInformer, 0)
for _, ns := range utils.GetNamespaces() {
factory := genInformer.NewFilteredSharedInformerFactory(fissionClient, time.Minute*30, ns, nil)
funcInformer[ns] = factory.Core().V1().Functions()
}
for _, ns := range utils.GetNamespaces() {
factory := genInformer.NewFilteredSharedInformerFactory(fissionClient, time.Minute*30, ns, nil)
envInformer[ns] = factory.Core().V1().Environments()
}
for _, ns := range utils.GetNamespaces() {
factory := genInformer.NewFilteredSharedInformerFactory(fissionClient, time.Minute*30, ns, nil)
pkgInformer[ns] = factory.Core().V1().Packages()
}
gpmInformerFactory, err := utils.GetInformerFactoryByExecutor(kubernetesClient, fv1.ExecutorTypePoolmgr, time.Minute*30)
if err != nil {
@@ -364,20 +379,29 @@ func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace st
cms := cms.MakeConfigSecretController(ctx, logger, fissionClient, kubernetesClient, executorTypes, configmapInformer, secretInformer)
fissionInformers := make([]k8sCache.SharedIndexInformer, 0)
for _, informer := range funcInformer {
fissionInformers = append(fissionInformers, informer.Informer())
}
for _, informer := range envInformer {
fissionInformers = append(fissionInformers, informer.Informer())
}
for _, informer := range pkgInformer {
fissionInformers = append(fissionInformers, informer.Informer())
}
fissionInformers = append(fissionInformers,
configmapInformer.Informer(),
secretInformer.Informer(),
gpmPodInformer.Informer(),
gpmRsInformer.Informer(),
ndmDeplInformer.Informer(),
ndmSvcInformer.Informer(),
cnmDeplInformer.Informer(),
cnmSvcInformer.Informer(),
)
api, err := MakeExecutor(ctx, logger, cms, fissionClient, executorTypes,
[]k8sCache.SharedIndexInformer{
funcInformer.Informer(),
pkgInformer.Informer(),
envInformer.Informer(),
configmapInformer.Informer(),
secretInformer.Informer(),
gpmPodInformer.Informer(),
gpmRsInformer.Informer(),
ndmDeplInformer.Informer(),
ndmSvcInformer.Informer(),
cnmDeplInformer.Informer(),
cnmSvcInformer.Informer(),
})
fissionInformers...,
)
if err != nil {
return err
}
pkg/executor/executortype/container/containermgr.go
+4 -2
View File
@@ -98,7 +98,7 @@ func MakeContainer(
kubernetesClient kubernetes.Interface,
namespace string,
instanceID string,
funcInformer finformerv1.FunctionInformer,
funcInformer map[string]finformerv1.FunctionInformer,
deplInformer appsinformers.DeploymentInformer,
svcInformer coreinformers.ServiceInformer,
) (executortype.ExecutorType, error) {
@@ -135,7 +135,9 @@ func MakeContainer(
caaf.svcLister = svcInformer.Lister()
caaf.svcListerSynced = svcInformer.Informer().HasSynced
funcInformer.Informer().AddEventHandler(caaf.FuncInformerHandler(ctx))
for _, informer := range funcInformer {
informer.Informer().AddEventHandler(caaf.FuncInformerHandler(ctx))
}
return caaf, nil
}
pkg/executor/executortype/newdeploy/newdeploymgr.go
+8 -5
View File
@@ -103,8 +103,8 @@ func MakeNewDeploy(
namespace string,
fetcherConfig *fetcherConfig.Config,
instanceID string,
funcInformer finformerv1.FunctionInformer,
envInformer finformerv1.EnvironmentInformer,
funcInformer map[string]finformerv1.FunctionInformer,
envInformer map[string]finformerv1.EnvironmentInformer,
deplInformer appsinformers.DeploymentInformer,
svcInformer coreinformers.ServiceInformer,
podSpecPatch *apiv1.PodSpec,
@@ -146,9 +146,12 @@ func MakeNewDeploy(
nd.svcLister = svcInformer.Lister()
nd.svcListerSynced = svcInformer.Informer().HasSynced
funcInformer.Informer().AddEventHandler(nd.FunctionEventHandlers(ctx))
envInformer.Informer().AddEventHandler(nd.EnvEventHandlers(ctx))
for _, fnInformer := range funcInformer {
fnInformer.Informer().AddEventHandler(nd.FunctionEventHandlers(ctx))
}
for _, envInformer := range envInformer {
envInformer.Informer().AddEventHandler(nd.EnvEventHandlers(ctx))
}
return nd, nil
}
pkg/executor/executortype/newdeploy/newdeploymgr_test.go
+9 -5
View File
@@ -22,6 +22,7 @@ import (
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
fClient "github.com/fission/fission/pkg/generated/clientset/versioned/fake"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
finformerv1 "github.com/fission/fission/pkg/generated/informers/externalversions/core/v1"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/loggerfactory"
)
@@ -47,9 +48,12 @@ func TestRefreshFuncPods(t *testing.T) {
kubernetesClient := fake.NewSimpleClientset()
fissionClient := fClient.NewSimpleClientset()
informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30)
funcInformer := informerFactory.Core().V1().Functions()
envInformer := informerFactory.Core().V1().Environments()
funcInformer := map[string]finformerv1.FunctionInformer{
metav1.NamespaceAll: informerFactory.Core().V1().Functions(),
}
envInformer := map[string]finformerv1.EnvironmentInformer{
metav1.NamespaceAll: informerFactory.Core().V1().Environments(),
}
newDeployInformerFactory, err := utils.GetInformerFactoryByExecutor(kubernetesClient, fv1.ExecutorTypeNewdeploy, time.Minute*30)
if err != nil {
t.Fatalf("Error creating informer factory: %s", err)
@@ -88,8 +92,8 @@ func TestRefreshFuncPods(t *testing.T) {
t.Log("New deploy manager started")
runInformers(ctx, []k8sCache.SharedIndexInformer{
envInformer.Informer(),
funcInformer.Informer(),
envInformer[metav1.NamespaceAll].Informer(),
funcInformer[metav1.NamespaceAll].Informer(),
deployInformer.Informer(),
svcInformer.Informer(),
})
pkg/executor/executortype/poolmgr/gpm.go
+8 -4
View File
@@ -119,9 +119,9 @@ func MakeGenericPoolManager(ctx context.Context,
functionNamespace string,
fetcherConfig *fetcherConfig.Config,
instanceID string,
funcInformer finformerv1.FunctionInformer,
pkgInformer finformerv1.PackageInformer,
envInformer finformerv1.EnvironmentInformer,
funcInformer map[string]finformerv1.FunctionInformer,
pkgInformer map[string]finformerv1.PackageInformer,
envInformer map[string]finformerv1.EnvironmentInformer,
podInformer coreinformers.PodInformer,
rsInformer appsinformers.ReplicaSetInformer,
podSpecPatch *apiv1.PodSpec,
@@ -554,7 +554,11 @@ func (gpm *GenericPoolManager) getFunctionEnv(ctx context.Context, fn *fv1.Funct
}
// Get env from controller
env, err = gpm.poolPodC.envLister.Environments(fn.Spec.Environment.Namespace).Get(fn.Spec.Environment.Name)
envLister, err := gpm.poolPodC.getEnvLister(fn.Spec.Environment.Namespace)
if err != nil {
return nil, err
}
env, err = envLister.Environments(fn.Spec.Environment.Namespace).Get(fn.Spec.Environment.Name)
if err != nil {
return nil, err
}
pkg/executor/executortype/poolmgr/poolpodcontroller.go
+55 -22
View File
@@ -17,6 +17,7 @@ package poolmgr
import (
"context"
"fmt"
"strings"
"time"
@@ -48,8 +49,8 @@ type (
namespace string
enableIstio bool
envLister flisterv1.EnvironmentLister
envListerSynced k8sCache.InformerSynced
envLister map[string]flisterv1.EnvironmentLister
envListerSynced map[string]k8sCache.InformerSynced
// podLister can list/get pods from the shared informer's store
podLister corelisters.PodLister
@@ -70,37 +71,44 @@ func NewPoolPodController(ctx context.Context, logger *zap.Logger,
kubernetesClient kubernetes.Interface,
namespace string,
enableIstio bool,
funcInformer finformerv1.FunctionInformer,
pkgInformer finformerv1.PackageInformer,
envInformer finformerv1.EnvironmentInformer,
funcInformer map[string]finformerv1.FunctionInformer,
pkgInformer map[string]finformerv1.PackageInformer,
envInformer map[string]finformerv1.EnvironmentInformer,
rsInformer appsinformers.ReplicaSetInformer,
podInformer coreinformers.PodInformer) *PoolPodController {
logger = logger.Named("pool_pod_controller")
p := &PoolPodController{
logger: logger,
kubernetesClient: kubernetesClient,
namespace: namespace,
enableIstio: enableIstio,
logger: logger,
kubernetesClient: kubernetesClient,
namespace: namespace,
enableIstio: enableIstio,
envLister: make(map[string]flisterv1.EnvironmentLister, 0),
envListerSynced: make(map[string]k8sCache.InformerSynced, 0),
envCreateUpdateQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "EnvAddUpdateQueue"),
envDeleteQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "EnvDeleteQueue"),
spCleanupPodQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "SpecializedPodCleanupQueue"),
}
funcInformer.Informer().AddEventHandler(FunctionEventHandlers(ctx, p.logger, p.kubernetesClient, p.namespace, p.enableIstio))
pkgInformer.Informer().AddEventHandler(PackageEventHandlers(ctx, p.logger, p.kubernetesClient, p.namespace))
envInformer.Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: p.enqueueEnvAdd,
UpdateFunc: p.enqueueEnvUpdate,
DeleteFunc: p.enqueueEnvDelete,
})
for _, informer := range funcInformer {
informer.Informer().AddEventHandler(FunctionEventHandlers(ctx, p.logger, p.kubernetesClient, p.namespace, p.enableIstio))
}
for _, informer := range pkgInformer {
informer.Informer().AddEventHandler(PackageEventHandlers(ctx, p.logger, p.kubernetesClient, p.namespace))
}
for ns, informer := range envInformer {
informer.Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: p.enqueueEnvAdd,
UpdateFunc: p.enqueueEnvUpdate,
DeleteFunc: p.enqueueEnvDelete,
})
p.envLister[ns] = informer.Lister()
p.envListerSynced[ns] = informer.Informer().HasSynced
}
rsInformer.Informer().AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: p.handleRSAdd,
UpdateFunc: p.handleRSUpdate,
DeleteFunc: p.handleRSDelete,
})
p.envLister = envInformer.Lister()
p.envListerSynced = envInformer.Informer().HasSynced
p.podLister = podInformer.Lister()
p.podListerSynced = podInformer.Informer().HasSynced
p.logger.Info("pool pod controller handlers registered")
@@ -221,10 +229,15 @@ func (p *PoolPodController) Run(ctx context.Context, stopCh <-chan struct{}) {
defer p.envCreateUpdateQueue.ShutDown()
defer p.envDeleteQueue.ShutDown()
defer p.spCleanupPodQueue.ShutDown()
// Wait for the caches to be synced before starting workers
p.logger.Info("Waiting for informer caches to sync")
if ok := k8sCache.WaitForCacheSync(stopCh, p.envListerSynced, p.podListerSynced); !ok {
waitSynced := make([]k8sCache.InformerSynced, 0)
waitSynced = append(waitSynced, p.podListerSynced)
for _, synced := range p.envListerSynced {
waitSynced = append(waitSynced, synced)
}
if ok := k8sCache.WaitForCacheSync(stopCh, waitSynced...); !ok {
p.logger.Fatal("failed to wait for caches to sync")
}
for i := 0; i < 4; i++ {
@@ -249,6 +262,20 @@ func (p *PoolPodController) workerRun(ctx context.Context, name string, processF
}
}
func (p *PoolPodController) getEnvLister(namespace string) (flisterv1.EnvironmentLister, error) {
lister, ok := p.envLister[metav1.NamespaceAll]
if ok {
return lister, nil
}
for ns, lister := range p.envLister {
if ns == namespace {
return lister, nil
}
}
p.logger.Error("no environment lister found for namespace", zap.String("namespace", namespace))
return nil, fmt.Errorf("no environment lister found for namespace %s", namespace)
}
func (p *PoolPodController) envCreateUpdateQueueProcessFunc(ctx context.Context) bool {
maxRetries := 3
handleEnv := func(ctx context.Context, env *fv1.Environment) error {
@@ -292,7 +319,13 @@ func (p *PoolPodController) envCreateUpdateQueueProcessFunc(ctx context.Context)
p.envCreateUpdateQueue.Forget(key)
return false
}
env, err := p.envLister.Environments(namespace).Get(name)
envLister, err := p.getEnvLister(namespace)
if err != nil {
p.logger.Error("error getting environment lister", zap.Error(err))
p.envCreateUpdateQueue.Forget(key)
return false
}
env, err := envLister.Environments(namespace).Get(name)
if apierrors.IsNotFound(err) {
p.logger.Info("env not found", zap.String("key", key))
p.envCreateUpdateQueue.Forget(key)
pkg/executor/executortype/poolmgr/poolpodcontroller_test.go
+13 -6
View File
@@ -32,6 +32,7 @@ import (
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
fClient "github.com/fission/fission/pkg/generated/clientset/versioned/fake"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
finformerv1 "github.com/fission/fission/pkg/generated/informers/externalversions/core/v1"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/loggerfactory"
)
@@ -50,9 +51,15 @@ func TestPoolPodControllerPodCleanup(t *testing.T) {
kubernetesClient := fake.NewSimpleClientset()
fissionClient := fClient.NewSimpleClientset()
informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30)
funcInformer := informerFactory.Core().V1().Functions()
pkgInformer := informerFactory.Core().V1().Packages()
envInformer := informerFactory.Core().V1().Environments()
funcInformer := map[string]finformerv1.FunctionInformer{
metav1.NamespaceAll: informerFactory.Core().V1().Functions(),
}
pkgInformer := map[string]finformerv1.PackageInformer{
metav1.NamespaceAll: informerFactory.Core().V1().Packages(),
}
envInformer := map[string]finformerv1.EnvironmentInformer{
metav1.NamespaceAll: informerFactory.Core().V1().Environments(),
}
gpmInformerFactory, err := utils.GetInformerFactoryByExecutor(kubernetesClient, fv1.ExecutorTypePoolmgr, time.Minute*30)
if err != nil {
@@ -92,9 +99,9 @@ func TestPoolPodControllerPodCleanup(t *testing.T) {
podInformer := gpmPodInformer.Informer()
runInformers(ctx, []k8sCache.SharedIndexInformer{
funcInformer.Informer(),
pkgInformer.Informer(),
envInformer.Informer(),
funcInformer[metav1.NamespaceAll].Informer(),
pkgInformer[metav1.NamespaceAll].Informer(),
envInformer[metav1.NamespaceAll].Informer(),
podInformer,
gpmRsInformer.Informer(),
})