Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shared informers (#2092)

Browse Source 
* Use shared informers across executor for fission CRs

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Add informer for configmap and secrets

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Use shared informers in router

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Add shared informer for canary config

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Add sharedinformer for ready pod check

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Refer informer instead of store in resolver

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Run informers before executors

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Add missing canary config handler calls

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Enable websocket test

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

* Run dump collection always

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
2021-07-05 20:28:26 +05:30
committed by GitHub
parent 43814450e5
commit f86fde81e6
22 changed files with 793 additions and 718 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/router/functionReferenceResolver.go
+8 -7
View File
@@ -33,8 +33,9 @@ type (
// reference into a resolveResult
functionReferenceResolver struct {
// FunctionReference -> function metadata
refCache *cache.Cache
store k8sCache.Store
refCache *cache.Cache
funcInformer *k8sCache.SharedIndexInformer
// store k8sCache.Store
}
resolveResultType int
@@ -68,10 +69,10 @@ const (
resolveResultMultipleFunctions
)
func makeFunctionReferenceResolver(store k8sCache.Store) *functionReferenceResolver {
func makeFunctionReferenceResolver(funcInformer *k8sCache.SharedIndexInformer) *functionReferenceResolver {
frr := &functionReferenceResolver{
refCache: cache.MakeCache(time.Minute, 0),
store: store,
refCache: cache.MakeCache(time.Minute, 0),
funcInformer: funcInformer,
}
return frr
}
@@ -120,7 +121,7 @@ func (frr *functionReferenceResolver) resolve(trigger fv1.HTTPTrigger) (*resolve
// resolveByName simply looks up function by name in a namespace.
func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) {
// get function from cache
obj, isExist, err := frr.store.Get(&fv1.Function{
obj, isExist, err := (*frr.funcInformer).GetStore().Get(&fv1.Function{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
@@ -154,7 +155,7 @@ func (frr *functionReferenceResolver) resolveByFunctionWeights(namespace string,
for functionName, functionWeight := range fr.FunctionWeights {
// get function from cache
obj, isExist, err := frr.store.Get(&fv1.Function{
obj, isExist, err := (*frr.funcInformer).GetStore().Get(&fv1.Function{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: functionName,
pkg/router/httpTriggers.go
+70 -83
View File
@@ -23,8 +23,6 @@ import (
"github.com/gorilla/mux"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@@ -33,6 +31,7 @@ import (
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/crd"
executorClient "github.com/fission/fission/pkg/executor/client"
genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
"github.com/fission/fission/pkg/throttler"
"github.com/fission/fission/pkg/utils"
)
@@ -49,11 +48,9 @@ type HTTPTriggerSet struct {
resolver *functionReferenceResolver
crdClient rest.Interface
triggers []fv1.HTTPTrigger
triggerStore k8sCache.Store
triggerController k8sCache.Controller
triggerInformer k8sCache.SharedIndexInformer
functions []fv1.Function
funcStore k8sCache.Store
funcController k8sCache.Controller
funcInformer k8sCache.SharedIndexInformer
updateRouterRequestChannel chan struct{}
tsRoundTripperParams *tsRoundTripperParams
isDebugEnv bool
@@ -62,7 +59,7 @@ type HTTPTriggerSet struct {
}
func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, fissionClient *crd.FissionClient,
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient rest.Interface, params *tsRoundTripperParams, isDebugEnv bool, unTapServiceTimeout time.Duration, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient rest.Interface, params *tsRoundTripperParams, isDebugEnv bool, unTapServiceTimeout time.Duration, actionThrottler *throttler.Throttler) *HTTPTriggerSet {
httpTriggerSet := &HTTPTriggerSet{
logger: logger.Named("http_trigger_set"),
@@ -78,21 +75,19 @@ func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, fissionCli
svcAddrUpdateThrottler: actionThrottler,
unTapServiceTimeout: unTapServiceTimeout,
}
var tStore, fnStore k8sCache.Store
var tController, fnController k8sCache.Controller
if httpTriggerSet.crdClient != nil {
tStore, tController = httpTriggerSet.initTriggerController()
httpTriggerSet.triggerStore = tStore
httpTriggerSet.triggerController = tController
fnStore, fnController = httpTriggerSet.initFunctionController()
httpTriggerSet.funcStore = fnStore
httpTriggerSet.funcController = fnController
informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Second*30)
httpTriggerSet.triggerInformer = informerFactory.Core().V1().HTTPTriggers().Informer()
httpTriggerSet.funcInformer = informerFactory.Core().V1().Functions().Informer()
httpTriggerSet.addTriggerHandlers()
httpTriggerSet.addFunctionHandlers()
}
return httpTriggerSet, tStore, fnStore
return httpTriggerSet
}
func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter, resolver *functionReferenceResolver) {
func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter) {
resolver := makeFunctionReferenceResolver(&ts.funcInformer)
ts.resolver = resolver
ts.mutableRouter = mr
@@ -104,8 +99,8 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter
}
go ts.updateRouter()
go ts.syncTriggers()
go ts.runWatcher(ctx, ts.funcController)
go ts.runWatcher(ctx, ts.triggerController)
go ts.runInformer(ctx, ts.funcInformer)
go ts.runInformer(ctx, ts.triggerInformer)
}
func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
@@ -234,78 +229,70 @@ func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *fv1.HTTPTrigger, err err
// TODO
}
func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Controller) {
resyncPeriod := 30 * time.Second
listWatch := k8sCache.NewListWatchFromClient(ts.crdClient, "httptriggers", metav1.NamespaceAll, fields.Everything())
store, controller := k8sCache.NewInformer(listWatch, &fv1.HTTPTrigger{}, resyncPeriod,
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
trigger := obj.(*fv1.HTTPTrigger)
go createIngress(ts.logger, trigger, ts.kubeClient)
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
trigger := obj.(*fv1.HTTPTrigger)
go deleteIngress(ts.logger, trigger, ts.kubeClient)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldTrigger := oldObj.(*fv1.HTTPTrigger)
newTrigger := newObj.(*fv1.HTTPTrigger)
func (ts *HTTPTriggerSet) addTriggerHandlers() {
ts.triggerInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
trigger := obj.(*fv1.HTTPTrigger)
go createIngress(ts.logger, trigger, ts.kubeClient)
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
trigger := obj.(*fv1.HTTPTrigger)
go deleteIngress(ts.logger, trigger, ts.kubeClient)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldTrigger := oldObj.(*fv1.HTTPTrigger)
newTrigger := newObj.(*fv1.HTTPTrigger)
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
return
}
if oldTrigger.ObjectMeta.ResourceVersion == newTrigger.ObjectMeta.ResourceVersion {
return
}
go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient)
ts.syncTriggers()
},
})
return store, controller
go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient)
ts.syncTriggers()
},
})
}
func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Controller) {
resyncPeriod := 30 * time.Second
listWatch := k8sCache.NewListWatchFromClient(ts.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
store, controller := k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod,
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldFn := oldObj.(*fv1.Function)
fn := newObj.(*fv1.Function)
func (ts *HTTPTriggerSet) addFunctionHandlers() {
ts.funcInformer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
oldFn := oldObj.(*fv1.Function)
fn := newObj.(*fv1.Function)
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
return
}
if oldFn.ObjectMeta.ResourceVersion == fn.ObjectMeta.ResourceVersion {
return
}
// update resolver function reference cache
for key, rr := range ts.resolver.copy() {
if key.namespace == fn.ObjectMeta.Namespace &&
rr.functionMap[fn.ObjectMeta.Name] != nil &&
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
// invalidate resolver cache
ts.logger.Debug("invalidating resolver cache")
err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
if err != nil {
ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
}
break
// update resolver function reference cache
for key, rr := range ts.resolver.copy() {
if key.namespace == fn.ObjectMeta.Namespace &&
rr.functionMap[fn.ObjectMeta.Name] != nil &&
rr.functionMap[fn.ObjectMeta.Name].ObjectMeta.ResourceVersion != fn.ObjectMeta.ResourceVersion {
// invalidate resolver cache
ts.logger.Debug("invalidating resolver cache")
err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
if err != nil {
ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
}
break
}
ts.syncTriggers()
},
})
return store, controller
}
ts.syncTriggers()
},
})
}
func (ts *HTTPTriggerSet) runWatcher(ctx context.Context, controller k8sCache.Controller) {
func (ts *HTTPTriggerSet) runInformer(ctx context.Context, informer k8sCache.SharedIndexInformer) {
go func() {
controller.Run(ctx.Done())
informer.Run(ctx.Done())
}()
}
@@ -316,7 +303,7 @@ func (ts *HTTPTriggerSet) syncTriggers() {
func (ts *HTTPTriggerSet) updateRouter() {
for range ts.updateRouterRequestChannel {
// get triggers
latestTriggers := ts.triggerStore.List()
latestTriggers := ts.triggerInformer.GetStore().List()
triggers := make([]fv1.HTTPTrigger, len(latestTriggers))
for _, t := range latestTriggers {
triggers = append(triggers, *t.(*fv1.HTTPTrigger))
@@ -324,7 +311,7 @@ func (ts *HTTPTriggerSet) updateRouter() {
ts.triggers = triggers
// get functions
latestFunctions := ts.funcStore.List()
latestFunctions := ts.funcInformer.GetStore().List()
functionTimeout := make(map[types.UID]int, len(latestFunctions))
functions := make([]fv1.Function, len(latestFunctions))
for _, f := range latestFunctions {
pkg/router/router.go
+6 -8
View File
@@ -63,7 +63,7 @@ import (
// request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url
func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter {
func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet) *mutableRouter {
var mr *mutableRouter
// see issue https://github.com/fission/fission/issues/1317
@@ -74,13 +74,13 @@ func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTrigger
mr = newMutableRouter(logger, mux.NewRouter())
}
httpTriggerSet.subscribeRouter(ctx, mr, resolver)
httpTriggerSet.subscribeRouter(ctx, mr)
return mr
}
func serve(ctx context.Context, logger *zap.Logger, port int, tracingSamplingRate float64,
httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver, displayAccessLog bool) {
mr := router(ctx, logger, httpTriggerSet, resolver)
httpTriggerSet *HTTPTriggerSet, displayAccessLog bool) {
mr := router(ctx, logger, httpTriggerSet)
url := fmt.Sprintf(":%v", port)
err := http.ListenAndServe(url, &ochttp.Handler{
@@ -236,7 +236,7 @@ func Start(logger *zap.Logger, port int, executorURL string) {
zap.Bool("default", displayAccessLog))
}
triggers, _, fnStore := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, fissionClient, kubeClient, executor, fissionClient.CoreV1().RESTClient(), &tsRoundTripperParams{
triggers := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, fissionClient, kubeClient, executor, fissionClient.CoreV1().RESTClient(), &tsRoundTripperParams{
timeout: timeout,
timeoutExponent: timeoutExponent,
disableKeepAlive: disableKeepAlive,
@@ -245,12 +245,10 @@ func Start(logger *zap.Logger, port int, executorURL string) {
svcAddrRetryCount: svcAddrRetryCount,
}, isDebugEnv, unTapServiceTimeout, throttler.MakeThrottler(svcAddrUpdateTimeout))
resolver := makeFunctionReferenceResolver(fnStore)
go serveMetric(logger)
logger.Info("starting router", zap.Int("port", port))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
serve(ctx, logger, port, tracingSamplingRate, triggers, resolver, displayAccessLog)
serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog)
}