Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use Kubernetes informer and cache in router (#376)

Browse Source 
Use Kubernetes client's informer and cache to watch functions and triggers in the router. This replaces the polling loop we were using so far.
This commit is contained in:
Ta-Ching Chen
2017-10-25 15:10:44 -07:00
committed by Soam Vasani
parent 8abb0005f6
commit 439e7d4535
7 changed files with 150 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/functionReferenceResolver.go
+51 -2
View File
@@ -21,6 +21,11 @@ import (
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/rest"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission"
"github.com/fission/fission/cache"
@@ -35,6 +40,9 @@ type (
// FunctionReference -> function metadata
refCache *cache.Cache
stopCh chan struct{}
store k8sCache.Store
}
resolveResultType int
@@ -61,10 +69,40 @@ const (
)
func makeFunctionReferenceResolver(fissionClient *tpr.FissionClient) *functionReferenceResolver {
return &functionReferenceResolver{
frr := &functionReferenceResolver{
fissionClient: fissionClient,
refCache: cache.MakeCache(time.Minute, 0),
}
return frr
}
// Sync starts syncing tpr function resources from k8s api server
func (frr *functionReferenceResolver) Sync(tprClient *rest.RESTClient) {
stopCh := make(chan struct{})
store, controller := makeK8SCache(tprClient)
frr.stopCh = stopCh
frr.store = store
go controller.Run(stopCh)
}
// Stop stops tpr resources syncing
func (frr *functionReferenceResolver) Stop() {
frr.stopCh <- struct{}{}
}
func makeK8SCache(tprClient *rest.RESTClient) (k8sCache.Store, k8sCache.Controller) {
watchlist := k8sCache.NewListWatchFromClient(tprClient, "functions", metav1.NamespaceDefault, fields.Everything())
listWatch := &k8sCache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return watchlist.List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return watchlist.Watch(options)
},
}
resyncPeriod := 30 * time.Second
return k8sCache.NewInformer(listWatch, &tpr.Function{}, resyncPeriod,
k8sCache.ResourceEventHandlerFuncs{})
}
// resolve translates a namespace and a function reference to resolveResult.
@@ -106,10 +144,21 @@ func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.Func
// resolveByName simply looks up function by name in a namespace.
func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) {
f, err := frr.fissionClient.Functions(namespace).Get(name)
// get function from cache
obj, isExist, err := frr.store.Get(&tpr.Function{
Metadata: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
},
})
if err != nil {
return nil, err
}
if !isExist {
return nil, fmt.Errorf("function %v does not exist", name)
}
f := obj.(*tpr.Function)
rr := resolveResult{
resolveResultType: resolveResultSingleFunction,
functionMetadata: &f.Metadata,
router/httpTriggers.go
+73 -64
View File
@@ -23,7 +23,11 @@ import (
"github.com/gorilla/mux"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/rest"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission"
poolmgrClient "github.com/fission/fission/poolmgr/client"
@@ -38,9 +42,11 @@ type HTTPTriggerSet struct {
resolver *functionReferenceResolver
triggers []tpr.Httptrigger
functions []tpr.Function
tprClient *rest.RESTClient
}
func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClient, poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver) *HTTPTriggerSet {
func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClient,
poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver, tprClient *rest.RESTClient) *HTTPTriggerSet {
triggers := make([]tpr.Httptrigger, 1)
return &HTTPTriggerSet{
functionServiceMap: fmap,
@@ -48,6 +54,7 @@ func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClie
fissionClient: fissionClient,
poolmgr: poolmgr,
resolver: resolver,
tprClient: tprClient,
}
}
@@ -135,75 +142,77 @@ func (ts *HTTPTriggerSet) watchTriggers() {
// sync all http triggers
ts.syncTriggers()
// Watch controller for updates to triggers and update the router accordingly.
rv := ""
for {
wi, err := ts.fissionClient.Httptriggers(metav1.NamespaceAll).Watch(metav1.ListOptions{
ResourceVersion: rv,
})
if err != nil {
log.Fatalf("Failed to watch http trigger list: %v", err)
}
for {
ev, more := <-wi.ResultChan()
if !more {
// restart watch from last rv
break
}
if ev.Type == watch.Error {
// restart watch from the start
rv = ""
time.Sleep(time.Second)
break
}
ht := ev.Object.(*tpr.Httptrigger)
rv = ht.Metadata.ResourceVersion
ts.syncTriggers()
}
watchlist := k8sCache.NewListWatchFromClient(ts.tprClient, "httptriggers", metav1.NamespaceDefault, fields.Everything())
listWatch := &k8sCache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return watchlist.List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return watchlist.Watch(options)
},
}
resyncPeriod := 30 * time.Second
_, controller := k8sCache.NewInformer(listWatch, &tpr.Httptrigger{}, resyncPeriod,
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
ts.syncTriggers()
},
})
stop := make(chan struct{})
defer func() {
stop <- struct{}{}
}()
controller.Run(stop)
}
func (ts *HTTPTriggerSet) watchFunctions() {
rv := ""
for {
wi, err := ts.fissionClient.Functions(metav1.NamespaceAll).Watch(metav1.ListOptions{
ResourceVersion: rv,
})
if err != nil {
log.Fatalf("Failed to watch function list: %v", err)
}
ts.syncTriggers()
for {
ev, more := <-wi.ResultChan()
if !more {
// restart watch from last rv
break
}
if ev.Type == watch.Error {
// restart watch from the start
rv = ""
time.Sleep(time.Second)
break
}
fn := ev.Object.(*tpr.Function)
rv = fn.Metadata.ResourceVersion
// update resolver function reference cache
for key, rr := range ts.resolver.copy() {
if key.functionReference.Name == fn.Metadata.Name &&
rr.functionMetadata.ResourceVersion != fn.Metadata.ResourceVersion {
err := ts.resolver.delete(key.namespace, &key.functionReference)
if err != nil {
log.Printf("Error deleting functionReferenceResolver cache: %v", err)
}
break
}
}
ts.syncTriggers()
}
watchlist := k8sCache.NewListWatchFromClient(ts.tprClient, "functions", metav1.NamespaceDefault, fields.Everything())
listWatch := &k8sCache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
return watchlist.List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return watchlist.Watch(options)
},
}
resyncPeriod := 30 * time.Second
_, controller := k8sCache.NewInformer(listWatch, &tpr.Function{}, resyncPeriod,
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
ts.syncTriggers()
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
fn := newObj.(*tpr.Function)
// update resolver function reference cache
for key, rr := range ts.resolver.copy() {
if key.functionReference.Name == fn.Metadata.Name &&
rr.functionMetadata.ResourceVersion != fn.Metadata.ResourceVersion {
err := ts.resolver.delete(key.namespace, &key.functionReference)
if err != nil {
log.Printf("Error deleting functionReferenceResolver cache: %v", err)
}
break
}
}
ts.syncTriggers()
},
})
stop := make(chan struct{})
defer func() {
stop <- struct{}{}
}()
controller.Run(stop)
}
func (ts *HTTPTriggerSet) syncTriggers() {
router/router.go
+6 -1
View File
@@ -77,9 +77,14 @@ func Start(port int, poolmgrUrl string) {
if err != nil {
log.Fatalf("Error connecting to kubernetes API: %v", err)
}
restClient := fissionClient.GetTprClient()
poolmgr := poolmgrClient.MakeClient(poolmgrUrl)
resolver := makeFunctionReferenceResolver(fissionClient)
triggers := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, resolver)
resolver.Sync(restClient)
defer func() {
resolver.Stop()
}()
triggers := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, resolver, restClient)
log.Printf("Starting router at port %v\n", port)
serve(port, triggers)
}
router/router_test.go
+1 -1
View File
@@ -58,7 +58,7 @@ func TestRouter(t *testing.T) {
frr.refCache.Set(nfr, rr)
// HTTP trigger set with a trigger for this function
triggers := makeHTTPTriggerSet(fmap, nil, nil, frr)
triggers := makeHTTPTriggerSet(fmap, nil, nil, frr, nil)
triggerUrl := "/foo"
triggers.triggers = append(triggers.triggers,
tpr.Httptrigger{