V2 types and TPR (#266)

This changes the core fission function, environment and trigger types. It also changes Fission's storage to use ThirdPartyResources.

 - Functions are now specified by packages. Functions can also have both source and deployment packages. A package can be specified by a literal, or by a URL.
 - Environments have a build and runtime component.
 - Triggers reference functions by a FunctionReference. This is a layer of indirection between triggers and functions, and will allow things like incremental function upgrades in future releases.

See Documentation/wip/env-v2.md for design discussion about points 1 and 2.

Changes:

* V2 Types

All types now have a spec, following the pattern of K8s objects.

Functions now have source and deployment packages. A Package can be
specified by literal, or by URL.

Environments now have a builder and runtime component.

All triggers use a new FunctionReference to specify the function. This
for now only uses a function name, but in the future can be extended
to be more flexible.

A new FunctionLoadRequest type is added for specialization requests to
the environment runtime.

* TPR types, TPR init code, and a "fission client"

Implements TPR types using the spec types in fission/types.go.

Adds code for adding creating TPR types, and convenient types for crud
operations on each of our resource types.

Adds code for connecting to K8s API and configuring a REST client with
fission types set up.

* Change old stateful controller into a thin apiserver

This apiserver is now simply a stateless api layer on top of the TPR
types. At the moment it doesn't do anything that couldn't be done by
simply talking to the TPR types. In the future we can have better
validation and potentially some higher level APIs (like versioning for
example) in here.

* Split controller client into files and update for v2 types.

* Update CLI for v2 types.

As far as possible we keep the CLI flags the same. We'll have to add
flags for source/deploy packages and builder/runtime
environments. That will come in the next change.

* Update poolmgr and fetcher for v2 types.

Also adds a poolmgr_test.

* Update router for new types.

Also adds a function reference resolver, which separates out the job
of resolving a FunctionReference to a function.

* Update kubewatcher and timer for v2 types.

* Update Message Queue trigger type for v2 types.

* Minor odds and ends.

* Fission bundle CLI updates

Remove controllerUrl flag, since we don't need it any more.

* Remove etcd deployment (replaced by storing state in TPR)

Also update the poolmgr commandline, and use an env var for the
fetcher image URL.

* Explicit ChecksumType and consts

* Clarify separation of environment interface types

router/functionHandler.go

@@ -26,20 +26,20 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"k8s.io/client-go/1.5/pkg/api"
 
-	"github.com/fission/fission"
 	poolmgrClient "github.com/fission/fission/poolmgr/client"
 )
 
 type functionHandler struct {
 	fmap     *functionServiceMap
 	poolmgr  *poolmgrClient.Client
-	Function fission.Metadata
+	function *api.ObjectMeta
 }
 
 func (fh *functionHandler) getServiceForFunction() (*url.URL, error) {
 	// call poolmgr, get a url for a function
-	svcName, err := fh.poolmgr.GetServiceForFunction(&fh.Function)
+	svcName, err := fh.poolmgr.GetServiceForFunction(fh.function)
 	if err != nil {
 		return nil, err
 	}
@@ -99,16 +99,15 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 	}
 
 	// cache lookup
-	serviceUrl, err := fh.fmap.lookup(&fh.Function)
+	serviceUrl, err := fh.fmap.lookup(fh.function)
 	if err != nil {
 		// Cache miss: request the Pool Manager to make a new service.
-		log.Printf("Not cached, getting new service for %v", fh.Function)
+		log.Printf("Not cached, getting new service for %v", fh.function)
 
 		var poolErr error
 		serviceUrl, poolErr = fh.getServiceForFunction()
 		if poolErr != nil {
-			log.Printf("Failed to get service for function (%v,%v): %v",
-				fh.Function.Name, fh.Function.Uid, poolErr)
+			log.Printf("Failed to get service for function %v: %v", fh.function.Name, poolErr)
 			// We might want a specific error code or header for fission
 			// failures as opposed to user function bugs.
 			http.Error(responseWriter, "Internal server error (fission)", 500)
@@ -116,7 +115,7 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 		}
 
 		// add it to the map
-		fh.fmap.assign(&fh.Function, serviceUrl)
+		fh.fmap.assign(fh.function, serviceUrl)
 	} else {
 		// if we're using our cache, asynchronously tell
 		// poolmgr we're using this service

router/functionHandler_test.go

@@ -23,7 +23,7 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/fission/fission"
+	"k8s.io/client-go/1.5/pkg/api"
 )
 
 func createBackendService(testResponseString string) *url.URL {
@@ -49,11 +49,12 @@ func TestFunctionProxying(t *testing.T) {
 	backendURL := createBackendService(testResponseString)
 	log.Printf("Created backend svc at %v", backendURL)
 
-	fn := &fission.Metadata{Name: "foo", Uid: "xxx"}
+	fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault}
+
 	fmap := makeFunctionServiceMap(0)
 	fmap.assign(fn, backendURL)
 
-	fh := &functionHandler{fmap: fmap, Function: *fn}
+	fh := &functionHandler{fmap: fmap, function: fn}
 	functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler))
 	fhURL := functionHandlerServer.URL
 

router/functionReferenceResolver.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/client-go/1.5/pkg/api"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/cache"
+	"github.com/fission/fission/tpr"
+)
+
+type (
+	// functionReferenceResolver provides a resolver to turn a function
+	// reference into a resolveResult
+	functionReferenceResolver struct {
+		fissionClient *tpr.FissionClient
+
+		// FunctionReference -> function metadata
+		refCache *cache.Cache
+	}
+
+	resolveResultType int
+
+	// resolveResult is the result of resolving a function reference; for now
+	// it's just the metadata of one function, but in the future could support
+	// a distribution of requests across two functions.
+	resolveResult struct {
+		resolveResultType
+		functionMetadata *api.ObjectMeta
+	}
+
+	// namespacedFunctionReference is just a function reference plus a
+	// namespace. Since a function reference works on names, it's only
+	// meaningful within a namespace.
+	namespacedFunctionReference struct {
+		namespace         string
+		functionReference fission.FunctionReference
+	}
+)
+
+const (
+	resolveResultSingleFunction = iota
+)
+
+func makeFunctionReferenceResolver(fissionClient *tpr.FissionClient) *functionReferenceResolver {
+	return &functionReferenceResolver{
+		fissionClient: fissionClient,
+		refCache:      cache.MakeCache(time.Minute, 0),
+	}
+}
+
+// resolve translates a namespace and a function reference to resolveResult.
+// The resolveResult for now is just a function's metadata. In the future, some
+// function ref types may resolve to two functions rather than just one
+// (e.g. for incremental deployment), which will make the resolveResult a bit
+// more complex.
+func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.FunctionReference) (*resolveResult, error) {
+	nfr := namespacedFunctionReference{
+		namespace:         namespace,
+		functionReference: *fr,
+	}
+
+	// check cache
+	rrInt, err := frr.refCache.Get(nfr)
+	if err == nil {
+		result := rrInt.(resolveResult)
+		return &result, nil
+	}
+
+	// resolve on cache miss
+	var rr *resolveResult
+
+	switch fr.Type {
+	case fission.FunctionReferenceTypeFunctionName:
+		rr, err = frr.resolveByName(namespace, fr.Name)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("Unrecognized function reference type %v", fr.Type)
+	}
+
+	// cache resolve result
+	frr.refCache.Set(nfr, *rr)
+
+	return rr, nil
+}
+
+// resolveByName simply looks up function by name in a namespace.
+func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) {
+	f, err := frr.fissionClient.Functions(namespace).Get(name)
+	if err != nil {
+		return nil, err
+	}
+	rr := resolveResult{
+		resolveResultType: resolveResultSingleFunction,
+		functionMetadata:  &f.Metadata,
+	}
+	return &rr, nil
+}

router/functionServiceMap.go

@@ -21,13 +21,24 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/fission/fission"
+	"k8s.io/client-go/1.5/pkg/api"
+
 	"github.com/fission/fission/cache"
 )
 
-type functionServiceMap struct {
-	cache *cache.Cache // map[fission.Metadata]*url.URL
-}
+type (
+	functionServiceMap struct {
+		cache *cache.Cache // map[metadataKey]*url.URL
+	}
+
+	// api.ObjectMeta is not hashable, so we make a hashable copy
+	// of the subset of its fields that are identifiable.
+	metadataKey struct {
+		Name            string
+		Namespace       string
+		ResourceVersion string
+	}
+)
 
 func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap {
 	return &functionServiceMap{
@@ -35,8 +46,17 @@ func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap {
 	}
 }
 
-func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) {
-	item, err := fmap.cache.Get(*f)
+func keyFromMetadata(m *api.ObjectMeta) *metadataKey {
+	return &metadataKey{
+		Name:            m.Name,
+		Namespace:       m.Namespace,
+		ResourceVersion: m.ResourceVersion,
+	}
+}
+
+func (fmap *functionServiceMap) lookup(f *api.ObjectMeta) (*url.URL, error) {
+	mk := keyFromMetadata(f)
+	item, err := fmap.cache.Get(*mk)
 	if err != nil {
 		return nil, err
 	}
@@ -44,8 +64,9 @@ func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) {
 	return u, nil
 }
 
-func (fmap *functionServiceMap) assign(f *fission.Metadata, serviceUrl *url.URL) {
-	err, old := fmap.cache.Set(*f, serviceUrl)
+func (fmap *functionServiceMap) assign(f *api.ObjectMeta, serviceUrl *url.URL) {
+	mk := keyFromMetadata(f)
+	err, old := fmap.cache.Set(*mk, serviceUrl)
 	if err != nil {
 		if *serviceUrl == *(old.(*url.URL)) {
 			return

router/functionServiceMap_test.go

@@ -20,12 +20,12 @@ import (
 	"net/url"
 	"testing"
 
-	"github.com/fission/fission"
+	"k8s.io/client-go/1.5/pkg/api"
 )
 
 func TestFunctionServiceMap(t *testing.T) {
 	m := makeFunctionServiceMap(0)
-	fn := &fission.Metadata{Name: "foo", Uid: "012"}
+	fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault}
 	u, err := url.Parse("/foo012")
 	if err != nil {
 		t.Errorf("can't parse url")

router/httpTriggers.go

@@ -22,34 +22,44 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"k8s.io/client-go/1.5/pkg/api"
+	"k8s.io/client-go/1.5/pkg/watch"
 
 	"github.com/fission/fission"
-	controllerClient "github.com/fission/fission/controller/client"
 	poolmgrClient "github.com/fission/fission/poolmgr/client"
+	"github.com/fission/fission/tpr"
 )
 
 type HTTPTriggerSet struct {
 	*functionServiceMap
 	*mutableRouter
-	controller *controllerClient.Client
-	poolmgr    *poolmgrClient.Client
-	triggers   []fission.HTTPTrigger
-	functions  []fission.Function
+	fissionClient *tpr.FissionClient
+	poolmgr       *poolmgrClient.Client
+	resolver      *functionReferenceResolver
+	triggers      []tpr.Httptrigger
+	functions     []tpr.Function
 }
 
-func makeHTTPTriggerSet(fmap *functionServiceMap, controller *controllerClient.Client, poolmgr *poolmgrClient.Client) *HTTPTriggerSet {
-	triggers := make([]fission.HTTPTrigger, 1)
+func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClient, poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver) *HTTPTriggerSet {
+	triggers := make([]tpr.Httptrigger, 1)
 	return &HTTPTriggerSet{
 		functionServiceMap: fmap,
 		triggers:           triggers,
-		controller:         controller,
+		fissionClient:      fissionClient,
 		poolmgr:            poolmgr,
+		resolver:           resolver,
 	}
 }
 
 func (ts *HTTPTriggerSet) subscribeRouter(mr *mutableRouter) {
 	ts.mutableRouter = mr
 	mr.updateRouter(ts.getRouter())
+
+	if ts.fissionClient == nil {
+		// Used in tests only.
+		log.Printf("Skipping continuous trigger updates")
+		return
+	}
 	go ts.watchTriggers()
 }
 
@@ -60,27 +70,33 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
 func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 	muxRouter := mux.NewRouter()
 
-	// make a function name -> latest version map
-	latestVersions := make(map[string]string)
-	for _, f := range ts.functions {
-		latestVersions[f.Metadata.Name] = f.Metadata.Uid
-	}
-
 	// HTTP triggers setup by the user
 	homeHandled := false
 	for _, trigger := range ts.triggers {
-		m := trigger.Function
-		if len(m.Uid) == 0 {
-			// explicitly use the latest function version
-			m.Uid = latestVersions[m.Name]
+
+		// resolve function reference
+		rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference)
+		if err != nil {
+			// Unresolvable function reference. Report the error via
+			// the trigger's status.
+			go ts.updateTriggerStatusFailed(&trigger, err)
+
+			// Ignore this route and let it 404.
+			continue
 		}
+
+		if rr.resolveResultType != resolveResultSingleFunction {
+			// not implemented yet
+			log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType)
+		}
+
 		fh := &functionHandler{
 			fmap:     ts.functionServiceMap,
-			Function: m,
+			function: rr.functionMetadata,
 			poolmgr:  ts.poolmgr,
 		}
-		muxRouter.HandleFunc(trigger.UrlPattern, fh.handler).Methods(trigger.Method)
-		if trigger.UrlPattern == "/" && trigger.Method == "GET" {
+		muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method)
+		if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" {
 			homeHandled = true
 		}
 	}
@@ -95,53 +111,74 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 		muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET")
 	}
 
-	// Internal triggers for (the latest version of) each function
+	// Internal triggers for each function by name. Non-http
+	// triggers route into these.
 	for _, function := range ts.functions {
-		m := fission.Metadata{Name: function.Metadata.Name}
 		fh := &functionHandler{
 			fmap:     ts.functionServiceMap,
-			Function: function.Metadata,
+			function: &function.Metadata,
 			poolmgr:  ts.poolmgr,
 		}
-		muxRouter.HandleFunc(fission.UrlForFunction(&m), fh.handler)
+		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
 	}
 
 	return muxRouter
 }
 
+func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *tpr.Httptrigger, err error) {
+	// TODO
+}
+
 func (ts *HTTPTriggerSet) watchTriggers() {
-	if ts.controller == nil {
-		return
-	}
-
-	// the number of connection failures we'll accept before quitting
-	maxFailures := 5
-
-	// amount of time to sleep between polling calls
-	pollSleepDuration := 3 * time.Second
+	// sync all http triggers
+	ts.syncTriggers()
 
 	// Watch controller for updates to triggers and update the router accordingly.
-	// TODO change this to use a watch API; or maybe even watch etcd directly.
-	failureCount := 0
+	rv := ""
 	for {
-		triggers, err := ts.controller.HTTPTriggerList()
+		wi, err := ts.fissionClient.Httptriggers(api.NamespaceAll).Watch(api.ListOptions{
+			ResourceVersion: rv,
+		})
 		if err != nil {
-			failureCount += 1
-			if failureCount >= maxFailures {
-				log.Fatalf("Failed to connect to controller after %v retries: %v", failureCount, err)
+			log.Fatalf("Failed to watch http trigger list: %v", err)
+		}
+
+		for {
+			ev, more := <-wi.ResultChan()
+			if !more {
+				// restart watch from last rv
+				break
 			}
-			time.Sleep(pollSleepDuration)
-			continue
+			if ev.Type == watch.Error {
+				// restart watch from the start
+				rv = ""
+				time.Sleep(time.Second)
+				break
+			}
+			ht := ev.Object.(*tpr.Httptrigger)
+			rv = ht.Metadata.ResourceVersion
+			ts.syncTriggers()
 		}
-		ts.triggers = triggers
-
-		functions, err := ts.controller.FunctionList()
-		if err != nil {
-			log.Fatalf("Failed to get function list")
-		}
-		ts.functions = functions
-
-		ts.mutableRouter.updateRouter(ts.getRouter())
-		time.Sleep(pollSleepDuration)
 	}
 }
+
+func (ts *HTTPTriggerSet) syncTriggers() {
+	log.Printf("Syncing http triggers")
+
+	// get triggers
+	triggers, err := ts.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{})
+	if err != nil {
+		log.Fatalf("Failed to get http trigger list: %v", err)
+	}
+	ts.triggers = triggers.Items
+
+	// get functions
+	functions, err := ts.fissionClient.Functions(api.NamespaceAll).List(api.ListOptions{})
+	if err != nil {
+		log.Fatalf("Failed to get function list: %v", err)
+	}
+	ts.functions = functions.Items
+
+	// make a new router and use it
+	ts.mutableRouter.updateRouter(ts.getRouter())
+}

router/router.go

@@ -49,8 +49,8 @@ import (
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 
-	controllerClient "github.com/fission/fission/controller/client"
 	poolmgrClient "github.com/fission/fission/poolmgr/client"
+	"github.com/fission/fission/tpr"
 )
 
 // request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url
@@ -70,12 +70,16 @@ func serve(port int, httpTriggerSet *HTTPTriggerSet) {
 	http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr))
 }
 
-func Start(port int, controllerUrl string, poolmgrUrl string) {
+func Start(port int, poolmgrUrl string) {
 	fmap := makeFunctionServiceMap(time.Minute)
-	controller := controllerClient.MakeClient(controllerUrl)
-	poolmgr := poolmgrClient.MakeClient(poolmgrUrl)
 
-	triggers := makeHTTPTriggerSet(fmap, controller, poolmgr)
+	fissionClient, _, err := tpr.MakeFissionClient()
+	if err != nil {
+		log.Fatalf("Error connecting to kubernetes API: %v", err)
+	}
+	poolmgr := poolmgrClient.MakeClient(poolmgrUrl)
+	resolver := makeFunctionReferenceResolver(fissionClient)
+	triggers := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, resolver)
 	log.Printf("Starting router at port %v\n", port)
 	serve(port, triggers)
 }

router/router_test.go

@@ -21,26 +21,64 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/client-go/1.5/pkg/api"
+
 	"github.com/fission/fission"
+	"github.com/fission/fission/tpr"
 )
 
 func TestRouter(t *testing.T) {
-	fmap := makeFunctionServiceMap(0)
-	fn := &fission.Metadata{Name: "foo", Uid: "xxx"}
+	// metadata for a fake function
+	fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault}
 
+	// and a reference to it
+	fr := fission.FunctionReference{
+		Type: fission.FunctionReferenceTypeFunctionName,
+		Name: fn.Name,
+	}
+
+	// start a fake service
 	testResponseString := "hi"
 	testServiceUrl := createBackendService(testResponseString)
 
+	// set up the cache with this fake service
+	fmap := makeFunctionServiceMap(0)
 	fmap.assign(fn, testServiceUrl)
 
-	triggers := makeHTTPTriggerSet(fmap, nil, nil)
-	triggerUrl := "/foo"
-	triggers.triggers = append(triggers.triggers, fission.HTTPTrigger{UrlPattern: triggerUrl, Function: *fn, Method: "GET"})
+	// set up the resolver's cache for this function
+	frr := makeFunctionReferenceResolver(nil)
+	nfr := namespacedFunctionReference{
+		namespace:         api.NamespaceDefault,
+		functionReference: fr,
+	}
+	rr := resolveResult{
+		resolveResultType: resolveResultSingleFunction,
+		functionMetadata:  fn,
+	}
+	frr.refCache.Set(nfr, rr)
 
+	// HTTP trigger set with a trigger for this function
+	triggers := makeHTTPTriggerSet(fmap, nil, nil, frr)
+	triggerUrl := "/foo"
+	triggers.triggers = append(triggers.triggers,
+		tpr.Httptrigger{
+			Metadata: api.ObjectMeta{
+				Name:      "xxx",
+				Namespace: api.NamespaceDefault,
+			},
+			Spec: fission.HTTPTriggerSpec{
+				RelativeURL:       triggerUrl,
+				FunctionReference: fr,
+				Method:            "GET",
+			},
+		})
+
+	// run the router
 	port := 4242
 	go serve(port, triggers)
 	time.Sleep(100 * time.Millisecond)
 
+	// hit the router
 	testUrl := fmt.Sprintf("http://localhost:%v%v", port, triggerUrl)
 	testRequest(testUrl, testResponseString)
 }