V2 types and TPR (#266)

This changes the core fission function, environment and trigger types. It also changes Fission's storage to use ThirdPartyResources.

 - Functions are now specified by packages. Functions can also have both source and deployment packages. A package can be specified by a literal, or by a URL.
 - Environments have a build and runtime component.
 - Triggers reference functions by a FunctionReference. This is a layer of indirection between triggers and functions, and will allow things like incremental function upgrades in future releases.

See Documentation/wip/env-v2.md for design discussion about points 1 and 2.

Changes:

* V2 Types

All types now have a spec, following the pattern of K8s objects.

Functions now have source and deployment packages. A Package can be
specified by literal, or by URL.

Environments now have a builder and runtime component.

All triggers use a new FunctionReference to specify the function. This
for now only uses a function name, but in the future can be extended
to be more flexible.

A new FunctionLoadRequest type is added for specialization requests to
the environment runtime.

* TPR types, TPR init code, and a "fission client"

Implements TPR types using the spec types in fission/types.go.

Adds code for adding creating TPR types, and convenient types for crud
operations on each of our resource types.

Adds code for connecting to K8s API and configuring a REST client with
fission types set up.

* Change old stateful controller into a thin apiserver

This apiserver is now simply a stateless api layer on top of the TPR
types. At the moment it doesn't do anything that couldn't be done by
simply talking to the TPR types. In the future we can have better
validation and potentially some higher level APIs (like versioning for
example) in here.

* Split controller client into files and update for v2 types.

* Update CLI for v2 types.

As far as possible we keep the CLI flags the same. We'll have to add
flags for source/deploy packages and builder/runtime
environments. That will come in the next change.

* Update poolmgr and fetcher for v2 types.

Also adds a poolmgr_test.

* Update router for new types.

Also adds a function reference resolver, which separates out the job
of resolving a FunctionReference to a function.

* Update kubewatcher and timer for v2 types.

* Update Message Queue trigger type for v2 types.

* Minor odds and ends.

* Fission bundle CLI updates

Remove controllerUrl flag, since we don't need it any more.

* Remove etcd deployment (replaced by storing state in TPR)

Also update the poolmgr commandline, and use an env var for the
fetcher image URL.

* Explicit ChecksumType and consts

* Clarify separation of environment interface types

kubewatcher/kubewatcher.go

@@ -33,10 +33,12 @@ import (
 	apierrs "k8s.io/client-go/1.5/pkg/api/errors"
 	"k8s.io/client-go/1.5/pkg/api/meta"
 	"k8s.io/client-go/1.5/pkg/runtime"
+	"k8s.io/client-go/1.5/pkg/types"
 	"k8s.io/client-go/1.5/pkg/watch"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/publisher"
+	"github.com/fission/fission/tpr"
 )
 
 type requestType int
@@ -47,7 +49,7 @@ const (
 
 type (
 	KubeWatcher struct {
-		watches          map[string]watchSubscription
+		watches          map[types.UID]watchSubscription
 		kubernetesClient *kubernetes.Clientset
 		requestChannel   chan *kubeWatcherRequest
 		publisher        publisher.Publisher
@@ -55,7 +57,7 @@ type (
 	}
 
 	watchSubscription struct {
-		fission.Watch
+		watch               tpr.Kuberneteswatchtrigger
 		kubeWatch           watch.Interface
 		lastResourceVersion string
 		stopped             *int32
@@ -65,7 +67,7 @@ type (
 
 	kubeWatcherRequest struct {
 		requestType
-		watches         []fission.Watch
+		watches         []tpr.Kuberneteswatchtrigger
 		responseChannel chan *kubeWatcherResponse
 	}
 	kubeWatcherResponse struct {
@@ -75,7 +77,7 @@ type (
 
 func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
 	kw := &KubeWatcher{
-		watches:          make(map[string]watchSubscription),
+		watches:          make(map[types.UID]watchSubscription),
 		kubernetesClient: kubernetesClient,
 		publisher:        publisher,
 		requestChannel:   make(chan *kubeWatcherRequest),
@@ -84,7 +86,7 @@ func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher
 	return kw
 }
 
-func (kw *KubeWatcher) Sync(watches []fission.Watch) error {
+func (kw *KubeWatcher) Sync(watches []tpr.Kuberneteswatchtrigger) error {
 	req := &kubeWatcherRequest{
 		requestType:     SYNC,
 		watches:         watches,
@@ -100,19 +102,19 @@ func (kw *KubeWatcher) svc() {
 		req := <-kw.requestChannel
 		switch req.requestType {
 		case SYNC:
-			newWatchUids := make(map[string]bool)
+			newWatchUids := make(map[types.UID]bool)
 			for _, w := range req.watches {
-				newWatchUids[w.Metadata.Uid] = true
+				newWatchUids[w.Metadata.UID] = true
 			}
 			// Remove old watches
 			for uid, ws := range kw.watches {
 				if _, ok := newWatchUids[uid]; !ok {
-					kw.removeWatch(&ws.Watch)
+					kw.removeWatch(&ws.watch)
 				}
 			}
 			// Add new watches
 			for _, w := range req.watches {
-				if _, ok := kw.watches[w.Metadata.Uid]; !ok {
+				if _, ok := kw.watches[w.Metadata.UID]; !ok {
 					kw.addWatch(&w)
 				}
 			}
@@ -144,7 +146,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error {
 	return err
 }
 
-func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, resourceVersion string) (watch.Interface, error) {
+func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *tpr.Kuberneteswatchtrigger, resourceVersion string) (watch.Interface, error) {
 	var wi watch.Interface
 	var err error
 	var watchTimeoutSec int64 = 120
@@ -156,41 +158,41 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, r
 	}
 
 	// TODO handle the full list of types
-	switch strings.ToUpper(w.ObjType) {
+	switch strings.ToUpper(w.Spec.Type) {
 	case "POD":
-		wi, err = kubeClient.Core().Pods(w.Namespace).Watch(listOptions)
+		wi, err = kubeClient.Core().Pods(w.Spec.Namespace).Watch(listOptions)
 	case "SERVICE":
-		wi, err = kubeClient.Core().Services(w.Namespace).Watch(listOptions)
+		wi, err = kubeClient.Core().Services(w.Spec.Namespace).Watch(listOptions)
 	case "REPLICATIONCONTROLLER":
-		wi, err = kubeClient.Core().ReplicationControllers(w.Namespace).Watch(listOptions)
+		wi, err = kubeClient.Core().ReplicationControllers(w.Spec.Namespace).Watch(listOptions)
 	case "JOB":
-		wi, err = kubeClient.Batch().Jobs(w.Namespace).Watch(listOptions)
+		wi, err = kubeClient.Batch().Jobs(w.Spec.Namespace).Watch(listOptions)
 	default:
-		msg := fmt.Sprintf("Error: unknown obj type '%v'", w.ObjType)
+		msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)
 		log.Println(msg)
 		err = errors.New(msg)
 	}
 	return wi, err
 }
 
-func (kw *KubeWatcher) addWatch(w *fission.Watch) error {
-	log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Function.Name)
+func (kw *KubeWatcher) addWatch(w *tpr.Kuberneteswatchtrigger) error {
+	log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference)
 	ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher)
 	if err != nil {
 		return err
 	}
-	kw.watches[w.Metadata.Uid] = *ws
+	kw.watches[w.Metadata.UID] = *ws
 	return nil
 }
 
-func (kw *KubeWatcher) removeWatch(w *fission.Watch) error {
-	log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Function.Name)
-	ws, ok := kw.watches[w.Metadata.Uid]
+func (kw *KubeWatcher) removeWatch(w *tpr.Kuberneteswatchtrigger) error {
+	log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference)
+	ws, ok := kw.watches[w.Metadata.UID]
 	if !ok {
 		return fission.MakeError(fission.ErrorNotFound,
 			fmt.Sprintf("watch doesn't exist: %v", w.Metadata))
 	}
-	delete(kw.watches, w.Metadata.Uid)
+	delete(kw.watches, w.Metadata.UID)
 	ws.stop()
 	return nil
 }
@@ -210,10 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *fission.Watch) error {
 // 	return nil
 // }
 
-func MakeWatchSubscription(w *fission.Watch, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
+func MakeWatchSubscription(w *tpr.Kuberneteswatchtrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
 	var stopped int32 = 0
 	ws := &watchSubscription{
-		Watch:               *w,
+		watch:               *w,
 		kubeWatch:           nil,
 		stopped:             &stopped,
 		kubernetesClient:    kubeClient,
@@ -234,8 +236,8 @@ func (ws *watchSubscription) restartWatch() error {
 	retries := 60
 	for {
 		log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v",
-			ws.Watch.Metadata, ws.Watch.Namespace, ws.Watch.ObjType, ws.lastResourceVersion)
-		wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.Watch, ws.lastResourceVersion)
+			ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion)
+		wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
 		if err != nil {
 			retries--
 			if retries > 0 {
@@ -259,7 +261,7 @@ func getResourceVersion(obj runtime.Object) (string, error) {
 }
 
 func (ws *watchSubscription) eventDispatchLoop() {
-	log.Println("Listening to watch ", ws.Watch.Metadata.Name)
+	log.Println("Listening to watch ", ws.watch.Metadata.Name)
 	for {
 		// check watchSubscription is stopped or not before waiting for event
 		// comes from the kubeWatch.ResultChan(). This fix the edge case that
@@ -272,11 +274,11 @@ func (ws *watchSubscription) eventDispatchLoop() {
 		if !more {
 			if ws.isStopped() {
 				// watch is removed by user.
-				log.Println("Watch stopped", ws.Watch.Metadata.Name)
+				log.Println("Watch stopped", ws.watch.Metadata.Name)
 				return
 			} else {
 				// watch closed due to timeout, restart it.
-				log.Printf("Watch %v timed out, restarting", ws.Watch.Metadata.Name)
+				log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name)
 				err := ws.restartWatch()
 				if err != nil {
 					log.Panicf("Failed to restart watch: %v", err)
@@ -318,7 +320,16 @@ func (ws *watchSubscription) eventDispatchLoop() {
 			"X-Kubernetes-Event-Type":  string(ev.Type),
 			"X-Kubernetes-Object-Type": reflect.TypeOf(ev.Object).Elem().Name(),
 		}
-		ws.publisher.Publish(buf.String(), headers, ws.Watch.Target)
+
+		// TODO support other function ref types. Or perhaps delegate to router?
+		if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
+			log.Printf("Error: unsupported function ref type: %v, can't publish event",
+				ws.watch.Spec.FunctionReference.Type)
+			continue
+		}
+
+		url := fission.UrlForFunction(ws.watch.Spec.FunctionReference.Name)
+		ws.publisher.Publish(buf.String(), headers, url)
 	}
 }
 

kubewatcher/main.go

@@ -17,43 +17,18 @@ limitations under the License.
 package kubewatcher
 
 import (
-	"log"
-
-	"k8s.io/client-go/1.5/kubernetes"
-	"k8s.io/client-go/1.5/rest"
-
-	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/publisher"
+	"github.com/fission/fission/tpr"
 )
 
-// Get a kubernetes client using the pod's service account.
-func getKubernetesClient() (*kubernetes.Clientset, error) {
-	// creates the in-cluster config
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		log.Printf("Error getting kubernetes client config: %v", err)
-		return nil, err
-	}
-
-	// creates the clientset
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		log.Printf("Error getting kubernetes client: %v", err)
-		return nil, err
-	}
-
-	return clientset, nil
-}
-
-func Start(controllerUrl string, routerUrl string) error {
-	kubeClient, err := getKubernetesClient()
+func Start(routerUrl string) error {
+	fissionClient, kubeClient, err := tpr.MakeFissionClient()
 	if err != nil {
 		return err
 	}
 	poster := publisher.MakeWebhookPublisher(routerUrl)
 	kubeWatch := MakeKubeWatcher(kubeClient, poster)
-
-	MakeWatchSync(client.MakeClient(controllerUrl), kubeWatch)
+	MakeWatchSync(fissionClient, kubeWatch)
 
 	return nil
 }

kubewatcher/watchSync.go

@@ -20,17 +20,19 @@ import (
 	"log"
 	"time"
 
-	"github.com/fission/fission/controller/client"
+	"k8s.io/client-go/1.5/pkg/api"
+
+	"github.com/fission/fission/tpr"
 )
 
 type (
 	WatchSync struct {
-		client      *client.Client
+		client      *tpr.FissionClient
 		kubeWatcher *KubeWatcher
 	}
 )
 
-func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync {
+func MakeWatchSync(client *tpr.FissionClient, kubeWatcher *KubeWatcher) *WatchSync {
 	ws := &WatchSync{
 		client:      client,
 		kubeWatcher: kubeWatcher,
@@ -40,20 +42,14 @@ func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync {
 }
 
 func (ws *WatchSync) syncSvc() {
-	failureCount := 0
-	maxFailures := 6
+	// TODO watch instead of polling
 	for {
-		watches, err := ws.client.WatchList()
+		watches, err := ws.client.Kuberneteswatchtriggers(api.NamespaceAll).List(api.ListOptions{})
 		if err != nil {
-			failureCount++
-			if failureCount > maxFailures {
-				log.Fatalf("Failed to connect to controller: %v", err)
-			}
-			time.Sleep(10 * time.Second)
-			continue
+			log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err)
 		}
 
-		ws.kubeWatcher.Sync(watches)
+		ws.kubeWatcher.Sync(watches.Items)
 		time.Sleep(3 * time.Second)
 	}
 }