V2 types and TPR (#266)

This changes the core fission function, environment and trigger types. It also changes Fission's storage to use ThirdPartyResources.

 - Functions are now specified by packages. Functions can also have both source and deployment packages. A package can be specified by a literal, or by a URL.
 - Environments have a build and runtime component.
 - Triggers reference functions by a FunctionReference. This is a layer of indirection between triggers and functions, and will allow things like incremental function upgrades in future releases.

See Documentation/wip/env-v2.md for design discussion about points 1 and 2.

Changes:

* V2 Types

All types now have a spec, following the pattern of K8s objects.

Functions now have source and deployment packages. A Package can be
specified by literal, or by URL.

Environments now have a builder and runtime component.

All triggers use a new FunctionReference to specify the function. This
for now only uses a function name, but in the future can be extended
to be more flexible.

A new FunctionLoadRequest type is added for specialization requests to
the environment runtime.

* TPR types, TPR init code, and a "fission client"

Implements TPR types using the spec types in fission/types.go.

Adds code for adding creating TPR types, and convenient types for crud
operations on each of our resource types.

Adds code for connecting to K8s API and configuring a REST client with
fission types set up.

* Change old stateful controller into a thin apiserver

This apiserver is now simply a stateless api layer on top of the TPR
types. At the moment it doesn't do anything that couldn't be done by
simply talking to the TPR types. In the future we can have better
validation and potentially some higher level APIs (like versioning for
example) in here.

* Split controller client into files and update for v2 types.

* Update CLI for v2 types.

As far as possible we keep the CLI flags the same. We'll have to add
flags for source/deploy packages and builder/runtime
environments. That will come in the next change.

* Update poolmgr and fetcher for v2 types.

Also adds a poolmgr_test.

* Update router for new types.

Also adds a function reference resolver, which separates out the job
of resolving a FunctionReference to a function.

* Update kubewatcher and timer for v2 types.

* Update Message Queue trigger type for v2 types.

* Minor odds and ends.

* Fission bundle CLI updates

Remove controllerUrl flag, since we don't need it any more.

* Remove etcd deployment (replaced by storing state in TPR)

Also update the poolmgr commandline, and use an env var for the
fetcher image URL.

* Explicit ChecksumType and consts

* Clarify separation of environment interface types

poolmgr/api.go

@@ -29,54 +29,46 @@ import (
 
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+	"k8s.io/client-go/1.5/pkg/api"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/cache"
-	controllerclient "github.com/fission/fission/controller/client"
+	"github.com/fission/fission/tpr"
 )
 
-type funcSvc struct {
-	function    *fission.Metadata    // function this pod/service is for
-	environment *fission.Environment // env it was obtained from
-	address     string               // Host:Port or IP:Port that the service can be reached at.
-	podName     string               // pod name (within the function namespace)
+type (
+	createFuncServiceRequest struct {
+		funcMeta *api.ObjectMeta
+		respChan chan *createFuncServiceResponse
+	}
 
-	ctime time.Time
-	atime time.Time
-}
+	createFuncServiceResponse struct {
+		address string
+		err     error
+	}
 
-type createFuncServiceRequest struct {
-	funcMeta *fission.Metadata
-	respChan chan *createFuncServiceResponse
-}
-type createFuncServiceResponse struct {
-	address string
-	err     error
-}
+	Poolmgr struct {
+		gpm           *GenericPoolManager
+		functionEnv   *cache.Cache // map[string]tpr.Environment
+		fsCache       *functionServiceCache
+		fissionClient *tpr.FissionClient
 
-type API struct {
-	poolMgr          *GenericPoolManager
-	functionEnv      *cache.Cache // map[fission.Metadata]fission.Environment
-	fsCache          *functionServiceCache
-	controller       *controllerclient.Client
-	fsCreateChannels map[fission.Metadata]*sync.WaitGroup
-	requestChan      chan *createFuncServiceRequest
+		fsCreateChannels map[string]*sync.WaitGroup // xxx no channels here, rename this
+		requestChan      chan *createFuncServiceRequest
+	}
+)
 
-	//functionService *cache.Cache // map[fission.Metadata]*funcSvc
-	//urlFuncSvc      *cache.Cache // map[string]*funcSvc
-}
-
-func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCache *functionServiceCache) *API {
-	api := API{
-		poolMgr:          gpm,
-		functionEnv:      cache.MakeCache(time.Minute, 0),
+func MakePoolmgr(gpm *GenericPoolManager, fissionClient *tpr.FissionClient, fissionNs string, fsCache *functionServiceCache) *Poolmgr {
+	poolMgr := &Poolmgr{
+		gpm:              gpm,
+		functionEnv:      cache.MakeCache(10*time.Second, 0),
 		fsCache:          fsCache,
-		controller:       controller,
-		fsCreateChannels: make(map[fission.Metadata]*sync.WaitGroup),
+		fissionClient:    fissionClient,
+		fsCreateChannels: make(map[string]*sync.WaitGroup),
 		requestChan:      make(chan *createFuncServiceRequest),
 	}
-	go api.serveCreateFuncServices()
-	return &api
+	go poolMgr.serveCreateFuncServices()
+	return poolMgr
 }
 
 // All non-cached function service requests go through this goroutine
@@ -85,29 +77,29 @@ func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCac
 // get specialized. In other words, it ensures that when there's an
 // ongoing request for a certain function, all other requests wait for
 // that request to complete.
-func (api *API) serveCreateFuncServices() {
+func (poolMgr *Poolmgr) serveCreateFuncServices() {
 	for {
-		req := <-api.requestChan
+		req := <-poolMgr.requestChan
 		m := req.funcMeta
 
 		// Cache miss -- is this first one to request the func?
-		wg, found := api.fsCreateChannels[*m]
+		wg, found := poolMgr.fsCreateChannels[tpr.CacheKey(m)]
 		if !found {
 			// create a waitgroup for other requests for
 			// the same function to wait on
 			wg := &sync.WaitGroup{}
 			wg.Add(1)
-			api.fsCreateChannels[*m] = wg
+			poolMgr.fsCreateChannels[tpr.CacheKey(m)] = wg
 
 			// launch a goroutine for each request, to parallelize
 			// the specialization of different functions
 			go func() {
-				address, err := api.createServiceForFunction(m)
+				address, err := poolMgr.createServiceForFunction(m)
 				req.respChan <- &createFuncServiceResponse{
 					address: address,
 					err:     err,
 				}
-				delete(api.fsCreateChannels, *m)
+				delete(poolMgr.fsCreateChannels, tpr.CacheKey(m))
 				wg.Done()
 			}()
 		} else {
@@ -117,7 +109,7 @@ func (api *API) serveCreateFuncServices() {
 				wg.Wait()
 
 				// get the function service from the cache
-				fsvc, err := api.fsCache.GetByFunction(m)
+				fsvc, err := poolMgr.fsCache.GetByFunction(m)
 				address := ""
 				if err == nil {
 					address = fsvc.address
@@ -131,7 +123,7 @@ func (api *API) serveCreateFuncServices() {
 	}
 }
 
-func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
+func (poolMgr *Poolmgr) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		http.Error(w, "Failed to read request", 500)
@@ -139,14 +131,14 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request)
 	}
 
 	// get function metadata
-	m := fission.Metadata{}
+	m := api.ObjectMeta{}
 	err = json.Unmarshal(body, &m)
 	if err != nil {
 		http.Error(w, "Failed to parse request", 400)
 		return
 	}
 
-	serviceName, err := api.getServiceForFunction(&m)
+	serviceName, err := poolMgr.getServiceForFunction(&m)
 	if err != nil {
 		code, msg := fission.GetHTTPError(err)
 		log.Printf("Error: %v: %v", code, msg)
@@ -156,55 +148,46 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request)
 	w.Write([]byte(serviceName))
 }
 
-func (api *API) getFunctionEnv(m *fission.Metadata) (*fission.Environment, error) {
-	var env *fission.Environment
+func (poolMgr *Poolmgr) getFunctionEnv(m *api.ObjectMeta) (*tpr.Environment, error) {
+	var env *tpr.Environment
 
 	// Cached ?
-	result, err := api.functionEnv.Get(*m)
+	result, err := poolMgr.functionEnv.Get(tpr.CacheKey(m))
 	if err == nil {
-		env = result.(*fission.Environment)
+		env = result.(*tpr.Environment)
 		return env, nil
 	}
 
 	// Cache miss -- get func from controller
-	log.Printf("[%v] getting function from controller", m)
-	f, err := api.controller.FunctionGet(m)
+	f, err := poolMgr.fissionClient.Functions(m.Namespace).Get(m.Name)
 	if err != nil {
 		return nil, err
 	}
 
 	// Get env from metadata
-	log.Printf("[%v] getting env from controller", m)
-	env, err = api.controller.EnvironmentGet(&f.Environment)
+	log.Printf("[%v] getting env", m)
+	env, err = poolMgr.fissionClient.Environments(f.Metadata.Namespace).Get(f.Spec.EnvironmentName)
 	if err != nil {
 		return nil, err
 	}
 
-	// cache for future
-	api.functionEnv.Set(*m, env)
+	// cache for future lookups
+	poolMgr.functionEnv.Set(tpr.CacheKey(m), env)
 
 	return env, nil
 }
 
-func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) {
-	// Make sure we have the full metadata.  This ensures that
-	// poolmgr does not implicitly interpret empty-UID as latest
-	// version.
-	if len(m.Uid) == 0 {
-		return "", fission.MakeError(fission.ErrorInvalidArgument,
-			fmt.Sprintf("invalid metadata for function %v", m.Name))
-	}
-
+func (poolMgr *Poolmgr) getServiceForFunction(m *api.ObjectMeta) (string, error) {
 	// Check function -> svc cache
 	log.Printf("[%v] Checking for cached function service", m.Name)
-	fsvc, err := api.fsCache.GetByFunction(m)
+	fsvc, err := poolMgr.fsCache.GetByFunction(m)
 	if err == nil {
 		// Cached, return svc address
 		return fsvc.address, nil
 	}
 
 	respChan := make(chan *createFuncServiceResponse)
-	api.requestChan <- &createFuncServiceRequest{
+	poolMgr.requestChan <- &createFuncServiceRequest{
 		funcMeta: m,
 		respChan: respChan,
 	}
@@ -212,20 +195,20 @@ func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) {
 	return resp.address, resp.err
 }
 
-func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) {
+func (poolMgr *Poolmgr) createServiceForFunction(m *api.ObjectMeta) (string, error) {
 	// None exists, so create a new funcSvc:
 	log.Printf("[%v] No cached function service found, creating one", m.Name)
 
 	// from Func -> get Env
 	log.Printf("[%v] getting environment for function", m.Name)
-	env, err := api.getFunctionEnv(m)
+	env, err := poolMgr.getFunctionEnv(m)
 	if err != nil {
 		return "", err
 	}
 
 	// from Env -> get GenericPool
 	log.Printf("[%v] getting generic pool for env", m.Name)
-	pool, err := api.poolMgr.GetPool(env)
+	pool, err := poolMgr.gpm.GetPool(env)
 	if err != nil {
 		return "", err
 	}
@@ -241,7 +224,7 @@ func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) {
 }
 
 // find funcSvc and update its atime
-func (api *API) tapService(w http.ResponseWriter, r *http.Request) {
+func (poolMgr *Poolmgr) tapService(w http.ResponseWriter, r *http.Request) {
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		http.Error(w, "Failed to read request", 500)
@@ -250,7 +233,7 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) {
 	svcName := string(body)
 	svcHost := strings.TrimPrefix(svcName, "http://")
 
-	err = api.fsCache.TouchByAddress(svcHost)
+	err = poolMgr.fsCache.TouchByAddress(svcHost)
 	if err != nil {
 		log.Printf("funcSvc tap error: %v", err)
 		http.Error(w, "Not found", 404)
@@ -259,11 +242,10 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
-func (api *API) Serve(port int) {
+func (poolMgr *Poolmgr) Serve(port int) {
 	r := mux.NewRouter()
-	r.HandleFunc("/v1/getServiceForFunction", api.getServiceForFunctionApi).Methods("POST")
-	r.HandleFunc("/v1/tapService", api.tapService).Methods("POST")
-
+	r.HandleFunc("/v2/getServiceForFunction", poolMgr.getServiceForFunctionApi).Methods("POST")
+	r.HandleFunc("/v2/tapService", poolMgr.tapService).Methods("POST")
 	address := fmt.Sprintf(":%v", port)
 	log.Printf("starting poolmgr at port %v", port)
 	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))

poolmgr/client/client.go

@@ -26,6 +26,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/client-go/1.5/pkg/api"
+
 	"github.com/fission/fission"
 )
 
@@ -45,8 +47,9 @@ func MakeClient(poolmgrUrl string) *Client {
 	return c
 }
 
-func (c *Client) GetServiceForFunction(metadata *fission.Metadata) (string, error) {
-	poolmgrUrl := c.poolmgrUrl + "/v1/getServiceForFunction"
+func (c *Client) GetServiceForFunction(metadata *api.ObjectMeta) (string, error) {
+	poolmgrUrl := c.poolmgrUrl + "/v2/getServiceForFunction"
+
 	body, err := json.Marshal(metadata)
 	if err != nil {
 		return "", err
@@ -97,7 +100,7 @@ func (c *Client) TapService(serviceUrl *url.URL) {
 }
 
 func (c *Client) _tapService(serviceUrlStr string) error {
-	poolmgrUrl := c.poolmgrUrl + "/v1/tapService"
+	poolmgrUrl := c.poolmgrUrl + "/v2/tapService"
 
 	resp, err := http.Post(poolmgrUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr)))
 	if err != nil {

poolmgr/functionServiceCache.go

@@ -20,8 +20,10 @@ import (
 	"log"
 	"time"
 
-	"github.com/fission/fission"
+	"k8s.io/client-go/1.5/pkg/api"
+
 	"github.com/fission/fission/cache"
+	"github.com/fission/fission/tpr"
 )
 
 type fscRequestType int
@@ -34,10 +36,20 @@ const (
 )
 
 type (
+	funcSvc struct {
+		function    *api.ObjectMeta  // function this pod/service is for
+		environment *tpr.Environment // function's environment
+		address     string           // Host:Port or IP:Port that the function's service can be reached at.
+		podName     string           // pod name (within the function namespace)
+
+		ctime time.Time
+		atime time.Time
+	}
+
 	functionServiceCache struct {
-		byFunction *cache.Cache // function -> funcSvc : map[fission.Metadata]*funcSvc
-		byAddress  *cache.Cache // address -> function : map[string]fission.Metadata
-		byPod      *cache.Cache // podname -> function : map[string]fission.Metadata
+		byFunction *cache.Cache // function-key -> funcSvc  : map[string]*funcSvc
+		byAddress  *cache.Cache // address      -> function : map[string]api.ObjectMeta
+		byPod      *cache.Cache // podname      -> function : map[string]api.ObjectMeta
 
 		requestChannel chan *fscRequest
 	}
@@ -79,8 +91,8 @@ func (fsc *functionServiceCache) service() {
 			byPodCopy := fsc.byPod.Copy()
 			pods := make([]string, 0)
 			for podNameI, mI := range byPodCopy {
-				m := mI.(fission.Metadata)
-				fsvcI, err := fsc.byFunction.Get(m)
+				m := mI.(api.ObjectMeta)
+				fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m))
 				if err != nil {
 					resp.error = err
 				} else {
@@ -95,10 +107,9 @@ func (fsc *functionServiceCache) service() {
 		case LOG:
 			funcCopy := fsc.byFunction.Copy()
 			log.Printf("Cache has %v entries", len(funcCopy))
-			for mI, fsvcI := range funcCopy {
-				m := mI.(fission.Metadata)
+			for key, fsvcI := range funcCopy {
 				fsvc := fsvcI.(*funcSvc)
-				log.Printf("%v:%v\t%v", m.Name, m.Uid, fsvc.podName)
+				log.Printf("%v\t%v", key, fsvc.podName)
 			}
 		case DELETE_BY_POD:
 			resp.deleted, resp.error = fsc._deleteByPod(req.podName, req.age)
@@ -107,11 +118,14 @@ func (fsc *functionServiceCache) service() {
 	}
 }
 
-func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, error) {
-	fsvcI, err := fsc.byFunction.Get(*m)
+func (fsc *functionServiceCache) GetByFunction(m *api.ObjectMeta) (*funcSvc, error) {
+	key := tpr.CacheKey(m)
+
+	fsvcI, err := fsc.byFunction.Get(key)
 	if err != nil {
 		return nil, err
 	}
+
 	// update atime
 	fsvc := fsvcI.(*funcSvc)
 	fsvc.atime = time.Now()
@@ -121,7 +135,7 @@ func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, e
 }
 
 func (fsc *functionServiceCache) Add(fsvc funcSvc) (error, *funcSvc) {
-	err, existing := fsc.byFunction.Set(*fsvc.function, &fsvc)
+	err, existing := fsc.byFunction.Set(tpr.CacheKey(fsvc.function), &fsvc)
 	if err != nil {
 		if existing != nil {
 			f := existing.(*funcSvc)
@@ -167,8 +181,8 @@ func (fsc *functionServiceCache) _touchByAddress(address string) error {
 	if err != nil {
 		return err
 	}
-	m := mI.(fission.Metadata)
-	fsvcI, err := fsc.byFunction.Get(m)
+	m := mI.(api.ObjectMeta)
+	fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m))
 	if err != nil {
 		return err
 	}
@@ -196,8 +210,8 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati
 	if err != nil {
 		return false, err
 	}
-	m := mI.(fission.Metadata)
-	fsvcI, err := fsc.byFunction.Get(m)
+	m := mI.(api.ObjectMeta)
+	fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m))
 	if err != nil {
 		return false, err
 	}
@@ -207,7 +221,7 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati
 		return false, nil
 	}
 
-	fsc.byFunction.Delete(m)
+	fsc.byFunction.Delete(tpr.CacheKey(&m))
 	fsc.byAddress.Delete(fsvc.address)
 	fsc.byPod.Delete(podName)
 	return true, nil

poolmgr/functionServiceCache_test.go

@@ -5,7 +5,10 @@ import (
 	"testing"
 	"time"
 
+	"k8s.io/client-go/1.5/pkg/api"
+
 	"github.com/fission/fission"
+	"github.com/fission/fission/tpr"
 )
 
 func TestFunctionServiceCache(t *testing.T) {
@@ -18,16 +21,22 @@ func TestFunctionServiceCache(t *testing.T) {
 	now := time.Now()
 
 	fsvc = &funcSvc{
-		function: &fission.Metadata{
+		function: &api.ObjectMeta{
 			Name: "foo",
-			Uid:  "1212",
+			UID:  "1212",
 		},
-		environment: &fission.Environment{
-			Metadata: fission.Metadata{
+		environment: &tpr.Environment{
+			Metadata: api.ObjectMeta{
 				Name: "foo-env",
-				Uid:  "2323",
+				UID:  "2323",
+			},
+			Spec: fission.EnvironmentSpec{
+				Version: 1,
+				Runtime: fission.Runtime{
+					Image: "fission/foo-env",
+				},
+				Builder: fission.Builder{},
 			},
-			RunContainerImageUrl: "fission/foo-env",
 		},
 		address: "xxx",
 		podName: "yyy",

poolmgr/gp.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -38,7 +39,10 @@ import (
 	"k8s.io/client-go/1.5/pkg/util/intstr"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/environments/fetcher"
+	fetcherClient "github.com/fission/fission/environments/fetcher/client"
 	"github.com/fission/fission/logger"
+	"github.com/fission/fission/tpr"
 )
 
 const POOLMGR_INSTANCEID_LABEL string = "poolmgrInstanceId"
@@ -46,16 +50,16 @@ const POD_PHASE_RUNNING string = "Running"
 
 type (
 	GenericPool struct {
-		env              *fission.Environment
-		replicas         int32               // num idle pods
-		deployment       *v1beta1.Deployment // kubernetes deployment
-		namespace        string              // namespace to keep our resources
-		podReadyTimeout  time.Duration       // timeout for generic pods to become ready
-		controllerUrl    string
+		env              *tpr.Environment
+		replicas         int32                 // num idle pods
+		deployment       *v1beta1.Deployment   // kubernetes deployment
+		namespace        string                // namespace to keep our resources
+		podReadyTimeout  time.Duration         // timeout for generic pods to become ready
 		idlePodReapTime  time.Duration         // pods unused for idlePodReapTime are deleted
 		fsCache          *functionServiceCache // cache funcSvc's by function, address and podname
 		useSvc           bool                  // create k8s service for specialized pods
 		poolInstanceId   string                // small random string to uniquify pod names
+		fetcherImage     string
 		kubernetesClient *kubernetes.Clientset
 		instanceId       string // poolmgr instance id
 		labelsForPool    map[string]string
@@ -74,15 +78,20 @@ type (
 )
 
 func MakeGenericPool(
-	controllerUrl string,
 	kubernetesClient *kubernetes.Clientset,
-	env *fission.Environment,
+	env *tpr.Environment,
 	initialReplicas int32,
 	namespace string,
 	fsCache *functionServiceCache,
 	instanceId string) (*GenericPool, error) {
 
 	log.Printf("Creating pool for environment %v", env.Metadata)
+
+	fetcherImage := os.Getenv("FETCHER_IMAGE")
+	if len(fetcherImage) == 0 {
+		fetcherImage = "fission/fetcher"
+	}
+
 	// TODO: in general we need to provide the user a way to configure pools.  Initial
 	// replicas, autoscaling params, various timeouts, etc.
 	gp := &GenericPool{
@@ -92,19 +101,18 @@ func MakeGenericPool(
 		kubernetesClient: kubernetesClient,
 		namespace:        namespace,
 		podReadyTimeout:  5 * time.Minute, // TODO make this an env param?
-		controllerUrl:    controllerUrl,
 		idlePodReapTime:  3 * time.Minute, // TODO make this configurable
 		fsCache:          fsCache,
 		poolInstanceId:   uniuri.NewLen(8),
 		instanceId:       instanceId,
-
-		useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start
+		fetcherImage:     fetcherImage,
+		useSvc:           false, // defaults off -- svc takes a second or more to become routable, slowing cold start
 	}
 
 	// Labels for generic deployment/RS/pods.
 	gp.labelsForPool = map[string]string{
 		"environmentName":        gp.env.Metadata.Name,
-		"environmentUid":         gp.env.Metadata.Uid,
+		"environmentUid":         string(gp.env.Metadata.UID),
 		POOLMGR_INSTANCEID_LABEL: gp.instanceId,
 	}
 
@@ -220,10 +228,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*v1.Pod, error)
 	}
 }
 
-func (gp *GenericPool) labelsForFunction(metadata *fission.Metadata) map[string]string {
+func (gp *GenericPool) labelsForFunction(metadata *api.ObjectMeta) map[string]string {
 	return map[string]string{
 		"functionName":           metadata.Name,
-		"functionUid":            metadata.Uid,
+		"functionUid":            string(metadata.UID),
 		"unmanaged":              "true", // this allows us to easily find pods not managed by the deployment
 		POOLMGR_INSTANCEID_LABEL: gp.instanceId,
 	}
@@ -240,10 +248,30 @@ func (gp *GenericPool) scheduleDeletePod(name string) {
 	}()
 }
 
+func (gp *GenericPool) getFetcherUrl(podIP string) string {
+	testUrl := os.Getenv("TEST_FETCHER_URL")
+	if len(testUrl) != 0 {
+		// it takes a second or so for the test service to
+		// become routable once a pod is relabeled. This is
+		// super hacky, but only runs in unit tests.
+		time.Sleep(5 * time.Second)
+		return testUrl
+	}
+	return fmt.Sprintf("http://%v:8000/", podIP)
+}
+
+func (gp *GenericPool) getSpecializeUrl(podIP string) string {
+	u := os.Getenv("TEST_SPECIALIZE_URL")
+	if len(u) != 0 {
+		return u
+	}
+	return fmt.Sprintf("http://%v:8888/specialize", podIP)
+}
+
 // specializePod chooses a pod, copies the required user-defined function to that pod
 // (via fetcher), and calls the function-run container to load it, resulting in a
 // specialized pod.
-func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) error {
+func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *api.ObjectMeta) error {
 	// for fetcher we don't need to create a service, just talk to the pod directly
 	podIP := pod.Status.PodIP
 	if len(podIP) == 0 {
@@ -251,28 +279,23 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er
 	}
 
 	// tell fetcher to get the function.
-	fetcherUrl := fmt.Sprintf("http://%v:8000/", podIP)
-	functionUrl := fmt.Sprintf("%v/v1/functions/%v?uid=%v&raw=1",
-		gp.controllerUrl, metadata.Name, metadata.Uid)
-	fetcherRequest := fmt.Sprintf("{\"url\": \"%v\", \"filename\": \"user\"}", functionUrl)
-
-	log.Printf("[%v] calling fetcher to copy function", metadata)
-	resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader([]byte(fetcherRequest)))
+	fetcherUrl := gp.getFetcherUrl(podIP)
+	log.Printf("[%v] calling fetcher to copy function", metadata.Name)
+	err := fetcherClient.DoFetchRequest(fetcherUrl, &fetcher.FetchRequest{
+		FetchType: fetcher.FETCH_DEPLOYMENT,
+		Function:  *metadata,
+		Filename:  "user", // XXX use function id instead
+	})
 	if err != nil {
-		// TODO we should retry this call in case fetcher hasn't come up yet
 		return err
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return errors.New(fmt.Sprintf("Error from fetcher: %v", resp.Status))
-	}
 
 	// Tell logging helper about this function invocation
 	gp.setupLogging(pod, metadata)
 
 	// get function run container to specialize
-	log.Printf("[%v] specializing pod", metadata)
-	specializeUrl := fmt.Sprintf("http://%v:8888/specialize", podIP)
+	log.Printf("[%v] specializing pod", metadata.Name)
+	specializeUrl := gp.getSpecializeUrl(podIP)
 
 	// retry the specialize call a few times in case the env server hasn't come up yet
 	maxRetries := 20
@@ -311,7 +334,7 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er
 // creates the pool but doesn't wait for any pods to be ready.
 func (gp *GenericPool) createPool() error {
 	poolDeploymentName := fmt.Sprintf("%v-%v-%v",
-		gp.env.Metadata.Name, gp.env.Metadata.Uid, strings.ToLower(gp.poolInstanceId))
+		gp.env.Metadata.Name, gp.env.Metadata.UID, strings.ToLower(gp.poolInstanceId))
 
 	sharedMountPath := "/userfunc"
 	deployment := &v1beta1.Deployment{
@@ -340,7 +363,7 @@ func (gp *GenericPool) createPool() error {
 					Containers: []v1.Container{
 						{
 							Name:                   gp.env.Metadata.Name,
-							Image:                  gp.env.RunContainerImageUrl,
+							Image:                  gp.env.Spec.Runtime.Image,
 							ImagePullPolicy:        v1.PullIfNotPresent,
 							TerminationMessagePath: "/dev/termination-log",
 							VolumeMounts: []v1.VolumeMount{
@@ -352,7 +375,7 @@ func (gp *GenericPool) createPool() error {
 						},
 						{
 							Name:                   "fetcher",
-							Image:                  "fission/fetcher",
+							Image:                  gp.fetcherImage,
 							ImagePullPolicy:        v1.PullIfNotPresent,
 							TerminationMessagePath: "/dev/termination-log",
 							VolumeMounts: []v1.VolumeMount{
@@ -418,9 +441,9 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*v1.Ser
 	return svc, err
 }
 
-func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) {
+func (gp *GenericPool) GetFuncSvc(m *api.ObjectMeta) (*funcSvc, error) {
 
-	log.Printf("[%v] Choosing pod from pool", m)
+	log.Printf("[%v] Choosing pod from pool", m.Name)
 	newLabels := gp.labelsForFunction(m)
 	pod, err := gp.choosePod(newLabels)
 	if err != nil {
@@ -437,8 +460,8 @@ func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) {
 	var svcHost string
 	if gp.useSvc {
 		svcName := fmt.Sprintf("svc-%v", m.Name)
-		if len(m.Uid) > 0 {
-			svcName += "-" + m.Uid
+		if len(m.UID) > 0 {
+			svcName += ("-" + string(m.UID))
 		}
 
 		labels := gp.labelsForFunction(m)
@@ -580,13 +603,13 @@ func (gp *GenericPool) destroy() error {
 
 // Calls the logging daemonset pod on the node where the given pod is
 // running.
-func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *fission.Metadata) {
+func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *api.ObjectMeta) {
 	logReq := logger.LogRequest{
 		Namespace: pod.Namespace,
 		Pod:       pod.Name,
 		Container: gp.env.Metadata.Name,
 		FuncName:  metadata.Name,
-		FuncUid:   metadata.Uid,
+		FuncUid:   string(metadata.UID),
 	}
 	reqbody, err := json.Marshal(logReq)
 	if err != nil {

poolmgr/gpm.go

@@ -21,9 +21,9 @@ import (
 	"time"
 
 	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
 
-	"github.com/fission/fission"
-	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/tpr"
 )
 
 type requestType int
@@ -35,19 +35,18 @@ const (
 
 type (
 	GenericPoolManager struct {
-		pools            map[fission.Environment]*GenericPool
+		pools            map[string]*GenericPool
 		kubernetesClient *kubernetes.Clientset
 		namespace        string
-		controllerUrl    string
-		controllerClient *client.Client
+		fissionClient    *tpr.FissionClient
 		fsCache          *functionServiceCache
 		instanceId       string
 		requestChannel   chan *request
 	}
 	request struct {
 		requestType
-		env             *fission.Environment
-		envList         []fission.Environment
+		env             *tpr.Environment
+		envList         []tpr.Environment
 		responseChannel chan *response
 	}
 	response struct {
@@ -57,18 +56,18 @@ type (
 )
 
 func MakeGenericPoolManager(
-	controllerUrl string,
+	fissionClient *tpr.FissionClient,
 	kubernetesClient *kubernetes.Clientset,
-	namespace string,
+	fissionNamespace string,
+	functionNamespace string,
 	fsCache *functionServiceCache,
 	instanceId string) *GenericPoolManager {
 
 	gpm := &GenericPoolManager{
-		pools:            make(map[fission.Environment]*GenericPool),
+		pools:            make(map[string]*GenericPool),
 		kubernetesClient: kubernetesClient,
-		namespace:        namespace,
-		controllerUrl:    controllerUrl,
-		controllerClient: client.MakeClient(controllerUrl),
+		namespace:        functionNamespace,
+		fissionClient:    fissionClient,
 		fsCache:          fsCache,
 		instanceId:       instanceId,
 		requestChannel:   make(chan *request),
@@ -85,30 +84,30 @@ func (gpm *GenericPoolManager) service() {
 		switch req.requestType {
 		case GET_POOL:
 			var err error
-			pool, ok := gpm.pools[*req.env]
+			pool, ok := gpm.pools[tpr.CacheKey(&req.env.Metadata)]
 			if !ok {
 				pool, err = MakeGenericPool(
-					gpm.controllerUrl, gpm.kubernetesClient, req.env,
+					gpm.kubernetesClient, req.env,
 					3, // TODO configurable/autoscalable
 					gpm.namespace, gpm.fsCache, gpm.instanceId)
 				if err != nil {
 					req.responseChannel <- &response{error: err}
 					continue
 				}
-				gpm.pools[*req.env] = pool
+				gpm.pools[tpr.CacheKey(&req.env.Metadata)] = pool
 			}
 			req.responseChannel <- &response{pool: pool}
 		case CLEANUP_POOLS:
-			uids := make(map[string]bool)
+			latestEnvSet := make(map[string]bool)
 			for _, env := range req.envList {
-				uids[env.Metadata.Uid] = true
+				latestEnvSet[tpr.CacheKey(&env.Metadata)] = true
 			}
-			for env, pool := range gpm.pools {
-				_, ok := uids[env.Metadata.Uid]
+			for key, pool := range gpm.pools {
+				_, ok := latestEnvSet[key]
 				if !ok {
 					// Env no longer exists -- remove our cache
-					log.Printf("Destroying generic pool for environment [%v]", env)
-					delete(gpm.pools, env)
+					log.Printf("Destroying generic pool for environment [%v]", key)
+					delete(gpm.pools, key)
 
 					// and delete the pool asynchronously.
 					go pool.destroy()
@@ -119,7 +118,7 @@ func (gpm *GenericPoolManager) service() {
 	}
 }
 
-func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool, error) {
+func (gpm *GenericPoolManager) GetPool(env *tpr.Environment) (*GenericPool, error) {
 	c := make(chan *response)
 	gpm.requestChannel <- &request{
 		requestType:     GET_POOL,
@@ -130,7 +129,7 @@ func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool,
 	return resp.pool, resp.error
 }
 
-func (gpm *GenericPoolManager) CleanupPools(envs []fission.Environment) {
+func (gpm *GenericPoolManager) CleanupPools(envs []tpr.Environment) {
 	gpm.requestChannel <- &request{
 		requestType: CLEANUP_POOLS,
 		envList:     envs,
@@ -145,11 +144,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
 		time.Sleep(pollSleep)
 
 		// get list of envs from controller
-		envs, err := gpm.controllerClient.EnvironmentList()
+		envs, err := gpm.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{})
 		if err != nil {
 			failureCount++
 			if failureCount >= maxFailures {
-				log.Fatalf("Failed to connect to controller %v times: %v", maxFailures, err)
+				log.Fatalf("Failed %v times: %v", maxFailures, err)
 			}
 		}
 
@@ -157,14 +156,14 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
 		// creating pools for envs that are actually used by functions.  Also we might want
 		// to keep these eagerly created pools smaller than the ones created when there are
 		// actual function calls.
-		for i := range envs {
-			_, err := gpm.GetPool(&envs[i])
+		for i := range envs.Items {
+			_, err := gpm.GetPool(&envs.Items[i])
 			if err != nil {
 				log.Printf("eager-create pool failed: %v", err)
 			}
 		}
 
 		// Clean up pools whose env was deleted
-		gpm.CleanupPools(envs)
+		gpm.CleanupPools(envs.Items)
 	}
 }

poolmgr/poolmgr.go

@@ -18,52 +18,29 @@ package poolmgr
 
 import (
 	"log"
-	"strings"
 
 	"github.com/dchest/uniuri"
-	"k8s.io/client-go/1.5/kubernetes"
-	"k8s.io/client-go/1.5/rest"
 
-	controllerclient "github.com/fission/fission/controller/client"
+	"github.com/fission/fission/tpr"
 )
 
-// Get a kubernetes client using the pod's service account.  This only
-// works when we're running inside a kubernetes cluster.
-func getKubernetesClient() (*kubernetes.Clientset, error) {
-	// creates the in-cluster config
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		log.Printf("Error getting kubernetes client config: %v", err)
-		return nil, err
-	}
-
-	// creates the clientset
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		log.Printf("Error getting kubernetes client: %v", err)
-		return nil, err
-	}
-
-	return clientset, nil
-}
-
-func StartPoolmgr(controllerUrl string, namespace string, port int) error {
-	controllerUrl = strings.TrimSuffix(controllerUrl, "/")
-	controllerClient := controllerclient.MakeClient(controllerUrl)
-
-	kubernetesClient, err := getKubernetesClient()
+// Start the poolmgr service.
+func StartPoolmgr(fissionNamespace string, functionNamespace string, port int) error {
+	fissionClient, kubernetesClient, err := tpr.MakeFissionClient()
 	if err != nil {
 		log.Printf("Failed to get kubernetes client: %v", err)
 		return err
 	}
 
 	instanceId := uniuri.NewLen(8)
-	cleanupOldPoolmgrResources(kubernetesClient, namespace, instanceId)
+	cleanupOldPoolmgrResources(kubernetesClient, functionNamespace, instanceId)
 
 	fsCache := MakeFunctionServiceCache()
-	gpm := MakeGenericPoolManager(controllerUrl, kubernetesClient, namespace, fsCache, instanceId)
+	gpm := MakeGenericPoolManager(
+		fissionClient, kubernetesClient, fissionNamespace,
+		functionNamespace, fsCache, instanceId)
 
-	api := MakeAPI(gpm, controllerClient, fsCache)
+	api := MakePoolmgr(gpm, fissionClient, fissionNamespace, fsCache)
 	go api.Serve(port)
 
 	return nil

poolmgr/poolmgr_test.go

@@ -0,0 +1,234 @@
+//
+// This test depends on several env vars:
+//
+//   KUBECONFIG has to point at a kube config with a cluster. The test
+//      will use the default context from that config. Be careful,
+//      don't point this at your production environment. The test is
+//      skipped if KUBECONFIG is undefined.
+//
+//   TEST_SPECIALIZE_URL
+//   TEST_FETCHER_URL
+//      These need to point at <node ip>:30001 and <node ip>:30002,
+//      where <node ip> is the address of any node in the test
+//      cluster.
+//
+//   FETCHER_IMAGE
+//      Optional. Set this to a fetcher image; otherwise uses the
+//      default.
+//
+
+// Here's how I run this on my setup, with minikube:
+// TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v .
+
+package poolmgr
+
+import (
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
+	"k8s.io/client-go/1.5/pkg/api/v1"
+	"k8s.io/client-go/1.5/pkg/labels"
+	"k8s.io/client-go/1.5/pkg/util/intstr"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/poolmgr/client"
+	"github.com/fission/fission/tpr"
+	"io/ioutil"
+)
+
+// return the number of pods in the given namespace matching the given labels
+func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int {
+	pods, err := kubeClient.Pods(ns).List(api.ListOptions{
+		LabelSelector: labels.Set(labelz).AsSelector(),
+	})
+	if err != nil {
+		log.Panicf("Failed to list pods: %v", err)
+	}
+	return len(pods.Items)
+}
+
+func createTestNamespace(kubeClient *kubernetes.Clientset, ns string) {
+	_, err := kubeClient.Namespaces().Create(&v1.Namespace{
+		ObjectMeta: v1.ObjectMeta{
+			Name: ns,
+		},
+	})
+	if err != nil {
+		log.Panicf("failed to create ns %v: %v", ns, err)
+	}
+}
+
+// create a nodeport service
+func createSvc(kubeClient *kubernetes.Clientset, ns string, name string, targetPort int, nodePort int32, labels map[string]string) *v1.Service {
+	svc, err := kubeClient.Services(ns).Create(&v1.Service{
+		ObjectMeta: v1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.ServiceSpec{
+			Type: v1.ServiceTypeNodePort,
+			Ports: []v1.ServicePort{
+				{
+					Protocol:   v1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(targetPort),
+					NodePort:   nodePort,
+				},
+			},
+			Selector: labels,
+		},
+	})
+	if err != nil {
+		log.Panicf("Failed to create svc: %v", err)
+	}
+	return svc
+}
+
+func httpGet(url string) string {
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Panicf("HTTP Get failed: URL %v: %v", url, err)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Panicf("HTTP Get failed to read body: URL %v: %v", url, err)
+	}
+	return string(body)
+}
+
+func TestPoolmgr(t *testing.T) {
+	// run in a random namespace so we can have concurrent tests
+	// on a given cluster
+	rand.Seed(time.Now().UTC().UnixNano())
+	testId := rand.Intn(999)
+	fissionNs := fmt.Sprintf("test-%v", testId)
+	functionNs := fmt.Sprintf("test-function-%v", testId)
+
+	// skip test if no cluster available for testing
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if len(kubeconfig) == 0 {
+		t.Skip("Skipping test, no kubernetes cluster")
+		return
+	}
+
+	// connect to k8s
+	// and get TPR client
+	fissionClient, kubeClient, err := tpr.MakeFissionClient()
+	if err != nil {
+		log.Panicf("failed to connect: %v", err)
+	}
+
+	// create the test's namespaces
+	createTestNamespace(kubeClient, fissionNs)
+	defer kubeClient.Namespaces().Delete(fissionNs, nil)
+
+	createTestNamespace(kubeClient, functionNs)
+	defer kubeClient.Namespaces().Delete(functionNs, nil)
+
+	// make sure TPR types exist on cluster
+	err = tpr.EnsureFissionTPRs(kubeClient)
+	if err != nil {
+		log.Panicf("failed to ensure tprs: %v", err)
+	}
+	fissionClient.WaitForTPRs()
+
+	// create an env on the cluster
+	env, err := fissionClient.Environments(fissionNs).Create(&tpr.Environment{
+		Metadata: api.ObjectMeta{
+			Name:      "nodejs",
+			Namespace: fissionNs,
+		},
+		Spec: fission.EnvironmentSpec{
+			Version: 1,
+			Runtime: fission.Runtime{
+				Image: "fission/node-env",
+			},
+			Builder: fission.Builder{},
+		},
+	})
+	if err != nil {
+		log.Panicf("failed to create env: %v", err)
+	}
+
+	// create poolmgr
+	port := 9999
+	err = StartPoolmgr(fissionNs, functionNs, port)
+	if err != nil {
+		log.Panicf("failed to start poolmgr: %v", err)
+	}
+
+	// connect poolmgr client
+	poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port))
+
+	// Wait for pool to be created (we don't actually need to do
+	// this, since the API should do the right thing in any case).
+	// waitForPool(functionNs, "nodejs")
+	time.Sleep(6 * time.Second)
+
+	// create a function
+	f := &tpr.Function{
+		Metadata: api.ObjectMeta{
+			Name:      "hello",
+			Namespace: fissionNs,
+		},
+		Spec: fission.FunctionSpec{
+			Source: fission.Package{},
+			Deployment: fission.Package{
+				Type:    fission.PackageTypeLiteral,
+				Literal: []byte(`module.exports = async function(context) { return { status: 200, body: "Hello, world!\n" }; }`),
+			},
+			EnvironmentName: env.Metadata.Name,
+		},
+	}
+	_, err = fissionClient.Functions(fissionNs).Create(f)
+	if err != nil {
+		log.Panicf("failed to create function: %v", err)
+	}
+
+	// create a service to call fetcher and the env container
+	labels := map[string]string{"functionName": f.Metadata.Name}
+	var fetcherPort int32 = 30001
+	fetcherSvc := createSvc(kubeClient, functionNs, fmt.Sprintf("%v-%v", f.Metadata.Name, "fetcher"), 8000, fetcherPort, labels)
+	defer kubeClient.Services(functionNs).Delete(fetcherSvc.ObjectMeta.Name, nil)
+
+	var funcSvcPort int32 = 30002
+	functionSvc := createSvc(kubeClient, functionNs, f.Metadata.Name, 8888, funcSvcPort, labels)
+	defer kubeClient.Services(functionNs).Delete(functionSvc.ObjectMeta.Name, nil)
+
+	// the main test: get a service for a given function
+	t1 := time.Now()
+	svc, err := poolmgrClient.GetServiceForFunction(&f.Metadata)
+	if err != nil {
+		log.Panicf("failed to get func svc: %v", err)
+	}
+	log.Printf("svc for function created at: %v (in %v)", svc, time.Now().Sub(t1))
+
+	// ensure that a pod with the label functionName=f.Metadata.Name exists
+	podCount := countPods(kubeClient, functionNs, map[string]string{"functionName": f.Metadata.Name})
+	if podCount != 1 {
+		log.Panicf("expected 1 function pod, found %v", podCount)
+	}
+
+	// call the service to ensure it works
+
+	// wait for a bit
+
+	// tap service to simulate calling it again
+
+	// make sure the same pod is still there
+
+	// wait for idleTimeout to ensure the pod is removed
+
+	// remove env
+
+	// wait for pool to be destroyed
+
+	// that's it
+}