From e238776bf73bf690e74a2b065d2c90d9ea8da44e Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Sat, 5 Aug 2017 01:18:30 -0700 Subject: [PATCH] V2 types and TPR (#266) This changes the core fission function, environment and trigger types. It also changes Fission's storage to use ThirdPartyResources. - Functions are now specified by packages. Functions can also have both source and deployment packages. A package can be specified by a literal, or by a URL. - Environments have a build and runtime component. - Triggers reference functions by a FunctionReference. This is a layer of indirection between triggers and functions, and will allow things like incremental function upgrades in future releases. See Documentation/wip/env-v2.md for design discussion about points 1 and 2. Changes: * V2 Types All types now have a spec, following the pattern of K8s objects. Functions now have source and deployment packages. A Package can be specified by literal, or by URL. Environments now have a builder and runtime component. All triggers use a new FunctionReference to specify the function. This for now only uses a function name, but in the future can be extended to be more flexible. A new FunctionLoadRequest type is added for specialization requests to the environment runtime. * TPR types, TPR init code, and a "fission client" Implements TPR types using the spec types in fission/types.go. Adds code for adding creating TPR types, and convenient types for crud operations on each of our resource types. Adds code for connecting to K8s API and configuring a REST client with fission types set up. * Change old stateful controller into a thin apiserver This apiserver is now simply a stateless api layer on top of the TPR types. At the moment it doesn't do anything that couldn't be done by simply talking to the TPR types. In the future we can have better validation and potentially some higher level APIs (like versioning for example) in here. * Split controller client into files and update for v2 types. * Update CLI for v2 types. As far as possible we keep the CLI flags the same. We'll have to add flags for source/deploy packages and builder/runtime environments. That will come in the next change. * Update poolmgr and fetcher for v2 types. Also adds a poolmgr_test. * Update router for new types. Also adds a function reference resolver, which separates out the job of resolving a FunctionReference to a function. * Update kubewatcher and timer for v2 types. * Update Message Queue trigger type for v2 types. * Minor odds and ends. * Fission bundle CLI updates Remove controllerUrl flag, since we don't need it any more. * Remove etcd deployment (replaced by storing state in TPR) Also update the poolmgr commandline, and use an env var for the fetcher image URL. * Explicit ChecksumType and consts * Clarify separation of environment interface types --- common.go | 8 +- controller/api.go | 89 ++- controller/api_test.go | 347 ++++----- controller/client/client.go | 667 +----------------- controller/client/environment.go | 132 ++++ controller/client/function.go | 155 ++++ controller/client/httptrigger.go | 131 ++++ controller/client/kuberneteswatchtrigger.go | 111 +++ controller/client/mqtrigger.go | 137 ++++ controller/client/timetrigger.go | 131 ++++ .../{jsonSerializer.go => controller.go} | 27 +- controller/environmentApi.go | 91 ++- controller/environmentStore.go | 79 --- controller/fileStore.go | 134 ---- controller/fileStore_test.go | 63 -- controller/functionApi.go | 129 ++-- controller/functionStore.go | 156 ---- controller/httpTriggerApi.go | 145 ++-- controller/httpTriggerStore.go | 79 --- controller/mqTriggerApi.go | 115 ++- controller/mqTriggerStore.go | 80 --- controller/resourceStore.go | 287 -------- controller/resourceStore_test.go | 152 ---- controller/timeTriggerApi.go | 115 ++- controller/timeTriggerStore.go | 79 --- controller/{types.go => tpr.go} | 19 +- controller/watchApi.go | 77 +- controller/watchStore.go | 79 --- environments/fetcher/client/client.go | 35 + environments/fetcher/{ => cmd}/Dockerfile | 0 environments/fetcher/{ => cmd}/build.sh | 1 + environments/fetcher/cmd/main.go | 26 + environments/fetcher/fetcher.go | 205 ++++-- fission-bundle/main.go | 66 +- fission-rbac.yaml | 42 +- fission.yaml | 43 +- fission/deployment.go | 181 ----- fission/environment.go | 45 +- fission/function.go | 212 +++--- fission/httptrigger.go | 49 +- fission/main.go | 36 +- fission/mqtrigger.go | 71 +- fission/timetrigger.go | 52 +- fission/watch.go | 42 +- kubewatcher/kubewatcher.go | 73 +- kubewatcher/main.go | 33 +- kubewatcher/watchSync.go | 22 +- mqtrigger/main.go | 12 +- mqtrigger/messageQueue/messageQueue.go | 107 +-- mqtrigger/messageQueue/nats.go | 30 +- poolmgr/api.go | 134 ++-- poolmgr/client/client.go | 9 +- poolmgr/functionServiceCache.go | 48 +- poolmgr/functionServiceCache_test.go | 21 +- poolmgr/gp.go | 99 ++- poolmgr/gpm.go | 57 +- poolmgr/poolmgr.go | 41 +- poolmgr/poolmgr_test.go | 234 ++++++ router/functionHandler.go | 15 +- router/functionHandler_test.go | 7 +- router/functionReferenceResolver.go | 118 ++++ router/functionServiceMap.go | 37 +- router/functionServiceMap_test.go | 4 +- router/httpTriggers.go | 141 ++-- router/router.go | 14 +- router/router_test.go | 48 +- timer/main.go | 12 +- timer/timer.go | 38 +- timer/timerSync.go | 29 +- tpr/client.go | 198 ++++++ tpr/environment.go | 119 ++++ tpr/function.go | 119 ++++ tpr/httptrigger.go | 119 ++++ resource.go => tpr/key.go | 34 +- tpr/kubernetesWatchTrigger.go | 119 ++++ tpr/messagequeuetrigger.go | 119 ++++ tpr/timetrigger.go | 119 ++++ tpr/tpr.go | 106 +++ tpr/tpr_test.go | 429 +++++++++++ tpr/types.go | 208 ++++++ types.go | 245 +++++-- 81 files changed, 4649 insertions(+), 3588 deletions(-) create mode 100644 controller/client/environment.go create mode 100644 controller/client/function.go create mode 100644 controller/client/httptrigger.go create mode 100644 controller/client/kuberneteswatchtrigger.go create mode 100644 controller/client/mqtrigger.go create mode 100644 controller/client/timetrigger.go rename controller/{jsonSerializer.go => controller.go} (59%) delete mode 100644 controller/environmentStore.go delete mode 100644 controller/fileStore.go delete mode 100644 controller/fileStore_test.go delete mode 100644 controller/functionStore.go delete mode 100644 controller/httpTriggerStore.go delete mode 100644 controller/mqTriggerStore.go delete mode 100644 controller/resourceStore.go delete mode 100644 controller/resourceStore_test.go delete mode 100644 controller/timeTriggerStore.go rename controller/{types.go => tpr.go} (73%) delete mode 100644 controller/watchStore.go create mode 100644 environments/fetcher/client/client.go rename environments/fetcher/{ => cmd}/Dockerfile (100%) rename environments/fetcher/{ => cmd}/build.sh (81%) create mode 100644 environments/fetcher/cmd/main.go delete mode 100644 fission/deployment.go create mode 100644 poolmgr/poolmgr_test.go create mode 100644 router/functionReferenceResolver.go create mode 100644 tpr/client.go create mode 100644 tpr/environment.go create mode 100644 tpr/function.go create mode 100644 tpr/httptrigger.go rename resource.go => tpr/key.go (52%) create mode 100644 tpr/kubernetesWatchTrigger.go create mode 100644 tpr/messagequeuetrigger.go create mode 100644 tpr/timetrigger.go create mode 100644 tpr/tpr.go create mode 100644 tpr/tpr_test.go create mode 100644 tpr/types.go diff --git a/common.go b/common.go index 5f9a2c56..9aa8caca 100644 --- a/common.go +++ b/common.go @@ -20,11 +20,7 @@ import ( "fmt" ) -func UrlForFunction(m *Metadata) string { +func UrlForFunction(name string) string { prefix := "/fission-function" - if len(m.Uid) > 0 { - return fmt.Sprintf("%v/%v/%v", prefix, m.Name, m.Uid) - } else { - return fmt.Sprintf("%v/%v", prefix, m.Name) - } + return fmt.Sprintf("%v/%v", prefix, name) } diff --git a/controller/api.go b/controller/api.go index 3504ba72..a601a405 100644 --- a/controller/api.go +++ b/controller/api.go @@ -26,19 +26,16 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + kerrors "k8s.io/client-go/1.5/pkg/api/errors" "github.com/fission/fission" "github.com/fission/fission/fission/logdb" + "github.com/fission/fission/tpr" ) type ( API struct { - FunctionStore - HTTPTriggerStore - TimeTriggerStore - MessageQueueTriggerStore - EnvironmentStore - WatchStore + fissionClient *tpr.FissionClient } logDBConfig struct { @@ -48,16 +45,8 @@ type ( } ) -func MakeAPI(rs *ResourceStore) *API { - api := &API{ - FunctionStore: FunctionStore{ResourceStore: *rs}, - HTTPTriggerStore: HTTPTriggerStore{ResourceStore: *rs}, - TimeTriggerStore: TimeTriggerStore{ResourceStore: *rs}, - MessageQueueTriggerStore: MessageQueueTriggerStore{ResourceStore: *rs}, - EnvironmentStore: EnvironmentStore{ResourceStore: *rs}, - WatchStore: WatchStore{ResourceStore: *rs}, - } - return api +func MakeAPI() (*API, error) { + return makeTPRBackedAPI() } func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) { @@ -71,6 +60,14 @@ func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) { func (api *API) respondWithError(w http.ResponseWriter, err error) { debug.PrintStack() + + // this error type comes with an HTTP code, so just use that + se, ok := err.(*kerrors.StatusError) + if ok { + http.Error(w, string(se.ErrStatus.Reason), int(se.ErrStatus.Code)) + return + } + code, msg := fission.GetHTTPError(err) log.Errorf("Error: %v: %v", code, msg) http.Error(w, msg, code) @@ -102,41 +99,41 @@ func (api *API) Serve(port int) { r := mux.NewRouter() r.HandleFunc("/", api.HomeHandler) - r.HandleFunc("/v1/functions", api.FunctionApiList).Methods("GET") - r.HandleFunc("/v1/functions", api.FunctionApiCreate).Methods("POST") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiGet).Methods("GET") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiUpdate).Methods("PUT") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiDelete).Methods("DELETE") + r.HandleFunc("/v2/functions", api.FunctionApiList).Methods("GET") + r.HandleFunc("/v2/functions", api.FunctionApiCreate).Methods("POST") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiGet).Methods("GET") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiUpdate).Methods("PUT") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/http", api.HTTPTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/http", api.HTTPTriggerApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE") - r.HandleFunc("/v1/environments", api.EnvironmentApiList).Methods("GET") - r.HandleFunc("/v1/environments", api.EnvironmentApiCreate).Methods("POST") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiGet).Methods("GET") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE") + r.HandleFunc("/v2/environments", api.EnvironmentApiList).Methods("GET") + r.HandleFunc("/v2/environments", api.EnvironmentApiCreate).Methods("POST") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiGet).Methods("GET") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE") - r.HandleFunc("/v1/watches", api.WatchApiList).Methods("GET") - r.HandleFunc("/v1/watches", api.WatchApiCreate).Methods("POST") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiGet).Methods("GET") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiUpdate).Methods("PUT") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiDelete).Methods("DELETE") + r.HandleFunc("/v2/watches", api.WatchApiList).Methods("GET") + r.HandleFunc("/v2/watches", api.WatchApiCreate).Methods("POST") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiGet).Methods("GET") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiUpdate).Methods("PUT") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/time", api.TimeTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/time", api.TimeTriggerApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiDelete).Methods("DELETE") r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST") diff --git a/controller/api_test.go b/controller/api_test.go index e4ae7ea5..8ff71270 100644 --- a/controller/api_test.go +++ b/controller/api_test.go @@ -19,35 +19,50 @@ package controller import ( "flag" "io/ioutil" + "log" "net/http" "os" "testing" "time" - etcdClient "github.com/coreos/etcd/client" - log "github.com/sirupsen/logrus" - "golang.org/x/net/context" + "k8s.io/client-go/1.5/pkg/api" + "bytes" "github.com/fission/fission" "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) var g struct { client *client.Client } -func assertNameReuseFails(err error, name string) { +func panicIf(err error) { + if err != nil { + log.Panicf("err: %v", err) + } +} + +func assert(c bool, msg string) { + if !c { + log.Fatalf("assert failed: %v", msg) + } +} + +func assertNameReuseFailure(err error, name string) { assert(err != nil, "recreating "+name+" with same name must fail") fe, ok := err.(fission.Error) assert(ok, "error must be a fission Error") assert(fe.Code == fission.ErrorNameExists, "error must be a name exists error") } -func assertNotFoundFails(err error, name string) { +func assertNotFoundFailure(err error, name string) { assert(err != nil, "requesting a non-existent "+name+" must fail") fe, ok := err.(fission.Error) assert(ok, "error must be a fission Error") - assert(fe.Code == fission.ErrorNotFound, "error must be a not found error") + if fe.Code != fission.ErrorNotFound { + log.Fatalf("error must be a not found error: %v", fe) + } } func assertCronSpecFails(err error) { @@ -58,67 +73,42 @@ func assertCronSpecFails(err error) { } func TestFunctionApi(t *testing.T) { - log.SetFormatter(&log.TextFormatter{DisableColors: true}) + name1 := "foo" + name2 := "bar" - testFunc := &fission.Function{ - Metadata: fission.Metadata{ - Name: "foo", - Uid: "", + testFunc := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: name1, + Namespace: api.NamespaceDefault, }, - Environment: fission.Metadata{ - Name: "nodejs", - Uid: "xxx", + Spec: fission.FunctionSpec{ + EnvironmentName: "nodejs", + Deployment: fission.Package{ + Literal: []byte("code1"), + }, }, - Code: "code1", } - _, err := g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "function") + _, err := g.client.FunctionGet(&api.ObjectMeta{ + Name: testFunc.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "function") m, err := g.client.FunctionCreate(testFunc) panicIf(err) - uid1 := m.Uid - //log.Printf("Created function %v: %v", m.Name, m.Uid) _, err = g.client.FunctionCreate(testFunc) - assertNameReuseFails(err, "function") + assertNameReuseFailure(err, "function") - code, err := g.client.FunctionGetRaw(m) + code, err := g.client.FunctionGetRawDeployment(m) panicIf(err) - assert(string(code) == testFunc.Code, "code from FunctionGetRaw must match created function") + assert(bytes.Compare(code, testFunc.Spec.Deployment.Literal) == 0, "code from FunctionGetRawDeployment must match created function") - testFunc.Code = "code2" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid2 := m.Uid - //log.Printf("Updated function %v: %v", m.Name, m.Uid) - - m.Uid = uid1 - testFunc.Code = "code1" - f, err := g.client.FunctionGet(m) + testFunc.Spec.Deployment.Literal = []byte("code2") + _, err = g.client.FunctionUpdate(testFunc) panicIf(err) - testFunc.Metadata.Uid = m.Uid - //log.Printf("f = %#v", f) - //log.Printf("testFunc = %#v", testFunc) - assert(*f == *testFunc, "first version should match when read by uid") - - m.Uid = uid2 - testFunc.Metadata.Uid = m.Uid - testFunc.Code = "code2" - f, err = g.client.FunctionGet(m) - panicIf(err) - - assert(*f == *testFunc, "second version should match when read by uid") - - m.Uid = "" - testFunc.Metadata.Uid = uid2 - testFunc.Code = "code2" - f, err = g.client.FunctionGet(m) - panicIf(err) - - assert(*f == *testFunc, "second version should match when read as latest") - - testFunc.Metadata.Name = "bar" + testFunc.Metadata.Name = name2 m, err = g.client.FunctionCreate(testFunc) panicIf(err) @@ -127,7 +117,7 @@ func TestFunctionApi(t *testing.T) { assert(len(funcs) == 2, "created two functions, but didn't find them") - funcs_url := g.client.Url + "/v1/functions" + funcs_url := g.client.Url + "/v2/functions" resp, err := http.Get(funcs_url) panicIf(err) defer resp.Body.Close() @@ -141,112 +131,55 @@ func TestFunctionApi(t *testing.T) { } assert(found, "incorrect response content type") - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo"}) + err = g.client.FunctionDelete(&api.ObjectMeta{Name: name1, Namespace: api.NamespaceDefault}) panicIf(err) - err = g.client.FunctionDelete(&fission.Metadata{Name: "bar"}) + err = g.client.FunctionDelete(&api.ObjectMeta{Name: name2, Namespace: api.NamespaceDefault}) panicIf(err) } -func TestFunctionVersionApi(t *testing.T) { - testFunc := &fission.Function{ - Metadata: fission.Metadata{ - Name: "foo", - Uid: "", - }, - Environment: fission.Metadata{ - Name: "nodejs", - Uid: "xxx", - }, - Code: "code1", - } - - testFunc.Code = "code1" - m, err := g.client.FunctionCreate(testFunc) - panicIf(err) - uid1 := m.Uid - - testFunc.Code = "code2" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid2 := m.Uid - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo", Uid: uid1}) - panicIf(err) - - f, err := g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - panicIf(err) - assert(f.Metadata.Uid == uid2, "deleted version1, but version2 does not exist") - - testFunc.Code = "code3" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid3 := m.Uid - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo", Uid: uid3}) - panicIf(err) - - f, err = g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - panicIf(err) - assert(f.Metadata.Uid == uid2, "deleted version3, but version2 does not exist") - - testFunc.Code = "code4" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo"}) - panicIf(err) - - funcs, err := g.client.FunctionList() - panicIf(err) - assert(len(funcs) == 0, - "created one function with two versions(2 and 4), delete without uid but cannot delete them all") -} - func TestHTTPTriggerApi(t *testing.T) { - testTrigger := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testTrigger := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, }, - UrlPattern: "/hello", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.HTTPTriggerSpec{ + RelativeURL: "/hello", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, }, } - _, err := g.client.HTTPTriggerGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "trigger") + _, err := g.client.HTTPTriggerGet(&api.ObjectMeta{ + Name: testTrigger.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "httptrigger") m, err := g.client.HTTPTriggerCreate(testTrigger) panicIf(err) defer g.client.HTTPTriggerDelete(m) _, err = g.client.HTTPTriggerCreate(testTrigger) - assertNameReuseFails(err, "trigger") + assertNameReuseFailure(err, "httptrigger") tr, err := g.client.HTTPTriggerGet(m) panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") + assert(testTrigger.Spec == tr.Spec, "trigger should match after reading") - testTrigger.UrlPattern = "/hi" - m2, err := g.client.HTTPTriggerUpdate(testTrigger) + testTrigger.Spec.RelativeURL = "/hi" + _, err = g.client.HTTPTriggerUpdate(testTrigger) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.HTTPTriggerGet(m) - panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") - testTrigger.Metadata.Name = "yyy" - m, err = g.client.HTTPTriggerCreate(testTrigger) + _, err = g.client.HTTPTriggerCreate(testTrigger) assert(err != nil, "duplicate trigger should not be allowed") - testTrigger.UrlPattern = "/hi2" - m, err = g.client.HTTPTriggerCreate(testTrigger) + testTrigger.Spec.RelativeURL = "/hi2" + m2, err := g.client.HTTPTriggerCreate(testTrigger) panicIf(err) - defer g.client.HTTPTriggerDelete(m) + defer g.client.HTTPTriggerDelete(m2) ts, err := g.client.HTTPTriggerList() panicIf(err) @@ -254,42 +187,42 @@ func TestHTTPTriggerApi(t *testing.T) { } func TestEnvironmentApi(t *testing.T) { - testEnv := &fission.Environment{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testEnv := &tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: fission.EnvironmentSpec{ + Runtime: fission.Runtime{ + Image: "gcr.io/xyz", + }, }, - RunContainerImageUrl: "gcr.io/xyz", } - _, err := g.client.EnvironmentGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "environment") + _, err := g.client.EnvironmentGet(&api.ObjectMeta{ + Name: testEnv.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "environment") m, err := g.client.EnvironmentCreate(testEnv) panicIf(err) defer g.client.EnvironmentDelete(m) _, err = g.client.EnvironmentCreate(testEnv) - assertNameReuseFails(err, "environment") + assertNameReuseFailure(err, "environment") - tr, err := g.client.EnvironmentGet(m) + e, err := g.client.EnvironmentGet(m) panicIf(err) - testEnv.Metadata.Uid = m.Uid - assert(*testEnv == *tr, "env should match after reading") + assert(testEnv.Spec == e.Spec, "env should match after reading") - testEnv.RunContainerImageUrl = "/hi" - m2, err := g.client.EnvironmentUpdate(testEnv) + testEnv.Spec.Runtime.Image = "another-img" + _, err = g.client.EnvironmentUpdate(testEnv) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.EnvironmentGet(m) + testEnv.Metadata.Name = "bar" + m2, err := g.client.EnvironmentCreate(testEnv) panicIf(err) - testEnv.Metadata.Uid = m.Uid - assert(*testEnv == *tr, "env should match after reading") - - testEnv.Metadata.Name = "yyy" - m, err = g.client.EnvironmentCreate(testEnv) - panicIf(err) - defer g.client.EnvironmentDelete(m) + defer g.client.EnvironmentDelete(m2) ts, err := g.client.EnvironmentList() panicIf(err) @@ -297,36 +230,38 @@ func TestEnvironmentApi(t *testing.T) { } func TestWatchApi(t *testing.T) { - testWatch := &fission.Watch{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testWatch := &tpr.Kuberneteswatchtrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, }, - Namespace: "default", - ObjType: "pod", - LabelSelector: "", - FieldSelector: "", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: "default", + Type: "pod", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, }, - Target: "", } - _, err := g.client.WatchGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "watch") + _, err := g.client.WatchGet(&api.ObjectMeta{ + Name: testWatch.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "watch") m, err := g.client.WatchCreate(testWatch) panicIf(err) defer g.client.WatchDelete(m) _, err = g.client.WatchCreate(testWatch) - assertNameReuseFails(err, "watch") + assertNameReuseFailure(err, "watch") w, err := g.client.WatchGet(m) panicIf(err) - testWatch.Metadata.Uid = m.Uid - w.Target = "" - assert(*testWatch == *w, "watch should match after reading") + assert((testWatch.Spec.Namespace == w.Spec.Namespace && + testWatch.Spec.Type == w.Spec.Type && + testWatch.Spec.FunctionReference == w.Spec.FunctionReference), "watch should match after reading") testWatch.Metadata.Name = "yyy" m2, err := g.client.WatchCreate(testWatch) @@ -339,45 +274,40 @@ func TestWatchApi(t *testing.T) { } func TestTimeTriggerApi(t *testing.T) { - testTrigger := &fission.TimeTrigger{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testTrigger := &tpr.Timetrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, }, - Cron: "0 30 * * * *", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.TimeTriggerSpec{ + Cron: "0 30 * * * *", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "asdf", + }, }, } - _, err := g.client.TimeTriggerGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "trigger") + _, err := g.client.TimeTriggerGet(&api.ObjectMeta{Name: testTrigger.Metadata.Name}) + assertNotFoundFailure(err, "trigger") m, err := g.client.TimeTriggerCreate(testTrigger) panicIf(err) defer g.client.TimeTriggerDelete(m) _, err = g.client.TimeTriggerCreate(testTrigger) - assertNameReuseFails(err, "trigger") + assertNameReuseFailure(err, "trigger") tr, err := g.client.TimeTriggerGet(m) panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") + assert(testTrigger.Spec == tr.Spec, "trigger should match after reading") - testTrigger.Cron = "@hourly" - m2, err := g.client.TimeTriggerUpdate(testTrigger) + testTrigger.Spec.Cron = "@hourly" + _, err = g.client.TimeTriggerUpdate(testTrigger) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.TimeTriggerGet(m) - panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") - testTrigger.Metadata.Name = "yyy" - testTrigger.Cron = "Not valid cron spec" - m, err = g.client.TimeTriggerCreate(testTrigger) + testTrigger.Spec.Cron = "Not valid cron spec" + _, err = g.client.TimeTriggerCreate(testTrigger) assertCronSpecFails(err) ts, err := g.client.TimeTriggerList() @@ -388,20 +318,17 @@ func TestTimeTriggerApi(t *testing.T) { func TestMain(m *testing.M) { flag.Parse() - fileStore, ks, rs := getTestResourceStore() - defer os.RemoveAll(fileStore.root) + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + log.Println("Skipping test, no kubernetes cluster") + return + } - api := MakeAPI(rs) - g.client = client.MakeClient("http://localhost:8888") + go Start(8888) - ks.Delete(context.Background(), "Function", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "HTTPTrigger", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "TimeTrigger", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "Environment", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "Watch", &etcdClient.DeleteOptions{Recursive: true}) - - go api.Serve(8888) time.Sleep(500 * time.Millisecond) + g.client = client.MakeClient("http://localhost:8888") resp, err := http.Get("http://localhost:8888/") panicIf(err) diff --git a/controller/client/client.go b/controller/client/client.go index 571ee086..0cd03e56 100644 --- a/controller/client/client.go +++ b/controller/client/client.go @@ -18,10 +18,7 @@ package client import ( "bytes" - "encoding/base64" - "encoding/json" "errors" - "fmt" "io/ioutil" "net/http" "strings" @@ -73,7 +70,7 @@ func (c *Client) put(relativeUrl string, contentType string, body []byte) (*http } func (c *Client) url(relativeUrl string) string { - return c.Url + "/v1/" + relativeUrl + return c.Url + "/v2/" + relativeUrl } func (c *Client) handleResponse(resp *http.Response) ([]byte, error) { @@ -91,665 +88,3 @@ func (c *Client) handleCreateResponse(resp *http.Response) ([]byte, error) { body, err := ioutil.ReadAll(resp.Body) return body, err } - -func (c *Client) FunctionCreate(f *fission.Function) (*fission.Metadata, error) { - orig := f.Code - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) - defer func() { f.Code = orig }() - - reqbody, err := json.Marshal(f) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) FunctionGet(m *fission.Metadata) (*fission.Function, error) { - relativeUrl := fmt.Sprintf("functions/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var f fission.Function - err = json.Unmarshal(body, &f) - if err != nil { - return nil, err - } - - dec, err := base64.StdEncoding.DecodeString(f.Code) - if err != nil { - return nil, err - } - f.Code = string(dec) - - return &f, nil -} - -func (c *Client) FunctionGetRaw(m *fission.Metadata) ([]byte, error) { - relativeUrl := fmt.Sprintf("functions/%v?raw=1", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("&uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - return c.handleResponse(resp) -} - -func (c *Client) FunctionUpdate(f *fission.Function) (*fission.Metadata, error) { - orig := f.Code - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) - defer func() { f.Code = orig }() - - reqbody, err := json.Marshal(f) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) FunctionDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("functions/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) FunctionList() ([]fission.Function, error) { - resp, err := http.Get(c.url("functions")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - funcs := make([]fission.Function, 0) - err = json.Unmarshal(body, &funcs) - if err != nil { - return nil, err - } - - return funcs, nil -} - -func (c *Client) HTTPTriggerCreate(t *fission.HTTPTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/http"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) HTTPTriggerGet(m *fission.Metadata) (*fission.HTTPTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.HTTPTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) HTTPTriggerUpdate(t *fission.HTTPTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/http/%v", t.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) HTTPTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) HTTPTriggerList() ([]fission.HTTPTrigger, error) { - resp, err := http.Get(c.url("triggers/http")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.HTTPTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} - -func (c *Client) EnvironmentCreate(env *fission.Environment) (*fission.Metadata, error) { - reqbody, err := json.Marshal(env) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) EnvironmentGet(m *fission.Metadata) (*fission.Environment, error) { - relativeUrl := fmt.Sprintf("environments/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var env fission.Environment - err = json.Unmarshal(body, &env) - if err != nil { - return nil, err - } - - return &env, nil -} - -func (c *Client) EnvironmentUpdate(env *fission.Environment) (*fission.Metadata, error) { - reqbody, err := json.Marshal(env) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) EnvironmentDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("environments/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) EnvironmentList() ([]fission.Environment, error) { - resp, err := http.Get(c.url("environments")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - envs := make([]fission.Environment, 0) - err = json.Unmarshal(body, &envs) - if err != nil { - return nil, err - } - - return envs, nil -} - -func (c *Client) WatchCreate(w *fission.Watch) (*fission.Metadata, error) { - reqbody, err := json.Marshal(w) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("watches"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) WatchGet(m *fission.Metadata) (*fission.Watch, error) { - relativeUrl := fmt.Sprintf("watches/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var w fission.Watch - err = json.Unmarshal(body, &w) - if err != nil { - return nil, err - } - - return &w, nil -} - -func (c *Client) WatchUpdate(w *fission.Watch) (*fission.Metadata, error) { - return nil, fission.MakeError(fission.ErrorNotImplmented, - "watch update not implemented") -} - -func (c *Client) WatchDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("watches/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err - -} - -func (c *Client) WatchList() ([]fission.Watch, error) { - resp, err := http.Get(c.url("watches")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - watches := make([]fission.Watch, 0) - err = json.Unmarshal(body, &watches) - if err != nil { - return nil, err - } - - return watches, err -} - -func (c *Client) TimeTriggerCreate(t *fission.TimeTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/time"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) TimeTriggerGet(m *fission.Metadata) (*fission.TimeTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.TimeTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) TimeTriggerUpdate(t *fission.TimeTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/time/%v", t.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) TimeTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) TimeTriggerList() ([]fission.TimeTrigger, error) { - resp, err := http.Get(c.url("triggers/time")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.TimeTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} - -func (c *Client) MessageQueueTriggerCreate(t *fission.MessageQueueTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) MessageQueueTriggerGet(m *fission.Metadata) (*fission.MessageQueueTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.MessageQueueTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) MessageQueueTriggerUpdate(mqTrigger *fission.MessageQueueTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(mqTrigger) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) MessageQueueTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) MessageQueueTriggerList(mqType string) ([]fission.MessageQueueTrigger, error) { - relativeUrl := "triggers/messagequeue" - if len(mqType) > 0 { - relativeUrl += fmt.Sprintf("?mqtype=%v", mqType) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.MessageQueueTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} diff --git a/controller/client/environment.go b/controller/client/environment.go new file mode 100644 index 00000000..d0d7ec6a --- /dev/null +++ b/controller/client/environment.go @@ -0,0 +1,132 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) EnvironmentCreate(env *tpr.Environment) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(env) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) EnvironmentGet(m *api.ObjectMeta) (*tpr.Environment, error) { + relativeUrl := fmt.Sprintf("environments/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var env tpr.Environment + err = json.Unmarshal(body, &env) + if err != nil { + return nil, err + } + + return &env, nil +} + +func (c *Client) EnvironmentUpdate(env *tpr.Environment) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(env) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) EnvironmentDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("environments/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + return c.delete(relativeUrl) +} + +func (c *Client) EnvironmentList() ([]tpr.Environment, error) { + resp, err := http.Get(c.url("environments")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + envs := make([]tpr.Environment, 0) + err = json.Unmarshal(body, &envs) + if err != nil { + return nil, err + } + + return envs, nil +} diff --git a/controller/client/function.go b/controller/client/function.go new file mode 100644 index 00000000..14d17431 --- /dev/null +++ b/controller/client/function.go @@ -0,0 +1,155 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" +) + +func (c *Client) FunctionCreate(f *tpr.Function) (*api.ObjectMeta, error) { + + // ensure size limits + if len(f.Spec.Source.Literal) > 256*1024 { + return nil, fission.MakeError(fission.ErrorSizeLimitExceeded, "Source package literal larger than 256k") + } + if len(f.Spec.Deployment.Literal) > 256*1024 { + return nil, fission.MakeError(fission.ErrorSizeLimitExceeded, "Deployment package literal larger than 256k") + } + + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) FunctionGet(m *api.ObjectMeta) (*tpr.Function, error) { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var f tpr.Function + err = json.Unmarshal(body, &f) + if err != nil { + return nil, err + } + + return &f, nil +} + +func (c *Client) FunctionGetRawDeployment(m *api.ObjectMeta) ([]byte, error) { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + relativeUrl += fmt.Sprintf("&deploymentraw=1") + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return c.handleResponse(resp) +} + +func (c *Client) FunctionUpdate(f *tpr.Function) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) FunctionDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) FunctionList() ([]tpr.Function, error) { + resp, err := http.Get(c.url("functions")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + funcs := make([]tpr.Function, 0) + err = json.Unmarshal(body, &funcs) + if err != nil { + return nil, err + } + + return funcs, nil +} diff --git a/controller/client/httptrigger.go b/controller/client/httptrigger.go new file mode 100644 index 00000000..a12e802a --- /dev/null +++ b/controller/client/httptrigger.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) HTTPTriggerCreate(t *tpr.Httptrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/http"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) HTTPTriggerGet(m *api.ObjectMeta) (*tpr.Httptrigger, error) { + relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Httptrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) HTTPTriggerUpdate(t *tpr.Httptrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/http/%v", t.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) HTTPTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) HTTPTriggerList() ([]tpr.Httptrigger, error) { + resp, err := http.Get(c.url("triggers/http")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Httptrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/client/kuberneteswatchtrigger.go b/controller/client/kuberneteswatchtrigger.go new file mode 100644 index 00000000..7daf1aa6 --- /dev/null +++ b/controller/client/kuberneteswatchtrigger.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" +) + +func (c *Client) WatchCreate(w *tpr.Kuberneteswatchtrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(w) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("watches"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) WatchGet(m *api.ObjectMeta) (*tpr.Kuberneteswatchtrigger, error) { + relativeUrl := fmt.Sprintf("watches/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var w tpr.Kuberneteswatchtrigger + err = json.Unmarshal(body, &w) + if err != nil { + return nil, err + } + + return &w, nil +} + +func (c *Client) WatchUpdate(w *tpr.Kuberneteswatchtrigger) (*api.ObjectMeta, error) { + return nil, fission.MakeError(fission.ErrorNotImplmented, + "watch update not implemented") +} + +func (c *Client) WatchDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("watches/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) WatchList() ([]tpr.Kuberneteswatchtrigger, error) { + resp, err := http.Get(c.url("watches")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + watches := make([]tpr.Kuberneteswatchtrigger, 0) + err = json.Unmarshal(body, &watches) + if err != nil { + return nil, err + } + + return watches, err +} diff --git a/controller/client/mqtrigger.go b/controller/client/mqtrigger.go new file mode 100644 index 00000000..59c16ead --- /dev/null +++ b/controller/client/mqtrigger.go @@ -0,0 +1,137 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) MessageQueueTriggerCreate(t *tpr.Messagequeuetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) MessageQueueTriggerGet(m *api.ObjectMeta) (*tpr.Messagequeuetrigger, error) { + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Messagequeuetrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) MessageQueueTriggerUpdate(mqTrigger *tpr.Messagequeuetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(mqTrigger) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) MessageQueueTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) MessageQueueTriggerList(mqType string) ([]tpr.Messagequeuetrigger, error) { + relativeUrl := "triggers/messagequeue" + if len(mqType) > 0 { + // TODO remove this, replace with field selector + relativeUrl += fmt.Sprintf("?mqtype=%v", mqType) + } + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Messagequeuetrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/client/timetrigger.go b/controller/client/timetrigger.go new file mode 100644 index 00000000..cd3a1493 --- /dev/null +++ b/controller/client/timetrigger.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) TimeTriggerCreate(t *tpr.Timetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/time"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) TimeTriggerGet(m *api.ObjectMeta) (*tpr.Timetrigger, error) { + relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Timetrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) TimeTriggerUpdate(t *tpr.Timetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/time/%v", t.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) TimeTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) TimeTriggerList() ([]tpr.Timetrigger, error) { + resp, err := http.Get(c.url("triggers/time")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Timetrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/jsonSerializer.go b/controller/controller.go similarity index 59% rename from controller/jsonSerializer.go rename to controller/controller.go index 008ba608..f6027d20 100644 --- a/controller/jsonSerializer.go +++ b/controller/controller.go @@ -17,16 +17,27 @@ limitations under the License. package controller import ( - "encoding/json" + "log" + + "github.com/fission/fission/tpr" ) -type JsonSerializer struct { -} +func Start(port int) { + fc, kc, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Failed to connect to K8s API: %v", err) + } -func (j JsonSerializer) serialize(r resource) ([]byte, error) { - return json.Marshal(r) -} + err = tpr.EnsureFissionTPRs(kc) + if err != nil { + log.Fatalf("Failed to create fission TPRs: %v", err) + } -func (j JsonSerializer) deserialize(buf []byte, r resource) error { - return json.Unmarshal(buf, r) + fc.WaitForTPRs() + + api, err := MakeAPI() + if err != nil { + log.Fatalf("Failed to start controller: %v", err) + } + api.Serve(port) } diff --git a/controller/environmentApi.go b/controller/environmentApi.go index 0aa413ca..cd50b96a 100644 --- a/controller/environmentApi.go +++ b/controller/environmentApi.go @@ -23,133 +23,132 @@ import ( "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) EnvironmentApiList(w http.ResponseWriter, r *http.Request) { - envs, err := api.EnvironmentStore.List() +func (a *API) EnvironmentApiList(w http.ResponseWriter, r *http.Request) { + envs, err := a.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(envs) + resp, err := json.Marshal(envs.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var env fission.Environment + var env tpr.Environment err = json.Unmarshal(body, &env) if err != nil { log.Printf("Failed to unmarshal request body: [%v]", body) - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.EnvironmentStore.Create(&env) + enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Create(&env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: env.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(enew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) EnvironmentApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["environment"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["environment"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - env, err := api.EnvironmentStore.Get(&m) + env, err := a.fissionClient.Environments(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["environment"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var env fission.Environment + var env tpr.Environment err = json.Unmarshal(body, &env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != env.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Environment name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.EnvironmentStore.Update(&env) + enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Update(&env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: env.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(enew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["environment"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"httpTrigger": m.Name}).Info("Deleting all versions") + name := vars["environment"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.EnvironmentStore.Delete(m) + err := a.fissionClient.Environments(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/environmentStore.go b/controller/environmentStore.go deleted file mode 100644 index 3a257af5..00000000 --- a/controller/environmentStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type EnvironmentStore struct { - ResourceStore -} - -func (es *EnvironmentStore) Create(e *fission.Environment) (string, error) { - e.Metadata.Uid = uuid.NewV4().String() - return e.Metadata.Uid, es.ResourceStore.create(e) -} - -func (es *EnvironmentStore) Get(m *fission.Metadata) (*fission.Environment, error) { - var e fission.Environment - err := es.ResourceStore.read(m.Name, &e) - if err != nil { - return nil, err - } - return &e, nil -} - -func (es *EnvironmentStore) Update(e *fission.Environment) (string, error) { - e.Metadata.Uid = uuid.NewV4().String() - return e.Metadata.Uid, es.ResourceStore.update(e) -} - -func (es *EnvironmentStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.Environment{}) - if err != nil { - return err - } - return es.ResourceStore.delete(typeName, m.Name) -} - -func (es *EnvironmentStore) List() ([]fission.Environment, error) { - typeName, err := getTypeName(fission.Environment{}) - if err != nil { - return nil, err - } - - bufs, err := es.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - envs := make([]fission.Environment, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var e fission.Environment - err = js.deserialize([]byte(buf), &e) - if err != nil { - return nil, err - } - envs = append(envs, e) - } - - return envs, nil -} diff --git a/controller/fileStore.go b/controller/fileStore.go deleted file mode 100644 index 1d7c4c79..00000000 --- a/controller/fileStore.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "os" - "path" - - log "github.com/sirupsen/logrus" -) - -type requestType int - -const ( - READ requestType = iota - WRITE - DELETE -) - -type ( - FileStore struct { - root string // abs path of root of filestore - requestChannel chan fileStoreRequest - } - - fileStoreRequest struct { - requestType - fileName string // relative path - fileContents []byte - responseChannel chan fileStoreResponse - } - - fileStoreResponse struct { - error - fileContents []byte - } -) - -func MakeFileStore(path string) *FileStore { - - // create directory if necessary - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(path, os.ModeDir|0700) - if err != nil { - log.Printf("Failed to create directory %v", path) - return nil - } - log.Printf("Created directory %v", path) - } else { - log.Printf("Unknown error initializing filestore: %v", err) - return nil - } - } - - fileStore := &FileStore{ - root: path, - requestChannel: make(chan fileStoreRequest), - } - go fileStore.fileStoreService() - return fileStore -} - -func (fs *FileStore) fileStoreService() { - for { - req := <-fs.requestChannel - response := &fileStoreResponse{} - - log.WithFields(log.Fields{"file": req.fileName, "type": req.requestType}).Debug("fileStore request") - switch req.requestType { - case READ: - response.fileContents, response.error = ioutil.ReadFile(path.Join(fs.root, req.fileName)) - case WRITE: - response.error = ioutil.WriteFile(path.Join(fs.root, req.fileName), req.fileContents, 0600) - case DELETE: - response.error = os.Remove(path.Join(fs.root, req.fileName)) - if os.IsNotExist(response.error) { - response.error = nil - } - default: - log.Panic("bad request") - } - req.responseChannel <- *response - } -} - -func (fs *FileStore) read(fileName string) ([]byte, error) { - req := fileStoreRequest{ - requestType: READ, - fileName: fileName, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.fileContents, response.error -} - -func (fs *FileStore) write(fileName string, contents []byte) error { - req := fileStoreRequest{ - requestType: WRITE, - fileName: fileName, - fileContents: contents, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.error -} - -func (fs *FileStore) delete(fileName string) error { - req := fileStoreRequest{ - requestType: DELETE, - fileName: fileName, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.error -} diff --git a/controller/fileStore_test.go b/controller/fileStore_test.go deleted file mode 100644 index 02f7c15f..00000000 --- a/controller/fileStore_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "log" - "os" - "testing" -) - -func TestFileStore(t *testing.T) { - // tmp dir - dir, err := ioutil.TempDir("", "testFileStore") - if err != nil { - t.Fatalf("error creating temp dir: %v", err) - } - defer os.RemoveAll(dir) - log.Printf("temp dir at %v", dir) - - // file store - fs := MakeFileStore(dir) - - _, err = fs.read("nonexistent") - if err == nil { - t.Fatalf("expected an error") - } - - path := "fileStoreTest" - contents := []byte("bar") - err = fs.write(path, contents) - if err != nil { - t.Fatalf("error: %v", err) - } - - observedContents, err := fs.read(path) - if err != nil { - t.Fatalf("error: %v", err) - } - - if string(observedContents) != string(contents) { - t.Fatalf("contents don't match") - } - - err = fs.delete(path) - if err != nil { - t.Fatalf("error: %v", err) - } -} diff --git a/controller/functionApi.go b/controller/functionApi.go index 6187e4ca..b6abc683 100644 --- a/controller/functionApi.go +++ b/controller/functionApi.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "encoding/base64" "encoding/json" "io/ioutil" "net/http" @@ -26,165 +25,161 @@ import ( "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) FunctionApiList(w http.ResponseWriter, r *http.Request) { - funcs, err := api.FunctionStore.List() +func (a *API) FunctionApiList(w http.ResponseWriter, r *http.Request) { + funcs, err := a.fissionClient.Functions(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(funcs) + resp, err := json.Marshal(funcs.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var f fission.Function + var f tpr.Function err = json.Unmarshal(body, &f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - dec, err := base64.StdEncoding.DecodeString(f.Code) - if err != nil { - api.respondWithError(w, err) + // Ensure size limits + if len(f.Spec.Source.Literal) > 256*1024 { + err := fission.MakeError(fission.ErrorInvalidArgument, "Source package literal larger than 256K") + a.respondWithError(w, err) return } - f.Code = string(dec) - - uid, err := api.FunctionStore.Create(&f) - if err != nil { - api.respondWithError(w, err) + if len(f.Spec.Deployment.Literal) > 256*1024 { + err := fission.MakeError(fission.ErrorInvalidArgument, "Deployment package literal larger than 256K") + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: f.Name, Uid: uid} - resp, err := json.Marshal(m) + fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Create(&f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) + return + } + + resp, err := json.Marshal(fnew.Metadata) + if err != nil { + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["function"] - m.Uid = r.FormValue("uid") // empty if uid is absent - raw := r.FormValue("raw") // just the code + name := vars["function"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } + raw := r.FormValue("deploymentraw") // just the deployment pkg - f, err := api.FunctionStore.Get(&m) + f, err := a.fissionClient.Functions(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } var resp []byte if raw != "" { - resp = []byte(f.Code) + resp = []byte(f.Spec.Deployment.Literal) } else { - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) resp, err = json.Marshal(f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - funcName := vars["function"] + name := vars["function"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var f fission.Function + var f tpr.Function err = json.Unmarshal(body, &f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - if funcName != f.Metadata.Name { + if name != f.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Function name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - dec, err := base64.StdEncoding.DecodeString(f.Code) + fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Update(&f) if err != nil { - api.respondWithError(w, err) - return - } - f.Code = string(dec) - - uid, err := api.FunctionStore.Update(&f) - if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: f.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(fnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["function"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"function": m.Name}).Info("Deleting all versions") + name := vars["function"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.FunctionStore.Delete(m) + err := a.fissionClient.Functions(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } // FunctionLogsApiPost establishes a proxy server to log database, and redirect // query command send from client to database then proxy back the db response. -func (api *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) // get dbType from url dbType := vars["dbType"] // find correspond db http url - dbCnf := api.getLogDBConfig(dbType) + dbCnf := a.getLogDBConfig(dbType) svcUrl, err := url.Parse(dbCnf.httpURL) if err != nil { diff --git a/controller/functionStore.go b/controller/functionStore.go deleted file mode 100644 index 34774781..00000000 --- a/controller/functionStore.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - log "github.com/sirupsen/logrus" - - "github.com/fission/fission" -) - -type FunctionStore struct { - ResourceStore -} - -func (fs *FunctionStore) Create(f *fission.Function) (string, error) { - code := []byte(f.Code) - _, uid, err := fs.ResourceStore.writeFile(f.Key(), code) - if err != nil { - return "", err - } - - f.Metadata.Uid = uid - f.Code = "" - - err = fs.ResourceStore.create(f) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore errors - return "", err - } - return f.Metadata.Uid, nil -} - -func (fs *FunctionStore) Get(m *fission.Metadata) (*fission.Function, error) { - var f fission.Function - err := fs.ResourceStore.read(m.Name, &f) - if err != nil { - return nil, err - } - - var code []byte - if len(m.Uid) > 0 { - log.WithFields(log.Fields{"Uid": m.Uid}).Info("fetching by uid") - code, err = fs.ResourceStore.readFile(m.Name, &m.Uid) - f.Metadata = *m - } else { - code, err = fs.ResourceStore.readFile(m.Name, nil) - } - if err != nil { - return nil, err - } - - f.Code = string(code) - return &f, nil -} - -func (fs *FunctionStore) Update(f *fission.Function) (string, error) { - code := []byte(f.Code) - _, uid, err := fs.ResourceStore.writeFile(f.Key(), code) - if err != nil { - return "", err - } - - var fnew fission.Function - err = fs.ResourceStore.read(f.Metadata.Name, &fnew) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore err - return "", err - } - - fnew.Metadata.Uid = uid - fnew.Environment = f.Environment - - err = fs.ResourceStore.update(fnew) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore err - return "", err - } - return uid, err -} - -func (fs *FunctionStore) Delete(m fission.Metadata) error { - if len(m.Uid) == 0 { - err := fs.ResourceStore.deleteAllFiles(m.Name) - if err != nil { - return err - } - } else { - err := fs.ResourceStore.deleteFile(m.Name, m.Uid) - if err != nil { - return err - } - } - typeName, err := getTypeName(fission.Function{}) - if err != nil { - return err - } - - bufs, err := fs.ResourceStore.getAll("file/" + m.Name) - if err != nil { - return err - } - if len(bufs) == 0 { - return fs.ResourceStore.delete(typeName, m.Name) - } - - fnew, err := fs.Get(&fission.Metadata{Name: m.Name}) - if err != nil { - return err - } - - latestUid := bufs[len(bufs)-1] // function always tracks the latest version of code - if latestUid == fnew.Uid { - return nil - } - fnew.Uid = latestUid - return fs.ResourceStore.update(fnew) -} - -func (fs *FunctionStore) List() ([]fission.Function, error) { - typeName, err := getTypeName(fission.Function{}) - if err != nil { - return nil, err - } - - bufs, err := fs.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - js := JsonSerializer{} - functions := make([]fission.Function, 0, len(bufs)) - for _, buf := range bufs { - var f fission.Function - err = js.deserialize([]byte(buf), &f) - if err != nil { - return nil, err - } - functions = append(functions, f) - } - - return functions, nil -} diff --git a/controller/httpTriggerApi.go b/controller/httpTriggerApi.go index c98e5c03..7e9b0c72 100644 --- a/controller/httpTriggerApi.go +++ b/controller/httpTriggerApi.go @@ -18,151 +18,164 @@ package controller import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) HTTPTriggerApiList(w http.ResponseWriter, r *http.Request) { - triggers, err := api.HTTPTriggerStore.List() +func (a *API) HTTPTriggerApiList(w http.ResponseWriter, r *http.Request) { + triggers, err := a.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiCreate(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) +func (a *API) checkHttpTriggerDuplicates(t *tpr.Httptrigger) error { + triggers, err := a.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) - return + return err } - - var t fission.HTTPTrigger - err = json.Unmarshal(body, &t) - if err != nil { - api.respondWithError(w, err) - return - } - - triggers, err := api.HTTPTriggerStore.List() - if err != nil { - api.respondWithError(w, err) - return - } - for _, url := range triggers { - if url.UrlPattern == t.UrlPattern && url.Method == t.Method { - err = fission.MakeError(fission.ErrorNameExists, - "HTTPTrigger with same URL & method already exists") - api.respondWithError(w, err) - return + for _, ht := range triggers.Items { + if ht.Spec.RelativeURL == t.Spec.RelativeURL && ht.Spec.Method == t.Spec.Method { + return fission.MakeError(fission.ErrorNameExists, + fmt.Sprintf("HTTPTrigger with same URL & method already exists (%v)", + ht.Metadata.Name)) } } + return nil +} - uid, err := api.HTTPTriggerStore.Create(&t) +func (a *API) HTTPTriggerApiCreate(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + var t tpr.Httptrigger + err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) + return + } + + // Ensure we don't have a duplicate HTTP route defined (same URL and method) + err = a.checkHttpTriggerDuplicates(&t) + if err != nil { + a.respondWithError(w, err) + return + } + + tnew, err := a.fissionClient.Httptriggers(t.Metadata.Namespace).Create(&t) + if err != nil { + a.respondWithError(w, err) + return + } + + resp, err := json.Marshal(tnew.Metadata) + if err != nil { + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) HTTPTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["httpTrigger"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["httpTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - t, err := api.HTTPTriggerStore.Get(&m) + t, err := a.fissionClient.Httptriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) HTTPTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["httpTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.HTTPTrigger + var t tpr.Httptrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != t.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "HTTPTrigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.HTTPTriggerStore.Update(&t) + err = a.checkHttpTriggerDuplicates(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + tnew, err := a.fissionClient.Httptriggers(t.Metadata.Namespace).Update(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + + resp, err := json.Marshal(tnew.Metadata) + if err != nil { + a.respondWithError(w, err) + return + } + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) HTTPTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["httpTrigger"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"httpTrigger": m.Name}).Info("Deleting all versions") + name := vars["httpTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.HTTPTriggerStore.Delete(m) + err := a.fissionClient.Httptriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/httpTriggerStore.go b/controller/httpTriggerStore.go deleted file mode 100644 index 092e9d45..00000000 --- a/controller/httpTriggerStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type HTTPTriggerStore struct { - ResourceStore -} - -func (hts *HTTPTriggerStore) Create(ht *fission.HTTPTrigger) (string, error) { - ht.Metadata.Uid = uuid.NewV4().String() - return ht.Metadata.Uid, hts.ResourceStore.create(ht) -} - -func (hts *HTTPTriggerStore) Get(m *fission.Metadata) (*fission.HTTPTrigger, error) { - var ht fission.HTTPTrigger - err := hts.ResourceStore.read(m.Name, &ht) - if err != nil { - return nil, err - } - return &ht, nil -} - -func (hts *HTTPTriggerStore) Update(ht *fission.HTTPTrigger) (string, error) { - ht.Metadata.Uid = uuid.NewV4().String() - return ht.Metadata.Uid, hts.ResourceStore.update(ht) -} - -func (hts *HTTPTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.HTTPTrigger{}) - if err != nil { - return err - } - return hts.ResourceStore.delete(typeName, m.Name) -} - -func (hts *HTTPTriggerStore) List() ([]fission.HTTPTrigger, error) { - typeName, err := getTypeName(fission.HTTPTrigger{}) - if err != nil { - return nil, err - } - - bufs, err := hts.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.HTTPTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var ht fission.HTTPTrigger - err = js.deserialize([]byte(buf), &ht) - if err != nil { - return nil, err - } - triggers = append(triggers, ht) - } - - return triggers, nil -} diff --git a/controller/mqTriggerApi.go b/controller/mqTriggerApi.go index 91fd9ea2..7740dcad 100644 --- a/controller/mqTriggerApi.go +++ b/controller/mqTriggerApi.go @@ -22,145 +22,126 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) { - mqType := r.FormValue("mqtype") - triggers, err := api.MessageQueueTriggerStore.List(mqType) +func (a *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) { + //mqType := r.FormValue("mqtype") // ignored for now + triggers, err := a.fissionClient.Messagequeuetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var mqTrigger fission.MessageQueueTrigger + var mqTrigger tpr.Messagequeuetrigger err = json.Unmarshal(body, &mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - // trigger name must not conflict with any other trigger - // even they are different message queue type - triggers, err := api.MessageQueueTriggerStore.List("") + tnew, err := a.fissionClient.Messagequeuetriggers(mqTrigger.Metadata.Namespace).Create(&mqTrigger) if err != nil { - api.respondWithError(w, err) - return - } - for _, trigger := range triggers { - if trigger.Name == mqTrigger.Name { - err = fission.MakeError(fission.ErrorNameExists, - "Message queue trigger with same name already exists") - api.respondWithError(w, err) - return - } - } - - // save trigger info - uid, err := api.MessageQueueTriggerStore.Create(&mqTrigger) - if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid} - resp, err := json.Marshal(mqTriggerMeta) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiGet(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqTriggerMeta := fission.Metadata{ - Name: vars["mqTrigger"], - Uid: r.FormValue("uid"), // empty if uid is absent + name := vars["mqTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - mqTrigger, err := api.MessageQueueTriggerStore.Get(&mqTriggerMeta) + + mqTrigger, err := a.fissionClient.Messagequeuetriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqtName := vars["mqTrigger"] + name := vars["mqTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var mqTrigger fission.MessageQueueTrigger + var mqTrigger tpr.Messagequeuetrigger err = json.Unmarshal(body, &mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - if mqtName != mqTrigger.Metadata.Name { + if name != mqTrigger.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Message queue trigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.MessageQueueTriggerStore.Update(&mqTrigger) + tnew, err := a.fissionClient.Messagequeuetriggers(mqTrigger.Metadata.Namespace).Update(&mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid} - resp, err := json.Marshal(mqTriggerMeta) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqTriggerMeta := fission.Metadata{ - Name: vars["mqTrigger"], - Uid: r.FormValue("uid"), // empty if uid is absent + name := vars["mqTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - if len(mqTriggerMeta.Uid) == 0 { - log.WithFields(log.Fields{"mqTrigger": mqTriggerMeta.Name}).Info("Deleting all versions") - } - - err := api.MessageQueueTriggerStore.Delete(mqTriggerMeta) + err := a.fissionClient.Messagequeuetriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/mqTriggerStore.go b/controller/mqTriggerStore.go deleted file mode 100644 index 80fb6bf2..00000000 --- a/controller/mqTriggerStore.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -CopyrigmqTrigger 2017 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - mqTriggertp://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/fission/fission" - uuid "github.com/satori/go.uuid" -) - -type MessageQueueTriggerStore struct { - ResourceStore -} - -func (mqs *MessageQueueTriggerStore) Create(mqTrigger *fission.MessageQueueTrigger) (string, error) { - mqTrigger.Metadata.Uid = uuid.NewV4().String() - return mqTrigger.Metadata.Uid, mqs.ResourceStore.create(mqTrigger) -} - -func (mqs *MessageQueueTriggerStore) Get(m *fission.Metadata) (*fission.MessageQueueTrigger, error) { - var mqTrigger fission.MessageQueueTrigger - err := mqs.ResourceStore.read(m.Name, &mqTrigger) - if err != nil { - return nil, err - } - return &mqTrigger, nil -} - -func (mqs *MessageQueueTriggerStore) Update(mqTrigger *fission.MessageQueueTrigger) (string, error) { - mqTrigger.Metadata.Uid = uuid.NewV4().String() - return mqTrigger.Metadata.Uid, mqs.ResourceStore.update(mqTrigger) -} - -func (mqs *MessageQueueTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.MessageQueueTrigger{}) - if err != nil { - return err - } - return mqs.ResourceStore.delete(typeName, m.Name) -} - -func (mqs *MessageQueueTriggerStore) List(mqType string) ([]fission.MessageQueueTrigger, error) { - typeName, err := getTypeName(fission.MessageQueueTrigger{}) - if err != nil { - return nil, err - } - bufs, err := mqs.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.MessageQueueTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var mqTrigger fission.MessageQueueTrigger - err = js.deserialize([]byte(buf), &mqTrigger) - if err != nil { - return nil, err - } - if len(mqType) > 0 && mqType != mqTrigger.MessageQueueType { - continue - } - triggers = append(triggers, mqTrigger) - } - - return triggers, nil -} diff --git a/controller/resourceStore.go b/controller/resourceStore.go deleted file mode 100644 index 26e95c38..00000000 --- a/controller/resourceStore.go +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "errors" - "fmt" - "reflect" - "strings" - "time" - - "github.com/coreos/etcd/client" - "github.com/satori/go.uuid" - log "github.com/sirupsen/logrus" - "golang.org/x/net/context" - - "github.com/fission/fission" -) - -type ( - ResourceStore struct { - *FileStore - client.KeysAPI - serializer - } -) - -func MakeResourceStore(fs *FileStore, etcdUrls []string) (*ResourceStore, error) { - ks, err := getEtcdKeyAPI(etcdUrls) - if err != nil { - return nil, err - } - s := JsonSerializer{} - return &ResourceStore{FileStore: fs, KeysAPI: ks, serializer: s}, nil -} - -func getEtcdKeyAPI(etcdUrls []string) (client.KeysAPI, error) { - cfg := client.Config{ - Endpoints: etcdUrls, - Transport: client.DefaultTransport, - // set timeout per request to fail fast when the target endpoint is unavailable - HeaderTimeoutPerRequest: time.Second, - } - c, err := client.New(cfg) - if err != nil { - log.Printf("failed to connect to etcd: %v", err) - return nil, err - } - return client.NewKeysAPI(c), nil -} - -func getTypeName(r resource) (string, error) { - typ := reflect.TypeOf(r) - if typ.Kind().String() == "ptr" { - typ = typ.Elem() - } - typName := typ.Name() - if len(typName) == 0 { - return "", errors.New("Failed to get type") - } - return typName, nil -} - -func getKey(r resource) (string, error) { - typName, err := getTypeName(r) - if err != nil { - return "", err - } - rkey := r.Key() - return (typName + "/" + rkey), nil -} - -func (rs *ResourceStore) create(r resource) error { - key, err := getKey(r) - if err != nil { - return err - } - - serialized, err := rs.serializer.serialize(r) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), - &client.SetOptions{PrevExist: client.PrevNoExist}) - return handleEtcdErrorForResource(err, r) -} - -func (rs *ResourceStore) read(rkey string, res resource) error { - typName, err := getTypeName(res) - if err != nil { - return err - } - key := typName + "/" + rkey - - resp, err := rs.KeysAPI.Get(context.Background(), key, nil) - if err != nil { - return handleEtcdError(err, typName, rkey) - } - return rs.serializer.deserialize([]byte(resp.Node.Value), res) -} - -func (rs *ResourceStore) update(r resource) error { - key, err := getKey(r) - if err != nil { - return err - } - - serialized, err := rs.serializer.serialize(r) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), - &client.SetOptions{PrevExist: client.PrevExist}) - return handleEtcdErrorForResource(err, r) -} - -func (rs *ResourceStore) delete(typename, rkey string) error { - key := typename + "/" + rkey - _, err := rs.KeysAPI.Delete(context.Background(), key, nil) // ignore response - return handleEtcdError(err, typename, rkey) -} - -// getAll finds all entries under key. If none or found or key -// doesn't exist, returns an empty slice. -func (rs *ResourceStore) getAll(key string) ([]string, error) { - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Recursive: true, Sort: true}) - if err != nil { - if client.IsKeyNotFound(err) { - return []string{}, nil - } - return nil, handleEtcdError(err, "", key) - } - - res := make([]string, 0, len(resp.Node.Nodes)) - for _, n := range resp.Node.Nodes { - res = append(res, n.Value) - } - return res, nil -} - -func (rs *ResourceStore) writeFile(parentKey string, contents []byte) (string, string, error) { - uid := uuid.NewV4().String() - - err := rs.FileStore.write(uid, contents) - if err != nil { - return "", "", err - } - - parentKey = "file/" + parentKey - resp, err := rs.KeysAPI.CreateInOrder(context.Background(), parentKey, uid, nil) - if err != nil { - _ = rs.FileStore.delete(uid) - return "", "", handleEtcdError(err, "file", parentKey) - } - - return resp.Node.Key, uid, nil -} - -func (rs *ResourceStore) readFile(key string, uid *string) ([]byte, error) { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return nil, handleEtcdError(err, "file", key) - } - - if uid == nil { - // get latest - n := resp.Node.Nodes - uid = &n[len(n)-1].Value - } else { - // validate uid is in the list - found := false - for _, u := range resp.Node.Nodes { - if *uid == u.Value { - found = true - break - } - } - if !found { - return nil, errors.New("Invalid UID " + *uid) - } - } - - contents, err := rs.FileStore.read(*uid) - return contents, err -} - -func (rs *ResourceStore) deleteFile(key string, uid string) error { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return handleEtcdError(err, "file", key) - } - - var node *client.Node - for _, u := range resp.Node.Nodes { - if u.Value == uid { - node = u - } - } - if node == nil { - log.WithFields(log.Fields{"key": key, "uid": uid}).Error("unreferenced file") - return errors.New("won't delete unreferenced file") - } - - err = rs.FileStore.delete(node.Value) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Delete(context.Background(), node.Key, nil) - if err != nil { - return handleEtcdError(err, "", node.Key) - } - - if len(resp.Node.Nodes) == 1 { - _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) - return handleEtcdError(err, "file", key) - } - return nil -} - -func (rs *ResourceStore) deleteAllFiles(key string) error { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return handleEtcdError(err, "file", key) - } - for _, u := range resp.Node.Nodes { - err = rs.FileStore.delete(u.Value) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Delete(context.Background(), u.Key, nil) - if err != nil { - return handleEtcdError(err, "", u.Key) - } - } - _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) - return handleEtcdError(err, "file", key) -} - -func handleEtcdErrorForResource(e error, r resource) error { - resourceType, _ := getTypeName(r) - return handleEtcdError(e, resourceType, r.Key()) -} - -func handleEtcdError(e error, resourceType string, resourceKey string) error { - ee, ok := e.(client.Error) - if !ok { - return e - } - code := fission.ErrorInternal - msg := ee.Error() - - if len(resourceType) > 0 { - resourceType = strings.ToLower(resourceType) + " " - } - - //TODO: handle any other etcd error codes we care about - switch ee.Code { - case client.ErrorCodeNodeExist: - code = fission.ErrorNameExists - msg = fmt.Sprintf("%s'%s' already exists", resourceType, resourceKey) - case client.ErrorCodeKeyNotFound: - code = fission.ErrorNotFound - msg = fmt.Sprintf("%s'%s' does not exist", resourceType, resourceKey) - } - return fission.MakeError(code, msg) -} diff --git a/controller/resourceStore_test.go b/controller/resourceStore_test.go deleted file mode 100644 index 808cf7fc..00000000 --- a/controller/resourceStore_test.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "log" - "os" - "testing" - - "github.com/coreos/etcd/client" - "golang.org/x/net/context" -) - -type TestResource struct { - A string - B int -} - -func (tr TestResource) Key() string { - return tr.A -} - -func panicIf(err error) { - if err != nil { - log.Panicf("err: %v", err) - } -} - -func assert(b bool, msg string) { - if !b { - log.Panic("assertion failed: " + msg) - } -} - -func getTestResourceStore() (*FileStore, client.KeysAPI, *ResourceStore) { - // make a tmp dir - dir, err := ioutil.TempDir("", "testFileStore") - panicIf(err) - fs := MakeFileStore(dir) - - rs, err := MakeResourceStore(fs, []string{"http://localhost:2379"}) - panicIf(err) - - return fs, rs.KeysAPI, rs -} - -func TestResourceStore(t *testing.T) { - fs, ks, rs := getTestResourceStore() - defer os.RemoveAll(fs.root) - - s := JsonSerializer{} - - tr := TestResource{A: "hello", B: 1} - - // Delete the key first, in case of a panic'd previous test run; ignore errors - _ = rs.delete("TestResource", tr.Key()) - ks.Delete(context.Background(), "/TestResource", &client.DeleteOptions{Dir: true}) - - // Verify getAll for empty db - trs, err := rs.getAll("TestResource") - panicIf(err) - if len(trs) != 0 { - log.Fatalf("Expected zero length slice, got %v", trs) - } - - // Create - err = rs.create(tr) - panicIf(err) - defer ks.Delete(context.Background(), "/TestResource", &client.DeleteOptions{Dir: true}) - defer rs.delete("TestResource", tr.Key()) - - // Etcd key /TestResource/hello should exist - _, err = ks.Get(context.Background(), "TestResource/hello", nil) - panicIf(err) - - // Read - tr1 := TestResource{} - err = rs.read(tr.Key(), &tr1) - panicIf(err) - assert(tr1 == tr, "retrieved value must equal created value") - - // Update and Read - tr.B += 1 - err = rs.update(tr) - panicIf(err) - err = rs.read(tr.Key(), &tr1) - panicIf(err) - assert(tr1 == tr, "retrieved value must equal updated value") - - // Get list - results, err := rs.getAll("TestResource") - panicIf(err) - res := make([]TestResource, 0, 0) - for _, r := range results { - tmp := TestResource{} - err = s.deserialize([]byte(r), &tmp) - panicIf(err) - res = append(res, tmp) - } - assert(res[0] == tr, "value from retrieved list must equal updated value") - - // file tests - fileKey := "ResourceStoreTest" - fileContents1 := []byte("hello") - fileContents2 := []byte("world") - key, uid1, err := rs.writeFile(fileKey, fileContents1) - panicIf(err) - defer rs.deleteFile(fileKey, uid1) - log.Printf("key = %v, uid = %v", key, uid1) - - // read latest - contents, err := rs.readFile(fileKey, nil) - panicIf(err) - assert(string(contents) == string(fileContents1), "retrieved file contents must match written value") - - // update-- same key new contents - _, uid2, err := rs.writeFile(fileKey, fileContents2) - panicIf(err) - defer rs.deleteFile(fileKey, uid2) - - // read latest - contents, err = rs.readFile(fileKey, nil) - panicIf(err) - assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") - - // read by uid - // 1 - contents, err = rs.readFile(fileKey, &uid1) - panicIf(err) - assert(string(contents) == string(fileContents1), "retrieved file contents must match updated value") - - // 2 - contents, err = rs.readFile(fileKey, &uid2) - panicIf(err) - assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") - -} diff --git a/controller/timeTriggerApi.go b/controller/timeTriggerApi.go index d7a28db7..689966e2 100644 --- a/controller/timeTriggerApi.go +++ b/controller/timeTriggerApi.go @@ -23,161 +23,146 @@ import ( "github.com/gorilla/mux" "github.com/robfig/cron" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) TimeTriggerApiList(w http.ResponseWriter, r *http.Request) { - triggers, err := api.TimeTriggerStore.List() +func (a *API) TimeTriggerApiList(w http.ResponseWriter, r *http.Request) { + triggers, err := a.fissionClient.Timetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.TimeTrigger + var t tpr.Timetrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - triggers, err := api.TimeTriggerStore.List() - if err != nil { - api.respondWithError(w, err) - return - } - for _, trigger := range triggers { - if trigger.Name == t.Name { - err = fission.MakeError(fission.ErrorNameExists, - "TimeTrigger with same name already exists") - api.respondWithError(w, err) - return - } - } - - _, err = cron.Parse(t.Cron) + // validate + _, err = cron.Parse(t.Spec.Cron) if err != nil { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger cron spec is not valid") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.TimeTriggerStore.Create(&t) + tnew, err := a.fissionClient.Timetriggers(t.Metadata.Namespace).Create(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) TimeTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["timeTrigger"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["timeTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - t, err := api.TimeTriggerStore.Get(&m) + t, err := a.fissionClient.Timetriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["timeTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.TimeTrigger + var t tpr.Timetrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != t.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - _, err = cron.Parse(t.Cron) + _, err = cron.Parse(t.Spec.Cron) if err != nil { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger cron spec is not valid") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.TimeTriggerStore.Update(&t) + tnew, err := a.fissionClient.Timetriggers(t.Metadata.Namespace).Update(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["timeTrigger"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"timeTrigger": m.Name}).Info("Deleting all versions") + name := vars["timeTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.TimeTriggerStore.Delete(m) + err := a.fissionClient.Timetriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/timeTriggerStore.go b/controller/timeTriggerStore.go deleted file mode 100644 index b3a44c0c..00000000 --- a/controller/timeTriggerStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyrigtt 2017 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - tttp://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type TimeTriggerStore struct { - ResourceStore -} - -func (tts *TimeTriggerStore) Create(tt *fission.TimeTrigger) (string, error) { - tt.Metadata.Uid = uuid.NewV4().String() - return tt.Metadata.Uid, tts.ResourceStore.create(tt) -} - -func (tts *TimeTriggerStore) Get(m *fission.Metadata) (*fission.TimeTrigger, error) { - var tt fission.TimeTrigger - err := tts.ResourceStore.read(m.Name, &tt) - if err != nil { - return nil, err - } - return &tt, nil -} - -func (tts *TimeTriggerStore) Update(tt *fission.TimeTrigger) (string, error) { - tt.Metadata.Uid = uuid.NewV4().String() - return tt.Metadata.Uid, tts.ResourceStore.update(tt) -} - -func (tts *TimeTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.TimeTrigger{}) - if err != nil { - return err - } - return tts.ResourceStore.delete(typeName, m.Name) -} - -func (tts *TimeTriggerStore) List() ([]fission.TimeTrigger, error) { - typeName, err := getTypeName(fission.TimeTrigger{}) - if err != nil { - return nil, err - } - - bufs, err := tts.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.TimeTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var tt fission.TimeTrigger - err = js.deserialize([]byte(buf), &tt) - if err != nil { - return nil, err - } - triggers = append(triggers, tt) - } - - return triggers, nil -} diff --git a/controller/types.go b/controller/tpr.go similarity index 73% rename from controller/types.go rename to controller/tpr.go index abc575ec..aaea49d3 100644 --- a/controller/types.go +++ b/controller/tpr.go @@ -16,13 +16,14 @@ limitations under the License. package controller -type ( - resource interface { - Key() string - } - - serializer interface { - serialize(r resource) ([]byte, error) - deserialize(buf []byte, r resource) error - } +import ( + "github.com/fission/fission/tpr" ) + +func makeTPRBackedAPI() (*API, error) { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + return nil, err + } + return &API{fissionClient: fissionClient}, nil +} diff --git a/controller/watchApi.go b/controller/watchApi.go index 18b15826..d141bf5c 100644 --- a/controller/watchApi.go +++ b/controller/watchApi.go @@ -22,102 +22,101 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) WatchApiList(w http.ResponseWriter, r *http.Request) { - watches, err := api.WatchStore.List() +func (a *API) WatchApiList(w http.ResponseWriter, r *http.Request) { + watches, err := a.fissionClient.Kuberneteswatchtriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(watches) + resp, err := json.Marshal(watches.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) WatchApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var watch fission.Watch + var watch tpr.Kuberneteswatchtrigger err = json.Unmarshal(body, &watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - watch.Target = fission.UrlForFunction(&watch.Function) + // TODO check for duplicate watches - uid, err := api.WatchStore.Create(&watch) + wnew, err := a.fissionClient.Kuberneteswatchtriggers(watch.Metadata.Namespace).Create(&watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: watch.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(wnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) WatchApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["watch"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["watch"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - watch, err := api.WatchStore.Get(&m) + watch, err := a.fissionClient.Kuberneteswatchtriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiUpdate(w http.ResponseWriter, r *http.Request) { - api.respondWithError(w, fission.MakeError(fission.ErrorNotImplmented, +func (a *API) WatchApiUpdate(w http.ResponseWriter, r *http.Request) { + a.respondWithError(w, fission.MakeError(fission.ErrorNotImplmented, "Not implemented")) } -func (api *API) WatchApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) WatchApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["watch"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"watch": m.Name}).Info("Deleting all versions") + name := vars["watch"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.WatchStore.Delete(m) + err := a.fissionClient.Kuberneteswatchtriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/watchStore.go b/controller/watchStore.go deleted file mode 100644 index 25ddf248..00000000 --- a/controller/watchStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type WatchStore struct { - ResourceStore -} - -func (ws *WatchStore) Create(w *fission.Watch) (string, error) { - w.Metadata.Uid = uuid.NewV4().String() - return w.Metadata.Uid, ws.ResourceStore.create(w) -} - -func (ws *WatchStore) Get(m *fission.Metadata) (*fission.Watch, error) { - var w fission.Watch - err := ws.ResourceStore.read(m.Name, &w) - if err != nil { - return nil, err - } - return &w, err -} - -func (ws *WatchStore) Update(w *fission.Watch) (string, error) { - w.Metadata.Uid = uuid.NewV4().String() - return w.Metadata.Uid, ws.ResourceStore.update(w) -} - -func (ws *WatchStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.Watch{}) - if err != nil { - return err - } - return ws.ResourceStore.delete(typeName, m.Name) -} - -func (ws *WatchStore) List() ([]fission.Watch, error) { - typeName, err := getTypeName(fission.Watch{}) - if err != nil { - return nil, err - } - - bufs, err := ws.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - watches := make([]fission.Watch, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var w fission.Watch - err = js.deserialize([]byte(buf), &w) - if err != nil { - return nil, err - } - watches = append(watches, w) - } - - return watches, nil -} diff --git a/environments/fetcher/client/client.go b/environments/fetcher/client/client.go new file mode 100644 index 00000000..e31897ca --- /dev/null +++ b/environments/fetcher/client/client.go @@ -0,0 +1,35 @@ +package client + +import ( + "bytes" + "encoding/json" + "net/http" + //"time" + + "github.com/fission/fission" + "github.com/fission/fission/environments/fetcher" + //"github.com/fission/fission/router" +) + +func DoFetchRequest(fetcherUrl string, fr *fetcher.FetchRequest) error { + body, err := json.Marshal(fr) + if err != nil { + return err + } + + // client := http.Client{ + // Transport: router.MakeRetryingRoundTripper(10, 50*time.Millisecond), + // } + + resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader(body)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fission.MakeErrorFromHTTP(resp) + } + + return nil +} diff --git a/environments/fetcher/Dockerfile b/environments/fetcher/cmd/Dockerfile similarity index 100% rename from environments/fetcher/Dockerfile rename to environments/fetcher/cmd/Dockerfile diff --git a/environments/fetcher/build.sh b/environments/fetcher/cmd/build.sh similarity index 81% rename from environments/fetcher/build.sh rename to environments/fetcher/cmd/build.sh index a0f34b7a..d4b6f514 100755 --- a/environments/fetcher/build.sh +++ b/environments/fetcher/cmd/build.sh @@ -1 +1,2 @@ +#!/bin/sh GOOS=linux GOARCH=386 go build -o fetcher . diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go new file mode 100644 index 00000000..05ccaafd --- /dev/null +++ b/environments/fetcher/cmd/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "log" + "net/http" + "os" + + "github.com/fission/fission/environments/fetcher" +) + +// Usage: fetcher +func main() { + dir := os.Args[1] + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, os.ModeDir|0700) + if err != nil { + log.Fatalf("Error creating directory: %v", err) + } + } + } + fetcher := fetcher.MakeFetcher(dir) + mux := http.NewServeMux() + mux.HandleFunc("/", fetcher.Handler) + http.ListenAndServe(":8000", mux) +} diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 1f8b6829..c389cbfe 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -1,6 +1,8 @@ -package main +package fetcher import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -8,27 +10,108 @@ import ( "net/http" "os" "path/filepath" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" + "io" ) -type FetchRequest struct { - Url string `json:"url"` - Filename string `json:"filename"` -} +type ( + FetchRequestType int -type Fetcher struct { - sharedVolumePath string -} + FetchRequest struct { + FetchType FetchRequestType `json:"fetchType"` + Function api.ObjectMeta `json:"function"` + Url string `json:"url"` + StorageSvcUrl string `json:"storagesvcurl"` + Filename string `json:"filename"` + } + + Fetcher struct { + sharedVolumePath string + fissionClient *tpr.FissionClient + kubeClient *kubernetes.Clientset + } +) + +const ( + FETCH_SOURCE = iota + FETCH_DEPLOYMENT + FETCH_URL // remove this? +) func MakeFetcher(sharedVolumePath string) *Fetcher { - return &Fetcher{sharedVolumePath: sharedVolumePath} + fissionClient, kubeClient, err := tpr.MakeFissionClient() + if err != nil { + return nil + } + return &Fetcher{ + sharedVolumePath: sharedVolumePath, + fissionClient: fissionClient, + kubeClient: kubeClient, + } } -func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { +func downloadUrl(url string, localPath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = ioutil.WriteFile(localPath, body, 0600) + if err != nil { + return err + } + + return nil +} + +func verifyChecksum(path string, checksum *fission.Checksum) error { + if checksum.Type != fission.ChecksumTypeSHA256 { + return fission.MakeError(fission.ErrorInvalidArgument, "Unsupported checksum type") + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + hasher := sha256.New() + _, err = io.Copy(hasher, f) + if err != nil { + return err + } + + c := hex.EncodeToString(hasher.Sum(nil)) + if c != checksum.Sum { + return fission.MakeError(fission.ErrorChecksumFail, "Checksum validation failed") + } + return nil +} + +func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { http.Error(w, "", 404) return } + startTime := time.Now() + defer func() { + elapsed := time.Now().Sub(startTime) + log.Printf("elapsed time in fetch request = %v", elapsed) + }() + // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { @@ -43,35 +126,69 @@ func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), 500) return } - log.Printf("fetcher request: %v", req) + log.Printf("fetcher received request: %v", req) - // fetch the file and save it to tmp path - resp, err := http.Get(req.Url) - if err != nil { - e := fmt.Sprintf("Failed to fetch from url: %v", err) - log.Printf(e) - http.Error(w, e, 400) - return - } - defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - e := fmt.Sprintf("Failed to read from url: %v", err) - log.Printf(e) - http.Error(w, e, 400) - return - } tmpFile := req.Filename + ".tmp" tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile) - err = ioutil.WriteFile(tmpPath, body, 0600) - if err != nil { - e := fmt.Sprintf("Failed to write file: %v", err) - log.Printf(e) - http.Error(w, e, 500) - return - } - // TODO: add signature verification + if req.FetchType == FETCH_URL { + // fetch the file and save it to the tmp path + err := downloadUrl(req.Url, tmpPath) + if err != nil { + e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) + log.Printf(e) + http.Error(w, e, 400) + return + } + } else { + // get function object + fn, err := fetcher.fissionClient.Functions(req.Function.Namespace).Get(req.Function.Name) + if err != nil { + e := fmt.Sprintf("Failed to get function: %v", err) + log.Printf(e) + http.Error(w, e, 500) + return + } + + // get pkg + var pkg *fission.Package + if req.FetchType == FETCH_SOURCE { + pkg = &fn.Spec.Source + } else if req.FetchType == FETCH_DEPLOYMENT { + pkg = &fn.Spec.Deployment + } + + // get package data as literal or by url + if len(pkg.Literal) > 0 { + // write pkg.Literal into tmpPath + err = ioutil.WriteFile(tmpPath, pkg.Literal, 0600) + if err != nil { + e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err) + log.Printf(e) + http.Error(w, e, 500) + return + } + } else { + // download and verify + + err = downloadUrl(pkg.URL, tmpPath) + if err != nil { + e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) + log.Printf(e) + http.Error(w, e, 400) + return + } + + err = verifyChecksum(tmpPath, &pkg.Checksum) + if err != nil { + e := fmt.Sprintf("Failed to verify checksum: %v", err) + log.Printf(e) + http.Error(w, e, 400) + return + } + } + + } // move tmp file to requested filename err = os.Rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) @@ -82,23 +199,7 @@ func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { return } + log.Printf("Completed fetch request") // all done w.WriteHeader(http.StatusOK) } - -// Usage: fetcher -func main() { - dir := os.Args[1] - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(dir, os.ModeDir|0700) - if err != nil { - log.Fatalf("Error creating directory: %v", err) - } - } - } - fetcher := MakeFetcher(dir) - mux := http.NewServeMux() - mux.HandleFunc("/", fetcher.handler) - http.ListenAndServe(":8000", mux) -} diff --git a/fission-bundle/main.go b/fission-bundle/main.go index 30e295c4..c3e7de27 100644 --- a/fission-bundle/main.go +++ b/fission-bundle/main.go @@ -14,37 +14,25 @@ import ( "github.com/fission/fission/timer" ) -func runController(port int, etcdUrl string, filepath string) { - // filePath will be created if it doesn't exist. - fileStore := controller.MakeFileStore(filepath) - if fileStore == nil { - log.Fatalf("Failed to initialize filestore") - } - - rs, err := controller.MakeResourceStore(fileStore, []string{etcdUrl}) - if err != nil { - log.Fatalf("Error: %v", err) - } - - api := controller.MakeAPI(rs) - api.Serve(port) +func runController(port int) { + controller.Start(port) log.Fatalf("Error: Controller exited.") } -func runRouter(port int, controllerUrl string, poolmgrUrl string) { - router.Start(port, controllerUrl, poolmgrUrl) +func runRouter(port int, poolmgrUrl string) { + router.Start(port, poolmgrUrl) log.Fatalf("Error: Router exited.") } -func runPoolmgr(port int, controllerUrl string, namespace string) { - err := poolmgr.StartPoolmgr(controllerUrl, namespace, port) +func runPoolmgr(port int, fissionNamespace, functionNamespace string) { + err := poolmgr.StartPoolmgr(fissionNamespace, functionNamespace, port) if err != nil { log.Fatalf("Error starting poolmgr: %v", err) } } -func runKubeWatcher(controllerUrl, routerUrl string) { - err := kubewatcher.Start(controllerUrl, routerUrl) +func runKubeWatcher(routerUrl string) { + err := kubewatcher.Start(routerUrl) if err != nil { log.Fatalf("Error starting kubewatcher: %v", err) } @@ -55,15 +43,15 @@ func runLogger() { log.Fatalf("Error: Logger exited.") } -func runTimer(controllerUrl, routerUrl string) { - err := timer.Start(controllerUrl, routerUrl) +func runTimer(routerUrl string) { + err := timer.Start(routerUrl) if err != nil { log.Fatalf("Error starting timer: %v", err) } } -func runMessageQueueMgr(controllerUrl, routerUrl string) { - err := messagequeue.Start(controllerUrl, routerUrl) +func runMessageQueueMgr(routerUrl string) { + err := messagequeue.Start(routerUrl) if err != nil { log.Fatalf("Error starting timer: %v", err) } @@ -98,18 +86,17 @@ Use it to start one or more of the fission servers: Router implements HTTP triggers: it routes to running instances, working with the controller and poolmgr. Usage: - fission-bundle --controllerPort= [--etcdUrl=] --filepath= - fission-bundle --routerPort= [--controllerUrl= --poolmgrUrl=] - fission-bundle --poolmgrPort= [--controllerUrl= --namespace=] - fission-bundle --kubewatcher [--controllerUrl= --routerUrl=] + fission-bundle --controllerPort= + fission-bundle --routerPort= [--poolmgrUrl=] + fission-bundle --poolmgrPort= [--namespace=] [--fission-namespace=] + fission-bundle --kubewatcher [--routerUrl=] fission-bundle --logger - fission-bundle --timer [--controllerUrl= --routerUrl=] - fission-bundle --mqt [--controllerUrl= --routerUrl=] + fission-bundle --timer [--routerUrl=] + fission-bundle --mqt [--routerUrl=] Options: --controllerPort= Port that the controller should listen on. --routerPort= Port that the router should listen on. --poolmgrPort= Port that the poolmgr should listen on. - --controllerUrl= Controller URL. Not required if --controllerPort is specified. --poolmgrUrl= Poolmgr URL. Not required if --poolmgrPort is specified. --routerUrl= Router URL. --etcdUrl= Etcd URL. @@ -125,30 +112,29 @@ Options: log.Fatalf("Error: %v", err) } - namespace := getStringArgWithDefault(arguments["--namespace"], "fission-function") + functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function") + fissionNs := getStringArgWithDefault(arguments["--fission-namespace"], "fission") - controllerUrl := getStringArgWithDefault(arguments["--controllerUrl"], "http://controller.fission") - etcdUrl := getStringArgWithDefault(arguments["--etcdUrl"], "http://etcd:2379") poolmgrUrl := getStringArgWithDefault(arguments["--poolmgrUrl"], "http://poolmgr.fission") routerUrl := getStringArgWithDefault(arguments["--routerUrl"], "http://router.fission") if arguments["--controllerPort"] != nil { port := getPort(arguments["--controllerPort"]) - runController(port, etcdUrl, arguments["--filepath"].(string)) + runController(port) } if arguments["--routerPort"] != nil { port := getPort(arguments["--routerPort"]) - runRouter(port, controllerUrl, poolmgrUrl) + runRouter(port, poolmgrUrl) } if arguments["--poolmgrPort"] != nil { port := getPort(arguments["--poolmgrPort"]) - runPoolmgr(port, controllerUrl, namespace) + runPoolmgr(port, fissionNs, functionNs) } if arguments["--kubewatcher"] == true { - runKubeWatcher(controllerUrl, routerUrl) + runKubeWatcher(routerUrl) } if arguments["--logger"] == true { @@ -156,11 +142,11 @@ Options: } if arguments["--timer"] == true { - runTimer(controllerUrl, routerUrl) + runTimer(routerUrl) } if arguments["--mqt"] == true { - runMessageQueueMgr(controllerUrl, routerUrl) + runMessageQueueMgr(routerUrl) } select {} diff --git a/fission-rbac.yaml b/fission-rbac.yaml index 9d853128..dc5ebc9f 100644 --- a/fission-rbac.yaml +++ b/fission-rbac.yaml @@ -62,7 +62,7 @@ spec: - name: controller image: fission/fission-bundle command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] + args: ["--controllerPort", "8888"] serviceAccountName: fission-svc --- apiVersion: extensions/v1beta1 @@ -115,6 +115,9 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--poolmgrPort", "8888"] + env: + - name: FETCHER_IMAGE + value: fission/fetcher:v2 serviceAccountName: fission-svc --- apiVersion: extensions/v1beta1 @@ -135,40 +138,3 @@ spec: command: ["/fission-bundle"] args: ["--kubewatcher"] serviceAccountName: fission-svc ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - imagePullPolicy: IfNotPresent - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 - serviceAccountName: fission-svc diff --git a/fission.yaml b/fission.yaml index f0e32b45..8424de1d 100644 --- a/fission.yaml +++ b/fission.yaml @@ -30,7 +30,7 @@ spec: - name: controller image: fission/fission-bundle command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] + args: ["--controllerPort", "8888"] --- apiVersion: extensions/v1beta1 @@ -84,6 +84,9 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--poolmgrPort", "8888"] + env: + - name: FETCHER_IMAGE + value: fission/fetcher:v2 --- apiVersion: extensions/v1beta1 @@ -122,41 +125,3 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--timer"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - imagePullPolicy: IfNotPresent - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 diff --git a/fission/deployment.go b/fission/deployment.go deleted file mode 100644 index 1d2e53b9..00000000 --- a/fission/deployment.go +++ /dev/null @@ -1,181 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "fmt" - -func getDeploymentYaml() { - fmt.Printf(` -apiVersion: v1 -kind: Namespace -metadata: - name: fission - labels: - name: fission - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: fission-function - labels: - name: fission-function - ---- -apiVersion: v1 -kind: Service -metadata: - name: controller - namespace: fission - labels: - svc: controller -spec: - type: NodePort - ports: - - port: 80 - targetPort: 8888 - nodePort: 31313 - selector: - svc: controller - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: controller - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: controller - spec: - containers: - - name: controller - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: router - namespace: fission - labels: - svc: router -spec: - type: NodePort - ports: - - port: 80 - targetPort: 8888 - nodePort: 31314 - selector: - svc: router - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: router - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: router - spec: - containers: - - name: router - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--routerPort", "8888"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: poolmgr - namespace: fission - labels: - svc: poolmgr -spec: - ports: - - port: 80 - targetPort: 8888 - selector: - svc: poolmgr - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: poolmgr - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: poolmgr - spec: - containers: - - name: poolmgr - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--poolmgrPort", "8888"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 -`) -} diff --git a/fission/environment.go b/fission/environment.go index bf6e9181..b50203b3 100644 --- a/fission/environment.go +++ b/fission/environment.go @@ -22,8 +22,10 @@ import ( "text/tabwriter" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func envCreate(c *cli.Context) error { @@ -39,11 +41,17 @@ func envCreate(c *cli.Context) error { fatal("Need an image, use --image.") } - env := &fission.Environment{ - Metadata: fission.Metadata{ - Name: envName, + env := &tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: envImg, + }, }, - RunContainerImageUrl: envImg, } _, err := client.EnvironmentCreate(env) @@ -61,14 +69,17 @@ func envGet(c *cli.Context) error { fatal("Need a name, use --name.") } - m := &fission.Metadata{Name: envName} + m := &api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + } env, err := client.EnvironmentGet(m) checkErr(err, "get environment") w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") fmt.Fprintf(w, "%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.Uid, env.RunContainerImageUrl) + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) w.Flush() return nil } @@ -86,14 +97,15 @@ func envUpdate(c *cli.Context) error { fatal("Need an image, use --image.") } - env := &fission.Environment{ - Metadata: fission.Metadata{ - Name: envName, - }, - RunContainerImageUrl: envImg, - } + env, err := client.EnvironmentGet(&api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + }) + checkErr(err, "find environment") - _, err := client.EnvironmentUpdate(env) + env.Spec.Runtime.Image = envImg + + _, err = client.EnvironmentUpdate(env) checkErr(err, "update environment") fmt.Printf("environment '%v' updated\n", envName) @@ -108,7 +120,10 @@ func envDelete(c *cli.Context) error { fatal("Need a name , use --name.") } - m := &fission.Metadata{Name: envName} + m := &api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + } err := client.EnvironmentDelete(m) checkErr(err, "delete environment") @@ -126,7 +141,7 @@ func envList(c *cli.Context) error { fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") for _, env := range envs { fmt.Fprintf(w, "%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.Uid, env.RunContainerImageUrl) + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) } w.Flush() diff --git a/fission/function.go b/fission/function.go index 35e5c57e..8b6dc6c5 100644 --- a/fission/function.go +++ b/fission/function.go @@ -20,45 +20,53 @@ import ( "context" "fmt" "io/ioutil" - "net/http" + //"net/http" "os" - "os/exec" - "strings" "text/tabwriter" "time" "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/fission/logdb" + "github.com/fission/fission/tpr" ) -func fnFetchCode(filePath string) []byte { +func downloadUrl(url string) { + // var resp *http.Response + // resp, err := http.Get(url) + // if err != nil { + // checkErr(err, fmt.Sprintf("download function")) + // } + + // defer resp.Body.Close() + // if resp.StatusCode != http.StatusOK { + // err = fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) + // checkErr(err, fmt.Sprintf("download function")) + // } + + // contents, err := ioutil.ReadAll(resp.Body) + // if err != nil { + // checkErr(err, fmt.Sprintf("download function body %v", url)) + // } + + // return +} + +func fileSize(filePath string) int64 { + info, err := os.Stat(filePath) + checkErr(err, fmt.Sprintf("stat %v", filePath)) + return info.Size() +} + +func getPackageContents(filePath string) []byte { var code []byte var err error - if strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") { - var resp *http.Response - resp, err = http.Get(filePath) - if err != nil { - checkErr(err, fmt.Sprintf("download function")) - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) - checkErr(err, fmt.Sprintf("download function")) - } - - code, err = ioutil.ReadAll(resp.Body) - if err != nil { - checkErr(err, fmt.Sprintf("download function body %v", filePath)) - } - } else { - code, err = ioutil.ReadFile(filePath) - checkErr(err, fmt.Sprintf("read %v", filePath)) - } + code, err = ioutil.ReadFile(filePath) + checkErr(err, fmt.Sprintf("read %v", filePath)) return code } @@ -83,12 +91,25 @@ func fnCreate(c *cli.Context) error { } } - code := fnFetchCode(fileName) + if fileSize(fileName) > fission.PackageLiteralSizeLimit { + // TODO fallback to uploading + setting a Package URL + fmt.Printf("File size not supported yet") + os.Exit(1) + } - function := &fission.Function{ - Metadata: fission.Metadata{Name: fnName}, - Environment: fission.Metadata{Name: envName}, - Code: string(code), + pkgContents := getPackageContents(fileName) + + function := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.FunctionSpec{ + EnvironmentName: envName, + Deployment: fission.Package{ + Literal: pkgContents, + }, + }, } _, err := client.FunctionCreate(function) @@ -106,14 +127,18 @@ func fnCreate(c *cli.Context) error { method = "GET" } triggerName := uuid.NewV4().String() - ht := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: triggerName, + ht := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: triggerName, + Namespace: api.NamespaceDefault, }, - UrlPattern: triggerUrl, - Method: getMethod(method), - Function: fission.Metadata{ - Name: fnName, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + Method: getMethod(method), + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } _, err = client.HTTPTriggerCreate(ht) @@ -130,13 +155,15 @@ func fnGet(c *cli.Context) error { if len(fnName) == 0 { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } - code, err := client.FunctionGetRaw(m) + code, err := client.FunctionGetRawDeployment(m) checkErr(err, "get function") - fmt.Println(string(code)) + os.Stdout.Write(code) return err } @@ -148,8 +175,10 @@ func fnGetMeta(c *cli.Context) error { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } f, err := client.FunctionGet(m) checkErr(err, "get function") @@ -157,7 +186,7 @@ func fnGetMeta(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV") fmt.Fprintf(w, "%v\t%v\t%v\n", - f.Metadata.Name, f.Metadata.Uid, f.Environment.Name) + f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName) w.Flush() return err } @@ -170,14 +199,13 @@ func fnUpdate(c *cli.Context) error { fatal("Need name of function, use --name") } - function, err := client.FunctionGet(&fission.Metadata{Name: fnName}) + function, err := client.FunctionGet(&api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + }) checkErr(err, fmt.Sprintf("read function '%v'", fnName)) envName := c.String("env") - if len(envName) > 0 { - function.Environment.Name = envName - } - fileName := c.String("code") if len(fileName) == 0 { fileName = c.String("package") @@ -188,8 +216,16 @@ func fnUpdate(c *cli.Context) error { } if len(fileName) > 0 { - code := fnFetchCode(fileName) - function.Code = string(code) + if fileSize(fileName) > fission.PackageLiteralSizeLimit { + // TODO fallback to uploading + setting a Package URL + fmt.Printf("File size >256k not supported yet") + os.Exit(1) + } + + function.Spec.Deployment.Literal = getPackageContents(fileName) + } + if len(envName) > 0 { + function.Spec.EnvironmentName = envName } _, err = client.FunctionUpdate(function) @@ -207,8 +243,10 @@ func fnDelete(c *cli.Context) error { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } err := client.FunctionDelete(m) checkErr(err, fmt.Sprintf("delete function '%v'", fnName)) @@ -228,65 +266,13 @@ func fnList(c *cli.Context) error { fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV") for _, f := range fns { fmt.Fprintf(w, "%v\t%v\t%v\n", - f.Metadata.Name, f.Metadata.Uid, f.Environment.Name) + f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName) } w.Flush() return err } -func fnEdit(c *cli.Context) error { - client := getClient(c.GlobalString("server")) - - fnName := c.String("name") - if len(fnName) == 0 { - fatal("Need name of function, use --name") - } - fnUid := c.String("uid") - - // get function meta - function, err := client.FunctionGet(&fission.Metadata{Name: fnName, Uid: fnUid}) - checkErr(err, fmt.Sprintf("read function '%v'", fnName)) - - // write to tmp file - tmpFile, err := ioutil.TempFile("", fnName) - checkErr(err, "create temp file") - defer os.Remove(tmpFile.Name()) - - _, err = tmpFile.Write([]byte(function.Code)) - checkErr(err, "write temp file") - tmpFile.Close() - - // invoke $EDITOR on tmp file and wait for it - editor := os.Getenv("EDITOR") - if len(editor) == 0 { - editor = "vi" - } - cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("%v %v", editor, tmpFile.Name())) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Start() - checkErr(err, "start editor") - - err = cmd.Wait() - checkErr(err, "wait for editor") - - // read new code out of the file - contents, err := ioutil.ReadFile(tmpFile.Name()) - checkErr(err, "read temp file") - - function.Code = string(contents) - - // upload the updated function - newfn, err := client.FunctionUpdate(function) - checkErr(err, "upload edited function") - - fmt.Printf("function %v updated, new uuid: %v\n", newfn.Name, newfn.Uid) - return nil -} - func fnLogs(c *cli.Context) error { client := getClient(c.GlobalString("server")) @@ -301,13 +287,15 @@ func fnLogs(c *cli.Context) error { } fnPod := c.String("pod") - m := &fission.Metadata{Name: fnName} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } f, err := client.FunctionGet(m) checkErr(err, "get function") - // client first send db query to controller, then controller will - // establishe a proxy server that bridges the client and the database. + // request the controller to establish a proxy server to the database. logDB, err := logdb.GetLogDB(dbType, c.GlobalString("server")) if err != nil { fatal("failed to connect log database") @@ -325,7 +313,7 @@ func fnLogs(c *cli.Context) error { logFilter := logdb.LogFilter{ Pod: fnPod, Function: f.Metadata.Name, - FuncUid: f.Metadata.Uid, + FuncUid: string(f.Metadata.UID), Since: t, } logEntries, err := logDB.GetLogs(logFilter) @@ -372,7 +360,7 @@ func fnPods(c *cli.Context) error { dbType = logdb.INFLUXDB } - m := &fission.Metadata{Name: fnName} + m := &api.ObjectMeta{Name: fnName} f, err := client.FunctionGet(m) checkErr(err, "get function") @@ -386,7 +374,7 @@ func fnPods(c *cli.Context) error { logFilter := logdb.LogFilter{ Function: f.Metadata.Name, - FuncUid: f.Metadata.Uid, + FuncUid: string(f.Metadata.UID), } pods, err := logDB.GetPods(logFilter) if err != nil { diff --git a/fission/httptrigger.go b/fission/httptrigger.go index 42afea5a..db0289f8 100644 --- a/fission/httptrigger.go +++ b/fission/httptrigger.go @@ -25,8 +25,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) // returns one of http.Method* @@ -62,7 +64,6 @@ func htCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") triggerUrl := c.String("url") if len(triggerUrl) == 0 { fatal("Need a trigger URL, use --url") @@ -75,15 +76,18 @@ func htCreate(c *cli.Context) error { // just name triggers by uuid. triggerName := uuid.NewV4().String() - ht := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: triggerName, + ht := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: triggerName, + Namespace: api.NamespaceDefault, }, - UrlPattern: triggerUrl, - Method: getMethod(method), - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + Method: getMethod(method), + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } @@ -105,11 +109,21 @@ func htUpdate(c *cli.Context) error { fatal("Need name of trigger, use --name") } - ht, err := client.HTTPTriggerGet(&fission.Metadata{Name: htName}) + // update function ref + newFn := c.String("function") + if len(newFn) == 0 { + fatal("Nothing to update. Use --function to specify a new function.") + } + + ht, err := client.HTTPTriggerGet(&api.ObjectMeta{ + Name: htName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "get HTTP trigger") - newUid := c.String("uid") - ht.Function.Uid = newUid + if len(newFn) > 0 { + ht.Spec.FunctionReference.Name = newFn + } _, err = client.HTTPTriggerUpdate(ht) checkErr(err, "update HTTP trigger") @@ -125,7 +139,10 @@ func htDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.HTTPTriggerDelete(&fission.Metadata{Name: htName}) + err := client.HTTPTriggerDelete(&api.ObjectMeta{ + Name: htName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", htName) @@ -140,10 +157,10 @@ func htList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", "NAME", "METHOD", "URL", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "NAME", "METHOD", "URL", "FUNCTION_NAME") for _, ht := range hts { - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", - ht.Metadata.Name, ht.Method, ht.UrlPattern, ht.Function.Name, ht.Function.Uid) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", + ht.Metadata.Name, ht.Spec.Method, ht.Spec.RelativeURL, ht.Spec.FunctionReference.Name) } w.Flush() diff --git a/fission/main.go b/fission/main.go index 614fb216..734d0e56 100644 --- a/fission/main.go +++ b/fission/main.go @@ -40,18 +40,16 @@ func main() { fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"} fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"} fnPackageFlag := cli.StringFlag{Name: "package", Usage: "local path or URL for binary package"} - fnUidFlag := cli.StringFlag{Name: "uid", Usage: "function uid, optional (use latest if unspecified)"} fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"} fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"} fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"} fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"} fnSubcommands := []cli.Command{ {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate}, - {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGet}, - {Name: "edit", Usage: "Edit function source code in $EDITOR", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnEdit}, - {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGetMeta}, + {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet}, + {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta}, {Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag}, Action: fnUpdate}, - {Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnDelete}, + {Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete}, {Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList}, {Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs}, {Name: "pods", Usage: "Display function pods", Flags: []cli.Flag{fnNameFlag, fnLogDBTypeFlag}, Action: fnPods}, @@ -60,11 +58,10 @@ func main() { // httptriggers htNameFlag := cli.StringFlag{Name: "name", Usage: "HTTP Trigger name"} htFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - htFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} htSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag, htFnUidFlag}, Action: htCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag}, Action: htCreate}, {Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag}, Action: htGet}, - {Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, htFnNameFlag, htFnUidFlag}, Action: htUpdate}, + {Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, htFnNameFlag}, Action: htUpdate}, {Name: "delete", Usage: "Delete HTTP trigger", Flags: []cli.Flag{htNameFlag}, Action: htDelete}, {Name: "list", Usage: "List HTTP triggers", Flags: []cli.Flag{}, Action: htList}, } @@ -73,11 +70,10 @@ func main() { ttNameFlag := cli.StringFlag{Name: "name", Usage: "Time Trigger name"} ttCronFlag := cli.StringFlag{Name: "cron", Usage: "Time Trigger cron spec ('0 30 * * *', '@every 5m', '@hourly')"} ttFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - ttFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} ttSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create Time trigger", Flags: []cli.Flag{ttNameFlag, ttFnNameFlag, ttFnUidFlag, ttCronFlag}, Action: ttCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create Time trigger", Flags: []cli.Flag{ttNameFlag, ttFnNameFlag, ttCronFlag}, Action: ttCreate}, {Name: "get", Usage: "Get Time trigger", Flags: []cli.Flag{}, Action: ttGet}, - {Name: "update", Usage: "Update Time trigger", Flags: []cli.Flag{ttNameFlag, ttCronFlag}, Action: ttUpdate}, + {Name: "update", Usage: "Update Time trigger", Flags: []cli.Flag{ttNameFlag, ttCronFlag, ttFnNameFlag}, Action: ttUpdate}, {Name: "delete", Usage: "Delete Time trigger", Flags: []cli.Flag{ttNameFlag}, Action: ttDelete}, {Name: "list", Usage: "List Time triggers", Flags: []cli.Flag{}, Action: ttList}, } @@ -85,14 +81,13 @@ func main() { // Message queue trigger mqtNameFlag := cli.StringFlag{Name: "name", Usage: "Message queue Trigger name"} mqtFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - mqtFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} mqtMQTypeFlag := cli.StringFlag{Name: "mqtype", Usage: "Message queue type, e.g. nats-streaming (optional; uses \"nats-streaming\" if unspecified)"} mqtTopicFlag := cli.StringFlag{Name: "topic", Usage: "Message queue Topic the trigger listens on"} mqtRespTopicFlag := cli.StringFlag{Name: "resptopic", Usage: "Topic that the function response is sent on (optional; response discarded if unspecified)"} mqtSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create Message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtFnNameFlag, mqtFnUidFlag, mqtMQTypeFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create Message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtFnNameFlag, mqtMQTypeFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtCreate}, {Name: "get", Usage: "Get message queue trigger", Flags: []cli.Flag{}, Action: mqtGet}, - {Name: "update", Usage: "Update message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtUpdate}, + {Name: "update", Usage: "Update message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtTopicFlag, mqtRespTopicFlag, mqtFnNameFlag}, Action: mqtUpdate}, {Name: "delete", Usage: "Delete message queue trigger", Flags: []cli.Flag{mqtNameFlag}, Action: mqtDelete}, {Name: "list", Usage: "List message queue triggers", Flags: []cli.Flag{mqtMQTypeFlag}, Action: mqtList}, } @@ -111,12 +106,11 @@ func main() { // watches wNameFlag := cli.StringFlag{Name: "name", Usage: "Watch name"} wFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - wFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} wNamespaceFlag := cli.StringFlag{Name: "ns", Usage: "Namespace of resource to watch"} wObjTypeFlag := cli.StringFlag{Name: "type", Usage: "Type of resource to watch (Pod, Service, etc.)"} wLabelsFlag := cli.StringFlag{Name: "labels", Usage: "Label selector of the form a=b,c=d"} wSubCommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create a watch", Flags: []cli.Flag{wFnNameFlag, wFnUidFlag, wNamespaceFlag, wObjTypeFlag, wLabelsFlag}, Action: wCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create a watch", Flags: []cli.Flag{wFnNameFlag, wNamespaceFlag, wObjTypeFlag, wLabelsFlag}, Action: wCreate}, {Name: "get", Usage: "Get details about a watch", Flags: []cli.Flag{wNameFlag}, Action: wGet}, // TODO add update flag when supported {Name: "delete", Usage: "Delete watch", Flags: []cli.Flag{wNameFlag}, Action: wDelete}, @@ -130,16 +124,6 @@ func main() { {Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands}, {Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands}, {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, - - // Misc commands - { - Name: "get-deployment-yaml", - Usage: "Get deployment yaml. Use it as 'fission get-deployment-yaml | kubectl create -f -'", - Action: func(c *cli.Context) error { - getDeploymentYaml() - return nil - }, - }, } app.Run(os.Args) diff --git a/fission/mqtrigger.go b/fission/mqtrigger.go index d0e7b4d8..3c0465f6 100644 --- a/fission/mqtrigger.go +++ b/fission/mqtrigger.go @@ -23,9 +23,11 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/mqtrigger/messageQueue" + "github.com/fission/fission/tpr" ) func mqtCreate(c *cli.Context) error { @@ -39,7 +41,7 @@ func mqtCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") + mqType := c.String("mqtype") switch mqType { case "": @@ -58,24 +60,27 @@ func mqtCreate(c *cli.Context) error { respTopic := c.String("resptopic") if topic == respTopic { + // TODO maybe this should just be a warning, perhaps + // allow it behind a --force flag fatal("Listen topic should not equal to response topic") } checkMQTopicAvailability(mqType, topic, respTopic) - fnMeta := fission.Metadata{ - Name: fnName, - Uid: fnUid, - } - - mqt := fission.MessageQueueTrigger{ - Metadata: fission.Metadata{ - Name: mqtName, + mqt := tpr.Messagequeuetrigger{ + Metadata: api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.MessageQueueTriggerSpec{ + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, + MessageQueueType: mqType, + Topic: topic, + ResponseTopic: respTopic, }, - Function: fnMeta, - MessageQueueType: mqType, - Topic: topic, - ResponseTopic: respTopic, } _, err := client.MessageQueueTriggerCreate(&mqt) @@ -97,14 +102,33 @@ func mqtUpdate(c *cli.Context) error { } topic := c.String("topic") respTopic := c.String("resptopic") + fnName := c.String("function") - mqt, err := client.MessageQueueTriggerGet(&fission.Metadata{Name: mqtName}) + mqt, err := client.MessageQueueTriggerGet(&api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "get Time trigger") - checkMQTopicAvailability(mqt.MessageQueueType, topic, respTopic) + checkMQTopicAvailability(mqt.Spec.MessageQueueType, topic, respTopic) - mqt.Topic = topic - mqt.ResponseTopic = respTopic + updated := false + if len(topic) > 0 { + mqt.Spec.Topic = topic + updated = true + } + if len(respTopic) > 0 { + mqt.Spec.ResponseTopic = respTopic + updated = true + } + if len(fnName) > 0 { + mqt.Spec.FunctionReference.Name = fnName + updated = true + } + + if !updated { + fatal("Nothing to update. Use --topic, --resptopic, or --function.") + } _, err = client.MessageQueueTriggerUpdate(mqt) checkErr(err, "update Time trigger") @@ -120,7 +144,10 @@ func mqtDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.MessageQueueTriggerDelete(&fission.Metadata{Name: mqtName}) + err := client.MessageQueueTriggerDelete(&api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", mqtName) @@ -135,11 +162,11 @@ func mqtList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - "NAME", "FUNCTION_NAME", "FUNCTION_UID", "MESSAGE_QUEUE_TYPE", "TOPIC", "RESPONSE_TOPIC") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + "NAME", "FUNCTION_NAME", "MESSAGE_QUEUE_TYPE", "TOPIC", "RESPONSE_TOPIC") for _, mqt := range mqts { - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - mqt.Metadata.Name, mqt.Function.Name, mqt.Function.Uid, mqt.MessageQueueType, mqt.Topic, mqt.ResponseTopic) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + mqt.Metadata.Name, mqt.Spec.FunctionReference.Name, mqt.Spec.MessageQueueType, mqt.Spec.Topic, mqt.Spec.ResponseTopic) } w.Flush() diff --git a/fission/timetrigger.go b/fission/timetrigger.go index a7776851..d359d4a9 100644 --- a/fission/timetrigger.go +++ b/fission/timetrigger.go @@ -23,8 +23,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func ttCreate(c *cli.Context) error { @@ -38,20 +40,22 @@ func ttCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") cron := c.String("cron") if len(cron) == 0 { - fatal("Need a cron spec like '0 30 * * *', '@every 1h30m', '@hourly', use --cron") + fatal("Need a cron spec like '0 30 * * *', '@every 1h30m', or '@hourly'; use --cron") } - tt := &fission.TimeTrigger{ - Metadata: fission.Metadata{ - Name: name, + tt := &tpr.Timetrigger{ + Metadata: api.ObjectMeta{ + Name: name, + Namespace: api.NamespaceDefault, }, - Cron: cron, - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.TimeTriggerSpec{ + Cron: cron, + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } @@ -73,12 +77,26 @@ func ttUpdate(c *cli.Context) error { fatal("Need name of trigger, use --name") } - tt, err := client.TimeTriggerGet(&fission.Metadata{Name: ttName}) - checkErr(err, "get Time trigger") + tt, err := client.TimeTriggerGet(&api.ObjectMeta{ + Name: ttName, + Namespace: api.NamespaceDefault, + }) + checkErr(err, "get time trigger") + updated := false newCron := c.String("cron") if len(newCron) != 0 { - tt.Cron = newCron + tt.Spec.Cron = newCron + updated = true + } + fnName := c.String("function") + if len(fnName) > 0 { + tt.Spec.FunctionReference.Name = fnName + updated = true + } + + if !updated { + fatal("Nothing to update. Use --cron or --function.") } _, err = client.TimeTriggerUpdate(tt) @@ -95,7 +113,10 @@ func ttDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.TimeTriggerDelete(&fission.Metadata{Name: ttName}) + err := client.TimeTriggerDelete(&api.ObjectMeta{ + Name: ttName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", ttName) @@ -110,11 +131,10 @@ func ttList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", - "NAME", "CRON", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "CRON", "FUNCTION_NAME") for _, tt := range tts { fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", - tt.Metadata.Name, tt.Cron, tt.Function.Name, tt.Function.Uid) + tt.Metadata.Name, tt.Spec.Cron, tt.Spec.FunctionReference.Name) } w.Flush() diff --git a/fission/watch.go b/fission/watch.go index 222841f0..bbe68d29 100644 --- a/fission/watch.go +++ b/fission/watch.go @@ -23,8 +23,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func wCreate(c *cli.Context) error { @@ -34,7 +36,6 @@ func wCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a watch, use --function") } - fnUid := c.String("uid") namespace := c.String("ns") if len(namespace) == 0 { @@ -52,23 +53,28 @@ func wCreate(c *cli.Context) error { // empty 'labels' selects everything if len(labels) == 0 { fmt.Printf("Watching all objects of type '%v', use --labels to refine selection.\n", objType) + } else { + // TODO + fmt.Printf("Label selector not implemented, watching all objects") } // automatically name watches watchName := uuid.NewV4().String() - w := &fission.Watch{ - Metadata: fission.Metadata{ - Name: watchName, + w := &tpr.Kuberneteswatchtrigger{ + Metadata: api.ObjectMeta{ + Name: watchName, + Namespace: api.NamespaceDefault, }, - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: namespace, + Type: objType, + //LabelSelector: labels, + FunctionReference: fission.FunctionReference{ + Name: fnName, + Type: fission.FunctionReferenceTypeFunctionName, + }, }, - Namespace: namespace, - ObjType: objType, - LabelSelector: labels, - FieldSelector: "", // TODO } _, err := client.WatchCreate(w) @@ -79,10 +85,14 @@ func wCreate(c *cli.Context) error { } func wGet(c *cli.Context) error { + // TODO + fatal("Not implemented") return nil } func wUpdate(c *cli.Context) error { + // TODO + fatal("Not implemented") return nil } @@ -94,7 +104,10 @@ func wDelete(c *cli.Context) error { fatal("Need name of watch to delete, use --name") } - err := client.WatchDelete(&fission.Metadata{Name: wName}) + err := client.WatchDelete(&api.ObjectMeta{ + Name: wName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete watch") fmt.Printf("watch '%v' deleted\n", wName) @@ -109,10 +122,11 @@ func wList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "NAMESPACE", "OBJTYPE", "LABELS", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + "NAME", "NAMESPACE", "OBJTYPE", "LABELS", "FUNCTION_NAME") for _, wa := range ws { fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - wa.Metadata.Name, wa.Namespace, wa.ObjType, wa.LabelSelector, wa.Function.Name, wa.Function.Uid) + wa.Metadata.Name, wa.Spec.Namespace, wa.Spec.Type, wa.Spec.LabelSelector, wa.Spec.FunctionReference.Name) } w.Flush() diff --git a/kubewatcher/kubewatcher.go b/kubewatcher/kubewatcher.go index f9c680dd..fe4bd937 100644 --- a/kubewatcher/kubewatcher.go +++ b/kubewatcher/kubewatcher.go @@ -33,10 +33,12 @@ import ( apierrs "k8s.io/client-go/1.5/pkg/api/errors" "k8s.io/client-go/1.5/pkg/api/meta" "k8s.io/client-go/1.5/pkg/runtime" + "k8s.io/client-go/1.5/pkg/types" "k8s.io/client-go/1.5/pkg/watch" "github.com/fission/fission" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) type requestType int @@ -47,7 +49,7 @@ const ( type ( KubeWatcher struct { - watches map[string]watchSubscription + watches map[types.UID]watchSubscription kubernetesClient *kubernetes.Clientset requestChannel chan *kubeWatcherRequest publisher publisher.Publisher @@ -55,7 +57,7 @@ type ( } watchSubscription struct { - fission.Watch + watch tpr.Kuberneteswatchtrigger kubeWatch watch.Interface lastResourceVersion string stopped *int32 @@ -65,7 +67,7 @@ type ( kubeWatcherRequest struct { requestType - watches []fission.Watch + watches []tpr.Kuberneteswatchtrigger responseChannel chan *kubeWatcherResponse } kubeWatcherResponse struct { @@ -75,7 +77,7 @@ type ( func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ - watches: make(map[string]watchSubscription), + watches: make(map[types.UID]watchSubscription), kubernetesClient: kubernetesClient, publisher: publisher, requestChannel: make(chan *kubeWatcherRequest), @@ -84,7 +86,7 @@ func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher return kw } -func (kw *KubeWatcher) Sync(watches []fission.Watch) error { +func (kw *KubeWatcher) Sync(watches []tpr.Kuberneteswatchtrigger) error { req := &kubeWatcherRequest{ requestType: SYNC, watches: watches, @@ -100,19 +102,19 @@ func (kw *KubeWatcher) svc() { req := <-kw.requestChannel switch req.requestType { case SYNC: - newWatchUids := make(map[string]bool) + newWatchUids := make(map[types.UID]bool) for _, w := range req.watches { - newWatchUids[w.Metadata.Uid] = true + newWatchUids[w.Metadata.UID] = true } // Remove old watches for uid, ws := range kw.watches { if _, ok := newWatchUids[uid]; !ok { - kw.removeWatch(&ws.Watch) + kw.removeWatch(&ws.watch) } } // Add new watches for _, w := range req.watches { - if _, ok := kw.watches[w.Metadata.Uid]; !ok { + if _, ok := kw.watches[w.Metadata.UID]; !ok { kw.addWatch(&w) } } @@ -144,7 +146,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error { return err } -func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, resourceVersion string) (watch.Interface, error) { +func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *tpr.Kuberneteswatchtrigger, resourceVersion string) (watch.Interface, error) { var wi watch.Interface var err error var watchTimeoutSec int64 = 120 @@ -156,41 +158,41 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, r } // TODO handle the full list of types - switch strings.ToUpper(w.ObjType) { + switch strings.ToUpper(w.Spec.Type) { case "POD": - wi, err = kubeClient.Core().Pods(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().Pods(w.Spec.Namespace).Watch(listOptions) case "SERVICE": - wi, err = kubeClient.Core().Services(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().Services(w.Spec.Namespace).Watch(listOptions) case "REPLICATIONCONTROLLER": - wi, err = kubeClient.Core().ReplicationControllers(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().ReplicationControllers(w.Spec.Namespace).Watch(listOptions) case "JOB": - wi, err = kubeClient.Batch().Jobs(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Batch().Jobs(w.Spec.Namespace).Watch(listOptions) default: - msg := fmt.Sprintf("Error: unknown obj type '%v'", w.ObjType) + msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type) log.Println(msg) err = errors.New(msg) } return wi, err } -func (kw *KubeWatcher) addWatch(w *fission.Watch) error { - log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Function.Name) +func (kw *KubeWatcher) addWatch(w *tpr.Kuberneteswatchtrigger) error { + log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher) if err != nil { return err } - kw.watches[w.Metadata.Uid] = *ws + kw.watches[w.Metadata.UID] = *ws return nil } -func (kw *KubeWatcher) removeWatch(w *fission.Watch) error { - log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Function.Name) - ws, ok := kw.watches[w.Metadata.Uid] +func (kw *KubeWatcher) removeWatch(w *tpr.Kuberneteswatchtrigger) error { + log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) + ws, ok := kw.watches[w.Metadata.UID] if !ok { return fission.MakeError(fission.ErrorNotFound, fmt.Sprintf("watch doesn't exist: %v", w.Metadata)) } - delete(kw.watches, w.Metadata.Uid) + delete(kw.watches, w.Metadata.UID) ws.stop() return nil } @@ -210,10 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *fission.Watch) error { // return nil // } -func MakeWatchSubscription(w *fission.Watch, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(w *tpr.Kuberneteswatchtrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ - Watch: *w, + watch: *w, kubeWatch: nil, stopped: &stopped, kubernetesClient: kubeClient, @@ -234,8 +236,8 @@ func (ws *watchSubscription) restartWatch() error { retries := 60 for { log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v", - ws.Watch.Metadata, ws.Watch.Namespace, ws.Watch.ObjType, ws.lastResourceVersion) - wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.Watch, ws.lastResourceVersion) + ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion) + wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- if retries > 0 { @@ -259,7 +261,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { } func (ws *watchSubscription) eventDispatchLoop() { - log.Println("Listening to watch ", ws.Watch.Metadata.Name) + log.Println("Listening to watch ", ws.watch.Metadata.Name) for { // check watchSubscription is stopped or not before waiting for event // comes from the kubeWatch.ResultChan(). This fix the edge case that @@ -272,11 +274,11 @@ func (ws *watchSubscription) eventDispatchLoop() { if !more { if ws.isStopped() { // watch is removed by user. - log.Println("Watch stopped", ws.Watch.Metadata.Name) + log.Println("Watch stopped", ws.watch.Metadata.Name) return } else { // watch closed due to timeout, restart it. - log.Printf("Watch %v timed out, restarting", ws.Watch.Metadata.Name) + log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name) err := ws.restartWatch() if err != nil { log.Panicf("Failed to restart watch: %v", err) @@ -318,7 +320,16 @@ func (ws *watchSubscription) eventDispatchLoop() { "X-Kubernetes-Event-Type": string(ev.Type), "X-Kubernetes-Object-Type": reflect.TypeOf(ev.Object).Elem().Name(), } - ws.publisher.Publish(buf.String(), headers, ws.Watch.Target) + + // TODO support other function ref types. Or perhaps delegate to router? + if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { + log.Printf("Error: unsupported function ref type: %v, can't publish event", + ws.watch.Spec.FunctionReference.Type) + continue + } + + url := fission.UrlForFunction(ws.watch.Spec.FunctionReference.Name) + ws.publisher.Publish(buf.String(), headers, url) } } diff --git a/kubewatcher/main.go b/kubewatcher/main.go index 28118fc5..601a0f55 100644 --- a/kubewatcher/main.go +++ b/kubewatcher/main.go @@ -17,43 +17,18 @@ limitations under the License. package kubewatcher import ( - "log" - - "k8s.io/client-go/1.5/kubernetes" - "k8s.io/client-go/1.5/rest" - - "github.com/fission/fission/controller/client" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) -// Get a kubernetes client using the pod's service account. -func getKubernetesClient() (*kubernetes.Clientset, error) { - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - log.Printf("Error getting kubernetes client config: %v", err) - return nil, err - } - - // creates the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Printf("Error getting kubernetes client: %v", err) - return nil, err - } - - return clientset, nil -} - -func Start(controllerUrl string, routerUrl string) error { - kubeClient, err := getKubernetesClient() +func Start(routerUrl string) error { + fissionClient, kubeClient, err := tpr.MakeFissionClient() if err != nil { return err } poster := publisher.MakeWebhookPublisher(routerUrl) kubeWatch := MakeKubeWatcher(kubeClient, poster) - - MakeWatchSync(client.MakeClient(controllerUrl), kubeWatch) + MakeWatchSync(fissionClient, kubeWatch) return nil } diff --git a/kubewatcher/watchSync.go b/kubewatcher/watchSync.go index d3157516..5e8a15bf 100644 --- a/kubewatcher/watchSync.go +++ b/kubewatcher/watchSync.go @@ -20,17 +20,19 @@ import ( "log" "time" - "github.com/fission/fission/controller/client" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" ) type ( WatchSync struct { - client *client.Client + client *tpr.FissionClient kubeWatcher *KubeWatcher } ) -func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync { +func MakeWatchSync(client *tpr.FissionClient, kubeWatcher *KubeWatcher) *WatchSync { ws := &WatchSync{ client: client, kubeWatcher: kubeWatcher, @@ -40,20 +42,14 @@ func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync { } func (ws *WatchSync) syncSvc() { - failureCount := 0 - maxFailures := 6 + // TODO watch instead of polling for { - watches, err := ws.client.WatchList() + watches, err := ws.client.Kuberneteswatchtriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - failureCount++ - if failureCount > maxFailures { - log.Fatalf("Failed to connect to controller: %v", err) - } - time.Sleep(10 * time.Second) - continue + log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err) } - ws.kubeWatcher.Sync(watches) + ws.kubeWatcher.Sync(watches.Items) time.Sleep(3 * time.Second) } } diff --git a/mqtrigger/main.go b/mqtrigger/main.go index 31c0a9f2..e5219ed5 100644 --- a/mqtrigger/main.go +++ b/mqtrigger/main.go @@ -19,12 +19,16 @@ package messagequeue import ( "os" - controllerClient "github.com/fission/fission/controller/client" "github.com/fission/fission/mqtrigger/messageQueue" + "github.com/fission/fission/tpr" + "log" ) -func Start(controllerUrl string, routerUrl string) error { - controller := controllerClient.MakeClient(controllerUrl) +func Start(routerUrl string) error { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Failed to get fission client: %v", err) + } // Message queue type: nats is the only supported one for now mqType := os.Getenv("MESSAGE_QUEUE_TYPE") @@ -33,6 +37,6 @@ func Start(controllerUrl string, routerUrl string) error { MQType: mqType, Url: mqUrl, } - messageQueue.MakeMessageQueueTriggerManager(controller, routerUrl, mqCfg) + messageQueue.MakeMessageQueueTriggerManager(fissionClient, routerUrl, mqCfg) return nil } diff --git a/mqtrigger/messageQueue/messageQueue.go b/mqtrigger/messageQueue/messageQueue.go index 14356b9c..c2d80d7c 100644 --- a/mqtrigger/messageQueue/messageQueue.go +++ b/mqtrigger/messageQueue/messageQueue.go @@ -21,9 +21,10 @@ import ( "time" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" - controllerClient "github.com/fission/fission/controller/client" + //"github.com/fission/fission" + "github.com/fission/fission/tpr" ) const ( @@ -47,21 +48,20 @@ type ( } MessageQueue interface { - subscribe(trigger fission.MessageQueueTrigger) (messageQueueSubscription, error) + subscribe(trigger *tpr.Messagequeuetrigger) (messageQueueSubscription, error) unsubscribe(triggerSub messageQueueSubscription) error } MessageQueueTriggerManager struct { - reqChan chan request - mqCfg MessageQueueConfig - triggers map[string]*triggerSubscription - controller *controllerClient.Client - messageQueue MessageQueue + reqChan chan request + mqCfg MessageQueueConfig + triggers map[string]*triggerSubscription + fissionClient *tpr.FissionClient + messageQueue MessageQueue } triggerSubscription struct { - fission.Metadata - funcMeta fission.Metadata + trigger tpr.Messagequeuetrigger subscription messageQueueSubscription } @@ -72,20 +72,18 @@ type ( } response struct { err error - triggers *map[string]messageQueueSubscription + triggers *map[string]*triggerSubscription } ) -func MakeMessageQueueTriggerManager(ctrlClient *controllerClient.Client, - routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { - +func MakeMessageQueueTriggerManager(fissionClient *tpr.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { var messageQueue MessageQueue var err error mqTriggerMgr := MessageQueueTriggerManager{ - reqChan: make(chan request), - triggers: make(map[string]*triggerSubscription), - controller: ctrlClient, + reqChan: make(chan request), + triggers: make(map[string]*triggerSubscription), + fissionClient: fissionClient, } switch mqConfig.MQType { case NATS: @@ -108,22 +106,21 @@ func (mqt *MessageQueueTriggerManager) service() { switch req.requestType { case ADD_TRIGGER: var err error - triggerUid := req.triggerSub.Uid - if _, ok := mqt.triggers[triggerUid]; ok { + k := tpr.CacheKey(&req.triggerSub.trigger.Metadata) + if _, ok := mqt.triggers[k]; ok { err = errors.New("Trigger already exists") } else { - mqt.triggers[triggerUid] = req.triggerSub + mqt.triggers[k] = req.triggerSub } req.respChan <- response{err: err} case GET_ALL_TRIGGERS: - copyTriggers := make(map[string]messageQueueSubscription) + copyTriggers := make(map[string]*triggerSubscription) for key, val := range mqt.triggers { copyTriggers[key] = val } req.respChan <- response{triggers: ©Triggers} case DELETE_TRIGGER: - triggerUid := req.triggerSub.Uid - delete(mqt.triggers, triggerUid) + delete(mqt.triggers, tpr.CacheKey(&req.triggerSub.trigger.Metadata)) } } } @@ -139,7 +136,7 @@ func (mqt *MessageQueueTriggerManager) addTrigger(triggerSub *triggerSubscriptio return r.err } -func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]messageQueueSubscription { +func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]*triggerSubscription { respChan := make(chan response) mqt.reqChan <- request{ requestType: GET_ALL_TRIGGERS, @@ -149,12 +146,12 @@ func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]messageQueue return r.triggers } -func (mqt *MessageQueueTriggerManager) delTrigger(triggerUid string) { +func (mqt *MessageQueueTriggerManager) delTrigger(m *api.ObjectMeta) { mqt.reqChan <- request{ requestType: DELETE_TRIGGER, triggerSub: &triggerSubscription{ - Metadata: fission.Metadata{ - Uid: triggerUid, + trigger: tpr.Messagequeuetrigger{ + Metadata: *m, }, }, } @@ -162,57 +159,61 @@ func (mqt *MessageQueueTriggerManager) delTrigger(triggerUid string) { func (mqt *MessageQueueTriggerManager) syncTriggers() { for { - // TODO: handle error - newTriggers, err := mqt.controller.MessageQueueTriggerList(mqt.mqCfg.MQType) + // get new set of triggers + newTriggers, err := mqt.fissionClient.Messagequeuetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - log.Warnf("Failed to sync message queue trigger from controller: %v", err) + log.Fatalf("Failed to read message queue trigger list: %v", err) } - // sync trigger from controller - newTriggerMap := map[string]fission.MessageQueueTrigger{} - for _, trigger := range newTriggers { - newTriggerMap[trigger.Uid] = trigger + newTriggerMap := make(map[string]*tpr.Messagequeuetrigger) + for _, trigger := range newTriggers.Items { + newTriggerMap[tpr.CacheKey(&trigger.Metadata)] = &trigger } - currentTriggerSubMap := mqt.getAllTriggers() + + // get current set of triggers + currentTriggers := mqt.getAllTriggers() // register new triggers for key, trigger := range newTriggerMap { - if _, ok := (*currentTriggerSubMap)[key]; ok { + if _, ok := (*currentTriggers)[key]; ok { continue } + + // actually subscribe using the message queue client impl sub, err := mqt.messageQueue.subscribe(trigger) if err != nil { - log.Warnf("Message queue trigger %s created failed: %v", trigger.Name, err) + log.Warnf("Failed to subscribe to message queue trigger %s: %v", trigger.Metadata.Name, err) continue } + triggerSub := triggerSubscription{ - Metadata: fission.Metadata{ - Name: trigger.Name, - Uid: trigger.Uid, - }, - funcMeta: trigger.Function, + trigger: *trigger, subscription: sub, } + + // add to our list err = mqt.addTrigger(&triggerSub) if err != nil { - log.Warnf("Message queue trigger %s created failed: %v", trigger.Name, err) - continue + log.Fatalf("Message queue trigger %s addition failed: %v", trigger.Metadata.Name, err) } - log.Infof("Message queue trigger %s created", trigger.Name) + + log.Infof("Message queue trigger %s created", trigger.Metadata.Name) } // remove old triggers - for _, ts := range *currentTriggerSubMap { - triggerSub := ts.(*triggerSubscription) - if _, ok := newTriggerMap[triggerSub.Uid]; ok { + for key, triggerSub := range *currentTriggers { + if _, ok := newTriggerMap[key]; ok { continue } - if err := mqt.messageQueue.unsubscribe(triggerSub.subscription); err != nil { - log.Warnf("Message queue trigger %s deleted failed: %v", triggerSub.Name, err) - } else { - mqt.delTrigger(triggerSub.Uid) - log.Infof("Message queue trigger %s deleted", triggerSub.Name) + err := mqt.messageQueue.unsubscribe(triggerSub.subscription) + if err != nil { + log.Warnf("Failed to unsubscribe to trigger %s: %v", triggerSub.trigger.Metadata.Name, err) + continue } + mqt.delTrigger(&triggerSub.trigger.Metadata) + log.Infof("Message queue trigger %s deleted", triggerSub.trigger.Metadata.Name) } + + // TODO replace with a watch time.Sleep(3 * time.Second) } } diff --git a/mqtrigger/messageQueue/nats.go b/mqtrigger/messageQueue/nats.go index 39f6185f..4373f4a0 100644 --- a/mqtrigger/messageQueue/nats.go +++ b/mqtrigger/messageQueue/nats.go @@ -29,6 +29,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) const ( @@ -57,17 +58,17 @@ func makeNatsMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQu return nats, nil } -func (nats Nats) subscribe(trigger fission.MessageQueueTrigger) (messageQueueSubscription, error) { - subj := trigger.Topic +func (nats Nats) subscribe(trigger *tpr.Messagequeuetrigger) (messageQueueSubscription, error) { + subj := trigger.Spec.Topic if !isTopicValidForNats(subj) { - return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Topic)) + return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Spec.Topic)) } opts := []ns.SubscriptionOption{ // Create a durable subscription to nats, so that triggers could retrieve last unack message. // https://github.com/nats-io/go-nats-streaming#durable-subscriptions - ns.DurableName(trigger.Uid), + ns.DurableName(string(trigger.Metadata.UID)), // Nats-streaming server is auto-ack mode by default. Since we want nats-streaming server to // resend a message if the trigger does not ack it, we need to enable the manual ack mode, so that @@ -90,14 +91,21 @@ func isTopicValidForNats(topic string) bool { return nsUtil.IsSubjectValid(topic) } -func msgHandler(nats *Nats, trigger fission.MessageQueueTrigger) func(*ns.Msg) { +func msgHandler(nats *Nats, trigger *tpr.Messagequeuetrigger) func(*ns.Msg) { return func(msg *ns.Msg) { - url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(&trigger.Function), "/") + + // Support other function ref types + if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { + log.Fatalf("Unsupported function reference type (%v) for trigger %v", + trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) + } + + url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name), "/") log.Printf("Making HTTP request to %v", url) headers := map[string]string{ - "X-Fission-MQTrigger-Topic": trigger.Topic, - "X-Fission-MQTrigger-RespTopic": trigger.ResponseTopic, + "X-Fission-MQTrigger-Topic": trigger.Spec.Topic, + "X-Fission-MQTrigger-RespTopic": trigger.Spec.ResponseTopic, } // Create request @@ -129,10 +137,10 @@ func msgHandler(nats *Nats, trigger fission.MessageQueueTrigger) func(*ns.Msg) { log.Warningf("Failed to ack message: %v", err) } - if len(trigger.ResponseTopic) > 0 { - err = nats.nsConn.Publish(trigger.ResponseTopic, body) + if len(trigger.Spec.ResponseTopic) > 0 { + err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body) if err != nil { - log.Warningf("Failed to publish message to topic %s: %v", trigger.ResponseTopic, err) + log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) } } } diff --git a/poolmgr/api.go b/poolmgr/api.go index 854a5ceb..e88af431 100644 --- a/poolmgr/api.go +++ b/poolmgr/api.go @@ -29,54 +29,46 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/cache" - controllerclient "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) -type funcSvc struct { - function *fission.Metadata // function this pod/service is for - environment *fission.Environment // env it was obtained from - address string // Host:Port or IP:Port that the service can be reached at. - podName string // pod name (within the function namespace) +type ( + createFuncServiceRequest struct { + funcMeta *api.ObjectMeta + respChan chan *createFuncServiceResponse + } - ctime time.Time - atime time.Time -} + createFuncServiceResponse struct { + address string + err error + } -type createFuncServiceRequest struct { - funcMeta *fission.Metadata - respChan chan *createFuncServiceResponse -} -type createFuncServiceResponse struct { - address string - err error -} + Poolmgr struct { + gpm *GenericPoolManager + functionEnv *cache.Cache // map[string]tpr.Environment + fsCache *functionServiceCache + fissionClient *tpr.FissionClient -type API struct { - poolMgr *GenericPoolManager - functionEnv *cache.Cache // map[fission.Metadata]fission.Environment - fsCache *functionServiceCache - controller *controllerclient.Client - fsCreateChannels map[fission.Metadata]*sync.WaitGroup - requestChan chan *createFuncServiceRequest + fsCreateChannels map[string]*sync.WaitGroup // xxx no channels here, rename this + requestChan chan *createFuncServiceRequest + } +) - //functionService *cache.Cache // map[fission.Metadata]*funcSvc - //urlFuncSvc *cache.Cache // map[string]*funcSvc -} - -func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCache *functionServiceCache) *API { - api := API{ - poolMgr: gpm, - functionEnv: cache.MakeCache(time.Minute, 0), +func MakePoolmgr(gpm *GenericPoolManager, fissionClient *tpr.FissionClient, fissionNs string, fsCache *functionServiceCache) *Poolmgr { + poolMgr := &Poolmgr{ + gpm: gpm, + functionEnv: cache.MakeCache(10*time.Second, 0), fsCache: fsCache, - controller: controller, - fsCreateChannels: make(map[fission.Metadata]*sync.WaitGroup), + fissionClient: fissionClient, + fsCreateChannels: make(map[string]*sync.WaitGroup), requestChan: make(chan *createFuncServiceRequest), } - go api.serveCreateFuncServices() - return &api + go poolMgr.serveCreateFuncServices() + return poolMgr } // All non-cached function service requests go through this goroutine @@ -85,29 +77,29 @@ func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCac // get specialized. In other words, it ensures that when there's an // ongoing request for a certain function, all other requests wait for // that request to complete. -func (api *API) serveCreateFuncServices() { +func (poolMgr *Poolmgr) serveCreateFuncServices() { for { - req := <-api.requestChan + req := <-poolMgr.requestChan m := req.funcMeta // Cache miss -- is this first one to request the func? - wg, found := api.fsCreateChannels[*m] + wg, found := poolMgr.fsCreateChannels[tpr.CacheKey(m)] if !found { // create a waitgroup for other requests for // the same function to wait on wg := &sync.WaitGroup{} wg.Add(1) - api.fsCreateChannels[*m] = wg + poolMgr.fsCreateChannels[tpr.CacheKey(m)] = wg // launch a goroutine for each request, to parallelize // the specialization of different functions go func() { - address, err := api.createServiceForFunction(m) + address, err := poolMgr.createServiceForFunction(m) req.respChan <- &createFuncServiceResponse{ address: address, err: err, } - delete(api.fsCreateChannels, *m) + delete(poolMgr.fsCreateChannels, tpr.CacheKey(m)) wg.Done() }() } else { @@ -117,7 +109,7 @@ func (api *API) serveCreateFuncServices() { wg.Wait() // get the function service from the cache - fsvc, err := api.fsCache.GetByFunction(m) + fsvc, err := poolMgr.fsCache.GetByFunction(m) address := "" if err == nil { address = fsvc.address @@ -131,7 +123,7 @@ func (api *API) serveCreateFuncServices() { } } -func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) { +func (poolMgr *Poolmgr) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request", 500) @@ -139,14 +131,14 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) } // get function metadata - m := fission.Metadata{} + m := api.ObjectMeta{} err = json.Unmarshal(body, &m) if err != nil { http.Error(w, "Failed to parse request", 400) return } - serviceName, err := api.getServiceForFunction(&m) + serviceName, err := poolMgr.getServiceForFunction(&m) if err != nil { code, msg := fission.GetHTTPError(err) log.Printf("Error: %v: %v", code, msg) @@ -156,55 +148,46 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) w.Write([]byte(serviceName)) } -func (api *API) getFunctionEnv(m *fission.Metadata) (*fission.Environment, error) { - var env *fission.Environment +func (poolMgr *Poolmgr) getFunctionEnv(m *api.ObjectMeta) (*tpr.Environment, error) { + var env *tpr.Environment // Cached ? - result, err := api.functionEnv.Get(*m) + result, err := poolMgr.functionEnv.Get(tpr.CacheKey(m)) if err == nil { - env = result.(*fission.Environment) + env = result.(*tpr.Environment) return env, nil } // Cache miss -- get func from controller - log.Printf("[%v] getting function from controller", m) - f, err := api.controller.FunctionGet(m) + f, err := poolMgr.fissionClient.Functions(m.Namespace).Get(m.Name) if err != nil { return nil, err } // Get env from metadata - log.Printf("[%v] getting env from controller", m) - env, err = api.controller.EnvironmentGet(&f.Environment) + log.Printf("[%v] getting env", m) + env, err = poolMgr.fissionClient.Environments(f.Metadata.Namespace).Get(f.Spec.EnvironmentName) if err != nil { return nil, err } - // cache for future - api.functionEnv.Set(*m, env) + // cache for future lookups + poolMgr.functionEnv.Set(tpr.CacheKey(m), env) return env, nil } -func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) { - // Make sure we have the full metadata. This ensures that - // poolmgr does not implicitly interpret empty-UID as latest - // version. - if len(m.Uid) == 0 { - return "", fission.MakeError(fission.ErrorInvalidArgument, - fmt.Sprintf("invalid metadata for function %v", m.Name)) - } - +func (poolMgr *Poolmgr) getServiceForFunction(m *api.ObjectMeta) (string, error) { // Check function -> svc cache log.Printf("[%v] Checking for cached function service", m.Name) - fsvc, err := api.fsCache.GetByFunction(m) + fsvc, err := poolMgr.fsCache.GetByFunction(m) if err == nil { // Cached, return svc address return fsvc.address, nil } respChan := make(chan *createFuncServiceResponse) - api.requestChan <- &createFuncServiceRequest{ + poolMgr.requestChan <- &createFuncServiceRequest{ funcMeta: m, respChan: respChan, } @@ -212,20 +195,20 @@ func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) { return resp.address, resp.err } -func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) { +func (poolMgr *Poolmgr) createServiceForFunction(m *api.ObjectMeta) (string, error) { // None exists, so create a new funcSvc: log.Printf("[%v] No cached function service found, creating one", m.Name) // from Func -> get Env log.Printf("[%v] getting environment for function", m.Name) - env, err := api.getFunctionEnv(m) + env, err := poolMgr.getFunctionEnv(m) if err != nil { return "", err } // from Env -> get GenericPool log.Printf("[%v] getting generic pool for env", m.Name) - pool, err := api.poolMgr.GetPool(env) + pool, err := poolMgr.gpm.GetPool(env) if err != nil { return "", err } @@ -241,7 +224,7 @@ func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) { } // find funcSvc and update its atime -func (api *API) tapService(w http.ResponseWriter, r *http.Request) { +func (poolMgr *Poolmgr) tapService(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request", 500) @@ -250,7 +233,7 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) { svcName := string(body) svcHost := strings.TrimPrefix(svcName, "http://") - err = api.fsCache.TouchByAddress(svcHost) + err = poolMgr.fsCache.TouchByAddress(svcHost) if err != nil { log.Printf("funcSvc tap error: %v", err) http.Error(w, "Not found", 404) @@ -259,11 +242,10 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (api *API) Serve(port int) { +func (poolMgr *Poolmgr) Serve(port int) { r := mux.NewRouter() - r.HandleFunc("/v1/getServiceForFunction", api.getServiceForFunctionApi).Methods("POST") - r.HandleFunc("/v1/tapService", api.tapService).Methods("POST") - + r.HandleFunc("/v2/getServiceForFunction", poolMgr.getServiceForFunctionApi).Methods("POST") + r.HandleFunc("/v2/tapService", poolMgr.tapService).Methods("POST") address := fmt.Sprintf(":%v", port) log.Printf("starting poolmgr at port %v", port) log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r))) diff --git a/poolmgr/client/client.go b/poolmgr/client/client.go index 166dd0a1..d3ee2a17 100644 --- a/poolmgr/client/client.go +++ b/poolmgr/client/client.go @@ -26,6 +26,8 @@ import ( "strings" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" ) @@ -45,8 +47,9 @@ func MakeClient(poolmgrUrl string) *Client { return c } -func (c *Client) GetServiceForFunction(metadata *fission.Metadata) (string, error) { - poolmgrUrl := c.poolmgrUrl + "/v1/getServiceForFunction" +func (c *Client) GetServiceForFunction(metadata *api.ObjectMeta) (string, error) { + poolmgrUrl := c.poolmgrUrl + "/v2/getServiceForFunction" + body, err := json.Marshal(metadata) if err != nil { return "", err @@ -97,7 +100,7 @@ func (c *Client) TapService(serviceUrl *url.URL) { } func (c *Client) _tapService(serviceUrlStr string) error { - poolmgrUrl := c.poolmgrUrl + "/v1/tapService" + poolmgrUrl := c.poolmgrUrl + "/v2/tapService" resp, err := http.Post(poolmgrUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr))) if err != nil { diff --git a/poolmgr/functionServiceCache.go b/poolmgr/functionServiceCache.go index f4fece70..25225298 100644 --- a/poolmgr/functionServiceCache.go +++ b/poolmgr/functionServiceCache.go @@ -20,8 +20,10 @@ import ( "log" "time" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission/cache" + "github.com/fission/fission/tpr" ) type fscRequestType int @@ -34,10 +36,20 @@ const ( ) type ( + funcSvc struct { + function *api.ObjectMeta // function this pod/service is for + environment *tpr.Environment // function's environment + address string // Host:Port or IP:Port that the function's service can be reached at. + podName string // pod name (within the function namespace) + + ctime time.Time + atime time.Time + } + functionServiceCache struct { - byFunction *cache.Cache // function -> funcSvc : map[fission.Metadata]*funcSvc - byAddress *cache.Cache // address -> function : map[string]fission.Metadata - byPod *cache.Cache // podname -> function : map[string]fission.Metadata + byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc + byAddress *cache.Cache // address -> function : map[string]api.ObjectMeta + byPod *cache.Cache // podname -> function : map[string]api.ObjectMeta requestChannel chan *fscRequest } @@ -79,8 +91,8 @@ func (fsc *functionServiceCache) service() { byPodCopy := fsc.byPod.Copy() pods := make([]string, 0) for podNameI, mI := range byPodCopy { - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { resp.error = err } else { @@ -95,10 +107,9 @@ func (fsc *functionServiceCache) service() { case LOG: funcCopy := fsc.byFunction.Copy() log.Printf("Cache has %v entries", len(funcCopy)) - for mI, fsvcI := range funcCopy { - m := mI.(fission.Metadata) + for key, fsvcI := range funcCopy { fsvc := fsvcI.(*funcSvc) - log.Printf("%v:%v\t%v", m.Name, m.Uid, fsvc.podName) + log.Printf("%v\t%v", key, fsvc.podName) } case DELETE_BY_POD: resp.deleted, resp.error = fsc._deleteByPod(req.podName, req.age) @@ -107,11 +118,14 @@ func (fsc *functionServiceCache) service() { } } -func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, error) { - fsvcI, err := fsc.byFunction.Get(*m) +func (fsc *functionServiceCache) GetByFunction(m *api.ObjectMeta) (*funcSvc, error) { + key := tpr.CacheKey(m) + + fsvcI, err := fsc.byFunction.Get(key) if err != nil { return nil, err } + // update atime fsvc := fsvcI.(*funcSvc) fsvc.atime = time.Now() @@ -121,7 +135,7 @@ func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, e } func (fsc *functionServiceCache) Add(fsvc funcSvc) (error, *funcSvc) { - err, existing := fsc.byFunction.Set(*fsvc.function, &fsvc) + err, existing := fsc.byFunction.Set(tpr.CacheKey(fsvc.function), &fsvc) if err != nil { if existing != nil { f := existing.(*funcSvc) @@ -167,8 +181,8 @@ func (fsc *functionServiceCache) _touchByAddress(address string) error { if err != nil { return err } - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { return err } @@ -196,8 +210,8 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati if err != nil { return false, err } - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { return false, err } @@ -207,7 +221,7 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati return false, nil } - fsc.byFunction.Delete(m) + fsc.byFunction.Delete(tpr.CacheKey(&m)) fsc.byAddress.Delete(fsvc.address) fsc.byPod.Delete(podName) return true, nil diff --git a/poolmgr/functionServiceCache_test.go b/poolmgr/functionServiceCache_test.go index d33e1b9c..5a4003a0 100644 --- a/poolmgr/functionServiceCache_test.go +++ b/poolmgr/functionServiceCache_test.go @@ -5,7 +5,10 @@ import ( "testing" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func TestFunctionServiceCache(t *testing.T) { @@ -18,16 +21,22 @@ func TestFunctionServiceCache(t *testing.T) { now := time.Now() fsvc = &funcSvc{ - function: &fission.Metadata{ + function: &api.ObjectMeta{ Name: "foo", - Uid: "1212", + UID: "1212", }, - environment: &fission.Environment{ - Metadata: fission.Metadata{ + environment: &tpr.Environment{ + Metadata: api.ObjectMeta{ Name: "foo-env", - Uid: "2323", + UID: "2323", + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: "fission/foo-env", + }, + Builder: fission.Builder{}, }, - RunContainerImageUrl: "fission/foo-env", }, address: "xxx", podName: "yyy", diff --git a/poolmgr/gp.go b/poolmgr/gp.go index 3036bc1c..9a0bdf57 100644 --- a/poolmgr/gp.go +++ b/poolmgr/gp.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "net/url" + "os" "strings" "time" @@ -38,7 +39,10 @@ import ( "k8s.io/client-go/1.5/pkg/util/intstr" "github.com/fission/fission" + "github.com/fission/fission/environments/fetcher" + fetcherClient "github.com/fission/fission/environments/fetcher/client" "github.com/fission/fission/logger" + "github.com/fission/fission/tpr" ) const POOLMGR_INSTANCEID_LABEL string = "poolmgrInstanceId" @@ -46,16 +50,16 @@ const POD_PHASE_RUNNING string = "Running" type ( GenericPool struct { - env *fission.Environment - replicas int32 // num idle pods - deployment *v1beta1.Deployment // kubernetes deployment - namespace string // namespace to keep our resources - podReadyTimeout time.Duration // timeout for generic pods to become ready - controllerUrl string + env *tpr.Environment + replicas int32 // num idle pods + deployment *v1beta1.Deployment // kubernetes deployment + namespace string // namespace to keep our resources + podReadyTimeout time.Duration // timeout for generic pods to become ready idlePodReapTime time.Duration // pods unused for idlePodReapTime are deleted fsCache *functionServiceCache // cache funcSvc's by function, address and podname useSvc bool // create k8s service for specialized pods poolInstanceId string // small random string to uniquify pod names + fetcherImage string kubernetesClient *kubernetes.Clientset instanceId string // poolmgr instance id labelsForPool map[string]string @@ -74,15 +78,20 @@ type ( ) func MakeGenericPool( - controllerUrl string, kubernetesClient *kubernetes.Clientset, - env *fission.Environment, + env *tpr.Environment, initialReplicas int32, namespace string, fsCache *functionServiceCache, instanceId string) (*GenericPool, error) { log.Printf("Creating pool for environment %v", env.Metadata) + + fetcherImage := os.Getenv("FETCHER_IMAGE") + if len(fetcherImage) == 0 { + fetcherImage = "fission/fetcher" + } + // TODO: in general we need to provide the user a way to configure pools. Initial // replicas, autoscaling params, various timeouts, etc. gp := &GenericPool{ @@ -92,19 +101,18 @@ func MakeGenericPool( kubernetesClient: kubernetesClient, namespace: namespace, podReadyTimeout: 5 * time.Minute, // TODO make this an env param? - controllerUrl: controllerUrl, idlePodReapTime: 3 * time.Minute, // TODO make this configurable fsCache: fsCache, poolInstanceId: uniuri.NewLen(8), instanceId: instanceId, - - useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start + fetcherImage: fetcherImage, + useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start } // Labels for generic deployment/RS/pods. gp.labelsForPool = map[string]string{ "environmentName": gp.env.Metadata.Name, - "environmentUid": gp.env.Metadata.Uid, + "environmentUid": string(gp.env.Metadata.UID), POOLMGR_INSTANCEID_LABEL: gp.instanceId, } @@ -220,10 +228,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*v1.Pod, error) } } -func (gp *GenericPool) labelsForFunction(metadata *fission.Metadata) map[string]string { +func (gp *GenericPool) labelsForFunction(metadata *api.ObjectMeta) map[string]string { return map[string]string{ "functionName": metadata.Name, - "functionUid": metadata.Uid, + "functionUid": string(metadata.UID), "unmanaged": "true", // this allows us to easily find pods not managed by the deployment POOLMGR_INSTANCEID_LABEL: gp.instanceId, } @@ -240,10 +248,30 @@ func (gp *GenericPool) scheduleDeletePod(name string) { }() } +func (gp *GenericPool) getFetcherUrl(podIP string) string { + testUrl := os.Getenv("TEST_FETCHER_URL") + if len(testUrl) != 0 { + // it takes a second or so for the test service to + // become routable once a pod is relabeled. This is + // super hacky, but only runs in unit tests. + time.Sleep(5 * time.Second) + return testUrl + } + return fmt.Sprintf("http://%v:8000/", podIP) +} + +func (gp *GenericPool) getSpecializeUrl(podIP string) string { + u := os.Getenv("TEST_SPECIALIZE_URL") + if len(u) != 0 { + return u + } + return fmt.Sprintf("http://%v:8888/specialize", podIP) +} + // specializePod chooses a pod, copies the required user-defined function to that pod // (via fetcher), and calls the function-run container to load it, resulting in a // specialized pod. -func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) error { +func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *api.ObjectMeta) error { // for fetcher we don't need to create a service, just talk to the pod directly podIP := pod.Status.PodIP if len(podIP) == 0 { @@ -251,28 +279,23 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er } // tell fetcher to get the function. - fetcherUrl := fmt.Sprintf("http://%v:8000/", podIP) - functionUrl := fmt.Sprintf("%v/v1/functions/%v?uid=%v&raw=1", - gp.controllerUrl, metadata.Name, metadata.Uid) - fetcherRequest := fmt.Sprintf("{\"url\": \"%v\", \"filename\": \"user\"}", functionUrl) - - log.Printf("[%v] calling fetcher to copy function", metadata) - resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader([]byte(fetcherRequest))) + fetcherUrl := gp.getFetcherUrl(podIP) + log.Printf("[%v] calling fetcher to copy function", metadata.Name) + err := fetcherClient.DoFetchRequest(fetcherUrl, &fetcher.FetchRequest{ + FetchType: fetcher.FETCH_DEPLOYMENT, + Function: *metadata, + Filename: "user", // XXX use function id instead + }) if err != nil { - // TODO we should retry this call in case fetcher hasn't come up yet return err } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return errors.New(fmt.Sprintf("Error from fetcher: %v", resp.Status)) - } // Tell logging helper about this function invocation gp.setupLogging(pod, metadata) // get function run container to specialize - log.Printf("[%v] specializing pod", metadata) - specializeUrl := fmt.Sprintf("http://%v:8888/specialize", podIP) + log.Printf("[%v] specializing pod", metadata.Name) + specializeUrl := gp.getSpecializeUrl(podIP) // retry the specialize call a few times in case the env server hasn't come up yet maxRetries := 20 @@ -311,7 +334,7 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er // creates the pool but doesn't wait for any pods to be ready. func (gp *GenericPool) createPool() error { poolDeploymentName := fmt.Sprintf("%v-%v-%v", - gp.env.Metadata.Name, gp.env.Metadata.Uid, strings.ToLower(gp.poolInstanceId)) + gp.env.Metadata.Name, gp.env.Metadata.UID, strings.ToLower(gp.poolInstanceId)) sharedMountPath := "/userfunc" deployment := &v1beta1.Deployment{ @@ -340,7 +363,7 @@ func (gp *GenericPool) createPool() error { Containers: []v1.Container{ { Name: gp.env.Metadata.Name, - Image: gp.env.RunContainerImageUrl, + Image: gp.env.Spec.Runtime.Image, ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", VolumeMounts: []v1.VolumeMount{ @@ -352,7 +375,7 @@ func (gp *GenericPool) createPool() error { }, { Name: "fetcher", - Image: "fission/fetcher", + Image: gp.fetcherImage, ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", VolumeMounts: []v1.VolumeMount{ @@ -418,9 +441,9 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*v1.Ser return svc, err } -func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) { +func (gp *GenericPool) GetFuncSvc(m *api.ObjectMeta) (*funcSvc, error) { - log.Printf("[%v] Choosing pod from pool", m) + log.Printf("[%v] Choosing pod from pool", m.Name) newLabels := gp.labelsForFunction(m) pod, err := gp.choosePod(newLabels) if err != nil { @@ -437,8 +460,8 @@ func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) { var svcHost string if gp.useSvc { svcName := fmt.Sprintf("svc-%v", m.Name) - if len(m.Uid) > 0 { - svcName += "-" + m.Uid + if len(m.UID) > 0 { + svcName += ("-" + string(m.UID)) } labels := gp.labelsForFunction(m) @@ -580,13 +603,13 @@ func (gp *GenericPool) destroy() error { // Calls the logging daemonset pod on the node where the given pod is // running. -func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *fission.Metadata) { +func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *api.ObjectMeta) { logReq := logger.LogRequest{ Namespace: pod.Namespace, Pod: pod.Name, Container: gp.env.Metadata.Name, FuncName: metadata.Name, - FuncUid: metadata.Uid, + FuncUid: string(metadata.UID), } reqbody, err := json.Marshal(logReq) if err != nil { diff --git a/poolmgr/gpm.go b/poolmgr/gpm.go index 3214a261..f25ccc13 100644 --- a/poolmgr/gpm.go +++ b/poolmgr/gpm.go @@ -21,9 +21,9 @@ import ( "time" "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" - "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) type requestType int @@ -35,19 +35,18 @@ const ( type ( GenericPoolManager struct { - pools map[fission.Environment]*GenericPool + pools map[string]*GenericPool kubernetesClient *kubernetes.Clientset namespace string - controllerUrl string - controllerClient *client.Client + fissionClient *tpr.FissionClient fsCache *functionServiceCache instanceId string requestChannel chan *request } request struct { requestType - env *fission.Environment - envList []fission.Environment + env *tpr.Environment + envList []tpr.Environment responseChannel chan *response } response struct { @@ -57,18 +56,18 @@ type ( ) func MakeGenericPoolManager( - controllerUrl string, + fissionClient *tpr.FissionClient, kubernetesClient *kubernetes.Clientset, - namespace string, + fissionNamespace string, + functionNamespace string, fsCache *functionServiceCache, instanceId string) *GenericPoolManager { gpm := &GenericPoolManager{ - pools: make(map[fission.Environment]*GenericPool), + pools: make(map[string]*GenericPool), kubernetesClient: kubernetesClient, - namespace: namespace, - controllerUrl: controllerUrl, - controllerClient: client.MakeClient(controllerUrl), + namespace: functionNamespace, + fissionClient: fissionClient, fsCache: fsCache, instanceId: instanceId, requestChannel: make(chan *request), @@ -85,30 +84,30 @@ func (gpm *GenericPoolManager) service() { switch req.requestType { case GET_POOL: var err error - pool, ok := gpm.pools[*req.env] + pool, ok := gpm.pools[tpr.CacheKey(&req.env.Metadata)] if !ok { pool, err = MakeGenericPool( - gpm.controllerUrl, gpm.kubernetesClient, req.env, + gpm.kubernetesClient, req.env, 3, // TODO configurable/autoscalable gpm.namespace, gpm.fsCache, gpm.instanceId) if err != nil { req.responseChannel <- &response{error: err} continue } - gpm.pools[*req.env] = pool + gpm.pools[tpr.CacheKey(&req.env.Metadata)] = pool } req.responseChannel <- &response{pool: pool} case CLEANUP_POOLS: - uids := make(map[string]bool) + latestEnvSet := make(map[string]bool) for _, env := range req.envList { - uids[env.Metadata.Uid] = true + latestEnvSet[tpr.CacheKey(&env.Metadata)] = true } - for env, pool := range gpm.pools { - _, ok := uids[env.Metadata.Uid] + for key, pool := range gpm.pools { + _, ok := latestEnvSet[key] if !ok { // Env no longer exists -- remove our cache - log.Printf("Destroying generic pool for environment [%v]", env) - delete(gpm.pools, env) + log.Printf("Destroying generic pool for environment [%v]", key) + delete(gpm.pools, key) // and delete the pool asynchronously. go pool.destroy() @@ -119,7 +118,7 @@ func (gpm *GenericPoolManager) service() { } } -func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool, error) { +func (gpm *GenericPoolManager) GetPool(env *tpr.Environment) (*GenericPool, error) { c := make(chan *response) gpm.requestChannel <- &request{ requestType: GET_POOL, @@ -130,7 +129,7 @@ func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool, return resp.pool, resp.error } -func (gpm *GenericPoolManager) CleanupPools(envs []fission.Environment) { +func (gpm *GenericPoolManager) CleanupPools(envs []tpr.Environment) { gpm.requestChannel <- &request{ requestType: CLEANUP_POOLS, envList: envs, @@ -145,11 +144,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { time.Sleep(pollSleep) // get list of envs from controller - envs, err := gpm.controllerClient.EnvironmentList() + envs, err := gpm.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{}) if err != nil { failureCount++ if failureCount >= maxFailures { - log.Fatalf("Failed to connect to controller %v times: %v", maxFailures, err) + log.Fatalf("Failed %v times: %v", maxFailures, err) } } @@ -157,14 +156,14 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { // creating pools for envs that are actually used by functions. Also we might want // to keep these eagerly created pools smaller than the ones created when there are // actual function calls. - for i := range envs { - _, err := gpm.GetPool(&envs[i]) + for i := range envs.Items { + _, err := gpm.GetPool(&envs.Items[i]) if err != nil { log.Printf("eager-create pool failed: %v", err) } } // Clean up pools whose env was deleted - gpm.CleanupPools(envs) + gpm.CleanupPools(envs.Items) } } diff --git a/poolmgr/poolmgr.go b/poolmgr/poolmgr.go index 896cac96..75c12ff9 100644 --- a/poolmgr/poolmgr.go +++ b/poolmgr/poolmgr.go @@ -18,52 +18,29 @@ package poolmgr import ( "log" - "strings" "github.com/dchest/uniuri" - "k8s.io/client-go/1.5/kubernetes" - "k8s.io/client-go/1.5/rest" - controllerclient "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) -// Get a kubernetes client using the pod's service account. This only -// works when we're running inside a kubernetes cluster. -func getKubernetesClient() (*kubernetes.Clientset, error) { - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - log.Printf("Error getting kubernetes client config: %v", err) - return nil, err - } - - // creates the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Printf("Error getting kubernetes client: %v", err) - return nil, err - } - - return clientset, nil -} - -func StartPoolmgr(controllerUrl string, namespace string, port int) error { - controllerUrl = strings.TrimSuffix(controllerUrl, "/") - controllerClient := controllerclient.MakeClient(controllerUrl) - - kubernetesClient, err := getKubernetesClient() +// Start the poolmgr service. +func StartPoolmgr(fissionNamespace string, functionNamespace string, port int) error { + fissionClient, kubernetesClient, err := tpr.MakeFissionClient() if err != nil { log.Printf("Failed to get kubernetes client: %v", err) return err } instanceId := uniuri.NewLen(8) - cleanupOldPoolmgrResources(kubernetesClient, namespace, instanceId) + cleanupOldPoolmgrResources(kubernetesClient, functionNamespace, instanceId) fsCache := MakeFunctionServiceCache() - gpm := MakeGenericPoolManager(controllerUrl, kubernetesClient, namespace, fsCache, instanceId) + gpm := MakeGenericPoolManager( + fissionClient, kubernetesClient, fissionNamespace, + functionNamespace, fsCache, instanceId) - api := MakeAPI(gpm, controllerClient, fsCache) + api := MakePoolmgr(gpm, fissionClient, fissionNamespace, fsCache) go api.Serve(port) return nil diff --git a/poolmgr/poolmgr_test.go b/poolmgr/poolmgr_test.go new file mode 100644 index 00000000..98e3427d --- /dev/null +++ b/poolmgr/poolmgr_test.go @@ -0,0 +1,234 @@ +// +// This test depends on several env vars: +// +// KUBECONFIG has to point at a kube config with a cluster. The test +// will use the default context from that config. Be careful, +// don't point this at your production environment. The test is +// skipped if KUBECONFIG is undefined. +// +// TEST_SPECIALIZE_URL +// TEST_FETCHER_URL +// These need to point at :30001 and :30002, +// where is the address of any node in the test +// cluster. +// +// FETCHER_IMAGE +// Optional. Set this to a fetcher image; otherwise uses the +// default. +// + +// Here's how I run this on my setup, with minikube: +// TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v . + +package poolmgr + +import ( + "fmt" + "log" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/v1" + "k8s.io/client-go/1.5/pkg/labels" + "k8s.io/client-go/1.5/pkg/util/intstr" + + "github.com/fission/fission" + "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" + "io/ioutil" +) + +// return the number of pods in the given namespace matching the given labels +func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int { + pods, err := kubeClient.Pods(ns).List(api.ListOptions{ + LabelSelector: labels.Set(labelz).AsSelector(), + }) + if err != nil { + log.Panicf("Failed to list pods: %v", err) + } + return len(pods.Items) +} + +func createTestNamespace(kubeClient *kubernetes.Clientset, ns string) { + _, err := kubeClient.Namespaces().Create(&v1.Namespace{ + ObjectMeta: v1.ObjectMeta{ + Name: ns, + }, + }) + if err != nil { + log.Panicf("failed to create ns %v: %v", ns, err) + } +} + +// create a nodeport service +func createSvc(kubeClient *kubernetes.Clientset, ns string, name string, targetPort int, nodePort int32, labels map[string]string) *v1.Service { + svc, err := kubeClient.Services(ns).Create(&v1.Service{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + { + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(targetPort), + NodePort: nodePort, + }, + }, + Selector: labels, + }, + }) + if err != nil { + log.Panicf("Failed to create svc: %v", err) + } + return svc +} + +func httpGet(url string) string { + resp, err := http.Get(url) + if err != nil { + log.Panicf("HTTP Get failed: URL %v: %v", url, err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Panicf("HTTP Get failed to read body: URL %v: %v", url, err) + } + return string(body) +} + +func TestPoolmgr(t *testing.T) { + // run in a random namespace so we can have concurrent tests + // on a given cluster + rand.Seed(time.Now().UTC().UnixNano()) + testId := rand.Intn(999) + fissionNs := fmt.Sprintf("test-%v", testId) + functionNs := fmt.Sprintf("test-function-%v", testId) + + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + t.Skip("Skipping test, no kubernetes cluster") + return + } + + // connect to k8s + // and get TPR client + fissionClient, kubeClient, err := tpr.MakeFissionClient() + if err != nil { + log.Panicf("failed to connect: %v", err) + } + + // create the test's namespaces + createTestNamespace(kubeClient, fissionNs) + defer kubeClient.Namespaces().Delete(fissionNs, nil) + + createTestNamespace(kubeClient, functionNs) + defer kubeClient.Namespaces().Delete(functionNs, nil) + + // make sure TPR types exist on cluster + err = tpr.EnsureFissionTPRs(kubeClient) + if err != nil { + log.Panicf("failed to ensure tprs: %v", err) + } + fissionClient.WaitForTPRs() + + // create an env on the cluster + env, err := fissionClient.Environments(fissionNs).Create(&tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: "nodejs", + Namespace: fissionNs, + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: "fission/node-env", + }, + Builder: fission.Builder{}, + }, + }) + if err != nil { + log.Panicf("failed to create env: %v", err) + } + + // create poolmgr + port := 9999 + err = StartPoolmgr(fissionNs, functionNs, port) + if err != nil { + log.Panicf("failed to start poolmgr: %v", err) + } + + // connect poolmgr client + poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port)) + + // Wait for pool to be created (we don't actually need to do + // this, since the API should do the right thing in any case). + // waitForPool(functionNs, "nodejs") + time.Sleep(6 * time.Second) + + // create a function + f := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: "hello", + Namespace: fissionNs, + }, + Spec: fission.FunctionSpec{ + Source: fission.Package{}, + Deployment: fission.Package{ + Type: fission.PackageTypeLiteral, + Literal: []byte(`module.exports = async function(context) { return { status: 200, body: "Hello, world!\n" }; }`), + }, + EnvironmentName: env.Metadata.Name, + }, + } + _, err = fissionClient.Functions(fissionNs).Create(f) + if err != nil { + log.Panicf("failed to create function: %v", err) + } + + // create a service to call fetcher and the env container + labels := map[string]string{"functionName": f.Metadata.Name} + var fetcherPort int32 = 30001 + fetcherSvc := createSvc(kubeClient, functionNs, fmt.Sprintf("%v-%v", f.Metadata.Name, "fetcher"), 8000, fetcherPort, labels) + defer kubeClient.Services(functionNs).Delete(fetcherSvc.ObjectMeta.Name, nil) + + var funcSvcPort int32 = 30002 + functionSvc := createSvc(kubeClient, functionNs, f.Metadata.Name, 8888, funcSvcPort, labels) + defer kubeClient.Services(functionNs).Delete(functionSvc.ObjectMeta.Name, nil) + + // the main test: get a service for a given function + t1 := time.Now() + svc, err := poolmgrClient.GetServiceForFunction(&f.Metadata) + if err != nil { + log.Panicf("failed to get func svc: %v", err) + } + log.Printf("svc for function created at: %v (in %v)", svc, time.Now().Sub(t1)) + + // ensure that a pod with the label functionName=f.Metadata.Name exists + podCount := countPods(kubeClient, functionNs, map[string]string{"functionName": f.Metadata.Name}) + if podCount != 1 { + log.Panicf("expected 1 function pod, found %v", podCount) + } + + // call the service to ensure it works + + // wait for a bit + + // tap service to simulate calling it again + + // make sure the same pod is still there + + // wait for idleTimeout to ensure the pod is removed + + // remove env + + // wait for pool to be destroyed + + // that's it +} diff --git a/router/functionHandler.go b/router/functionHandler.go index 0b32a441..b899c3d5 100644 --- a/router/functionHandler.go +++ b/router/functionHandler.go @@ -26,20 +26,20 @@ import ( "time" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" poolmgrClient "github.com/fission/fission/poolmgr/client" ) type functionHandler struct { fmap *functionServiceMap poolmgr *poolmgrClient.Client - Function fission.Metadata + function *api.ObjectMeta } func (fh *functionHandler) getServiceForFunction() (*url.URL, error) { // call poolmgr, get a url for a function - svcName, err := fh.poolmgr.GetServiceForFunction(&fh.Function) + svcName, err := fh.poolmgr.GetServiceForFunction(fh.function) if err != nil { return nil, err } @@ -99,16 +99,15 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request * } // cache lookup - serviceUrl, err := fh.fmap.lookup(&fh.Function) + serviceUrl, err := fh.fmap.lookup(fh.function) if err != nil { // Cache miss: request the Pool Manager to make a new service. - log.Printf("Not cached, getting new service for %v", fh.Function) + log.Printf("Not cached, getting new service for %v", fh.function) var poolErr error serviceUrl, poolErr = fh.getServiceForFunction() if poolErr != nil { - log.Printf("Failed to get service for function (%v,%v): %v", - fh.Function.Name, fh.Function.Uid, poolErr) + log.Printf("Failed to get service for function %v: %v", fh.function.Name, poolErr) // We might want a specific error code or header for fission // failures as opposed to user function bugs. http.Error(responseWriter, "Internal server error (fission)", 500) @@ -116,7 +115,7 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request * } // add it to the map - fh.fmap.assign(&fh.Function, serviceUrl) + fh.fmap.assign(fh.function, serviceUrl) } else { // if we're using our cache, asynchronously tell // poolmgr we're using this service diff --git a/router/functionHandler_test.go b/router/functionHandler_test.go index 1cb9b3dd..76d11981 100644 --- a/router/functionHandler_test.go +++ b/router/functionHandler_test.go @@ -23,7 +23,7 @@ import ( "net/url" "testing" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" ) func createBackendService(testResponseString string) *url.URL { @@ -49,11 +49,12 @@ func TestFunctionProxying(t *testing.T) { backendURL := createBackendService(testResponseString) log.Printf("Created backend svc at %v", backendURL) - fn := &fission.Metadata{Name: "foo", Uid: "xxx"} + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} + fmap := makeFunctionServiceMap(0) fmap.assign(fn, backendURL) - fh := &functionHandler{fmap: fmap, Function: *fn} + fh := &functionHandler{fmap: fmap, function: fn} functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler)) fhURL := functionHandlerServer.URL diff --git a/router/functionReferenceResolver.go b/router/functionReferenceResolver.go new file mode 100644 index 00000000..ffb18660 --- /dev/null +++ b/router/functionReferenceResolver.go @@ -0,0 +1,118 @@ +/* +Copyright 2017 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package router + +import ( + "fmt" + "time" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/cache" + "github.com/fission/fission/tpr" +) + +type ( + // functionReferenceResolver provides a resolver to turn a function + // reference into a resolveResult + functionReferenceResolver struct { + fissionClient *tpr.FissionClient + + // FunctionReference -> function metadata + refCache *cache.Cache + } + + resolveResultType int + + // resolveResult is the result of resolving a function reference; for now + // it's just the metadata of one function, but in the future could support + // a distribution of requests across two functions. + resolveResult struct { + resolveResultType + functionMetadata *api.ObjectMeta + } + + // namespacedFunctionReference is just a function reference plus a + // namespace. Since a function reference works on names, it's only + // meaningful within a namespace. + namespacedFunctionReference struct { + namespace string + functionReference fission.FunctionReference + } +) + +const ( + resolveResultSingleFunction = iota +) + +func makeFunctionReferenceResolver(fissionClient *tpr.FissionClient) *functionReferenceResolver { + return &functionReferenceResolver{ + fissionClient: fissionClient, + refCache: cache.MakeCache(time.Minute, 0), + } +} + +// resolve translates a namespace and a function reference to resolveResult. +// The resolveResult for now is just a function's metadata. In the future, some +// function ref types may resolve to two functions rather than just one +// (e.g. for incremental deployment), which will make the resolveResult a bit +// more complex. +func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.FunctionReference) (*resolveResult, error) { + nfr := namespacedFunctionReference{ + namespace: namespace, + functionReference: *fr, + } + + // check cache + rrInt, err := frr.refCache.Get(nfr) + if err == nil { + result := rrInt.(resolveResult) + return &result, nil + } + + // resolve on cache miss + var rr *resolveResult + + switch fr.Type { + case fission.FunctionReferenceTypeFunctionName: + rr, err = frr.resolveByName(namespace, fr.Name) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Unrecognized function reference type %v", fr.Type) + } + + // cache resolve result + frr.refCache.Set(nfr, *rr) + + return rr, nil +} + +// resolveByName simply looks up function by name in a namespace. +func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) { + f, err := frr.fissionClient.Functions(namespace).Get(name) + if err != nil { + return nil, err + } + rr := resolveResult{ + resolveResultType: resolveResultSingleFunction, + functionMetadata: &f.Metadata, + } + return &rr, nil +} diff --git a/router/functionServiceMap.go b/router/functionServiceMap.go index 6b08adf7..c0548e53 100644 --- a/router/functionServiceMap.go +++ b/router/functionServiceMap.go @@ -21,13 +21,24 @@ import ( "net/url" "time" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission/cache" ) -type functionServiceMap struct { - cache *cache.Cache // map[fission.Metadata]*url.URL -} +type ( + functionServiceMap struct { + cache *cache.Cache // map[metadataKey]*url.URL + } + + // api.ObjectMeta is not hashable, so we make a hashable copy + // of the subset of its fields that are identifiable. + metadataKey struct { + Name string + Namespace string + ResourceVersion string + } +) func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { return &functionServiceMap{ @@ -35,8 +46,17 @@ func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { } } -func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) { - item, err := fmap.cache.Get(*f) +func keyFromMetadata(m *api.ObjectMeta) *metadataKey { + return &metadataKey{ + Name: m.Name, + Namespace: m.Namespace, + ResourceVersion: m.ResourceVersion, + } +} + +func (fmap *functionServiceMap) lookup(f *api.ObjectMeta) (*url.URL, error) { + mk := keyFromMetadata(f) + item, err := fmap.cache.Get(*mk) if err != nil { return nil, err } @@ -44,8 +64,9 @@ func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) { return u, nil } -func (fmap *functionServiceMap) assign(f *fission.Metadata, serviceUrl *url.URL) { - err, old := fmap.cache.Set(*f, serviceUrl) +func (fmap *functionServiceMap) assign(f *api.ObjectMeta, serviceUrl *url.URL) { + mk := keyFromMetadata(f) + err, old := fmap.cache.Set(*mk, serviceUrl) if err != nil { if *serviceUrl == *(old.(*url.URL)) { return diff --git a/router/functionServiceMap_test.go b/router/functionServiceMap_test.go index a48221f8..61b3d930 100644 --- a/router/functionServiceMap_test.go +++ b/router/functionServiceMap_test.go @@ -20,12 +20,12 @@ import ( "net/url" "testing" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" ) func TestFunctionServiceMap(t *testing.T) { m := makeFunctionServiceMap(0) - fn := &fission.Metadata{Name: "foo", Uid: "012"} + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} u, err := url.Parse("/foo012") if err != nil { t.Errorf("can't parse url") diff --git a/router/httpTriggers.go b/router/httpTriggers.go index e8b0744f..2b8eec89 100644 --- a/router/httpTriggers.go +++ b/router/httpTriggers.go @@ -22,34 +22,44 @@ import ( "time" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" "github.com/fission/fission" - controllerClient "github.com/fission/fission/controller/client" poolmgrClient "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" ) type HTTPTriggerSet struct { *functionServiceMap *mutableRouter - controller *controllerClient.Client - poolmgr *poolmgrClient.Client - triggers []fission.HTTPTrigger - functions []fission.Function + fissionClient *tpr.FissionClient + poolmgr *poolmgrClient.Client + resolver *functionReferenceResolver + triggers []tpr.Httptrigger + functions []tpr.Function } -func makeHTTPTriggerSet(fmap *functionServiceMap, controller *controllerClient.Client, poolmgr *poolmgrClient.Client) *HTTPTriggerSet { - triggers := make([]fission.HTTPTrigger, 1) +func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClient, poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver) *HTTPTriggerSet { + triggers := make([]tpr.Httptrigger, 1) return &HTTPTriggerSet{ functionServiceMap: fmap, triggers: triggers, - controller: controller, + fissionClient: fissionClient, poolmgr: poolmgr, + resolver: resolver, } } func (ts *HTTPTriggerSet) subscribeRouter(mr *mutableRouter) { ts.mutableRouter = mr mr.updateRouter(ts.getRouter()) + + if ts.fissionClient == nil { + // Used in tests only. + log.Printf("Skipping continuous trigger updates") + return + } go ts.watchTriggers() } @@ -60,27 +70,33 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) { func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter := mux.NewRouter() - // make a function name -> latest version map - latestVersions := make(map[string]string) - for _, f := range ts.functions { - latestVersions[f.Metadata.Name] = f.Metadata.Uid - } - // HTTP triggers setup by the user homeHandled := false for _, trigger := range ts.triggers { - m := trigger.Function - if len(m.Uid) == 0 { - // explicitly use the latest function version - m.Uid = latestVersions[m.Name] + + // resolve function reference + rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference) + if err != nil { + // Unresolvable function reference. Report the error via + // the trigger's status. + go ts.updateTriggerStatusFailed(&trigger, err) + + // Ignore this route and let it 404. + continue } + + if rr.resolveResultType != resolveResultSingleFunction { + // not implemented yet + log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType) + } + fh := &functionHandler{ fmap: ts.functionServiceMap, - Function: m, + function: rr.functionMetadata, poolmgr: ts.poolmgr, } - muxRouter.HandleFunc(trigger.UrlPattern, fh.handler).Methods(trigger.Method) - if trigger.UrlPattern == "/" && trigger.Method == "GET" { + muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method) + if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" { homeHandled = true } } @@ -95,53 +111,74 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET") } - // Internal triggers for (the latest version of) each function + // Internal triggers for each function by name. Non-http + // triggers route into these. for _, function := range ts.functions { - m := fission.Metadata{Name: function.Metadata.Name} fh := &functionHandler{ fmap: ts.functionServiceMap, - Function: function.Metadata, + function: &function.Metadata, poolmgr: ts.poolmgr, } - muxRouter.HandleFunc(fission.UrlForFunction(&m), fh.handler) + muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler) } return muxRouter } +func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *tpr.Httptrigger, err error) { + // TODO +} + func (ts *HTTPTriggerSet) watchTriggers() { - if ts.controller == nil { - return - } - - // the number of connection failures we'll accept before quitting - maxFailures := 5 - - // amount of time to sleep between polling calls - pollSleepDuration := 3 * time.Second + // sync all http triggers + ts.syncTriggers() // Watch controller for updates to triggers and update the router accordingly. - // TODO change this to use a watch API; or maybe even watch etcd directly. - failureCount := 0 + rv := "" for { - triggers, err := ts.controller.HTTPTriggerList() + wi, err := ts.fissionClient.Httptriggers(api.NamespaceAll).Watch(api.ListOptions{ + ResourceVersion: rv, + }) if err != nil { - failureCount += 1 - if failureCount >= maxFailures { - log.Fatalf("Failed to connect to controller after %v retries: %v", failureCount, err) + log.Fatalf("Failed to watch http trigger list: %v", err) + } + + for { + ev, more := <-wi.ResultChan() + if !more { + // restart watch from last rv + break } - time.Sleep(pollSleepDuration) - continue + if ev.Type == watch.Error { + // restart watch from the start + rv = "" + time.Sleep(time.Second) + break + } + ht := ev.Object.(*tpr.Httptrigger) + rv = ht.Metadata.ResourceVersion + ts.syncTriggers() } - ts.triggers = triggers - - functions, err := ts.controller.FunctionList() - if err != nil { - log.Fatalf("Failed to get function list") - } - ts.functions = functions - - ts.mutableRouter.updateRouter(ts.getRouter()) - time.Sleep(pollSleepDuration) } } + +func (ts *HTTPTriggerSet) syncTriggers() { + log.Printf("Syncing http triggers") + + // get triggers + triggers, err := ts.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) + if err != nil { + log.Fatalf("Failed to get http trigger list: %v", err) + } + ts.triggers = triggers.Items + + // get functions + functions, err := ts.fissionClient.Functions(api.NamespaceAll).List(api.ListOptions{}) + if err != nil { + log.Fatalf("Failed to get function list: %v", err) + } + ts.functions = functions.Items + + // make a new router and use it + ts.mutableRouter.updateRouter(ts.getRouter()) +} diff --git a/router/router.go b/router/router.go index 01cccf56..bc2ef636 100644 --- a/router/router.go +++ b/router/router.go @@ -49,8 +49,8 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" - controllerClient "github.com/fission/fission/controller/client" poolmgrClient "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" ) // request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url @@ -70,12 +70,16 @@ func serve(port int, httpTriggerSet *HTTPTriggerSet) { http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr)) } -func Start(port int, controllerUrl string, poolmgrUrl string) { +func Start(port int, poolmgrUrl string) { fmap := makeFunctionServiceMap(time.Minute) - controller := controllerClient.MakeClient(controllerUrl) - poolmgr := poolmgrClient.MakeClient(poolmgrUrl) - triggers := makeHTTPTriggerSet(fmap, controller, poolmgr) + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Error connecting to kubernetes API: %v", err) + } + poolmgr := poolmgrClient.MakeClient(poolmgrUrl) + resolver := makeFunctionReferenceResolver(fissionClient) + triggers := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, resolver) log.Printf("Starting router at port %v\n", port) serve(port, triggers) } diff --git a/router/router_test.go b/router/router_test.go index 59b5c7db..5a848e66 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -21,26 +21,64 @@ import ( "testing" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func TestRouter(t *testing.T) { - fmap := makeFunctionServiceMap(0) - fn := &fission.Metadata{Name: "foo", Uid: "xxx"} + // metadata for a fake function + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} + // and a reference to it + fr := fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fn.Name, + } + + // start a fake service testResponseString := "hi" testServiceUrl := createBackendService(testResponseString) + // set up the cache with this fake service + fmap := makeFunctionServiceMap(0) fmap.assign(fn, testServiceUrl) - triggers := makeHTTPTriggerSet(fmap, nil, nil) - triggerUrl := "/foo" - triggers.triggers = append(triggers.triggers, fission.HTTPTrigger{UrlPattern: triggerUrl, Function: *fn, Method: "GET"}) + // set up the resolver's cache for this function + frr := makeFunctionReferenceResolver(nil) + nfr := namespacedFunctionReference{ + namespace: api.NamespaceDefault, + functionReference: fr, + } + rr := resolveResult{ + resolveResultType: resolveResultSingleFunction, + functionMetadata: fn, + } + frr.refCache.Set(nfr, rr) + // HTTP trigger set with a trigger for this function + triggers := makeHTTPTriggerSet(fmap, nil, nil, frr) + triggerUrl := "/foo" + triggers.triggers = append(triggers.triggers, + tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, + }, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + FunctionReference: fr, + Method: "GET", + }, + }) + + // run the router port := 4242 go serve(port, triggers) time.Sleep(100 * time.Millisecond) + // hit the router testUrl := fmt.Sprintf("http://localhost:%v%v", port, triggerUrl) testRequest(testUrl, testResponseString) } diff --git a/timer/main.go b/timer/main.go index aff393e8..301464d2 100644 --- a/timer/main.go +++ b/timer/main.go @@ -17,14 +17,18 @@ limitations under the License. package timer import ( - controllerClient "github.com/fission/fission/controller/client" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) -func Start(controllerUrl string, routerUrl string) error { - controller := controllerClient.MakeClient(controllerUrl) +func Start(routerUrl string) error { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + return err + } + poster := publisher.MakeWebhookPublisher(routerUrl) - MakeTimerSync(controller, MakeTimer(poster)) + MakeTimerSync(fissionClient, MakeTimer(poster)) return nil } diff --git a/timer/timer.go b/timer/timer.go index 1b61544d..3118cfc0 100644 --- a/timer/timer.go +++ b/timer/timer.go @@ -20,9 +20,11 @@ import ( "log" "github.com/robfig/cron" + // "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) type requestType int @@ -40,14 +42,14 @@ type ( timerRequest struct { requestType - triggers []fission.TimeTrigger + triggers []tpr.Timetrigger responseChannel chan *timerResponse } timerResponse struct { error } timerTriggerWithCron struct { - trigger fission.TimeTrigger + trigger tpr.Timetrigger cron *cron.Cron } ) @@ -62,7 +64,7 @@ func MakeTimer(publisher publisher.Publisher) *Timer { return timer } -func (timer *Timer) Sync(triggers []fission.TimeTrigger) error { +func (timer *Timer) Sync(triggers []tpr.Timetrigger) error { req := &timerRequest{ requestType: SYNC, triggers: triggers, @@ -84,16 +86,21 @@ func (timer *Timer) svc() { } } -func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { +func (timer *Timer) syncCron(triggers []tpr.Timetrigger) error { + // add new triggers or update existing ones for _, t := range triggers { - if item, ok := timer.triggers[t.Name]; ok { + if item, ok := timer.triggers[t.Metadata.Name]; ok { // the item exists, update the item if needed - if item.trigger.Uid == t.Uid { + + // if both UID and ResourceVersion match, the + // two triggers are identical + if item.trigger.Metadata.UID == t.Metadata.UID && + item.trigger.Metadata.ResourceVersion == t.Metadata.ResourceVersion { continue } // update cron if the cron spec changed - if item.trigger.Cron != t.Cron { + if item.trigger.Spec.Cron != t.Spec.Cron { // if there is an cron running, stop it if item.cron != nil { item.cron.Stop() @@ -103,17 +110,18 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { item.trigger = t } else { - timer.triggers[t.Name] = &timerTriggerWithCron{ + timer.triggers[t.Metadata.Name] = &timerTriggerWithCron{ trigger: t, cron: timer.newCron(t), } } } + // process removed triggers for k, v := range timer.triggers { found := false for _, t := range triggers { - if t.Name == k { + if t.Metadata.Name == k { found = true break } @@ -121,7 +129,7 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { if !found { if v.cron != nil { v.cron.Stop() - log.Printf("Cron for time trigger %s stopped", v.trigger.Name) + log.Printf("Cron for time trigger %s stopped", v.trigger.Metadata.Name) } delete(timer.triggers, k) } @@ -130,15 +138,15 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { return nil } -func (timer *Timer) newCron(t fission.TimeTrigger) *cron.Cron { +func (timer *Timer) newCron(t tpr.Timetrigger) *cron.Cron { c := cron.New() - c.AddFunc(t.Cron, func() { + c.AddFunc(t.Spec.Cron, func() { headers := map[string]string{ - "X-Fission-Timer-Name": t.Name, + "X-Fission-Timer-Name": t.Metadata.Name, } - (*timer.publisher).Publish("", headers, fission.UrlForFunction(&t.Function)) + (*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name)) }) c.Start() - log.Printf("Add new cron for time trigger %v", t.Name) + log.Printf("Add new cron for time trigger %v", t.Metadata.Name) return c } diff --git a/timer/timerSync.go b/timer/timerSync.go index 692d37b9..0aa018b4 100644 --- a/timer/timerSync.go +++ b/timer/timerSync.go @@ -20,39 +20,36 @@ import ( "log" "time" - controllerClient "github.com/fission/fission/controller/client" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" ) type ( TimerSync struct { - controller *controllerClient.Client - timer *Timer + fissionClient *tpr.FissionClient + timer *Timer } ) -func MakeTimerSync(controller *controllerClient.Client, timer *Timer) *TimerSync { +func MakeTimerSync(fissionClient *tpr.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ - controller: controller, - timer: timer, + fissionClient: fissionClient, + timer: timer, } go ws.syncSvc() return ws } func (ws *TimerSync) syncSvc() { - failureCount := 0 - maxFailures := 6 for { - triggers, err := ws.controller.TimeTriggerList() + triggers, err := ws.fissionClient.Timetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - failureCount++ - if failureCount > maxFailures { - log.Fatalf("Failed to connect to controller: %v", err) - } - time.Sleep(10 * time.Second) - continue + log.Fatalf("Failed get time trigger list: %v", err) } - ws.timer.Sync(triggers) + ws.timer.Sync(triggers.Items) + + // TODO switch to watches time.Sleep(3 * time.Second) } } diff --git a/tpr/client.go b/tpr/client.go new file mode 100644 index 00000000..f5f41066 --- /dev/null +++ b/tpr/client.go @@ -0,0 +1,198 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "errors" + "os" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/unversioned" + "k8s.io/client-go/1.5/pkg/runtime" + "k8s.io/client-go/1.5/pkg/runtime/serializer" + "k8s.io/client-go/1.5/rest" + "k8s.io/client-go/1.5/tools/clientcmd" +) + +type ( + FissionClient struct { + tprClient *rest.RESTClient + } +) + +// Get a kubernetes client using the kubeconfig file at the +// environment var $KUBECONFIG, or an in-cluster config if that's +// undefined. +func GetKubernetesClient() (*rest.Config, *kubernetes.Clientset, error) { + var config *rest.Config + var err error + + // get the config, either from kubeconfig or using our + // in-cluster service account + kubeConfig := os.Getenv("KUBECONFIG") + if len(kubeConfig) != 0 { + config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return nil, nil, err + } + } else { + config, err = rest.InClusterConfig() + if err != nil { + return nil, nil, err + } + } + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, nil, err + } + + return config, clientset, nil +} + +// GetTprClient gets a TPR client config +func GetTprClient(config *rest.Config) (*rest.RESTClient, error) { + // mutate config to add our types + configureClient(config) + + // make a REST client with that config + return rest.RESTClientFor(config) +} + +// configureClient sets up a REST client for Fission TPR types. +// +// This is copied from the client-go TPR example. (I don't understand +// all of it completely.) It registers our types with the global API +// "scheme" (api.Scheme), which keeps a directory of types [I guess so +// it can use the string in the Kind field to make a Go object?]. It +// also puts the fission TPR types under a "group version" which we +// create for our TPRs types. +func configureClient(config *rest.Config) { + groupversion := unversioned.GroupVersion{ + Group: "fission.io", + Version: "v1", + } + config.GroupVersion = &groupversion + config.APIPath = "/apis" + config.ContentType = runtime.ContentTypeJSON + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs} + + schemeBuilder := runtime.NewSchemeBuilder( + func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes( + groupversion, + &Function{}, + &FunctionList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Environment{}, + &EnvironmentList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Httptrigger{}, + &HttptriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Kuberneteswatchtrigger{}, + &KuberneteswatchtriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Timetrigger{}, + &TimetriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Messagequeuetrigger{}, + &MessagequeuetriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + return nil + }) + schemeBuilder.AddToScheme(api.Scheme) +} + +func waitForTPRs(tprClient *rest.RESTClient) error { + start := time.Now() + for { + fi := MakeFunctionInterface(tprClient, api.NamespaceDefault) + _, err := fi.List(api.ListOptions{}) + if err != nil { + time.Sleep(100 * time.Millisecond) + } else { + return nil + } + + if time.Now().Sub(start) > 30*time.Second { + return errors.New("timeout waiting for TPRs") + } + } +} + +func MakeFissionClient() (*FissionClient, *kubernetes.Clientset, error) { + config, kubeClient, err := GetKubernetesClient() + if err != nil { + return nil, nil, err + } + tprClient, err := GetTprClient(config) + if err != nil { + return nil, nil, err + } + fc := &FissionClient{ + tprClient: tprClient, + } + return fc, kubeClient, nil +} + +func (fc *FissionClient) Functions(ns string) FunctionInterface { + return MakeFunctionInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Environments(ns string) EnvironmentInterface { + return MakeEnvironmentInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Httptriggers(ns string) HttptriggerInterface { + return MakeHttptriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Kuberneteswatchtriggers(ns string) KuberneteswatchtriggerInterface { + return MakeKuberneteswatchtriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Timetriggers(ns string) TimetriggerInterface { + return MakeTimetriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Messagequeuetriggers(ns string) MessagequeuetriggerInterface { + return MakeMessagequeuetriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) WaitForTPRs() { + waitForTPRs(fc.tprClient) +} diff --git a/tpr/environment.go b/tpr/environment.go new file mode 100644 index 00000000..4367693d --- /dev/null +++ b/tpr/environment.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + EnvironmentInterface interface { + Create(*Environment) (*Environment, error) + Get(name string) (*Environment, error) + Update(*Environment) (*Environment, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*EnvironmentList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + environmentClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeEnvironmentInterface(tprClient *rest.RESTClient, namespace string) EnvironmentInterface { + return &environmentClient{ + client: tprClient, + namespace: namespace, + } +} + +func (ec *environmentClient) Create(e *Environment) (*Environment, error) { + var result Environment + err := ec.client.Post(). + Resource("environments"). + Namespace(ec.namespace). + Body(e). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Get(name string) (*Environment, error) { + var result Environment + err := ec.client.Get(). + Resource("environments"). + Namespace(ec.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Update(e *Environment) (*Environment, error) { + var result Environment + err := ec.client.Put(). + Resource("environments"). + Namespace(ec.namespace). + Name(e.Metadata.Name). + Body(e). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Delete(name string, opts *api.DeleteOptions) error { + return ec.client.Delete(). + Namespace(ec.namespace). + Resource("environments"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (ec *environmentClient) List(opts api.ListOptions) (*EnvironmentList, error) { + var result EnvironmentList + err := ec.client.Get(). + Namespace(ec.namespace). + Resource("environments"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return ec.client.Get(). + Prefix("watch"). + Namespace(ec.namespace). + Resource("environments"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/function.go b/tpr/function.go new file mode 100644 index 00000000..186301ac --- /dev/null +++ b/tpr/function.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + FunctionInterface interface { + Create(*Function) (*Function, error) + Get(name string) (*Function, error) + Update(*Function) (*Function, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*FunctionList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + functionClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeFunctionInterface(tprClient *rest.RESTClient, namespace string) FunctionInterface { + return &functionClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *functionClient) Create(f *Function) (*Function, error) { + var result Function + err := fc.client.Post(). + Resource("functions"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Get(name string) (*Function, error) { + var result Function + err := fc.client.Get(). + Resource("functions"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Update(f *Function) (*Function, error) { + var result Function + err := fc.client.Put(). + Resource("functions"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("functions"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *functionClient) List(opts api.ListOptions) (*FunctionList, error) { + var result FunctionList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("functions"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("functions"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/httptrigger.go b/tpr/httptrigger.go new file mode 100644 index 00000000..c70cf36e --- /dev/null +++ b/tpr/httptrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + HttptriggerInterface interface { + Create(*Httptrigger) (*Httptrigger, error) + Get(name string) (*Httptrigger, error) + Update(*Httptrigger) (*Httptrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*HttptriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + httpTriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeHttptriggerInterface(tprClient *rest.RESTClient, namespace string) HttptriggerInterface { + return &httpTriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (c *httpTriggerClient) Create(obj *Httptrigger) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Post(). + Resource("httptriggers"). + Namespace(c.namespace). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Get(name string) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Get(). + Resource("httptriggers"). + Namespace(c.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Update(obj *Httptrigger) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Put(). + Resource("httptriggers"). + Namespace(c.namespace). + Name(obj.Metadata.Name). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.namespace). + Resource("httptriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (c *httpTriggerClient) List(opts api.ListOptions) (*HttptriggerList, error) { + var result HttptriggerList + err := c.client.Get(). + Namespace(c.namespace). + Resource("httptriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return c.client.Get(). + Prefix("watch"). + Namespace(c.namespace). + Resource("httptriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/resource.go b/tpr/key.go similarity index 52% rename from resource.go rename to tpr/key.go index ccfbf35f..0cb64407 100644 --- a/resource.go +++ b/tpr/key.go @@ -14,28 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fission +package tpr -func (f Function) Key() string { - return f.Metadata.Name -} +import ( + "fmt" -func (e Environment) Key() string { - return e.Metadata.Name -} + "k8s.io/client-go/1.5/pkg/api" +) -func (ht HTTPTrigger) Key() string { - return ht.Metadata.Name -} - -func (mqt MessageQueueTrigger) Key() string { - return mqt.Metadata.Name -} - -func (tt TimeTrigger) Key() string { - return tt.Metadata.Name -} - -func (w Watch) Key() string { - return w.Metadata.Name +// Given metadata, create a key that uniquely identifies the contents +// of the object. Since resourceVersion changes on every update and +// UIDs are unique, uid+resourceVersion identifies the +// content. (ResourceVersion may also update on status updates, so +// this will result in some unnecessary cache misses. That should be +// ok.) +func CacheKey(metadata *api.ObjectMeta) string { + return fmt.Sprintf("%v_%v", metadata.UID, metadata.ResourceVersion) } diff --git a/tpr/kubernetesWatchTrigger.go b/tpr/kubernetesWatchTrigger.go new file mode 100644 index 00000000..3255fd21 --- /dev/null +++ b/tpr/kubernetesWatchTrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + KuberneteswatchtriggerInterface interface { + Create(*Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) + Get(name string) (*Kuberneteswatchtrigger, error) + Update(*Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*KuberneteswatchtriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + kubernetesWatchTriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeKuberneteswatchtriggerInterface(tprClient *rest.RESTClient, namespace string) KuberneteswatchtriggerInterface { + return &kubernetesWatchTriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (c *kubernetesWatchTriggerClient) Create(obj *Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Post(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Get(name string) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Get(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Update(obj *Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Put(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Name(obj.Metadata.Name). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (c *kubernetesWatchTriggerClient) List(opts api.ListOptions) (*KuberneteswatchtriggerList, error) { + var result KuberneteswatchtriggerList + err := c.client.Get(). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return c.client.Get(). + Prefix("watch"). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/messagequeuetrigger.go b/tpr/messagequeuetrigger.go new file mode 100644 index 00000000..61f08e8e --- /dev/null +++ b/tpr/messagequeuetrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + MessagequeuetriggerInterface interface { + Create(*Messagequeuetrigger) (*Messagequeuetrigger, error) + Get(name string) (*Messagequeuetrigger, error) + Update(*Messagequeuetrigger) (*Messagequeuetrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*MessagequeuetriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + messagequeuetriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeMessagequeuetriggerInterface(tprClient *rest.RESTClient, namespace string) MessagequeuetriggerInterface { + return &messagequeuetriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *messagequeuetriggerClient) Create(f *Messagequeuetrigger) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Post(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Get(name string) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Get(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Update(f *Messagequeuetrigger) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Put(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *messagequeuetriggerClient) List(opts api.ListOptions) (*MessagequeuetriggerList, error) { + var result MessagequeuetriggerList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/timetrigger.go b/tpr/timetrigger.go new file mode 100644 index 00000000..600bfe56 --- /dev/null +++ b/tpr/timetrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + TimetriggerInterface interface { + Create(*Timetrigger) (*Timetrigger, error) + Get(name string) (*Timetrigger, error) + Update(*Timetrigger) (*Timetrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*TimetriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + timetriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeTimetriggerInterface(tprClient *rest.RESTClient, namespace string) TimetriggerInterface { + return &timetriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *timetriggerClient) Create(f *Timetrigger) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Post(). + Resource("timetriggers"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Get(name string) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Get(). + Resource("timetriggers"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Update(f *Timetrigger) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Put(). + Resource("timetriggers"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("timetriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *timetriggerClient) List(opts api.ListOptions) (*TimetriggerList, error) { + var result TimetriggerList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("timetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("timetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/tpr.go b/tpr/tpr.go new file mode 100644 index 00000000..8620a855 --- /dev/null +++ b/tpr/tpr.go @@ -0,0 +1,106 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api/errors" + "k8s.io/client-go/1.5/pkg/api/v1" + "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" +) + +// ensureTPR checks if the given TPR type exists, and creates it if +// needed. (Note that this creates the TPR type; it doesn't create any +// _instances_ of that type.) +func ensureTPR(clientset *kubernetes.Clientset, tpr *v1beta1.ThirdPartyResource) error { + _, err := clientset.Extensions().ThirdPartyResources().Get(tpr.ObjectMeta.Name) + if err != nil { + if errors.IsNotFound(err) { + _, err := clientset.Extensions().ThirdPartyResources().Create(tpr) + if err != nil { + return err + } + } + } + return nil +} + +func EnsureFissionTPRs(clientset *kubernetes.Clientset) error { + tprs := []v1beta1.ThirdPartyResource{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "function.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "environment.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Environments (function containers)", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "httptrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "HTTP triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "kuberneteswatchtrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Kubernetes watch triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "timetrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Time-based triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "messagequeuetrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Messqge queue triggers for functions", + }, + } + for _, tpr := range tprs { + err := ensureTPR(clientset, &tpr) + if err != nil { + return err + } + } + return nil +} diff --git a/tpr/tpr_test.go b/tpr/tpr_test.go new file mode 100644 index 00000000..cda5646b --- /dev/null +++ b/tpr/tpr_test.go @@ -0,0 +1,429 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "log" + "os" + "testing" + "time" + + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/unversioned" + "k8s.io/client-go/1.5/rest" + + "github.com/fission/fission" +) + +func panicIf(err error) { + if err != nil { + log.Panicf("err: %v", err) + } +} + +func functionTests(tprClient *rest.RESTClient) { + // sample function object + function := &Function{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Function", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.FunctionSpec{ + Source: fission.Package{}, + Deployment: fission.Package{ + Literal: []byte("hi"), + }, + EnvironmentName: "xxx", + }, + } + + // Test function CRUD + fi := MakeFunctionInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + fi.Delete(function.Metadata.Name, nil) + + // create + f, err := fi.Create(function) + panicIf(err) + if f.Metadata.Name != function.Metadata.Name { + log.Panicf("Bad result from create: %v", f) + } + + // read + f, err = fi.Get(function.Metadata.Name) + panicIf(err) + if len(f.Spec.Deployment.Literal) != len(function.Spec.Deployment.Literal) { + log.Panicf("Bad result from Get: %v", f) + } + + log.Printf("f.Metadata = %#v", f.Metadata) + + // update + function.Spec.EnvironmentName = "yyy" + f, err = fi.Update(function) + panicIf(err) + + log.Printf("f.Metadata = %#v", f.Metadata) + + // list + fl, err := fi.List(api.ListOptions{}) + panicIf(err) + if len(fl.Items) != 1 { + log.Panicf("wrong count from list: %v", fl) + } + if fl.Items[0].Spec.EnvironmentName != function.Spec.EnvironmentName { + log.Panicf("bad object from list: %v", fl.Items[0]) + } + + // delete + err = fi.Delete(f.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := fi.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + f, err = fi.Create(function) + panicIf(err) + defer fi.Delete(f.Metadata.Name, nil) + + // assert that we get a watch event for the new function + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + wf, ok := ev.Object.(*Function) + if !ok { + log.Panicf("Can't cast to Function") + } + if wf.Spec.EnvironmentName != function.Spec.EnvironmentName { + log.Panicf("Bad object from watch: %#v", wf) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func environmentTests(tprClient *rest.RESTClient) { + // sample environment object + environment := &Environment{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Environment", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.EnvironmentSpec{ + Runtime: fission.Runtime{ + Image: "xxx", + }, + Builder: fission.Builder{ + Image: "yyy", + Command: "zzz", + }, + }, + } + + // Test environment CRUD + ei := MakeEnvironmentInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(environment.Metadata.Name, nil) + + // create + e, err := ei.Create(environment) + panicIf(err) + if e.Metadata.Name != environment.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(environment.Metadata.Name) + panicIf(err) + if len(e.Spec.Runtime.Image) != len(environment.Spec.Runtime.Image) { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + environment.Spec.Runtime.Image = "www" + e, err = ei.Update(environment) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Runtime.Image != environment.Spec.Runtime.Image { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(environment) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new environment + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Environment) + if !ok { + log.Panicf("Can't cast to Environment") + } + if obj.Spec.Runtime.Image != environment.Spec.Runtime.Image { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func httpTriggerTests(tprClient *rest.RESTClient) { + // sample httpTrigger object + httpTrigger := &Httptrigger{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Httptrigger", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: "/hi", + Method: "GET", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "hello", + }, + }, + } + + // Test httpTrigger CRUD + ei := MakeHttptriggerInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(httpTrigger.Metadata.Name, nil) + + // create + e, err := ei.Create(httpTrigger) + panicIf(err) + if e.Metadata.Name != httpTrigger.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(httpTrigger.Metadata.Name) + panicIf(err) + if len(e.Spec.Method) != len(httpTrigger.Spec.Method) { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + httpTrigger.Spec.Method = "POST" + e, err = ei.Update(httpTrigger) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Method != httpTrigger.Spec.Method { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(httpTrigger) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new httpTrigger + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Httptrigger) + if !ok { + log.Panicf("Can't cast to Httptrigger") + } + if obj.Spec.Method != httpTrigger.Spec.Method { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func kubernetesWatchTriggerTests(tprClient *rest.RESTClient) { + // sample kubernetesWatchTrigger object + kubernetesWatchTrigger := &Kuberneteswatchtrigger{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Kuberneteswatchtrigger", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: "foo", + Type: "pod", + LabelSelector: map[string]string{ + "x": "y", + }, + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, + }, + } + + // Test kubernetesWatchTrigger CRUD + ei := MakeKuberneteswatchtriggerInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(kubernetesWatchTrigger.Metadata.Name, nil) + + // create + e, err := ei.Create(kubernetesWatchTrigger) + panicIf(err) + if e.Metadata.Name != kubernetesWatchTrigger.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(kubernetesWatchTrigger.Metadata.Name) + panicIf(err) + if e.Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + kubernetesWatchTrigger.Spec.Type = "service" + e, err = ei.Update(kubernetesWatchTrigger) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(kubernetesWatchTrigger) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new kubernetesWatchTrigger + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Kuberneteswatchtrigger) + if !ok { + log.Panicf("Can't cast to Kuberneteswatchtrigger") + } + if obj.Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func TestTpr(t *testing.T) { + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + log.Println("Skipping test, no kubernetes cluster") + return + } + + // Create the client config. Needs the KUBECONFIG env var to + // point at a valid kubeconfig. + config, clientset, err := GetKubernetesClient() + panicIf(err) + + // init our types + err = EnsureFissionTPRs(clientset) + panicIf(err) + + // rest client with knowledge about our tpr types + tprClient, err := GetTprClient(config) + panicIf(err) + + err = waitForTPRs(tprClient) + panicIf(err) + + functionTests(tprClient) + environmentTests(tprClient) + httpTriggerTests(tprClient) + kubernetesWatchTriggerTests(tprClient) +} diff --git a/tpr/types.go b/tpr/types.go new file mode 100644 index 00000000..84c04b2e --- /dev/null +++ b/tpr/types.go @@ -0,0 +1,208 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/meta" + "k8s.io/client-go/1.5/pkg/api/unversioned" + + "github.com/fission/fission" +) + +// +// To add a Fission TPR type: +// 1. Create a "spec" type, for everything in the type except metadata +// 2. Create the type with metadata + the spec +// 3. Create a list type (for example see FunctionList and Function, below) +// 4. Add methods at the bottom of this file for satisfying Object and List interfaces +// 5. Add the type to configureClient in client.go +// 6. Add the type to EnsureFissionTPRs in tpr.go +// 7. Add tests to tpr_test.go +// + +type ( + // Functions. + Function struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.FunctionSpec `json:"spec"` + } + FunctionList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Function `json:"items"` + } + + // Environments. + Environment struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.EnvironmentSpec `json:"spec"` + } + EnvironmentList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Environment `json:"items"` + } + + // HTTP Triggers. (Something in the TPR reflection stuff wants + // it to be spelled "Httptrigger" not "HTTPTrigger" or even + // "HttpTrigger". Bleh.) + Httptrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.HTTPTriggerSpec `json:"spec"` + } + HttptriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Httptrigger `json:"items"` + } + + // Kubernetes Watches as triggers + Kuberneteswatchtrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.KubernetesWatchTriggerSpec `json:"spec"` + } + KuberneteswatchtriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Kuberneteswatchtrigger `json:"items"` + } + + // Time triggers + Timetrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.TimeTriggerSpec `json:"spec"` + } + TimetriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Timetrigger `json:"items"` + } + + // Message Queue triggers + Messagequeuetrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.MessageQueueTriggerSpec `json:"spec"` + } + MessagequeuetriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Messagequeuetrigger `json:"items"` + } +) + +// Each TPR type needs: +// GetObjectKind (to satisfy the Object interface) +// +// In addition, each singular TPR type needs: +// GetObjectMeta (to satisfy the ObjectMetaAccessor interface) +// +// And each list TPR type needs: +// GetListMeta (to satisfy the ListMetaAccessor interface) + +func (f *Function) GetObjectKind() unversioned.ObjectKind { + return &f.TypeMeta +} +func (e *Environment) GetObjectKind() unversioned.ObjectKind { + return &e.TypeMeta +} +func (ht *Httptrigger) GetObjectKind() unversioned.ObjectKind { + return &ht.TypeMeta +} +func (w *Kuberneteswatchtrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} +func (w *Timetrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} +func (w *Messagequeuetrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} + +func (f *Function) GetObjectMeta() meta.Object { + return &f.Metadata +} +func (e *Environment) GetObjectMeta() meta.Object { + return &e.Metadata +} +func (ht *Httptrigger) GetObjectMeta() meta.Object { + return &ht.Metadata +} +func (w *Kuberneteswatchtrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} +func (w *Timetrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} +func (w *Messagequeuetrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} + +func (fl *FunctionList) GetObjectKind() unversioned.ObjectKind { + return &fl.TypeMeta +} +func (el *EnvironmentList) GetObjectKind() unversioned.ObjectKind { + return &el.TypeMeta +} +func (hl *HttptriggerList) GetObjectKind() unversioned.ObjectKind { + return &hl.TypeMeta +} +func (wl *KuberneteswatchtriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} +func (wl *TimetriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} +func (wl *MessagequeuetriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} + +func (fl *FunctionList) GetListMeta() unversioned.List { + return &fl.Metadata +} +func (el *EnvironmentList) GetListMeta() unversioned.List { + return &el.Metadata +} +func (hl *HttptriggerList) GetListMeta() unversioned.List { + return &hl.Metadata +} +func (wl *KuberneteswatchtriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} +func (wl *TimetriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} +func (wl *MessagequeuetriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} + +// In the client-go TPR example, UnmarshalJSON is defined here for the +// singular and list types. That's supposed to be a workaround for +// some ugorji bug. But we don't seem to need it, and all our tests +// pass without it, so we don't define any UnmarshalJSON methods. diff --git a/types.go b/types.go index 91575c8f..5f70a455 100644 --- a/types.go +++ b/types.go @@ -17,72 +17,155 @@ limitations under the License. package fission type ( - // Metadata is used as the general identifier for all kinds of - // resources managed by the controller. - Metadata struct { + // + // Functions and packages + // + + // ChecksumType specifies the checksum algorithm, such as + // sha256, used for a checksum. + ChecksumType string + + // Checksum of package contents when the contents are stored + // outside the Package struct. Type is the checksum algorithm; + // "sha256" is the only currently supported one. Sum is hex + // encoded. + Checksum struct { + Type ChecksumType `json:"type"` + Sum string `json:"sum"` + } + + // PackageType is either literal or URL, indicating whether + // the package is specified in the Package struct or + // externally. + PackageType string + + // Package contains or references a collection of source or + // binary files. + Package struct { + // Type defines how the package is specified: literal or URL. + Type PackageType `json:"type"` + + // Literal contents of the package. Can be used for + // encoding packages below TODO (256KB?) size. + Literal []byte `json:"literal"` + + // URL references a package. + URL string `json:"url"` + + // Checksum ensures the integrity of packages + // refereced by URL. Ignored for literals. + Checksum Checksum `json:"checksum"` + + // EntryPoint optionally specifies an entry point in + // the package. Each environment defines a default + // entry point, but that can be overridden here. + EntryPoint string `json:"entrypoint"` + } + + // FunctionSpec describes the contents of the function. + FunctionSpec struct { + // EnvironmentName is the name of the environment that this function is associated + // with. An Environment with this name should exist, otherwise the function cannot + // be invoked. + EnvironmentName string `json:"environmentName"` + + // Source is an source package for this function; it's used for the build step if + // the environment defines a build container. + Source Package `json:"source"` + + // Deployment is a deployable package for this function. This is the package that's + // loaded into the environment's runtime container. + Deployment Package `json:"deployment"` + } + + FunctionReferenceType string + + FunctionReference struct { + // Type indicates whether this function reference is by name or selector. For now, + // the only supported reference type is by name. Future reference types: + // * Function by label or annotation + // * Branch or tag of a versioned function + // * A "rolling upgrade" from one version of a function to another + Type FunctionReferenceType `json:"type"` + + // Name of the function. Name string `json:"name"` - Uid string `json:"uid,omitempty"` } - // Function is a unit of executable code. Though it's called - // a function, the code may have more than one function; it's - // usually some sort of module or package. - Function struct { - Metadata `json:"metadata"` - Environment Metadata `json:"environment"` - Code string `json:"code"` + // + // Environments + // + + Runtime struct { + // Image for containing the language runtime. + Image string `json:"image"` + + // LoadEndpointPort defines the port on which the + // server listens for function load + // requests. Optional; default 8888. + LoadEndpointPort int32 `json:"loadendpointport"` + + // LoadEndpointPath defines the relative URL on which + // the server listens for function load + // requests. Optional; default "/specialize". + LoadEndpointPath string `json:"loadendpointpath"` + + // FunctionEndpointPort defines the port on which the + // server listens for function requests. Optional; + // default 8888. + FunctionEndpointPort int32 `json:"functionendpointport"` + } + Builder struct { + Image string `json:"image"` + Command string `json:"command"` + } + EnvironmentSpec struct { + // Environment API version + Version int `json:"version"` + + // Runtime container image etc.; required + Runtime Runtime `json:"runtime"` + + // Optional + Builder Builder `json:"builder"` + + // Optional, but strongly encouraged. Used to populate + // links from UI, CLI, etc. + DocumentationURL string `json:"documentationurl"` } - // Environment identifies the language and OS specific - // resources that a function depends on. For now this - // includes only the function run container image. Later, - // this will also include build containers, as well as support - // tools like debuggers, profilers, etc. - Environment struct { - Metadata `json:"metadata"` - RunContainerImageUrl string `json:"runContainerImageUrl"` + // + // Triggers + // + + HTTPTriggerSpec struct { + Host string `json:"host"` + RelativeURL string `json:"relativeurl"` + Method string `json:"method"` + FunctionReference FunctionReference `json:"functionref"` } - // HTTPTrigger maps URL patterns to functions. Function.UID - // is optional; if absent, the latest version of the function - // will automatically be selected. - HTTPTrigger struct { - Metadata `json:"metadata"` - UrlPattern string `json:"urlpattern"` - Method string `json:"method"` - Function Metadata `json:"function"` + KubernetesWatchTriggerSpec struct { + Namespace string `json:"namespace"` + Type string `json:"type"` + LabelSelector map[string]string `json:"labelselector"` + FunctionReference FunctionReference `json:"functionref"` } - MessageQueueTrigger struct { - Metadata `json:"metadata"` - Function Metadata `json:"function"` - MessageQueueType string `json:"messageQueueType"` - Topic string `json:"topic"` - ResponseTopic string `json:"respTopic,omitempty"` - } - - // Watch is a specification of Kubernetes watch along with a URL to post events to. - Watch struct { - Metadata `json:"metadata"` - - Namespace string `json:"namespace"` - ObjType string `json:"objtype"` - LabelSelector string `json:"labelselector"` - FieldSelector string `json:"fieldselector"` - - Function Metadata `json:"function"` - - Target string `json:"target"` // Watch publish target (URL, NATS stream, etc) + // MessageQueueTriggerSpec defines a binding from a topic in a + // message queue to a function. + MessageQueueTriggerSpec struct { + FunctionReference FunctionReference `json:"functionref"` + MessageQueueType string `json:"messageQueueType"` + Topic string `json:"topic"` + ResponseTopic string `json:"respTopic,omitempty"` } // TimeTrigger invokes the specific function at a time or // times specified by a cron string. - TimeTrigger struct { - Metadata `json:"metadata"` - - Cron string `json:"cron"` - - Function Metadata `json:"function"` + TimeTriggerSpec struct { + Cron string `json:"cron"` + FunctionReference `json:"functionref"` } // Errors returned by the Fission API. @@ -94,6 +177,54 @@ type ( errorCode int ) +// +// Fission-Environment interface. The following types are not +// exposed in the Fission API, but rather used by Fission to +// talk to environments. +// +type ( + FunctionLoadRequest struct { + // FilePath is an absolute filesystem path to the + // function. What exactly is stored here is + // env-specific. Optional. + FilePath string `json:"filepath"` + + // Entrypoint has an environment-specific meaning; + // usually, it defines a function within a module + // containing multiple functions. Optional; default is + // environment-specific. + EntryPoint string `json:"entrypoint"` + + // URL to expose this function at. Optional; defaults + // to "/". + URL string `json:"url"` + } +) + +const ( + ChecksumTypeSHA256 ChecksumType = "sha256" +) + +const ( + // PackageTypeLiteral means the package contents are specified in the Literal field of + // resource itself. + PackageTypeLiteral PackageType = "literal" + + // PackageTypeUrl means the package contents are at the specified URL. + PackageTypeUrl PackageType = "url" +) + +const ( + // FunctionReferenceFunctionName means that the function + // reference is simply by function name. + FunctionReferenceTypeFunctionName = "name" + + // Other function reference types we'd like to support: + // Versioned function, latest version + // Versioned function. by semver "latest compatible" + // Set of function references (recursively), by percentage of traffic +) + const ( ErrorInternal = iota @@ -103,6 +234,8 @@ const ( ErrorInvalidArgument ErrorNoSpace ErrorNotImplmented + ErrorChecksumFail + ErrorSizeLimitExceeded ) // must match order and len of the above const @@ -114,4 +247,10 @@ var errorDescriptions = []string{ "Invalid argument", "No space", "Not implemented", + "Checksum verification failed", + "Size limit exceeded", } + +const ( + PackageLiteralSizeLimit int64 = 256 * 1024 +)