diff --git a/common.go b/common.go index 5f9a2c56..9aa8caca 100644 --- a/common.go +++ b/common.go @@ -20,11 +20,7 @@ import ( "fmt" ) -func UrlForFunction(m *Metadata) string { +func UrlForFunction(name string) string { prefix := "/fission-function" - if len(m.Uid) > 0 { - return fmt.Sprintf("%v/%v/%v", prefix, m.Name, m.Uid) - } else { - return fmt.Sprintf("%v/%v", prefix, m.Name) - } + return fmt.Sprintf("%v/%v", prefix, name) } diff --git a/controller/api.go b/controller/api.go index 3504ba72..a601a405 100644 --- a/controller/api.go +++ b/controller/api.go @@ -26,19 +26,16 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + kerrors "k8s.io/client-go/1.5/pkg/api/errors" "github.com/fission/fission" "github.com/fission/fission/fission/logdb" + "github.com/fission/fission/tpr" ) type ( API struct { - FunctionStore - HTTPTriggerStore - TimeTriggerStore - MessageQueueTriggerStore - EnvironmentStore - WatchStore + fissionClient *tpr.FissionClient } logDBConfig struct { @@ -48,16 +45,8 @@ type ( } ) -func MakeAPI(rs *ResourceStore) *API { - api := &API{ - FunctionStore: FunctionStore{ResourceStore: *rs}, - HTTPTriggerStore: HTTPTriggerStore{ResourceStore: *rs}, - TimeTriggerStore: TimeTriggerStore{ResourceStore: *rs}, - MessageQueueTriggerStore: MessageQueueTriggerStore{ResourceStore: *rs}, - EnvironmentStore: EnvironmentStore{ResourceStore: *rs}, - WatchStore: WatchStore{ResourceStore: *rs}, - } - return api +func MakeAPI() (*API, error) { + return makeTPRBackedAPI() } func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) { @@ -71,6 +60,14 @@ func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) { func (api *API) respondWithError(w http.ResponseWriter, err error) { debug.PrintStack() + + // this error type comes with an HTTP code, so just use that + se, ok := err.(*kerrors.StatusError) + if ok { + http.Error(w, string(se.ErrStatus.Reason), int(se.ErrStatus.Code)) + return + } + code, msg := fission.GetHTTPError(err) log.Errorf("Error: %v: %v", code, msg) http.Error(w, msg, code) @@ -102,41 +99,41 @@ func (api *API) Serve(port int) { r := mux.NewRouter() r.HandleFunc("/", api.HomeHandler) - r.HandleFunc("/v1/functions", api.FunctionApiList).Methods("GET") - r.HandleFunc("/v1/functions", api.FunctionApiCreate).Methods("POST") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiGet).Methods("GET") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiUpdate).Methods("PUT") - r.HandleFunc("/v1/functions/{function}", api.FunctionApiDelete).Methods("DELETE") + r.HandleFunc("/v2/functions", api.FunctionApiList).Methods("GET") + r.HandleFunc("/v2/functions", api.FunctionApiCreate).Methods("POST") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiGet).Methods("GET") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiUpdate).Methods("PUT") + r.HandleFunc("/v2/functions/{function}", api.FunctionApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/http", api.HTTPTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/http", api.HTTPTriggerApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE") - r.HandleFunc("/v1/environments", api.EnvironmentApiList).Methods("GET") - r.HandleFunc("/v1/environments", api.EnvironmentApiCreate).Methods("POST") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiGet).Methods("GET") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT") - r.HandleFunc("/v1/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE") + r.HandleFunc("/v2/environments", api.EnvironmentApiList).Methods("GET") + r.HandleFunc("/v2/environments", api.EnvironmentApiCreate).Methods("POST") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiGet).Methods("GET") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT") + r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE") - r.HandleFunc("/v1/watches", api.WatchApiList).Methods("GET") - r.HandleFunc("/v1/watches", api.WatchApiCreate).Methods("POST") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiGet).Methods("GET") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiUpdate).Methods("PUT") - r.HandleFunc("/v1/watches/{watch}", api.WatchApiDelete).Methods("DELETE") + r.HandleFunc("/v2/watches", api.WatchApiList).Methods("GET") + r.HandleFunc("/v2/watches", api.WatchApiCreate).Methods("POST") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiGet).Methods("GET") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiUpdate).Methods("PUT") + r.HandleFunc("/v2/watches/{watch}", api.WatchApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/time", api.TimeTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/time", api.TimeTriggerApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE") - r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET") - r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueApiCreate).Methods("POST") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiGet).Methods("GET") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiUpdate).Methods("PUT") - r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiDelete).Methods("DELETE") + r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET") + r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiCreate).Methods("POST") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiGet).Methods("GET") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiUpdate).Methods("PUT") + r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiDelete).Methods("DELETE") r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST") diff --git a/controller/api_test.go b/controller/api_test.go index e4ae7ea5..8ff71270 100644 --- a/controller/api_test.go +++ b/controller/api_test.go @@ -19,35 +19,50 @@ package controller import ( "flag" "io/ioutil" + "log" "net/http" "os" "testing" "time" - etcdClient "github.com/coreos/etcd/client" - log "github.com/sirupsen/logrus" - "golang.org/x/net/context" + "k8s.io/client-go/1.5/pkg/api" + "bytes" "github.com/fission/fission" "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) var g struct { client *client.Client } -func assertNameReuseFails(err error, name string) { +func panicIf(err error) { + if err != nil { + log.Panicf("err: %v", err) + } +} + +func assert(c bool, msg string) { + if !c { + log.Fatalf("assert failed: %v", msg) + } +} + +func assertNameReuseFailure(err error, name string) { assert(err != nil, "recreating "+name+" with same name must fail") fe, ok := err.(fission.Error) assert(ok, "error must be a fission Error") assert(fe.Code == fission.ErrorNameExists, "error must be a name exists error") } -func assertNotFoundFails(err error, name string) { +func assertNotFoundFailure(err error, name string) { assert(err != nil, "requesting a non-existent "+name+" must fail") fe, ok := err.(fission.Error) assert(ok, "error must be a fission Error") - assert(fe.Code == fission.ErrorNotFound, "error must be a not found error") + if fe.Code != fission.ErrorNotFound { + log.Fatalf("error must be a not found error: %v", fe) + } } func assertCronSpecFails(err error) { @@ -58,67 +73,42 @@ func assertCronSpecFails(err error) { } func TestFunctionApi(t *testing.T) { - log.SetFormatter(&log.TextFormatter{DisableColors: true}) + name1 := "foo" + name2 := "bar" - testFunc := &fission.Function{ - Metadata: fission.Metadata{ - Name: "foo", - Uid: "", + testFunc := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: name1, + Namespace: api.NamespaceDefault, }, - Environment: fission.Metadata{ - Name: "nodejs", - Uid: "xxx", + Spec: fission.FunctionSpec{ + EnvironmentName: "nodejs", + Deployment: fission.Package{ + Literal: []byte("code1"), + }, }, - Code: "code1", } - _, err := g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "function") + _, err := g.client.FunctionGet(&api.ObjectMeta{ + Name: testFunc.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "function") m, err := g.client.FunctionCreate(testFunc) panicIf(err) - uid1 := m.Uid - //log.Printf("Created function %v: %v", m.Name, m.Uid) _, err = g.client.FunctionCreate(testFunc) - assertNameReuseFails(err, "function") + assertNameReuseFailure(err, "function") - code, err := g.client.FunctionGetRaw(m) + code, err := g.client.FunctionGetRawDeployment(m) panicIf(err) - assert(string(code) == testFunc.Code, "code from FunctionGetRaw must match created function") + assert(bytes.Compare(code, testFunc.Spec.Deployment.Literal) == 0, "code from FunctionGetRawDeployment must match created function") - testFunc.Code = "code2" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid2 := m.Uid - //log.Printf("Updated function %v: %v", m.Name, m.Uid) - - m.Uid = uid1 - testFunc.Code = "code1" - f, err := g.client.FunctionGet(m) + testFunc.Spec.Deployment.Literal = []byte("code2") + _, err = g.client.FunctionUpdate(testFunc) panicIf(err) - testFunc.Metadata.Uid = m.Uid - //log.Printf("f = %#v", f) - //log.Printf("testFunc = %#v", testFunc) - assert(*f == *testFunc, "first version should match when read by uid") - - m.Uid = uid2 - testFunc.Metadata.Uid = m.Uid - testFunc.Code = "code2" - f, err = g.client.FunctionGet(m) - panicIf(err) - - assert(*f == *testFunc, "second version should match when read by uid") - - m.Uid = "" - testFunc.Metadata.Uid = uid2 - testFunc.Code = "code2" - f, err = g.client.FunctionGet(m) - panicIf(err) - - assert(*f == *testFunc, "second version should match when read as latest") - - testFunc.Metadata.Name = "bar" + testFunc.Metadata.Name = name2 m, err = g.client.FunctionCreate(testFunc) panicIf(err) @@ -127,7 +117,7 @@ func TestFunctionApi(t *testing.T) { assert(len(funcs) == 2, "created two functions, but didn't find them") - funcs_url := g.client.Url + "/v1/functions" + funcs_url := g.client.Url + "/v2/functions" resp, err := http.Get(funcs_url) panicIf(err) defer resp.Body.Close() @@ -141,112 +131,55 @@ func TestFunctionApi(t *testing.T) { } assert(found, "incorrect response content type") - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo"}) + err = g.client.FunctionDelete(&api.ObjectMeta{Name: name1, Namespace: api.NamespaceDefault}) panicIf(err) - err = g.client.FunctionDelete(&fission.Metadata{Name: "bar"}) + err = g.client.FunctionDelete(&api.ObjectMeta{Name: name2, Namespace: api.NamespaceDefault}) panicIf(err) } -func TestFunctionVersionApi(t *testing.T) { - testFunc := &fission.Function{ - Metadata: fission.Metadata{ - Name: "foo", - Uid: "", - }, - Environment: fission.Metadata{ - Name: "nodejs", - Uid: "xxx", - }, - Code: "code1", - } - - testFunc.Code = "code1" - m, err := g.client.FunctionCreate(testFunc) - panicIf(err) - uid1 := m.Uid - - testFunc.Code = "code2" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid2 := m.Uid - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo", Uid: uid1}) - panicIf(err) - - f, err := g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - panicIf(err) - assert(f.Metadata.Uid == uid2, "deleted version1, but version2 does not exist") - - testFunc.Code = "code3" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - uid3 := m.Uid - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo", Uid: uid3}) - panicIf(err) - - f, err = g.client.FunctionGet(&fission.Metadata{Name: "foo"}) - panicIf(err) - assert(f.Metadata.Uid == uid2, "deleted version3, but version2 does not exist") - - testFunc.Code = "code4" - m, err = g.client.FunctionUpdate(testFunc) - panicIf(err) - - err = g.client.FunctionDelete(&fission.Metadata{Name: "foo"}) - panicIf(err) - - funcs, err := g.client.FunctionList() - panicIf(err) - assert(len(funcs) == 0, - "created one function with two versions(2 and 4), delete without uid but cannot delete them all") -} - func TestHTTPTriggerApi(t *testing.T) { - testTrigger := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testTrigger := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, }, - UrlPattern: "/hello", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.HTTPTriggerSpec{ + RelativeURL: "/hello", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, }, } - _, err := g.client.HTTPTriggerGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "trigger") + _, err := g.client.HTTPTriggerGet(&api.ObjectMeta{ + Name: testTrigger.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "httptrigger") m, err := g.client.HTTPTriggerCreate(testTrigger) panicIf(err) defer g.client.HTTPTriggerDelete(m) _, err = g.client.HTTPTriggerCreate(testTrigger) - assertNameReuseFails(err, "trigger") + assertNameReuseFailure(err, "httptrigger") tr, err := g.client.HTTPTriggerGet(m) panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") + assert(testTrigger.Spec == tr.Spec, "trigger should match after reading") - testTrigger.UrlPattern = "/hi" - m2, err := g.client.HTTPTriggerUpdate(testTrigger) + testTrigger.Spec.RelativeURL = "/hi" + _, err = g.client.HTTPTriggerUpdate(testTrigger) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.HTTPTriggerGet(m) - panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") - testTrigger.Metadata.Name = "yyy" - m, err = g.client.HTTPTriggerCreate(testTrigger) + _, err = g.client.HTTPTriggerCreate(testTrigger) assert(err != nil, "duplicate trigger should not be allowed") - testTrigger.UrlPattern = "/hi2" - m, err = g.client.HTTPTriggerCreate(testTrigger) + testTrigger.Spec.RelativeURL = "/hi2" + m2, err := g.client.HTTPTriggerCreate(testTrigger) panicIf(err) - defer g.client.HTTPTriggerDelete(m) + defer g.client.HTTPTriggerDelete(m2) ts, err := g.client.HTTPTriggerList() panicIf(err) @@ -254,42 +187,42 @@ func TestHTTPTriggerApi(t *testing.T) { } func TestEnvironmentApi(t *testing.T) { - testEnv := &fission.Environment{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testEnv := &tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: fission.EnvironmentSpec{ + Runtime: fission.Runtime{ + Image: "gcr.io/xyz", + }, }, - RunContainerImageUrl: "gcr.io/xyz", } - _, err := g.client.EnvironmentGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "environment") + _, err := g.client.EnvironmentGet(&api.ObjectMeta{ + Name: testEnv.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "environment") m, err := g.client.EnvironmentCreate(testEnv) panicIf(err) defer g.client.EnvironmentDelete(m) _, err = g.client.EnvironmentCreate(testEnv) - assertNameReuseFails(err, "environment") + assertNameReuseFailure(err, "environment") - tr, err := g.client.EnvironmentGet(m) + e, err := g.client.EnvironmentGet(m) panicIf(err) - testEnv.Metadata.Uid = m.Uid - assert(*testEnv == *tr, "env should match after reading") + assert(testEnv.Spec == e.Spec, "env should match after reading") - testEnv.RunContainerImageUrl = "/hi" - m2, err := g.client.EnvironmentUpdate(testEnv) + testEnv.Spec.Runtime.Image = "another-img" + _, err = g.client.EnvironmentUpdate(testEnv) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.EnvironmentGet(m) + testEnv.Metadata.Name = "bar" + m2, err := g.client.EnvironmentCreate(testEnv) panicIf(err) - testEnv.Metadata.Uid = m.Uid - assert(*testEnv == *tr, "env should match after reading") - - testEnv.Metadata.Name = "yyy" - m, err = g.client.EnvironmentCreate(testEnv) - panicIf(err) - defer g.client.EnvironmentDelete(m) + defer g.client.EnvironmentDelete(m2) ts, err := g.client.EnvironmentList() panicIf(err) @@ -297,36 +230,38 @@ func TestEnvironmentApi(t *testing.T) { } func TestWatchApi(t *testing.T) { - testWatch := &fission.Watch{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testWatch := &tpr.Kuberneteswatchtrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, }, - Namespace: "default", - ObjType: "pod", - LabelSelector: "", - FieldSelector: "", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: "default", + Type: "pod", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, }, - Target: "", } - _, err := g.client.WatchGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "watch") + _, err := g.client.WatchGet(&api.ObjectMeta{ + Name: testWatch.Metadata.Name, + Namespace: api.NamespaceDefault, + }) + assertNotFoundFailure(err, "watch") m, err := g.client.WatchCreate(testWatch) panicIf(err) defer g.client.WatchDelete(m) _, err = g.client.WatchCreate(testWatch) - assertNameReuseFails(err, "watch") + assertNameReuseFailure(err, "watch") w, err := g.client.WatchGet(m) panicIf(err) - testWatch.Metadata.Uid = m.Uid - w.Target = "" - assert(*testWatch == *w, "watch should match after reading") + assert((testWatch.Spec.Namespace == w.Spec.Namespace && + testWatch.Spec.Type == w.Spec.Type && + testWatch.Spec.FunctionReference == w.Spec.FunctionReference), "watch should match after reading") testWatch.Metadata.Name = "yyy" m2, err := g.client.WatchCreate(testWatch) @@ -339,45 +274,40 @@ func TestWatchApi(t *testing.T) { } func TestTimeTriggerApi(t *testing.T) { - testTrigger := &fission.TimeTrigger{ - Metadata: fission.Metadata{ - Name: "xxx", - Uid: "yyy", + testTrigger := &tpr.Timetrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, }, - Cron: "0 30 * * * *", - Function: fission.Metadata{ - Name: "foo", - Uid: "", + Spec: fission.TimeTriggerSpec{ + Cron: "0 30 * * * *", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "asdf", + }, }, } - _, err := g.client.TimeTriggerGet(&fission.Metadata{Name: "foo"}) - assertNotFoundFails(err, "trigger") + _, err := g.client.TimeTriggerGet(&api.ObjectMeta{Name: testTrigger.Metadata.Name}) + assertNotFoundFailure(err, "trigger") m, err := g.client.TimeTriggerCreate(testTrigger) panicIf(err) defer g.client.TimeTriggerDelete(m) _, err = g.client.TimeTriggerCreate(testTrigger) - assertNameReuseFails(err, "trigger") + assertNameReuseFailure(err, "trigger") tr, err := g.client.TimeTriggerGet(m) panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") + assert(testTrigger.Spec == tr.Spec, "trigger should match after reading") - testTrigger.Cron = "@hourly" - m2, err := g.client.TimeTriggerUpdate(testTrigger) + testTrigger.Spec.Cron = "@hourly" + _, err = g.client.TimeTriggerUpdate(testTrigger) panicIf(err) - m.Uid = m2.Uid - tr, err = g.client.TimeTriggerGet(m) - panicIf(err) - testTrigger.Metadata.Uid = m.Uid - assert(*testTrigger == *tr, "trigger should match after reading") - testTrigger.Metadata.Name = "yyy" - testTrigger.Cron = "Not valid cron spec" - m, err = g.client.TimeTriggerCreate(testTrigger) + testTrigger.Spec.Cron = "Not valid cron spec" + _, err = g.client.TimeTriggerCreate(testTrigger) assertCronSpecFails(err) ts, err := g.client.TimeTriggerList() @@ -388,20 +318,17 @@ func TestTimeTriggerApi(t *testing.T) { func TestMain(m *testing.M) { flag.Parse() - fileStore, ks, rs := getTestResourceStore() - defer os.RemoveAll(fileStore.root) + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + log.Println("Skipping test, no kubernetes cluster") + return + } - api := MakeAPI(rs) - g.client = client.MakeClient("http://localhost:8888") + go Start(8888) - ks.Delete(context.Background(), "Function", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "HTTPTrigger", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "TimeTrigger", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "Environment", &etcdClient.DeleteOptions{Recursive: true}) - ks.Delete(context.Background(), "Watch", &etcdClient.DeleteOptions{Recursive: true}) - - go api.Serve(8888) time.Sleep(500 * time.Millisecond) + g.client = client.MakeClient("http://localhost:8888") resp, err := http.Get("http://localhost:8888/") panicIf(err) diff --git a/controller/client/client.go b/controller/client/client.go index 571ee086..0cd03e56 100644 --- a/controller/client/client.go +++ b/controller/client/client.go @@ -18,10 +18,7 @@ package client import ( "bytes" - "encoding/base64" - "encoding/json" "errors" - "fmt" "io/ioutil" "net/http" "strings" @@ -73,7 +70,7 @@ func (c *Client) put(relativeUrl string, contentType string, body []byte) (*http } func (c *Client) url(relativeUrl string) string { - return c.Url + "/v1/" + relativeUrl + return c.Url + "/v2/" + relativeUrl } func (c *Client) handleResponse(resp *http.Response) ([]byte, error) { @@ -91,665 +88,3 @@ func (c *Client) handleCreateResponse(resp *http.Response) ([]byte, error) { body, err := ioutil.ReadAll(resp.Body) return body, err } - -func (c *Client) FunctionCreate(f *fission.Function) (*fission.Metadata, error) { - orig := f.Code - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) - defer func() { f.Code = orig }() - - reqbody, err := json.Marshal(f) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) FunctionGet(m *fission.Metadata) (*fission.Function, error) { - relativeUrl := fmt.Sprintf("functions/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var f fission.Function - err = json.Unmarshal(body, &f) - if err != nil { - return nil, err - } - - dec, err := base64.StdEncoding.DecodeString(f.Code) - if err != nil { - return nil, err - } - f.Code = string(dec) - - return &f, nil -} - -func (c *Client) FunctionGetRaw(m *fission.Metadata) ([]byte, error) { - relativeUrl := fmt.Sprintf("functions/%v?raw=1", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("&uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - return c.handleResponse(resp) -} - -func (c *Client) FunctionUpdate(f *fission.Function) (*fission.Metadata, error) { - orig := f.Code - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) - defer func() { f.Code = orig }() - - reqbody, err := json.Marshal(f) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) FunctionDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("functions/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) FunctionList() ([]fission.Function, error) { - resp, err := http.Get(c.url("functions")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - funcs := make([]fission.Function, 0) - err = json.Unmarshal(body, &funcs) - if err != nil { - return nil, err - } - - return funcs, nil -} - -func (c *Client) HTTPTriggerCreate(t *fission.HTTPTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/http"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) HTTPTriggerGet(m *fission.Metadata) (*fission.HTTPTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.HTTPTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) HTTPTriggerUpdate(t *fission.HTTPTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/http/%v", t.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) HTTPTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) HTTPTriggerList() ([]fission.HTTPTrigger, error) { - resp, err := http.Get(c.url("triggers/http")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.HTTPTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} - -func (c *Client) EnvironmentCreate(env *fission.Environment) (*fission.Metadata, error) { - reqbody, err := json.Marshal(env) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) EnvironmentGet(m *fission.Metadata) (*fission.Environment, error) { - relativeUrl := fmt.Sprintf("environments/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var env fission.Environment - err = json.Unmarshal(body, &env) - if err != nil { - return nil, err - } - - return &env, nil -} - -func (c *Client) EnvironmentUpdate(env *fission.Environment) (*fission.Metadata, error) { - reqbody, err := json.Marshal(env) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) EnvironmentDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("environments/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) EnvironmentList() ([]fission.Environment, error) { - resp, err := http.Get(c.url("environments")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - envs := make([]fission.Environment, 0) - err = json.Unmarshal(body, &envs) - if err != nil { - return nil, err - } - - return envs, nil -} - -func (c *Client) WatchCreate(w *fission.Watch) (*fission.Metadata, error) { - reqbody, err := json.Marshal(w) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("watches"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) WatchGet(m *fission.Metadata) (*fission.Watch, error) { - relativeUrl := fmt.Sprintf("watches/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var w fission.Watch - err = json.Unmarshal(body, &w) - if err != nil { - return nil, err - } - - return &w, nil -} - -func (c *Client) WatchUpdate(w *fission.Watch) (*fission.Metadata, error) { - return nil, fission.MakeError(fission.ErrorNotImplmented, - "watch update not implemented") -} - -func (c *Client) WatchDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("watches/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err - -} - -func (c *Client) WatchList() ([]fission.Watch, error) { - resp, err := http.Get(c.url("watches")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - watches := make([]fission.Watch, 0) - err = json.Unmarshal(body, &watches) - if err != nil { - return nil, err - } - - return watches, err -} - -func (c *Client) TimeTriggerCreate(t *fission.TimeTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/time"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) TimeTriggerGet(m *fission.Metadata) (*fission.TimeTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.TimeTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) TimeTriggerUpdate(t *fission.TimeTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/time/%v", t.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) TimeTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) TimeTriggerList() ([]fission.TimeTrigger, error) { - resp, err := http.Get(c.url("triggers/time")) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.TimeTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} - -func (c *Client) MessageQueueTriggerCreate(t *fission.MessageQueueTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(t) - if err != nil { - return nil, err - } - - resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleCreateResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - - return &m, nil -} - -func (c *Client) MessageQueueTriggerGet(m *fission.Metadata) (*fission.MessageQueueTrigger, error) { - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var t fission.MessageQueueTrigger - err = json.Unmarshal(body, &t) - if err != nil { - return nil, err - } - - return &t, nil -} - -func (c *Client) MessageQueueTriggerUpdate(mqTrigger *fission.MessageQueueTrigger) (*fission.Metadata, error) { - reqbody, err := json.Marshal(mqTrigger) - if err != nil { - return nil, err - } - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name) - - resp, err := c.put(relativeUrl, "application/json", reqbody) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - var m fission.Metadata - err = json.Unmarshal(body, &m) - if err != nil { - return nil, err - } - return &m, nil -} - -func (c *Client) MessageQueueTriggerDelete(m *fission.Metadata) error { - relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) - if len(m.Uid) > 0 { - relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) - } - err := c.delete(relativeUrl) - return err -} - -func (c *Client) MessageQueueTriggerList(mqType string) ([]fission.MessageQueueTrigger, error) { - relativeUrl := "triggers/messagequeue" - if len(mqType) > 0 { - relativeUrl += fmt.Sprintf("?mqtype=%v", mqType) - } - - resp, err := http.Get(c.url(relativeUrl)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := c.handleResponse(resp) - if err != nil { - return nil, err - } - - triggers := make([]fission.MessageQueueTrigger, 0) - err = json.Unmarshal(body, &triggers) - if err != nil { - return nil, err - } - - return triggers, nil -} diff --git a/controller/client/environment.go b/controller/client/environment.go new file mode 100644 index 00000000..d0d7ec6a --- /dev/null +++ b/controller/client/environment.go @@ -0,0 +1,132 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) EnvironmentCreate(env *tpr.Environment) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(env) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) EnvironmentGet(m *api.ObjectMeta) (*tpr.Environment, error) { + relativeUrl := fmt.Sprintf("environments/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var env tpr.Environment + err = json.Unmarshal(body, &env) + if err != nil { + return nil, err + } + + return &env, nil +} + +func (c *Client) EnvironmentUpdate(env *tpr.Environment) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(env) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) EnvironmentDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("environments/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + return c.delete(relativeUrl) +} + +func (c *Client) EnvironmentList() ([]tpr.Environment, error) { + resp, err := http.Get(c.url("environments")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + envs := make([]tpr.Environment, 0) + err = json.Unmarshal(body, &envs) + if err != nil { + return nil, err + } + + return envs, nil +} diff --git a/controller/client/function.go b/controller/client/function.go new file mode 100644 index 00000000..14d17431 --- /dev/null +++ b/controller/client/function.go @@ -0,0 +1,155 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" +) + +func (c *Client) FunctionCreate(f *tpr.Function) (*api.ObjectMeta, error) { + + // ensure size limits + if len(f.Spec.Source.Literal) > 256*1024 { + return nil, fission.MakeError(fission.ErrorSizeLimitExceeded, "Source package literal larger than 256k") + } + if len(f.Spec.Deployment.Literal) > 256*1024 { + return nil, fission.MakeError(fission.ErrorSizeLimitExceeded, "Deployment package literal larger than 256k") + } + + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) FunctionGet(m *api.ObjectMeta) (*tpr.Function, error) { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var f tpr.Function + err = json.Unmarshal(body, &f) + if err != nil { + return nil, err + } + + return &f, nil +} + +func (c *Client) FunctionGetRawDeployment(m *api.ObjectMeta) ([]byte, error) { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + relativeUrl += fmt.Sprintf("&deploymentraw=1") + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return c.handleResponse(resp) +} + +func (c *Client) FunctionUpdate(f *tpr.Function) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) FunctionDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) FunctionList() ([]tpr.Function, error) { + resp, err := http.Get(c.url("functions")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + funcs := make([]tpr.Function, 0) + err = json.Unmarshal(body, &funcs) + if err != nil { + return nil, err + } + + return funcs, nil +} diff --git a/controller/client/httptrigger.go b/controller/client/httptrigger.go new file mode 100644 index 00000000..a12e802a --- /dev/null +++ b/controller/client/httptrigger.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) HTTPTriggerCreate(t *tpr.Httptrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/http"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) HTTPTriggerGet(m *api.ObjectMeta) (*tpr.Httptrigger, error) { + relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Httptrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) HTTPTriggerUpdate(t *tpr.Httptrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/http/%v", t.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) HTTPTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) HTTPTriggerList() ([]tpr.Httptrigger, error) { + resp, err := http.Get(c.url("triggers/http")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Httptrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/client/kuberneteswatchtrigger.go b/controller/client/kuberneteswatchtrigger.go new file mode 100644 index 00000000..7daf1aa6 --- /dev/null +++ b/controller/client/kuberneteswatchtrigger.go @@ -0,0 +1,111 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" +) + +func (c *Client) WatchCreate(w *tpr.Kuberneteswatchtrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(w) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("watches"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) WatchGet(m *api.ObjectMeta) (*tpr.Kuberneteswatchtrigger, error) { + relativeUrl := fmt.Sprintf("watches/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var w tpr.Kuberneteswatchtrigger + err = json.Unmarshal(body, &w) + if err != nil { + return nil, err + } + + return &w, nil +} + +func (c *Client) WatchUpdate(w *tpr.Kuberneteswatchtrigger) (*api.ObjectMeta, error) { + return nil, fission.MakeError(fission.ErrorNotImplmented, + "watch update not implemented") +} + +func (c *Client) WatchDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("watches/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) WatchList() ([]tpr.Kuberneteswatchtrigger, error) { + resp, err := http.Get(c.url("watches")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + watches := make([]tpr.Kuberneteswatchtrigger, 0) + err = json.Unmarshal(body, &watches) + if err != nil { + return nil, err + } + + return watches, err +} diff --git a/controller/client/mqtrigger.go b/controller/client/mqtrigger.go new file mode 100644 index 00000000..59c16ead --- /dev/null +++ b/controller/client/mqtrigger.go @@ -0,0 +1,137 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) MessageQueueTriggerCreate(t *tpr.Messagequeuetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) MessageQueueTriggerGet(m *api.ObjectMeta) (*tpr.Messagequeuetrigger, error) { + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Messagequeuetrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) MessageQueueTriggerUpdate(mqTrigger *tpr.Messagequeuetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(mqTrigger) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) MessageQueueTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) MessageQueueTriggerList(mqType string) ([]tpr.Messagequeuetrigger, error) { + relativeUrl := "triggers/messagequeue" + if len(mqType) > 0 { + // TODO remove this, replace with field selector + relativeUrl += fmt.Sprintf("?mqtype=%v", mqType) + } + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Messagequeuetrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/client/timetrigger.go b/controller/client/timetrigger.go new file mode 100644 index 00000000..cd3a1493 --- /dev/null +++ b/controller/client/timetrigger.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" +) + +func (c *Client) TimeTriggerCreate(t *tpr.Timetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("triggers/time"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleCreateResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) TimeTriggerGet(m *api.ObjectMeta) (*tpr.Timetrigger, error) { + relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var t tpr.Timetrigger + err = json.Unmarshal(body, &t) + if err != nil { + return nil, err + } + + return &t, nil +} + +func (c *Client) TimeTriggerUpdate(t *tpr.Timetrigger) (*api.ObjectMeta, error) { + reqbody, err := json.Marshal(t) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("triggers/time/%v", t.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + var m api.ObjectMeta + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) TimeTriggerDelete(m *api.ObjectMeta) error { + relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name) + relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace) + return c.delete(relativeUrl) +} + +func (c *Client) TimeTriggerList() ([]tpr.Timetrigger, error) { + resp, err := http.Get(c.url("triggers/time")) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := c.handleResponse(resp) + if err != nil { + return nil, err + } + + triggers := make([]tpr.Timetrigger, 0) + err = json.Unmarshal(body, &triggers) + if err != nil { + return nil, err + } + + return triggers, nil +} diff --git a/controller/jsonSerializer.go b/controller/controller.go similarity index 59% rename from controller/jsonSerializer.go rename to controller/controller.go index 008ba608..f6027d20 100644 --- a/controller/jsonSerializer.go +++ b/controller/controller.go @@ -17,16 +17,27 @@ limitations under the License. package controller import ( - "encoding/json" + "log" + + "github.com/fission/fission/tpr" ) -type JsonSerializer struct { -} +func Start(port int) { + fc, kc, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Failed to connect to K8s API: %v", err) + } -func (j JsonSerializer) serialize(r resource) ([]byte, error) { - return json.Marshal(r) -} + err = tpr.EnsureFissionTPRs(kc) + if err != nil { + log.Fatalf("Failed to create fission TPRs: %v", err) + } -func (j JsonSerializer) deserialize(buf []byte, r resource) error { - return json.Unmarshal(buf, r) + fc.WaitForTPRs() + + api, err := MakeAPI() + if err != nil { + log.Fatalf("Failed to start controller: %v", err) + } + api.Serve(port) } diff --git a/controller/environmentApi.go b/controller/environmentApi.go index 0aa413ca..cd50b96a 100644 --- a/controller/environmentApi.go +++ b/controller/environmentApi.go @@ -23,133 +23,132 @@ import ( "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) EnvironmentApiList(w http.ResponseWriter, r *http.Request) { - envs, err := api.EnvironmentStore.List() +func (a *API) EnvironmentApiList(w http.ResponseWriter, r *http.Request) { + envs, err := a.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(envs) + resp, err := json.Marshal(envs.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var env fission.Environment + var env tpr.Environment err = json.Unmarshal(body, &env) if err != nil { log.Printf("Failed to unmarshal request body: [%v]", body) - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.EnvironmentStore.Create(&env) + enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Create(&env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: env.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(enew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) EnvironmentApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["environment"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["environment"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - env, err := api.EnvironmentStore.Get(&m) + env, err := a.fissionClient.Environments(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["environment"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var env fission.Environment + var env tpr.Environment err = json.Unmarshal(body, &env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != env.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Environment name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.EnvironmentStore.Update(&env) + enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Update(&env) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: env.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(enew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) EnvironmentApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) EnvironmentApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["environment"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"httpTrigger": m.Name}).Info("Deleting all versions") + name := vars["environment"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.EnvironmentStore.Delete(m) + err := a.fissionClient.Environments(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/environmentStore.go b/controller/environmentStore.go deleted file mode 100644 index 3a257af5..00000000 --- a/controller/environmentStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type EnvironmentStore struct { - ResourceStore -} - -func (es *EnvironmentStore) Create(e *fission.Environment) (string, error) { - e.Metadata.Uid = uuid.NewV4().String() - return e.Metadata.Uid, es.ResourceStore.create(e) -} - -func (es *EnvironmentStore) Get(m *fission.Metadata) (*fission.Environment, error) { - var e fission.Environment - err := es.ResourceStore.read(m.Name, &e) - if err != nil { - return nil, err - } - return &e, nil -} - -func (es *EnvironmentStore) Update(e *fission.Environment) (string, error) { - e.Metadata.Uid = uuid.NewV4().String() - return e.Metadata.Uid, es.ResourceStore.update(e) -} - -func (es *EnvironmentStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.Environment{}) - if err != nil { - return err - } - return es.ResourceStore.delete(typeName, m.Name) -} - -func (es *EnvironmentStore) List() ([]fission.Environment, error) { - typeName, err := getTypeName(fission.Environment{}) - if err != nil { - return nil, err - } - - bufs, err := es.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - envs := make([]fission.Environment, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var e fission.Environment - err = js.deserialize([]byte(buf), &e) - if err != nil { - return nil, err - } - envs = append(envs, e) - } - - return envs, nil -} diff --git a/controller/fileStore.go b/controller/fileStore.go deleted file mode 100644 index 1d7c4c79..00000000 --- a/controller/fileStore.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "os" - "path" - - log "github.com/sirupsen/logrus" -) - -type requestType int - -const ( - READ requestType = iota - WRITE - DELETE -) - -type ( - FileStore struct { - root string // abs path of root of filestore - requestChannel chan fileStoreRequest - } - - fileStoreRequest struct { - requestType - fileName string // relative path - fileContents []byte - responseChannel chan fileStoreResponse - } - - fileStoreResponse struct { - error - fileContents []byte - } -) - -func MakeFileStore(path string) *FileStore { - - // create directory if necessary - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(path, os.ModeDir|0700) - if err != nil { - log.Printf("Failed to create directory %v", path) - return nil - } - log.Printf("Created directory %v", path) - } else { - log.Printf("Unknown error initializing filestore: %v", err) - return nil - } - } - - fileStore := &FileStore{ - root: path, - requestChannel: make(chan fileStoreRequest), - } - go fileStore.fileStoreService() - return fileStore -} - -func (fs *FileStore) fileStoreService() { - for { - req := <-fs.requestChannel - response := &fileStoreResponse{} - - log.WithFields(log.Fields{"file": req.fileName, "type": req.requestType}).Debug("fileStore request") - switch req.requestType { - case READ: - response.fileContents, response.error = ioutil.ReadFile(path.Join(fs.root, req.fileName)) - case WRITE: - response.error = ioutil.WriteFile(path.Join(fs.root, req.fileName), req.fileContents, 0600) - case DELETE: - response.error = os.Remove(path.Join(fs.root, req.fileName)) - if os.IsNotExist(response.error) { - response.error = nil - } - default: - log.Panic("bad request") - } - req.responseChannel <- *response - } -} - -func (fs *FileStore) read(fileName string) ([]byte, error) { - req := fileStoreRequest{ - requestType: READ, - fileName: fileName, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.fileContents, response.error -} - -func (fs *FileStore) write(fileName string, contents []byte) error { - req := fileStoreRequest{ - requestType: WRITE, - fileName: fileName, - fileContents: contents, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.error -} - -func (fs *FileStore) delete(fileName string) error { - req := fileStoreRequest{ - requestType: DELETE, - fileName: fileName, - responseChannel: make(chan fileStoreResponse), - } - fs.requestChannel <- req - response := <-req.responseChannel - return response.error -} diff --git a/controller/fileStore_test.go b/controller/fileStore_test.go deleted file mode 100644 index 02f7c15f..00000000 --- a/controller/fileStore_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "log" - "os" - "testing" -) - -func TestFileStore(t *testing.T) { - // tmp dir - dir, err := ioutil.TempDir("", "testFileStore") - if err != nil { - t.Fatalf("error creating temp dir: %v", err) - } - defer os.RemoveAll(dir) - log.Printf("temp dir at %v", dir) - - // file store - fs := MakeFileStore(dir) - - _, err = fs.read("nonexistent") - if err == nil { - t.Fatalf("expected an error") - } - - path := "fileStoreTest" - contents := []byte("bar") - err = fs.write(path, contents) - if err != nil { - t.Fatalf("error: %v", err) - } - - observedContents, err := fs.read(path) - if err != nil { - t.Fatalf("error: %v", err) - } - - if string(observedContents) != string(contents) { - t.Fatalf("contents don't match") - } - - err = fs.delete(path) - if err != nil { - t.Fatalf("error: %v", err) - } -} diff --git a/controller/functionApi.go b/controller/functionApi.go index 6187e4ca..b6abc683 100644 --- a/controller/functionApi.go +++ b/controller/functionApi.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "encoding/base64" "encoding/json" "io/ioutil" "net/http" @@ -26,165 +25,161 @@ import ( "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) FunctionApiList(w http.ResponseWriter, r *http.Request) { - funcs, err := api.FunctionStore.List() +func (a *API) FunctionApiList(w http.ResponseWriter, r *http.Request) { + funcs, err := a.fissionClient.Functions(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(funcs) + resp, err := json.Marshal(funcs.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var f fission.Function + var f tpr.Function err = json.Unmarshal(body, &f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - dec, err := base64.StdEncoding.DecodeString(f.Code) - if err != nil { - api.respondWithError(w, err) + // Ensure size limits + if len(f.Spec.Source.Literal) > 256*1024 { + err := fission.MakeError(fission.ErrorInvalidArgument, "Source package literal larger than 256K") + a.respondWithError(w, err) return } - f.Code = string(dec) - - uid, err := api.FunctionStore.Create(&f) - if err != nil { - api.respondWithError(w, err) + if len(f.Spec.Deployment.Literal) > 256*1024 { + err := fission.MakeError(fission.ErrorInvalidArgument, "Deployment package literal larger than 256K") + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: f.Name, Uid: uid} - resp, err := json.Marshal(m) + fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Create(&f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) + return + } + + resp, err := json.Marshal(fnew.Metadata) + if err != nil { + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["function"] - m.Uid = r.FormValue("uid") // empty if uid is absent - raw := r.FormValue("raw") // just the code + name := vars["function"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } + raw := r.FormValue("deploymentraw") // just the deployment pkg - f, err := api.FunctionStore.Get(&m) + f, err := a.fissionClient.Functions(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } var resp []byte if raw != "" { - resp = []byte(f.Code) + resp = []byte(f.Spec.Deployment.Literal) } else { - f.Code = base64.StdEncoding.EncodeToString([]byte(f.Code)) resp, err = json.Marshal(f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - funcName := vars["function"] + name := vars["function"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var f fission.Function + var f tpr.Function err = json.Unmarshal(body, &f) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - if funcName != f.Metadata.Name { + if name != f.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Function name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - dec, err := base64.StdEncoding.DecodeString(f.Code) + fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Update(&f) if err != nil { - api.respondWithError(w, err) - return - } - f.Code = string(dec) - - uid, err := api.FunctionStore.Update(&f) - if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: f.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(fnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["function"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"function": m.Name}).Info("Deleting all versions") + name := vars["function"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.FunctionStore.Delete(m) + err := a.fissionClient.Functions(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } // FunctionLogsApiPost establishes a proxy server to log database, and redirect // query command send from client to database then proxy back the db response. -func (api *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) { +func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) // get dbType from url dbType := vars["dbType"] // find correspond db http url - dbCnf := api.getLogDBConfig(dbType) + dbCnf := a.getLogDBConfig(dbType) svcUrl, err := url.Parse(dbCnf.httpURL) if err != nil { diff --git a/controller/functionStore.go b/controller/functionStore.go deleted file mode 100644 index 34774781..00000000 --- a/controller/functionStore.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - log "github.com/sirupsen/logrus" - - "github.com/fission/fission" -) - -type FunctionStore struct { - ResourceStore -} - -func (fs *FunctionStore) Create(f *fission.Function) (string, error) { - code := []byte(f.Code) - _, uid, err := fs.ResourceStore.writeFile(f.Key(), code) - if err != nil { - return "", err - } - - f.Metadata.Uid = uid - f.Code = "" - - err = fs.ResourceStore.create(f) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore errors - return "", err - } - return f.Metadata.Uid, nil -} - -func (fs *FunctionStore) Get(m *fission.Metadata) (*fission.Function, error) { - var f fission.Function - err := fs.ResourceStore.read(m.Name, &f) - if err != nil { - return nil, err - } - - var code []byte - if len(m.Uid) > 0 { - log.WithFields(log.Fields{"Uid": m.Uid}).Info("fetching by uid") - code, err = fs.ResourceStore.readFile(m.Name, &m.Uid) - f.Metadata = *m - } else { - code, err = fs.ResourceStore.readFile(m.Name, nil) - } - if err != nil { - return nil, err - } - - f.Code = string(code) - return &f, nil -} - -func (fs *FunctionStore) Update(f *fission.Function) (string, error) { - code := []byte(f.Code) - _, uid, err := fs.ResourceStore.writeFile(f.Key(), code) - if err != nil { - return "", err - } - - var fnew fission.Function - err = fs.ResourceStore.read(f.Metadata.Name, &fnew) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore err - return "", err - } - - fnew.Metadata.Uid = uid - fnew.Environment = f.Environment - - err = fs.ResourceStore.update(fnew) - if err != nil { - fs.ResourceStore.deleteFile(f.Key(), uid) // ignore err - return "", err - } - return uid, err -} - -func (fs *FunctionStore) Delete(m fission.Metadata) error { - if len(m.Uid) == 0 { - err := fs.ResourceStore.deleteAllFiles(m.Name) - if err != nil { - return err - } - } else { - err := fs.ResourceStore.deleteFile(m.Name, m.Uid) - if err != nil { - return err - } - } - typeName, err := getTypeName(fission.Function{}) - if err != nil { - return err - } - - bufs, err := fs.ResourceStore.getAll("file/" + m.Name) - if err != nil { - return err - } - if len(bufs) == 0 { - return fs.ResourceStore.delete(typeName, m.Name) - } - - fnew, err := fs.Get(&fission.Metadata{Name: m.Name}) - if err != nil { - return err - } - - latestUid := bufs[len(bufs)-1] // function always tracks the latest version of code - if latestUid == fnew.Uid { - return nil - } - fnew.Uid = latestUid - return fs.ResourceStore.update(fnew) -} - -func (fs *FunctionStore) List() ([]fission.Function, error) { - typeName, err := getTypeName(fission.Function{}) - if err != nil { - return nil, err - } - - bufs, err := fs.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - js := JsonSerializer{} - functions := make([]fission.Function, 0, len(bufs)) - for _, buf := range bufs { - var f fission.Function - err = js.deserialize([]byte(buf), &f) - if err != nil { - return nil, err - } - functions = append(functions, f) - } - - return functions, nil -} diff --git a/controller/httpTriggerApi.go b/controller/httpTriggerApi.go index c98e5c03..7e9b0c72 100644 --- a/controller/httpTriggerApi.go +++ b/controller/httpTriggerApi.go @@ -18,151 +18,164 @@ package controller import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) HTTPTriggerApiList(w http.ResponseWriter, r *http.Request) { - triggers, err := api.HTTPTriggerStore.List() +func (a *API) HTTPTriggerApiList(w http.ResponseWriter, r *http.Request) { + triggers, err := a.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiCreate(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) +func (a *API) checkHttpTriggerDuplicates(t *tpr.Httptrigger) error { + triggers, err := a.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) - return + return err } - - var t fission.HTTPTrigger - err = json.Unmarshal(body, &t) - if err != nil { - api.respondWithError(w, err) - return - } - - triggers, err := api.HTTPTriggerStore.List() - if err != nil { - api.respondWithError(w, err) - return - } - for _, url := range triggers { - if url.UrlPattern == t.UrlPattern && url.Method == t.Method { - err = fission.MakeError(fission.ErrorNameExists, - "HTTPTrigger with same URL & method already exists") - api.respondWithError(w, err) - return + for _, ht := range triggers.Items { + if ht.Spec.RelativeURL == t.Spec.RelativeURL && ht.Spec.Method == t.Spec.Method { + return fission.MakeError(fission.ErrorNameExists, + fmt.Sprintf("HTTPTrigger with same URL & method already exists (%v)", + ht.Metadata.Name)) } } + return nil +} - uid, err := api.HTTPTriggerStore.Create(&t) +func (a *API) HTTPTriggerApiCreate(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + var t tpr.Httptrigger + err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) + return + } + + // Ensure we don't have a duplicate HTTP route defined (same URL and method) + err = a.checkHttpTriggerDuplicates(&t) + if err != nil { + a.respondWithError(w, err) + return + } + + tnew, err := a.fissionClient.Httptriggers(t.Metadata.Namespace).Create(&t) + if err != nil { + a.respondWithError(w, err) + return + } + + resp, err := json.Marshal(tnew.Metadata) + if err != nil { + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) HTTPTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["httpTrigger"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["httpTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - t, err := api.HTTPTriggerStore.Get(&m) + t, err := a.fissionClient.Httptriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) HTTPTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["httpTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.HTTPTrigger + var t tpr.Httptrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != t.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "HTTPTrigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.HTTPTriggerStore.Update(&t) + err = a.checkHttpTriggerDuplicates(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + tnew, err := a.fissionClient.Httptriggers(t.Metadata.Namespace).Update(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + + resp, err := json.Marshal(tnew.Metadata) + if err != nil { + a.respondWithError(w, err) + return + } + a.respondWithSuccess(w, resp) } -func (api *API) HTTPTriggerApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) HTTPTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["httpTrigger"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"httpTrigger": m.Name}).Info("Deleting all versions") + name := vars["httpTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.HTTPTriggerStore.Delete(m) + err := a.fissionClient.Httptriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/httpTriggerStore.go b/controller/httpTriggerStore.go deleted file mode 100644 index 092e9d45..00000000 --- a/controller/httpTriggerStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type HTTPTriggerStore struct { - ResourceStore -} - -func (hts *HTTPTriggerStore) Create(ht *fission.HTTPTrigger) (string, error) { - ht.Metadata.Uid = uuid.NewV4().String() - return ht.Metadata.Uid, hts.ResourceStore.create(ht) -} - -func (hts *HTTPTriggerStore) Get(m *fission.Metadata) (*fission.HTTPTrigger, error) { - var ht fission.HTTPTrigger - err := hts.ResourceStore.read(m.Name, &ht) - if err != nil { - return nil, err - } - return &ht, nil -} - -func (hts *HTTPTriggerStore) Update(ht *fission.HTTPTrigger) (string, error) { - ht.Metadata.Uid = uuid.NewV4().String() - return ht.Metadata.Uid, hts.ResourceStore.update(ht) -} - -func (hts *HTTPTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.HTTPTrigger{}) - if err != nil { - return err - } - return hts.ResourceStore.delete(typeName, m.Name) -} - -func (hts *HTTPTriggerStore) List() ([]fission.HTTPTrigger, error) { - typeName, err := getTypeName(fission.HTTPTrigger{}) - if err != nil { - return nil, err - } - - bufs, err := hts.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.HTTPTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var ht fission.HTTPTrigger - err = js.deserialize([]byte(buf), &ht) - if err != nil { - return nil, err - } - triggers = append(triggers, ht) - } - - return triggers, nil -} diff --git a/controller/mqTriggerApi.go b/controller/mqTriggerApi.go index 91fd9ea2..7740dcad 100644 --- a/controller/mqTriggerApi.go +++ b/controller/mqTriggerApi.go @@ -22,145 +22,126 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) { - mqType := r.FormValue("mqtype") - triggers, err := api.MessageQueueTriggerStore.List(mqType) +func (a *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) { + //mqType := r.FormValue("mqtype") // ignored for now + triggers, err := a.fissionClient.Messagequeuetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var mqTrigger fission.MessageQueueTrigger + var mqTrigger tpr.Messagequeuetrigger err = json.Unmarshal(body, &mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - // trigger name must not conflict with any other trigger - // even they are different message queue type - triggers, err := api.MessageQueueTriggerStore.List("") + tnew, err := a.fissionClient.Messagequeuetriggers(mqTrigger.Metadata.Namespace).Create(&mqTrigger) if err != nil { - api.respondWithError(w, err) - return - } - for _, trigger := range triggers { - if trigger.Name == mqTrigger.Name { - err = fission.MakeError(fission.ErrorNameExists, - "Message queue trigger with same name already exists") - api.respondWithError(w, err) - return - } - } - - // save trigger info - uid, err := api.MessageQueueTriggerStore.Create(&mqTrigger) - if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid} - resp, err := json.Marshal(mqTriggerMeta) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiGet(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqTriggerMeta := fission.Metadata{ - Name: vars["mqTrigger"], - Uid: r.FormValue("uid"), // empty if uid is absent + name := vars["mqTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - mqTrigger, err := api.MessageQueueTriggerStore.Get(&mqTriggerMeta) + + mqTrigger, err := a.fissionClient.Messagequeuetriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqtName := vars["mqTrigger"] + name := vars["mqTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var mqTrigger fission.MessageQueueTrigger + var mqTrigger tpr.Messagequeuetrigger err = json.Unmarshal(body, &mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - if mqtName != mqTrigger.Metadata.Name { + if name != mqTrigger.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "Message queue trigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.MessageQueueTriggerStore.Update(&mqTrigger) + tnew, err := a.fissionClient.Messagequeuetriggers(mqTrigger.Metadata.Namespace).Update(&mqTrigger) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid} - resp, err := json.Marshal(mqTriggerMeta) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) MessageQueueApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) MessageQueueTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - mqTriggerMeta := fission.Metadata{ - Name: vars["mqTrigger"], - Uid: r.FormValue("uid"), // empty if uid is absent + name := vars["mqTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - if len(mqTriggerMeta.Uid) == 0 { - log.WithFields(log.Fields{"mqTrigger": mqTriggerMeta.Name}).Info("Deleting all versions") - } - - err := api.MessageQueueTriggerStore.Delete(mqTriggerMeta) + err := a.fissionClient.Messagequeuetriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/mqTriggerStore.go b/controller/mqTriggerStore.go deleted file mode 100644 index 80fb6bf2..00000000 --- a/controller/mqTriggerStore.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -CopyrigmqTrigger 2017 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - mqTriggertp://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/fission/fission" - uuid "github.com/satori/go.uuid" -) - -type MessageQueueTriggerStore struct { - ResourceStore -} - -func (mqs *MessageQueueTriggerStore) Create(mqTrigger *fission.MessageQueueTrigger) (string, error) { - mqTrigger.Metadata.Uid = uuid.NewV4().String() - return mqTrigger.Metadata.Uid, mqs.ResourceStore.create(mqTrigger) -} - -func (mqs *MessageQueueTriggerStore) Get(m *fission.Metadata) (*fission.MessageQueueTrigger, error) { - var mqTrigger fission.MessageQueueTrigger - err := mqs.ResourceStore.read(m.Name, &mqTrigger) - if err != nil { - return nil, err - } - return &mqTrigger, nil -} - -func (mqs *MessageQueueTriggerStore) Update(mqTrigger *fission.MessageQueueTrigger) (string, error) { - mqTrigger.Metadata.Uid = uuid.NewV4().String() - return mqTrigger.Metadata.Uid, mqs.ResourceStore.update(mqTrigger) -} - -func (mqs *MessageQueueTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.MessageQueueTrigger{}) - if err != nil { - return err - } - return mqs.ResourceStore.delete(typeName, m.Name) -} - -func (mqs *MessageQueueTriggerStore) List(mqType string) ([]fission.MessageQueueTrigger, error) { - typeName, err := getTypeName(fission.MessageQueueTrigger{}) - if err != nil { - return nil, err - } - bufs, err := mqs.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.MessageQueueTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var mqTrigger fission.MessageQueueTrigger - err = js.deserialize([]byte(buf), &mqTrigger) - if err != nil { - return nil, err - } - if len(mqType) > 0 && mqType != mqTrigger.MessageQueueType { - continue - } - triggers = append(triggers, mqTrigger) - } - - return triggers, nil -} diff --git a/controller/resourceStore.go b/controller/resourceStore.go deleted file mode 100644 index 26e95c38..00000000 --- a/controller/resourceStore.go +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "errors" - "fmt" - "reflect" - "strings" - "time" - - "github.com/coreos/etcd/client" - "github.com/satori/go.uuid" - log "github.com/sirupsen/logrus" - "golang.org/x/net/context" - - "github.com/fission/fission" -) - -type ( - ResourceStore struct { - *FileStore - client.KeysAPI - serializer - } -) - -func MakeResourceStore(fs *FileStore, etcdUrls []string) (*ResourceStore, error) { - ks, err := getEtcdKeyAPI(etcdUrls) - if err != nil { - return nil, err - } - s := JsonSerializer{} - return &ResourceStore{FileStore: fs, KeysAPI: ks, serializer: s}, nil -} - -func getEtcdKeyAPI(etcdUrls []string) (client.KeysAPI, error) { - cfg := client.Config{ - Endpoints: etcdUrls, - Transport: client.DefaultTransport, - // set timeout per request to fail fast when the target endpoint is unavailable - HeaderTimeoutPerRequest: time.Second, - } - c, err := client.New(cfg) - if err != nil { - log.Printf("failed to connect to etcd: %v", err) - return nil, err - } - return client.NewKeysAPI(c), nil -} - -func getTypeName(r resource) (string, error) { - typ := reflect.TypeOf(r) - if typ.Kind().String() == "ptr" { - typ = typ.Elem() - } - typName := typ.Name() - if len(typName) == 0 { - return "", errors.New("Failed to get type") - } - return typName, nil -} - -func getKey(r resource) (string, error) { - typName, err := getTypeName(r) - if err != nil { - return "", err - } - rkey := r.Key() - return (typName + "/" + rkey), nil -} - -func (rs *ResourceStore) create(r resource) error { - key, err := getKey(r) - if err != nil { - return err - } - - serialized, err := rs.serializer.serialize(r) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), - &client.SetOptions{PrevExist: client.PrevNoExist}) - return handleEtcdErrorForResource(err, r) -} - -func (rs *ResourceStore) read(rkey string, res resource) error { - typName, err := getTypeName(res) - if err != nil { - return err - } - key := typName + "/" + rkey - - resp, err := rs.KeysAPI.Get(context.Background(), key, nil) - if err != nil { - return handleEtcdError(err, typName, rkey) - } - return rs.serializer.deserialize([]byte(resp.Node.Value), res) -} - -func (rs *ResourceStore) update(r resource) error { - key, err := getKey(r) - if err != nil { - return err - } - - serialized, err := rs.serializer.serialize(r) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), - &client.SetOptions{PrevExist: client.PrevExist}) - return handleEtcdErrorForResource(err, r) -} - -func (rs *ResourceStore) delete(typename, rkey string) error { - key := typename + "/" + rkey - _, err := rs.KeysAPI.Delete(context.Background(), key, nil) // ignore response - return handleEtcdError(err, typename, rkey) -} - -// getAll finds all entries under key. If none or found or key -// doesn't exist, returns an empty slice. -func (rs *ResourceStore) getAll(key string) ([]string, error) { - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Recursive: true, Sort: true}) - if err != nil { - if client.IsKeyNotFound(err) { - return []string{}, nil - } - return nil, handleEtcdError(err, "", key) - } - - res := make([]string, 0, len(resp.Node.Nodes)) - for _, n := range resp.Node.Nodes { - res = append(res, n.Value) - } - return res, nil -} - -func (rs *ResourceStore) writeFile(parentKey string, contents []byte) (string, string, error) { - uid := uuid.NewV4().String() - - err := rs.FileStore.write(uid, contents) - if err != nil { - return "", "", err - } - - parentKey = "file/" + parentKey - resp, err := rs.KeysAPI.CreateInOrder(context.Background(), parentKey, uid, nil) - if err != nil { - _ = rs.FileStore.delete(uid) - return "", "", handleEtcdError(err, "file", parentKey) - } - - return resp.Node.Key, uid, nil -} - -func (rs *ResourceStore) readFile(key string, uid *string) ([]byte, error) { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return nil, handleEtcdError(err, "file", key) - } - - if uid == nil { - // get latest - n := resp.Node.Nodes - uid = &n[len(n)-1].Value - } else { - // validate uid is in the list - found := false - for _, u := range resp.Node.Nodes { - if *uid == u.Value { - found = true - break - } - } - if !found { - return nil, errors.New("Invalid UID " + *uid) - } - } - - contents, err := rs.FileStore.read(*uid) - return contents, err -} - -func (rs *ResourceStore) deleteFile(key string, uid string) error { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return handleEtcdError(err, "file", key) - } - - var node *client.Node - for _, u := range resp.Node.Nodes { - if u.Value == uid { - node = u - } - } - if node == nil { - log.WithFields(log.Fields{"key": key, "uid": uid}).Error("unreferenced file") - return errors.New("won't delete unreferenced file") - } - - err = rs.FileStore.delete(node.Value) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Delete(context.Background(), node.Key, nil) - if err != nil { - return handleEtcdError(err, "", node.Key) - } - - if len(resp.Node.Nodes) == 1 { - _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) - return handleEtcdError(err, "file", key) - } - return nil -} - -func (rs *ResourceStore) deleteAllFiles(key string) error { - key = "file/" + key - resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) - if err != nil { - return handleEtcdError(err, "file", key) - } - for _, u := range resp.Node.Nodes { - err = rs.FileStore.delete(u.Value) - if err != nil { - return err - } - - _, err = rs.KeysAPI.Delete(context.Background(), u.Key, nil) - if err != nil { - return handleEtcdError(err, "", u.Key) - } - } - _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) - return handleEtcdError(err, "file", key) -} - -func handleEtcdErrorForResource(e error, r resource) error { - resourceType, _ := getTypeName(r) - return handleEtcdError(e, resourceType, r.Key()) -} - -func handleEtcdError(e error, resourceType string, resourceKey string) error { - ee, ok := e.(client.Error) - if !ok { - return e - } - code := fission.ErrorInternal - msg := ee.Error() - - if len(resourceType) > 0 { - resourceType = strings.ToLower(resourceType) + " " - } - - //TODO: handle any other etcd error codes we care about - switch ee.Code { - case client.ErrorCodeNodeExist: - code = fission.ErrorNameExists - msg = fmt.Sprintf("%s'%s' already exists", resourceType, resourceKey) - case client.ErrorCodeKeyNotFound: - code = fission.ErrorNotFound - msg = fmt.Sprintf("%s'%s' does not exist", resourceType, resourceKey) - } - return fission.MakeError(code, msg) -} diff --git a/controller/resourceStore_test.go b/controller/resourceStore_test.go deleted file mode 100644 index 808cf7fc..00000000 --- a/controller/resourceStore_test.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "io/ioutil" - "log" - "os" - "testing" - - "github.com/coreos/etcd/client" - "golang.org/x/net/context" -) - -type TestResource struct { - A string - B int -} - -func (tr TestResource) Key() string { - return tr.A -} - -func panicIf(err error) { - if err != nil { - log.Panicf("err: %v", err) - } -} - -func assert(b bool, msg string) { - if !b { - log.Panic("assertion failed: " + msg) - } -} - -func getTestResourceStore() (*FileStore, client.KeysAPI, *ResourceStore) { - // make a tmp dir - dir, err := ioutil.TempDir("", "testFileStore") - panicIf(err) - fs := MakeFileStore(dir) - - rs, err := MakeResourceStore(fs, []string{"http://localhost:2379"}) - panicIf(err) - - return fs, rs.KeysAPI, rs -} - -func TestResourceStore(t *testing.T) { - fs, ks, rs := getTestResourceStore() - defer os.RemoveAll(fs.root) - - s := JsonSerializer{} - - tr := TestResource{A: "hello", B: 1} - - // Delete the key first, in case of a panic'd previous test run; ignore errors - _ = rs.delete("TestResource", tr.Key()) - ks.Delete(context.Background(), "/TestResource", &client.DeleteOptions{Dir: true}) - - // Verify getAll for empty db - trs, err := rs.getAll("TestResource") - panicIf(err) - if len(trs) != 0 { - log.Fatalf("Expected zero length slice, got %v", trs) - } - - // Create - err = rs.create(tr) - panicIf(err) - defer ks.Delete(context.Background(), "/TestResource", &client.DeleteOptions{Dir: true}) - defer rs.delete("TestResource", tr.Key()) - - // Etcd key /TestResource/hello should exist - _, err = ks.Get(context.Background(), "TestResource/hello", nil) - panicIf(err) - - // Read - tr1 := TestResource{} - err = rs.read(tr.Key(), &tr1) - panicIf(err) - assert(tr1 == tr, "retrieved value must equal created value") - - // Update and Read - tr.B += 1 - err = rs.update(tr) - panicIf(err) - err = rs.read(tr.Key(), &tr1) - panicIf(err) - assert(tr1 == tr, "retrieved value must equal updated value") - - // Get list - results, err := rs.getAll("TestResource") - panicIf(err) - res := make([]TestResource, 0, 0) - for _, r := range results { - tmp := TestResource{} - err = s.deserialize([]byte(r), &tmp) - panicIf(err) - res = append(res, tmp) - } - assert(res[0] == tr, "value from retrieved list must equal updated value") - - // file tests - fileKey := "ResourceStoreTest" - fileContents1 := []byte("hello") - fileContents2 := []byte("world") - key, uid1, err := rs.writeFile(fileKey, fileContents1) - panicIf(err) - defer rs.deleteFile(fileKey, uid1) - log.Printf("key = %v, uid = %v", key, uid1) - - // read latest - contents, err := rs.readFile(fileKey, nil) - panicIf(err) - assert(string(contents) == string(fileContents1), "retrieved file contents must match written value") - - // update-- same key new contents - _, uid2, err := rs.writeFile(fileKey, fileContents2) - panicIf(err) - defer rs.deleteFile(fileKey, uid2) - - // read latest - contents, err = rs.readFile(fileKey, nil) - panicIf(err) - assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") - - // read by uid - // 1 - contents, err = rs.readFile(fileKey, &uid1) - panicIf(err) - assert(string(contents) == string(fileContents1), "retrieved file contents must match updated value") - - // 2 - contents, err = rs.readFile(fileKey, &uid2) - panicIf(err) - assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") - -} diff --git a/controller/timeTriggerApi.go b/controller/timeTriggerApi.go index d7a28db7..689966e2 100644 --- a/controller/timeTriggerApi.go +++ b/controller/timeTriggerApi.go @@ -23,161 +23,146 @@ import ( "github.com/gorilla/mux" "github.com/robfig/cron" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) TimeTriggerApiList(w http.ResponseWriter, r *http.Request) { - triggers, err := api.TimeTriggerStore.List() +func (a *API) TimeTriggerApiList(w http.ResponseWriter, r *http.Request) { + triggers, err := a.fissionClient.Timetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(triggers) + resp, err := json.Marshal(triggers.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.TimeTrigger + var t tpr.Timetrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - triggers, err := api.TimeTriggerStore.List() - if err != nil { - api.respondWithError(w, err) - return - } - for _, trigger := range triggers { - if trigger.Name == t.Name { - err = fission.MakeError(fission.ErrorNameExists, - "TimeTrigger with same name already exists") - api.respondWithError(w, err) - return - } - } - - _, err = cron.Parse(t.Cron) + // validate + _, err = cron.Parse(t.Spec.Cron) if err != nil { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger cron spec is not valid") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.TimeTriggerStore.Create(&t) + tnew, err := a.fissionClient.Timetriggers(t.Metadata.Namespace).Create(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) TimeTriggerApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["timeTrigger"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["timeTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - t, err := api.TimeTriggerStore.Get(&m) + t, err := a.fissionClient.Timetriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiUpdate(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name := vars["timeTrigger"] body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var t fission.TimeTrigger + var t tpr.Timetrigger err = json.Unmarshal(body, &t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } if name != t.Metadata.Name { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger name doesn't match URL") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - _, err = cron.Parse(t.Cron) + _, err = cron.Parse(t.Spec.Cron) if err != nil { err = fission.MakeError(fission.ErrorInvalidArgument, "TimeTrigger cron spec is not valid") - api.respondWithError(w, err) + a.respondWithError(w, err) return } - uid, err := api.TimeTriggerStore.Update(&t) + tnew, err := a.fissionClient.Timetriggers(t.Metadata.Namespace).Update(&t) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: t.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(tnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) TimeTriggerApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) TimeTriggerApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["timeTrigger"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"timeTrigger": m.Name}).Info("Deleting all versions") + name := vars["timeTrigger"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.TimeTriggerStore.Delete(m) + err := a.fissionClient.Timetriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/timeTriggerStore.go b/controller/timeTriggerStore.go deleted file mode 100644 index b3a44c0c..00000000 --- a/controller/timeTriggerStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyrigtt 2017 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - tttp://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type TimeTriggerStore struct { - ResourceStore -} - -func (tts *TimeTriggerStore) Create(tt *fission.TimeTrigger) (string, error) { - tt.Metadata.Uid = uuid.NewV4().String() - return tt.Metadata.Uid, tts.ResourceStore.create(tt) -} - -func (tts *TimeTriggerStore) Get(m *fission.Metadata) (*fission.TimeTrigger, error) { - var tt fission.TimeTrigger - err := tts.ResourceStore.read(m.Name, &tt) - if err != nil { - return nil, err - } - return &tt, nil -} - -func (tts *TimeTriggerStore) Update(tt *fission.TimeTrigger) (string, error) { - tt.Metadata.Uid = uuid.NewV4().String() - return tt.Metadata.Uid, tts.ResourceStore.update(tt) -} - -func (tts *TimeTriggerStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.TimeTrigger{}) - if err != nil { - return err - } - return tts.ResourceStore.delete(typeName, m.Name) -} - -func (tts *TimeTriggerStore) List() ([]fission.TimeTrigger, error) { - typeName, err := getTypeName(fission.TimeTrigger{}) - if err != nil { - return nil, err - } - - bufs, err := tts.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - triggers := make([]fission.TimeTrigger, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var tt fission.TimeTrigger - err = js.deserialize([]byte(buf), &tt) - if err != nil { - return nil, err - } - triggers = append(triggers, tt) - } - - return triggers, nil -} diff --git a/controller/types.go b/controller/tpr.go similarity index 73% rename from controller/types.go rename to controller/tpr.go index abc575ec..aaea49d3 100644 --- a/controller/types.go +++ b/controller/tpr.go @@ -16,13 +16,14 @@ limitations under the License. package controller -type ( - resource interface { - Key() string - } - - serializer interface { - serialize(r resource) ([]byte, error) - deserialize(buf []byte, r resource) error - } +import ( + "github.com/fission/fission/tpr" ) + +func makeTPRBackedAPI() (*API, error) { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + return nil, err + } + return &API{fissionClient: fissionClient}, nil +} diff --git a/controller/watchApi.go b/controller/watchApi.go index 18b15826..d141bf5c 100644 --- a/controller/watchApi.go +++ b/controller/watchApi.go @@ -22,102 +22,101 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) -func (api *API) WatchApiList(w http.ResponseWriter, r *http.Request) { - watches, err := api.WatchStore.List() +func (a *API) WatchApiList(w http.ResponseWriter, r *http.Request) { + watches, err := a.fissionClient.Kuberneteswatchtriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - resp, err := json.Marshal(watches) + resp, err := json.Marshal(watches.Items) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiCreate(w http.ResponseWriter, r *http.Request) { +func (a *API) WatchApiCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - var watch fission.Watch + var watch tpr.Kuberneteswatchtrigger err = json.Unmarshal(body, &watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - watch.Target = fission.UrlForFunction(&watch.Function) + // TODO check for duplicate watches - uid, err := api.WatchStore.Create(&watch) + wnew, err := a.fissionClient.Kuberneteswatchtriggers(watch.Metadata.Namespace).Create(&watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - m := &fission.Metadata{Name: watch.Metadata.Name, Uid: uid} - resp, err := json.Marshal(m) + resp, err := json.Marshal(wnew.Metadata) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } w.WriteHeader(http.StatusCreated) - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiGet(w http.ResponseWriter, r *http.Request) { - var m fission.Metadata - +func (a *API) WatchApiGet(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - m.Name = vars["watch"] - m.Uid = r.FormValue("uid") // empty if uid is absent + name := vars["watch"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault + } - watch, err := api.WatchStore.Get(&m) + watch, err := a.fissionClient.Kuberneteswatchtriggers(ns).Get(name) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } resp, err := json.Marshal(watch) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, resp) + a.respondWithSuccess(w, resp) } -func (api *API) WatchApiUpdate(w http.ResponseWriter, r *http.Request) { - api.respondWithError(w, fission.MakeError(fission.ErrorNotImplmented, +func (a *API) WatchApiUpdate(w http.ResponseWriter, r *http.Request) { + a.respondWithError(w, fission.MakeError(fission.ErrorNotImplmented, "Not implemented")) } -func (api *API) WatchApiDelete(w http.ResponseWriter, r *http.Request) { +func (a *API) WatchApiDelete(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var m fission.Metadata - m.Name = vars["watch"] - - m.Uid = r.FormValue("uid") // empty if uid is absent - if len(m.Uid) == 0 { - log.WithFields(log.Fields{"watch": m.Name}).Info("Deleting all versions") + name := vars["watch"] + ns := vars["namespace"] + if len(ns) == 0 { + ns = api.NamespaceDefault } - err := api.WatchStore.Delete(m) + err := a.fissionClient.Kuberneteswatchtriggers(ns).Delete(name, &api.DeleteOptions{}) if err != nil { - api.respondWithError(w, err) + a.respondWithError(w, err) return } - api.respondWithSuccess(w, []byte("")) + a.respondWithSuccess(w, []byte("")) } diff --git a/controller/watchStore.go b/controller/watchStore.go deleted file mode 100644 index 25ddf248..00000000 --- a/controller/watchStore.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "github.com/satori/go.uuid" - - "github.com/fission/fission" -) - -type WatchStore struct { - ResourceStore -} - -func (ws *WatchStore) Create(w *fission.Watch) (string, error) { - w.Metadata.Uid = uuid.NewV4().String() - return w.Metadata.Uid, ws.ResourceStore.create(w) -} - -func (ws *WatchStore) Get(m *fission.Metadata) (*fission.Watch, error) { - var w fission.Watch - err := ws.ResourceStore.read(m.Name, &w) - if err != nil { - return nil, err - } - return &w, err -} - -func (ws *WatchStore) Update(w *fission.Watch) (string, error) { - w.Metadata.Uid = uuid.NewV4().String() - return w.Metadata.Uid, ws.ResourceStore.update(w) -} - -func (ws *WatchStore) Delete(m fission.Metadata) error { - typeName, err := getTypeName(fission.Watch{}) - if err != nil { - return err - } - return ws.ResourceStore.delete(typeName, m.Name) -} - -func (ws *WatchStore) List() ([]fission.Watch, error) { - typeName, err := getTypeName(fission.Watch{}) - if err != nil { - return nil, err - } - - bufs, err := ws.ResourceStore.getAll(typeName) - if err != nil { - return nil, err - } - - watches := make([]fission.Watch, 0, len(bufs)) - js := JsonSerializer{} - for _, buf := range bufs { - var w fission.Watch - err = js.deserialize([]byte(buf), &w) - if err != nil { - return nil, err - } - watches = append(watches, w) - } - - return watches, nil -} diff --git a/environments/fetcher/client/client.go b/environments/fetcher/client/client.go new file mode 100644 index 00000000..e31897ca --- /dev/null +++ b/environments/fetcher/client/client.go @@ -0,0 +1,35 @@ +package client + +import ( + "bytes" + "encoding/json" + "net/http" + //"time" + + "github.com/fission/fission" + "github.com/fission/fission/environments/fetcher" + //"github.com/fission/fission/router" +) + +func DoFetchRequest(fetcherUrl string, fr *fetcher.FetchRequest) error { + body, err := json.Marshal(fr) + if err != nil { + return err + } + + // client := http.Client{ + // Transport: router.MakeRetryingRoundTripper(10, 50*time.Millisecond), + // } + + resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader(body)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fission.MakeErrorFromHTTP(resp) + } + + return nil +} diff --git a/environments/fetcher/Dockerfile b/environments/fetcher/cmd/Dockerfile similarity index 100% rename from environments/fetcher/Dockerfile rename to environments/fetcher/cmd/Dockerfile diff --git a/environments/fetcher/build.sh b/environments/fetcher/cmd/build.sh similarity index 81% rename from environments/fetcher/build.sh rename to environments/fetcher/cmd/build.sh index a0f34b7a..d4b6f514 100755 --- a/environments/fetcher/build.sh +++ b/environments/fetcher/cmd/build.sh @@ -1 +1,2 @@ +#!/bin/sh GOOS=linux GOARCH=386 go build -o fetcher . diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go new file mode 100644 index 00000000..05ccaafd --- /dev/null +++ b/environments/fetcher/cmd/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "log" + "net/http" + "os" + + "github.com/fission/fission/environments/fetcher" +) + +// Usage: fetcher +func main() { + dir := os.Args[1] + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, os.ModeDir|0700) + if err != nil { + log.Fatalf("Error creating directory: %v", err) + } + } + } + fetcher := fetcher.MakeFetcher(dir) + mux := http.NewServeMux() + mux.HandleFunc("/", fetcher.Handler) + http.ListenAndServe(":8000", mux) +} diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 1f8b6829..c389cbfe 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -1,6 +1,8 @@ -package main +package fetcher import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -8,27 +10,108 @@ import ( "net/http" "os" "path/filepath" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" + "io" ) -type FetchRequest struct { - Url string `json:"url"` - Filename string `json:"filename"` -} +type ( + FetchRequestType int -type Fetcher struct { - sharedVolumePath string -} + FetchRequest struct { + FetchType FetchRequestType `json:"fetchType"` + Function api.ObjectMeta `json:"function"` + Url string `json:"url"` + StorageSvcUrl string `json:"storagesvcurl"` + Filename string `json:"filename"` + } + + Fetcher struct { + sharedVolumePath string + fissionClient *tpr.FissionClient + kubeClient *kubernetes.Clientset + } +) + +const ( + FETCH_SOURCE = iota + FETCH_DEPLOYMENT + FETCH_URL // remove this? +) func MakeFetcher(sharedVolumePath string) *Fetcher { - return &Fetcher{sharedVolumePath: sharedVolumePath} + fissionClient, kubeClient, err := tpr.MakeFissionClient() + if err != nil { + return nil + } + return &Fetcher{ + sharedVolumePath: sharedVolumePath, + fissionClient: fissionClient, + kubeClient: kubeClient, + } } -func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { +func downloadUrl(url string, localPath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = ioutil.WriteFile(localPath, body, 0600) + if err != nil { + return err + } + + return nil +} + +func verifyChecksum(path string, checksum *fission.Checksum) error { + if checksum.Type != fission.ChecksumTypeSHA256 { + return fission.MakeError(fission.ErrorInvalidArgument, "Unsupported checksum type") + } + + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + hasher := sha256.New() + _, err = io.Copy(hasher, f) + if err != nil { + return err + } + + c := hex.EncodeToString(hasher.Sum(nil)) + if c != checksum.Sum { + return fission.MakeError(fission.ErrorChecksumFail, "Checksum validation failed") + } + return nil +} + +func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { http.Error(w, "", 404) return } + startTime := time.Now() + defer func() { + elapsed := time.Now().Sub(startTime) + log.Printf("elapsed time in fetch request = %v", elapsed) + }() + // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { @@ -43,35 +126,69 @@ func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), 500) return } - log.Printf("fetcher request: %v", req) + log.Printf("fetcher received request: %v", req) - // fetch the file and save it to tmp path - resp, err := http.Get(req.Url) - if err != nil { - e := fmt.Sprintf("Failed to fetch from url: %v", err) - log.Printf(e) - http.Error(w, e, 400) - return - } - defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - e := fmt.Sprintf("Failed to read from url: %v", err) - log.Printf(e) - http.Error(w, e, 400) - return - } tmpFile := req.Filename + ".tmp" tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile) - err = ioutil.WriteFile(tmpPath, body, 0600) - if err != nil { - e := fmt.Sprintf("Failed to write file: %v", err) - log.Printf(e) - http.Error(w, e, 500) - return - } - // TODO: add signature verification + if req.FetchType == FETCH_URL { + // fetch the file and save it to the tmp path + err := downloadUrl(req.Url, tmpPath) + if err != nil { + e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) + log.Printf(e) + http.Error(w, e, 400) + return + } + } else { + // get function object + fn, err := fetcher.fissionClient.Functions(req.Function.Namespace).Get(req.Function.Name) + if err != nil { + e := fmt.Sprintf("Failed to get function: %v", err) + log.Printf(e) + http.Error(w, e, 500) + return + } + + // get pkg + var pkg *fission.Package + if req.FetchType == FETCH_SOURCE { + pkg = &fn.Spec.Source + } else if req.FetchType == FETCH_DEPLOYMENT { + pkg = &fn.Spec.Deployment + } + + // get package data as literal or by url + if len(pkg.Literal) > 0 { + // write pkg.Literal into tmpPath + err = ioutil.WriteFile(tmpPath, pkg.Literal, 0600) + if err != nil { + e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err) + log.Printf(e) + http.Error(w, e, 500) + return + } + } else { + // download and verify + + err = downloadUrl(pkg.URL, tmpPath) + if err != nil { + e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) + log.Printf(e) + http.Error(w, e, 400) + return + } + + err = verifyChecksum(tmpPath, &pkg.Checksum) + if err != nil { + e := fmt.Sprintf("Failed to verify checksum: %v", err) + log.Printf(e) + http.Error(w, e, 400) + return + } + } + + } // move tmp file to requested filename err = os.Rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) @@ -82,23 +199,7 @@ func (fetcher *Fetcher) handler(w http.ResponseWriter, r *http.Request) { return } + log.Printf("Completed fetch request") // all done w.WriteHeader(http.StatusOK) } - -// Usage: fetcher -func main() { - dir := os.Args[1] - if _, err := os.Stat(dir); err != nil { - if os.IsNotExist(err) { - err = os.MkdirAll(dir, os.ModeDir|0700) - if err != nil { - log.Fatalf("Error creating directory: %v", err) - } - } - } - fetcher := MakeFetcher(dir) - mux := http.NewServeMux() - mux.HandleFunc("/", fetcher.handler) - http.ListenAndServe(":8000", mux) -} diff --git a/fission-bundle/main.go b/fission-bundle/main.go index 30e295c4..c3e7de27 100644 --- a/fission-bundle/main.go +++ b/fission-bundle/main.go @@ -14,37 +14,25 @@ import ( "github.com/fission/fission/timer" ) -func runController(port int, etcdUrl string, filepath string) { - // filePath will be created if it doesn't exist. - fileStore := controller.MakeFileStore(filepath) - if fileStore == nil { - log.Fatalf("Failed to initialize filestore") - } - - rs, err := controller.MakeResourceStore(fileStore, []string{etcdUrl}) - if err != nil { - log.Fatalf("Error: %v", err) - } - - api := controller.MakeAPI(rs) - api.Serve(port) +func runController(port int) { + controller.Start(port) log.Fatalf("Error: Controller exited.") } -func runRouter(port int, controllerUrl string, poolmgrUrl string) { - router.Start(port, controllerUrl, poolmgrUrl) +func runRouter(port int, poolmgrUrl string) { + router.Start(port, poolmgrUrl) log.Fatalf("Error: Router exited.") } -func runPoolmgr(port int, controllerUrl string, namespace string) { - err := poolmgr.StartPoolmgr(controllerUrl, namespace, port) +func runPoolmgr(port int, fissionNamespace, functionNamespace string) { + err := poolmgr.StartPoolmgr(fissionNamespace, functionNamespace, port) if err != nil { log.Fatalf("Error starting poolmgr: %v", err) } } -func runKubeWatcher(controllerUrl, routerUrl string) { - err := kubewatcher.Start(controllerUrl, routerUrl) +func runKubeWatcher(routerUrl string) { + err := kubewatcher.Start(routerUrl) if err != nil { log.Fatalf("Error starting kubewatcher: %v", err) } @@ -55,15 +43,15 @@ func runLogger() { log.Fatalf("Error: Logger exited.") } -func runTimer(controllerUrl, routerUrl string) { - err := timer.Start(controllerUrl, routerUrl) +func runTimer(routerUrl string) { + err := timer.Start(routerUrl) if err != nil { log.Fatalf("Error starting timer: %v", err) } } -func runMessageQueueMgr(controllerUrl, routerUrl string) { - err := messagequeue.Start(controllerUrl, routerUrl) +func runMessageQueueMgr(routerUrl string) { + err := messagequeue.Start(routerUrl) if err != nil { log.Fatalf("Error starting timer: %v", err) } @@ -98,18 +86,17 @@ Use it to start one or more of the fission servers: Router implements HTTP triggers: it routes to running instances, working with the controller and poolmgr. Usage: - fission-bundle --controllerPort= [--etcdUrl=] --filepath= - fission-bundle --routerPort= [--controllerUrl= --poolmgrUrl=] - fission-bundle --poolmgrPort= [--controllerUrl= --namespace=] - fission-bundle --kubewatcher [--controllerUrl= --routerUrl=] + fission-bundle --controllerPort= + fission-bundle --routerPort= [--poolmgrUrl=] + fission-bundle --poolmgrPort= [--namespace=] [--fission-namespace=] + fission-bundle --kubewatcher [--routerUrl=] fission-bundle --logger - fission-bundle --timer [--controllerUrl= --routerUrl=] - fission-bundle --mqt [--controllerUrl= --routerUrl=] + fission-bundle --timer [--routerUrl=] + fission-bundle --mqt [--routerUrl=] Options: --controllerPort= Port that the controller should listen on. --routerPort= Port that the router should listen on. --poolmgrPort= Port that the poolmgr should listen on. - --controllerUrl= Controller URL. Not required if --controllerPort is specified. --poolmgrUrl= Poolmgr URL. Not required if --poolmgrPort is specified. --routerUrl= Router URL. --etcdUrl= Etcd URL. @@ -125,30 +112,29 @@ Options: log.Fatalf("Error: %v", err) } - namespace := getStringArgWithDefault(arguments["--namespace"], "fission-function") + functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function") + fissionNs := getStringArgWithDefault(arguments["--fission-namespace"], "fission") - controllerUrl := getStringArgWithDefault(arguments["--controllerUrl"], "http://controller.fission") - etcdUrl := getStringArgWithDefault(arguments["--etcdUrl"], "http://etcd:2379") poolmgrUrl := getStringArgWithDefault(arguments["--poolmgrUrl"], "http://poolmgr.fission") routerUrl := getStringArgWithDefault(arguments["--routerUrl"], "http://router.fission") if arguments["--controllerPort"] != nil { port := getPort(arguments["--controllerPort"]) - runController(port, etcdUrl, arguments["--filepath"].(string)) + runController(port) } if arguments["--routerPort"] != nil { port := getPort(arguments["--routerPort"]) - runRouter(port, controllerUrl, poolmgrUrl) + runRouter(port, poolmgrUrl) } if arguments["--poolmgrPort"] != nil { port := getPort(arguments["--poolmgrPort"]) - runPoolmgr(port, controllerUrl, namespace) + runPoolmgr(port, fissionNs, functionNs) } if arguments["--kubewatcher"] == true { - runKubeWatcher(controllerUrl, routerUrl) + runKubeWatcher(routerUrl) } if arguments["--logger"] == true { @@ -156,11 +142,11 @@ Options: } if arguments["--timer"] == true { - runTimer(controllerUrl, routerUrl) + runTimer(routerUrl) } if arguments["--mqt"] == true { - runMessageQueueMgr(controllerUrl, routerUrl) + runMessageQueueMgr(routerUrl) } select {} diff --git a/fission-rbac.yaml b/fission-rbac.yaml index 9d853128..dc5ebc9f 100644 --- a/fission-rbac.yaml +++ b/fission-rbac.yaml @@ -62,7 +62,7 @@ spec: - name: controller image: fission/fission-bundle command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] + args: ["--controllerPort", "8888"] serviceAccountName: fission-svc --- apiVersion: extensions/v1beta1 @@ -115,6 +115,9 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--poolmgrPort", "8888"] + env: + - name: FETCHER_IMAGE + value: fission/fetcher:v2 serviceAccountName: fission-svc --- apiVersion: extensions/v1beta1 @@ -135,40 +138,3 @@ spec: command: ["/fission-bundle"] args: ["--kubewatcher"] serviceAccountName: fission-svc ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - imagePullPolicy: IfNotPresent - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 - serviceAccountName: fission-svc diff --git a/fission.yaml b/fission.yaml index f0e32b45..8424de1d 100644 --- a/fission.yaml +++ b/fission.yaml @@ -30,7 +30,7 @@ spec: - name: controller image: fission/fission-bundle command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] + args: ["--controllerPort", "8888"] --- apiVersion: extensions/v1beta1 @@ -84,6 +84,9 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--poolmgrPort", "8888"] + env: + - name: FETCHER_IMAGE + value: fission/fetcher:v2 --- apiVersion: extensions/v1beta1 @@ -122,41 +125,3 @@ spec: image: fission/fission-bundle command: ["/fission-bundle"] args: ["--timer"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - imagePullPolicy: IfNotPresent - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 diff --git a/fission/deployment.go b/fission/deployment.go deleted file mode 100644 index 1d2e53b9..00000000 --- a/fission/deployment.go +++ /dev/null @@ -1,181 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "fmt" - -func getDeploymentYaml() { - fmt.Printf(` -apiVersion: v1 -kind: Namespace -metadata: - name: fission - labels: - name: fission - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: fission-function - labels: - name: fission-function - ---- -apiVersion: v1 -kind: Service -metadata: - name: controller - namespace: fission - labels: - svc: controller -spec: - type: NodePort - ports: - - port: 80 - targetPort: 8888 - nodePort: 31313 - selector: - svc: controller - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: controller - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: controller - spec: - containers: - - name: controller - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--controllerPort", "8888", "--filepath", "/filestore"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: router - namespace: fission - labels: - svc: router -spec: - type: NodePort - ports: - - port: 80 - targetPort: 8888 - nodePort: 31314 - selector: - svc: router - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: router - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: router - spec: - containers: - - name: router - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--routerPort", "8888"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: poolmgr - namespace: fission - labels: - svc: poolmgr -spec: - ports: - - port: 80 - targetPort: 8888 - selector: - svc: poolmgr - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: poolmgr - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: poolmgr - spec: - containers: - - name: poolmgr - image: fission/fission-bundle - command: ["/fission-bundle"] - args: ["--poolmgrPort", "8888"] - ---- -apiVersion: v1 -kind: Service -metadata: - name: etcd - namespace: fission - labels: - svc: etcd -spec: - ports: - - port: 2379 - targetPort: 2379 - selector: - svc: etcd - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: etcd - namespace: fission -spec: - replicas: 1 - template: - metadata: - labels: - svc: etcd - spec: - containers: - - name: etcd - image: quay.io/coreos/etcd - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: http://0.0.0.0:2379 - - name: ETCD_ADVERTISE_CLIENT_URLS - value: http://etcd:2379 -`) -} diff --git a/fission/environment.go b/fission/environment.go index bf6e9181..b50203b3 100644 --- a/fission/environment.go +++ b/fission/environment.go @@ -22,8 +22,10 @@ import ( "text/tabwriter" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func envCreate(c *cli.Context) error { @@ -39,11 +41,17 @@ func envCreate(c *cli.Context) error { fatal("Need an image, use --image.") } - env := &fission.Environment{ - Metadata: fission.Metadata{ - Name: envName, + env := &tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: envImg, + }, }, - RunContainerImageUrl: envImg, } _, err := client.EnvironmentCreate(env) @@ -61,14 +69,17 @@ func envGet(c *cli.Context) error { fatal("Need a name, use --name.") } - m := &fission.Metadata{Name: envName} + m := &api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + } env, err := client.EnvironmentGet(m) checkErr(err, "get environment") w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") fmt.Fprintf(w, "%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.Uid, env.RunContainerImageUrl) + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) w.Flush() return nil } @@ -86,14 +97,15 @@ func envUpdate(c *cli.Context) error { fatal("Need an image, use --image.") } - env := &fission.Environment{ - Metadata: fission.Metadata{ - Name: envName, - }, - RunContainerImageUrl: envImg, - } + env, err := client.EnvironmentGet(&api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + }) + checkErr(err, "find environment") - _, err := client.EnvironmentUpdate(env) + env.Spec.Runtime.Image = envImg + + _, err = client.EnvironmentUpdate(env) checkErr(err, "update environment") fmt.Printf("environment '%v' updated\n", envName) @@ -108,7 +120,10 @@ func envDelete(c *cli.Context) error { fatal("Need a name , use --name.") } - m := &fission.Metadata{Name: envName} + m := &api.ObjectMeta{ + Name: envName, + Namespace: api.NamespaceDefault, + } err := client.EnvironmentDelete(m) checkErr(err, "delete environment") @@ -126,7 +141,7 @@ func envList(c *cli.Context) error { fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") for _, env := range envs { fmt.Fprintf(w, "%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.Uid, env.RunContainerImageUrl) + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) } w.Flush() diff --git a/fission/function.go b/fission/function.go index 35e5c57e..8b6dc6c5 100644 --- a/fission/function.go +++ b/fission/function.go @@ -20,45 +20,53 @@ import ( "context" "fmt" "io/ioutil" - "net/http" + //"net/http" "os" - "os/exec" - "strings" "text/tabwriter" "time" "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/fission/logdb" + "github.com/fission/fission/tpr" ) -func fnFetchCode(filePath string) []byte { +func downloadUrl(url string) { + // var resp *http.Response + // resp, err := http.Get(url) + // if err != nil { + // checkErr(err, fmt.Sprintf("download function")) + // } + + // defer resp.Body.Close() + // if resp.StatusCode != http.StatusOK { + // err = fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) + // checkErr(err, fmt.Sprintf("download function")) + // } + + // contents, err := ioutil.ReadAll(resp.Body) + // if err != nil { + // checkErr(err, fmt.Sprintf("download function body %v", url)) + // } + + // return +} + +func fileSize(filePath string) int64 { + info, err := os.Stat(filePath) + checkErr(err, fmt.Sprintf("stat %v", filePath)) + return info.Size() +} + +func getPackageContents(filePath string) []byte { var code []byte var err error - if strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") { - var resp *http.Response - resp, err = http.Get(filePath) - if err != nil { - checkErr(err, fmt.Sprintf("download function")) - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) - checkErr(err, fmt.Sprintf("download function")) - } - - code, err = ioutil.ReadAll(resp.Body) - if err != nil { - checkErr(err, fmt.Sprintf("download function body %v", filePath)) - } - } else { - code, err = ioutil.ReadFile(filePath) - checkErr(err, fmt.Sprintf("read %v", filePath)) - } + code, err = ioutil.ReadFile(filePath) + checkErr(err, fmt.Sprintf("read %v", filePath)) return code } @@ -83,12 +91,25 @@ func fnCreate(c *cli.Context) error { } } - code := fnFetchCode(fileName) + if fileSize(fileName) > fission.PackageLiteralSizeLimit { + // TODO fallback to uploading + setting a Package URL + fmt.Printf("File size not supported yet") + os.Exit(1) + } - function := &fission.Function{ - Metadata: fission.Metadata{Name: fnName}, - Environment: fission.Metadata{Name: envName}, - Code: string(code), + pkgContents := getPackageContents(fileName) + + function := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.FunctionSpec{ + EnvironmentName: envName, + Deployment: fission.Package{ + Literal: pkgContents, + }, + }, } _, err := client.FunctionCreate(function) @@ -106,14 +127,18 @@ func fnCreate(c *cli.Context) error { method = "GET" } triggerName := uuid.NewV4().String() - ht := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: triggerName, + ht := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: triggerName, + Namespace: api.NamespaceDefault, }, - UrlPattern: triggerUrl, - Method: getMethod(method), - Function: fission.Metadata{ - Name: fnName, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + Method: getMethod(method), + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } _, err = client.HTTPTriggerCreate(ht) @@ -130,13 +155,15 @@ func fnGet(c *cli.Context) error { if len(fnName) == 0 { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } - code, err := client.FunctionGetRaw(m) + code, err := client.FunctionGetRawDeployment(m) checkErr(err, "get function") - fmt.Println(string(code)) + os.Stdout.Write(code) return err } @@ -148,8 +175,10 @@ func fnGetMeta(c *cli.Context) error { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } f, err := client.FunctionGet(m) checkErr(err, "get function") @@ -157,7 +186,7 @@ func fnGetMeta(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV") fmt.Fprintf(w, "%v\t%v\t%v\n", - f.Metadata.Name, f.Metadata.Uid, f.Environment.Name) + f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName) w.Flush() return err } @@ -170,14 +199,13 @@ func fnUpdate(c *cli.Context) error { fatal("Need name of function, use --name") } - function, err := client.FunctionGet(&fission.Metadata{Name: fnName}) + function, err := client.FunctionGet(&api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + }) checkErr(err, fmt.Sprintf("read function '%v'", fnName)) envName := c.String("env") - if len(envName) > 0 { - function.Environment.Name = envName - } - fileName := c.String("code") if len(fileName) == 0 { fileName = c.String("package") @@ -188,8 +216,16 @@ func fnUpdate(c *cli.Context) error { } if len(fileName) > 0 { - code := fnFetchCode(fileName) - function.Code = string(code) + if fileSize(fileName) > fission.PackageLiteralSizeLimit { + // TODO fallback to uploading + setting a Package URL + fmt.Printf("File size >256k not supported yet") + os.Exit(1) + } + + function.Spec.Deployment.Literal = getPackageContents(fileName) + } + if len(envName) > 0 { + function.Spec.EnvironmentName = envName } _, err = client.FunctionUpdate(function) @@ -207,8 +243,10 @@ func fnDelete(c *cli.Context) error { fatal("Need name of function, use --name") } - fnUid := c.String("uid") - m := &fission.Metadata{Name: fnName, Uid: fnUid} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } err := client.FunctionDelete(m) checkErr(err, fmt.Sprintf("delete function '%v'", fnName)) @@ -228,65 +266,13 @@ func fnList(c *cli.Context) error { fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV") for _, f := range fns { fmt.Fprintf(w, "%v\t%v\t%v\n", - f.Metadata.Name, f.Metadata.Uid, f.Environment.Name) + f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName) } w.Flush() return err } -func fnEdit(c *cli.Context) error { - client := getClient(c.GlobalString("server")) - - fnName := c.String("name") - if len(fnName) == 0 { - fatal("Need name of function, use --name") - } - fnUid := c.String("uid") - - // get function meta - function, err := client.FunctionGet(&fission.Metadata{Name: fnName, Uid: fnUid}) - checkErr(err, fmt.Sprintf("read function '%v'", fnName)) - - // write to tmp file - tmpFile, err := ioutil.TempFile("", fnName) - checkErr(err, "create temp file") - defer os.Remove(tmpFile.Name()) - - _, err = tmpFile.Write([]byte(function.Code)) - checkErr(err, "write temp file") - tmpFile.Close() - - // invoke $EDITOR on tmp file and wait for it - editor := os.Getenv("EDITOR") - if len(editor) == 0 { - editor = "vi" - } - cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("%v %v", editor, tmpFile.Name())) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Start() - checkErr(err, "start editor") - - err = cmd.Wait() - checkErr(err, "wait for editor") - - // read new code out of the file - contents, err := ioutil.ReadFile(tmpFile.Name()) - checkErr(err, "read temp file") - - function.Code = string(contents) - - // upload the updated function - newfn, err := client.FunctionUpdate(function) - checkErr(err, "upload edited function") - - fmt.Printf("function %v updated, new uuid: %v\n", newfn.Name, newfn.Uid) - return nil -} - func fnLogs(c *cli.Context) error { client := getClient(c.GlobalString("server")) @@ -301,13 +287,15 @@ func fnLogs(c *cli.Context) error { } fnPod := c.String("pod") - m := &fission.Metadata{Name: fnName} + m := &api.ObjectMeta{ + Name: fnName, + Namespace: api.NamespaceDefault, + } f, err := client.FunctionGet(m) checkErr(err, "get function") - // client first send db query to controller, then controller will - // establishe a proxy server that bridges the client and the database. + // request the controller to establish a proxy server to the database. logDB, err := logdb.GetLogDB(dbType, c.GlobalString("server")) if err != nil { fatal("failed to connect log database") @@ -325,7 +313,7 @@ func fnLogs(c *cli.Context) error { logFilter := logdb.LogFilter{ Pod: fnPod, Function: f.Metadata.Name, - FuncUid: f.Metadata.Uid, + FuncUid: string(f.Metadata.UID), Since: t, } logEntries, err := logDB.GetLogs(logFilter) @@ -372,7 +360,7 @@ func fnPods(c *cli.Context) error { dbType = logdb.INFLUXDB } - m := &fission.Metadata{Name: fnName} + m := &api.ObjectMeta{Name: fnName} f, err := client.FunctionGet(m) checkErr(err, "get function") @@ -386,7 +374,7 @@ func fnPods(c *cli.Context) error { logFilter := logdb.LogFilter{ Function: f.Metadata.Name, - FuncUid: f.Metadata.Uid, + FuncUid: string(f.Metadata.UID), } pods, err := logDB.GetPods(logFilter) if err != nil { diff --git a/fission/httptrigger.go b/fission/httptrigger.go index 42afea5a..db0289f8 100644 --- a/fission/httptrigger.go +++ b/fission/httptrigger.go @@ -25,8 +25,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) // returns one of http.Method* @@ -62,7 +64,6 @@ func htCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") triggerUrl := c.String("url") if len(triggerUrl) == 0 { fatal("Need a trigger URL, use --url") @@ -75,15 +76,18 @@ func htCreate(c *cli.Context) error { // just name triggers by uuid. triggerName := uuid.NewV4().String() - ht := &fission.HTTPTrigger{ - Metadata: fission.Metadata{ - Name: triggerName, + ht := &tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: triggerName, + Namespace: api.NamespaceDefault, }, - UrlPattern: triggerUrl, - Method: getMethod(method), - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + Method: getMethod(method), + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } @@ -105,11 +109,21 @@ func htUpdate(c *cli.Context) error { fatal("Need name of trigger, use --name") } - ht, err := client.HTTPTriggerGet(&fission.Metadata{Name: htName}) + // update function ref + newFn := c.String("function") + if len(newFn) == 0 { + fatal("Nothing to update. Use --function to specify a new function.") + } + + ht, err := client.HTTPTriggerGet(&api.ObjectMeta{ + Name: htName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "get HTTP trigger") - newUid := c.String("uid") - ht.Function.Uid = newUid + if len(newFn) > 0 { + ht.Spec.FunctionReference.Name = newFn + } _, err = client.HTTPTriggerUpdate(ht) checkErr(err, "update HTTP trigger") @@ -125,7 +139,10 @@ func htDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.HTTPTriggerDelete(&fission.Metadata{Name: htName}) + err := client.HTTPTriggerDelete(&api.ObjectMeta{ + Name: htName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", htName) @@ -140,10 +157,10 @@ func htList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", "NAME", "METHOD", "URL", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "NAME", "METHOD", "URL", "FUNCTION_NAME") for _, ht := range hts { - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", - ht.Metadata.Name, ht.Method, ht.UrlPattern, ht.Function.Name, ht.Function.Uid) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", + ht.Metadata.Name, ht.Spec.Method, ht.Spec.RelativeURL, ht.Spec.FunctionReference.Name) } w.Flush() diff --git a/fission/main.go b/fission/main.go index 614fb216..734d0e56 100644 --- a/fission/main.go +++ b/fission/main.go @@ -40,18 +40,16 @@ func main() { fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"} fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"} fnPackageFlag := cli.StringFlag{Name: "package", Usage: "local path or URL for binary package"} - fnUidFlag := cli.StringFlag{Name: "uid", Usage: "function uid, optional (use latest if unspecified)"} fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"} fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"} fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"} fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"} fnSubcommands := []cli.Command{ {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate}, - {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGet}, - {Name: "edit", Usage: "Edit function source code in $EDITOR", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnEdit}, - {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGetMeta}, + {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet}, + {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta}, {Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag}, Action: fnUpdate}, - {Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnDelete}, + {Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete}, {Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList}, {Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs}, {Name: "pods", Usage: "Display function pods", Flags: []cli.Flag{fnNameFlag, fnLogDBTypeFlag}, Action: fnPods}, @@ -60,11 +58,10 @@ func main() { // httptriggers htNameFlag := cli.StringFlag{Name: "name", Usage: "HTTP Trigger name"} htFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - htFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} htSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag, htFnUidFlag}, Action: htCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag}, Action: htCreate}, {Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag}, Action: htGet}, - {Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, htFnNameFlag, htFnUidFlag}, Action: htUpdate}, + {Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, htFnNameFlag}, Action: htUpdate}, {Name: "delete", Usage: "Delete HTTP trigger", Flags: []cli.Flag{htNameFlag}, Action: htDelete}, {Name: "list", Usage: "List HTTP triggers", Flags: []cli.Flag{}, Action: htList}, } @@ -73,11 +70,10 @@ func main() { ttNameFlag := cli.StringFlag{Name: "name", Usage: "Time Trigger name"} ttCronFlag := cli.StringFlag{Name: "cron", Usage: "Time Trigger cron spec ('0 30 * * *', '@every 5m', '@hourly')"} ttFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - ttFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} ttSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create Time trigger", Flags: []cli.Flag{ttNameFlag, ttFnNameFlag, ttFnUidFlag, ttCronFlag}, Action: ttCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create Time trigger", Flags: []cli.Flag{ttNameFlag, ttFnNameFlag, ttCronFlag}, Action: ttCreate}, {Name: "get", Usage: "Get Time trigger", Flags: []cli.Flag{}, Action: ttGet}, - {Name: "update", Usage: "Update Time trigger", Flags: []cli.Flag{ttNameFlag, ttCronFlag}, Action: ttUpdate}, + {Name: "update", Usage: "Update Time trigger", Flags: []cli.Flag{ttNameFlag, ttCronFlag, ttFnNameFlag}, Action: ttUpdate}, {Name: "delete", Usage: "Delete Time trigger", Flags: []cli.Flag{ttNameFlag}, Action: ttDelete}, {Name: "list", Usage: "List Time triggers", Flags: []cli.Flag{}, Action: ttList}, } @@ -85,14 +81,13 @@ func main() { // Message queue trigger mqtNameFlag := cli.StringFlag{Name: "name", Usage: "Message queue Trigger name"} mqtFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - mqtFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} mqtMQTypeFlag := cli.StringFlag{Name: "mqtype", Usage: "Message queue type, e.g. nats-streaming (optional; uses \"nats-streaming\" if unspecified)"} mqtTopicFlag := cli.StringFlag{Name: "topic", Usage: "Message queue Topic the trigger listens on"} mqtRespTopicFlag := cli.StringFlag{Name: "resptopic", Usage: "Topic that the function response is sent on (optional; response discarded if unspecified)"} mqtSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create Message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtFnNameFlag, mqtFnUidFlag, mqtMQTypeFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create Message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtFnNameFlag, mqtMQTypeFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtCreate}, {Name: "get", Usage: "Get message queue trigger", Flags: []cli.Flag{}, Action: mqtGet}, - {Name: "update", Usage: "Update message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtTopicFlag, mqtRespTopicFlag}, Action: mqtUpdate}, + {Name: "update", Usage: "Update message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtTopicFlag, mqtRespTopicFlag, mqtFnNameFlag}, Action: mqtUpdate}, {Name: "delete", Usage: "Delete message queue trigger", Flags: []cli.Flag{mqtNameFlag}, Action: mqtDelete}, {Name: "list", Usage: "List message queue triggers", Flags: []cli.Flag{mqtMQTypeFlag}, Action: mqtList}, } @@ -111,12 +106,11 @@ func main() { // watches wNameFlag := cli.StringFlag{Name: "name", Usage: "Watch name"} wFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"} - wFnUidFlag := cli.StringFlag{Name: "uid", Usage: "Function UID (optional; uses latest if unspecified)"} wNamespaceFlag := cli.StringFlag{Name: "ns", Usage: "Namespace of resource to watch"} wObjTypeFlag := cli.StringFlag{Name: "type", Usage: "Type of resource to watch (Pod, Service, etc.)"} wLabelsFlag := cli.StringFlag{Name: "labels", Usage: "Label selector of the form a=b,c=d"} wSubCommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Create a watch", Flags: []cli.Flag{wFnNameFlag, wFnUidFlag, wNamespaceFlag, wObjTypeFlag, wLabelsFlag}, Action: wCreate}, + {Name: "create", Aliases: []string{"add"}, Usage: "Create a watch", Flags: []cli.Flag{wFnNameFlag, wNamespaceFlag, wObjTypeFlag, wLabelsFlag}, Action: wCreate}, {Name: "get", Usage: "Get details about a watch", Flags: []cli.Flag{wNameFlag}, Action: wGet}, // TODO add update flag when supported {Name: "delete", Usage: "Delete watch", Flags: []cli.Flag{wNameFlag}, Action: wDelete}, @@ -130,16 +124,6 @@ func main() { {Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands}, {Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands}, {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, - - // Misc commands - { - Name: "get-deployment-yaml", - Usage: "Get deployment yaml. Use it as 'fission get-deployment-yaml | kubectl create -f -'", - Action: func(c *cli.Context) error { - getDeploymentYaml() - return nil - }, - }, } app.Run(os.Args) diff --git a/fission/mqtrigger.go b/fission/mqtrigger.go index d0e7b4d8..3c0465f6 100644 --- a/fission/mqtrigger.go +++ b/fission/mqtrigger.go @@ -23,9 +23,11 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/mqtrigger/messageQueue" + "github.com/fission/fission/tpr" ) func mqtCreate(c *cli.Context) error { @@ -39,7 +41,7 @@ func mqtCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") + mqType := c.String("mqtype") switch mqType { case "": @@ -58,24 +60,27 @@ func mqtCreate(c *cli.Context) error { respTopic := c.String("resptopic") if topic == respTopic { + // TODO maybe this should just be a warning, perhaps + // allow it behind a --force flag fatal("Listen topic should not equal to response topic") } checkMQTopicAvailability(mqType, topic, respTopic) - fnMeta := fission.Metadata{ - Name: fnName, - Uid: fnUid, - } - - mqt := fission.MessageQueueTrigger{ - Metadata: fission.Metadata{ - Name: mqtName, + mqt := tpr.Messagequeuetrigger{ + Metadata: api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }, + Spec: fission.MessageQueueTriggerSpec{ + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, + MessageQueueType: mqType, + Topic: topic, + ResponseTopic: respTopic, }, - Function: fnMeta, - MessageQueueType: mqType, - Topic: topic, - ResponseTopic: respTopic, } _, err := client.MessageQueueTriggerCreate(&mqt) @@ -97,14 +102,33 @@ func mqtUpdate(c *cli.Context) error { } topic := c.String("topic") respTopic := c.String("resptopic") + fnName := c.String("function") - mqt, err := client.MessageQueueTriggerGet(&fission.Metadata{Name: mqtName}) + mqt, err := client.MessageQueueTriggerGet(&api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "get Time trigger") - checkMQTopicAvailability(mqt.MessageQueueType, topic, respTopic) + checkMQTopicAvailability(mqt.Spec.MessageQueueType, topic, respTopic) - mqt.Topic = topic - mqt.ResponseTopic = respTopic + updated := false + if len(topic) > 0 { + mqt.Spec.Topic = topic + updated = true + } + if len(respTopic) > 0 { + mqt.Spec.ResponseTopic = respTopic + updated = true + } + if len(fnName) > 0 { + mqt.Spec.FunctionReference.Name = fnName + updated = true + } + + if !updated { + fatal("Nothing to update. Use --topic, --resptopic, or --function.") + } _, err = client.MessageQueueTriggerUpdate(mqt) checkErr(err, "update Time trigger") @@ -120,7 +144,10 @@ func mqtDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.MessageQueueTriggerDelete(&fission.Metadata{Name: mqtName}) + err := client.MessageQueueTriggerDelete(&api.ObjectMeta{ + Name: mqtName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", mqtName) @@ -135,11 +162,11 @@ func mqtList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - "NAME", "FUNCTION_NAME", "FUNCTION_UID", "MESSAGE_QUEUE_TYPE", "TOPIC", "RESPONSE_TOPIC") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + "NAME", "FUNCTION_NAME", "MESSAGE_QUEUE_TYPE", "TOPIC", "RESPONSE_TOPIC") for _, mqt := range mqts { - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - mqt.Metadata.Name, mqt.Function.Name, mqt.Function.Uid, mqt.MessageQueueType, mqt.Topic, mqt.ResponseTopic) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + mqt.Metadata.Name, mqt.Spec.FunctionReference.Name, mqt.Spec.MessageQueueType, mqt.Spec.Topic, mqt.Spec.ResponseTopic) } w.Flush() diff --git a/fission/timetrigger.go b/fission/timetrigger.go index a7776851..d359d4a9 100644 --- a/fission/timetrigger.go +++ b/fission/timetrigger.go @@ -23,8 +23,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func ttCreate(c *cli.Context) error { @@ -38,20 +40,22 @@ func ttCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a trigger, use --function") } - fnUid := c.String("uid") cron := c.String("cron") if len(cron) == 0 { - fatal("Need a cron spec like '0 30 * * *', '@every 1h30m', '@hourly', use --cron") + fatal("Need a cron spec like '0 30 * * *', '@every 1h30m', or '@hourly'; use --cron") } - tt := &fission.TimeTrigger{ - Metadata: fission.Metadata{ - Name: name, + tt := &tpr.Timetrigger{ + Metadata: api.ObjectMeta{ + Name: name, + Namespace: api.NamespaceDefault, }, - Cron: cron, - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.TimeTriggerSpec{ + Cron: cron, + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fnName, + }, }, } @@ -73,12 +77,26 @@ func ttUpdate(c *cli.Context) error { fatal("Need name of trigger, use --name") } - tt, err := client.TimeTriggerGet(&fission.Metadata{Name: ttName}) - checkErr(err, "get Time trigger") + tt, err := client.TimeTriggerGet(&api.ObjectMeta{ + Name: ttName, + Namespace: api.NamespaceDefault, + }) + checkErr(err, "get time trigger") + updated := false newCron := c.String("cron") if len(newCron) != 0 { - tt.Cron = newCron + tt.Spec.Cron = newCron + updated = true + } + fnName := c.String("function") + if len(fnName) > 0 { + tt.Spec.FunctionReference.Name = fnName + updated = true + } + + if !updated { + fatal("Nothing to update. Use --cron or --function.") } _, err = client.TimeTriggerUpdate(tt) @@ -95,7 +113,10 @@ func ttDelete(c *cli.Context) error { fatal("Need name of trigger to delete, use --name") } - err := client.TimeTriggerDelete(&fission.Metadata{Name: ttName}) + err := client.TimeTriggerDelete(&api.ObjectMeta{ + Name: ttName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete trigger") fmt.Printf("trigger '%v' deleted\n", ttName) @@ -110,11 +131,10 @@ func ttList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", - "NAME", "CRON", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "CRON", "FUNCTION_NAME") for _, tt := range tts { fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", - tt.Metadata.Name, tt.Cron, tt.Function.Name, tt.Function.Uid) + tt.Metadata.Name, tt.Spec.Cron, tt.Spec.FunctionReference.Name) } w.Flush() diff --git a/fission/watch.go b/fission/watch.go index 222841f0..bbe68d29 100644 --- a/fission/watch.go +++ b/fission/watch.go @@ -23,8 +23,10 @@ import ( "github.com/satori/go.uuid" "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func wCreate(c *cli.Context) error { @@ -34,7 +36,6 @@ func wCreate(c *cli.Context) error { if len(fnName) == 0 { fatal("Need a function name to create a watch, use --function") } - fnUid := c.String("uid") namespace := c.String("ns") if len(namespace) == 0 { @@ -52,23 +53,28 @@ func wCreate(c *cli.Context) error { // empty 'labels' selects everything if len(labels) == 0 { fmt.Printf("Watching all objects of type '%v', use --labels to refine selection.\n", objType) + } else { + // TODO + fmt.Printf("Label selector not implemented, watching all objects") } // automatically name watches watchName := uuid.NewV4().String() - w := &fission.Watch{ - Metadata: fission.Metadata{ - Name: watchName, + w := &tpr.Kuberneteswatchtrigger{ + Metadata: api.ObjectMeta{ + Name: watchName, + Namespace: api.NamespaceDefault, }, - Function: fission.Metadata{ - Name: fnName, - Uid: fnUid, + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: namespace, + Type: objType, + //LabelSelector: labels, + FunctionReference: fission.FunctionReference{ + Name: fnName, + Type: fission.FunctionReferenceTypeFunctionName, + }, }, - Namespace: namespace, - ObjType: objType, - LabelSelector: labels, - FieldSelector: "", // TODO } _, err := client.WatchCreate(w) @@ -79,10 +85,14 @@ func wCreate(c *cli.Context) error { } func wGet(c *cli.Context) error { + // TODO + fatal("Not implemented") return nil } func wUpdate(c *cli.Context) error { + // TODO + fatal("Not implemented") return nil } @@ -94,7 +104,10 @@ func wDelete(c *cli.Context) error { fatal("Need name of watch to delete, use --name") } - err := client.WatchDelete(&fission.Metadata{Name: wName}) + err := client.WatchDelete(&api.ObjectMeta{ + Name: wName, + Namespace: api.NamespaceDefault, + }) checkErr(err, "delete watch") fmt.Printf("watch '%v' deleted\n", wName) @@ -109,10 +122,11 @@ func wList(c *cli.Context) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "NAMESPACE", "OBJTYPE", "LABELS", "FUNCTION_NAME", "FUNCTION_UID") + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", + "NAME", "NAMESPACE", "OBJTYPE", "LABELS", "FUNCTION_NAME") for _, wa := range ws { fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", - wa.Metadata.Name, wa.Namespace, wa.ObjType, wa.LabelSelector, wa.Function.Name, wa.Function.Uid) + wa.Metadata.Name, wa.Spec.Namespace, wa.Spec.Type, wa.Spec.LabelSelector, wa.Spec.FunctionReference.Name) } w.Flush() diff --git a/kubewatcher/kubewatcher.go b/kubewatcher/kubewatcher.go index f9c680dd..fe4bd937 100644 --- a/kubewatcher/kubewatcher.go +++ b/kubewatcher/kubewatcher.go @@ -33,10 +33,12 @@ import ( apierrs "k8s.io/client-go/1.5/pkg/api/errors" "k8s.io/client-go/1.5/pkg/api/meta" "k8s.io/client-go/1.5/pkg/runtime" + "k8s.io/client-go/1.5/pkg/types" "k8s.io/client-go/1.5/pkg/watch" "github.com/fission/fission" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) type requestType int @@ -47,7 +49,7 @@ const ( type ( KubeWatcher struct { - watches map[string]watchSubscription + watches map[types.UID]watchSubscription kubernetesClient *kubernetes.Clientset requestChannel chan *kubeWatcherRequest publisher publisher.Publisher @@ -55,7 +57,7 @@ type ( } watchSubscription struct { - fission.Watch + watch tpr.Kuberneteswatchtrigger kubeWatch watch.Interface lastResourceVersion string stopped *int32 @@ -65,7 +67,7 @@ type ( kubeWatcherRequest struct { requestType - watches []fission.Watch + watches []tpr.Kuberneteswatchtrigger responseChannel chan *kubeWatcherResponse } kubeWatcherResponse struct { @@ -75,7 +77,7 @@ type ( func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ - watches: make(map[string]watchSubscription), + watches: make(map[types.UID]watchSubscription), kubernetesClient: kubernetesClient, publisher: publisher, requestChannel: make(chan *kubeWatcherRequest), @@ -84,7 +86,7 @@ func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher return kw } -func (kw *KubeWatcher) Sync(watches []fission.Watch) error { +func (kw *KubeWatcher) Sync(watches []tpr.Kuberneteswatchtrigger) error { req := &kubeWatcherRequest{ requestType: SYNC, watches: watches, @@ -100,19 +102,19 @@ func (kw *KubeWatcher) svc() { req := <-kw.requestChannel switch req.requestType { case SYNC: - newWatchUids := make(map[string]bool) + newWatchUids := make(map[types.UID]bool) for _, w := range req.watches { - newWatchUids[w.Metadata.Uid] = true + newWatchUids[w.Metadata.UID] = true } // Remove old watches for uid, ws := range kw.watches { if _, ok := newWatchUids[uid]; !ok { - kw.removeWatch(&ws.Watch) + kw.removeWatch(&ws.watch) } } // Add new watches for _, w := range req.watches { - if _, ok := kw.watches[w.Metadata.Uid]; !ok { + if _, ok := kw.watches[w.Metadata.UID]; !ok { kw.addWatch(&w) } } @@ -144,7 +146,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error { return err } -func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, resourceVersion string) (watch.Interface, error) { +func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *tpr.Kuberneteswatchtrigger, resourceVersion string) (watch.Interface, error) { var wi watch.Interface var err error var watchTimeoutSec int64 = 120 @@ -156,41 +158,41 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fission.Watch, r } // TODO handle the full list of types - switch strings.ToUpper(w.ObjType) { + switch strings.ToUpper(w.Spec.Type) { case "POD": - wi, err = kubeClient.Core().Pods(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().Pods(w.Spec.Namespace).Watch(listOptions) case "SERVICE": - wi, err = kubeClient.Core().Services(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().Services(w.Spec.Namespace).Watch(listOptions) case "REPLICATIONCONTROLLER": - wi, err = kubeClient.Core().ReplicationControllers(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Core().ReplicationControllers(w.Spec.Namespace).Watch(listOptions) case "JOB": - wi, err = kubeClient.Batch().Jobs(w.Namespace).Watch(listOptions) + wi, err = kubeClient.Batch().Jobs(w.Spec.Namespace).Watch(listOptions) default: - msg := fmt.Sprintf("Error: unknown obj type '%v'", w.ObjType) + msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type) log.Println(msg) err = errors.New(msg) } return wi, err } -func (kw *KubeWatcher) addWatch(w *fission.Watch) error { - log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Function.Name) +func (kw *KubeWatcher) addWatch(w *tpr.Kuberneteswatchtrigger) error { + log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher) if err != nil { return err } - kw.watches[w.Metadata.Uid] = *ws + kw.watches[w.Metadata.UID] = *ws return nil } -func (kw *KubeWatcher) removeWatch(w *fission.Watch) error { - log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Function.Name) - ws, ok := kw.watches[w.Metadata.Uid] +func (kw *KubeWatcher) removeWatch(w *tpr.Kuberneteswatchtrigger) error { + log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) + ws, ok := kw.watches[w.Metadata.UID] if !ok { return fission.MakeError(fission.ErrorNotFound, fmt.Sprintf("watch doesn't exist: %v", w.Metadata)) } - delete(kw.watches, w.Metadata.Uid) + delete(kw.watches, w.Metadata.UID) ws.stop() return nil } @@ -210,10 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *fission.Watch) error { // return nil // } -func MakeWatchSubscription(w *fission.Watch, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(w *tpr.Kuberneteswatchtrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ - Watch: *w, + watch: *w, kubeWatch: nil, stopped: &stopped, kubernetesClient: kubeClient, @@ -234,8 +236,8 @@ func (ws *watchSubscription) restartWatch() error { retries := 60 for { log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v", - ws.Watch.Metadata, ws.Watch.Namespace, ws.Watch.ObjType, ws.lastResourceVersion) - wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.Watch, ws.lastResourceVersion) + ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion) + wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- if retries > 0 { @@ -259,7 +261,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { } func (ws *watchSubscription) eventDispatchLoop() { - log.Println("Listening to watch ", ws.Watch.Metadata.Name) + log.Println("Listening to watch ", ws.watch.Metadata.Name) for { // check watchSubscription is stopped or not before waiting for event // comes from the kubeWatch.ResultChan(). This fix the edge case that @@ -272,11 +274,11 @@ func (ws *watchSubscription) eventDispatchLoop() { if !more { if ws.isStopped() { // watch is removed by user. - log.Println("Watch stopped", ws.Watch.Metadata.Name) + log.Println("Watch stopped", ws.watch.Metadata.Name) return } else { // watch closed due to timeout, restart it. - log.Printf("Watch %v timed out, restarting", ws.Watch.Metadata.Name) + log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name) err := ws.restartWatch() if err != nil { log.Panicf("Failed to restart watch: %v", err) @@ -318,7 +320,16 @@ func (ws *watchSubscription) eventDispatchLoop() { "X-Kubernetes-Event-Type": string(ev.Type), "X-Kubernetes-Object-Type": reflect.TypeOf(ev.Object).Elem().Name(), } - ws.publisher.Publish(buf.String(), headers, ws.Watch.Target) + + // TODO support other function ref types. Or perhaps delegate to router? + if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { + log.Printf("Error: unsupported function ref type: %v, can't publish event", + ws.watch.Spec.FunctionReference.Type) + continue + } + + url := fission.UrlForFunction(ws.watch.Spec.FunctionReference.Name) + ws.publisher.Publish(buf.String(), headers, url) } } diff --git a/kubewatcher/main.go b/kubewatcher/main.go index 28118fc5..601a0f55 100644 --- a/kubewatcher/main.go +++ b/kubewatcher/main.go @@ -17,43 +17,18 @@ limitations under the License. package kubewatcher import ( - "log" - - "k8s.io/client-go/1.5/kubernetes" - "k8s.io/client-go/1.5/rest" - - "github.com/fission/fission/controller/client" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) -// Get a kubernetes client using the pod's service account. -func getKubernetesClient() (*kubernetes.Clientset, error) { - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - log.Printf("Error getting kubernetes client config: %v", err) - return nil, err - } - - // creates the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Printf("Error getting kubernetes client: %v", err) - return nil, err - } - - return clientset, nil -} - -func Start(controllerUrl string, routerUrl string) error { - kubeClient, err := getKubernetesClient() +func Start(routerUrl string) error { + fissionClient, kubeClient, err := tpr.MakeFissionClient() if err != nil { return err } poster := publisher.MakeWebhookPublisher(routerUrl) kubeWatch := MakeKubeWatcher(kubeClient, poster) - - MakeWatchSync(client.MakeClient(controllerUrl), kubeWatch) + MakeWatchSync(fissionClient, kubeWatch) return nil } diff --git a/kubewatcher/watchSync.go b/kubewatcher/watchSync.go index d3157516..5e8a15bf 100644 --- a/kubewatcher/watchSync.go +++ b/kubewatcher/watchSync.go @@ -20,17 +20,19 @@ import ( "log" "time" - "github.com/fission/fission/controller/client" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" ) type ( WatchSync struct { - client *client.Client + client *tpr.FissionClient kubeWatcher *KubeWatcher } ) -func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync { +func MakeWatchSync(client *tpr.FissionClient, kubeWatcher *KubeWatcher) *WatchSync { ws := &WatchSync{ client: client, kubeWatcher: kubeWatcher, @@ -40,20 +42,14 @@ func MakeWatchSync(client *client.Client, kubeWatcher *KubeWatcher) *WatchSync { } func (ws *WatchSync) syncSvc() { - failureCount := 0 - maxFailures := 6 + // TODO watch instead of polling for { - watches, err := ws.client.WatchList() + watches, err := ws.client.Kuberneteswatchtriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - failureCount++ - if failureCount > maxFailures { - log.Fatalf("Failed to connect to controller: %v", err) - } - time.Sleep(10 * time.Second) - continue + log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err) } - ws.kubeWatcher.Sync(watches) + ws.kubeWatcher.Sync(watches.Items) time.Sleep(3 * time.Second) } } diff --git a/mqtrigger/main.go b/mqtrigger/main.go index 31c0a9f2..e5219ed5 100644 --- a/mqtrigger/main.go +++ b/mqtrigger/main.go @@ -19,12 +19,16 @@ package messagequeue import ( "os" - controllerClient "github.com/fission/fission/controller/client" "github.com/fission/fission/mqtrigger/messageQueue" + "github.com/fission/fission/tpr" + "log" ) -func Start(controllerUrl string, routerUrl string) error { - controller := controllerClient.MakeClient(controllerUrl) +func Start(routerUrl string) error { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Failed to get fission client: %v", err) + } // Message queue type: nats is the only supported one for now mqType := os.Getenv("MESSAGE_QUEUE_TYPE") @@ -33,6 +37,6 @@ func Start(controllerUrl string, routerUrl string) error { MQType: mqType, Url: mqUrl, } - messageQueue.MakeMessageQueueTriggerManager(controller, routerUrl, mqCfg) + messageQueue.MakeMessageQueueTriggerManager(fissionClient, routerUrl, mqCfg) return nil } diff --git a/mqtrigger/messageQueue/messageQueue.go b/mqtrigger/messageQueue/messageQueue.go index 14356b9c..c2d80d7c 100644 --- a/mqtrigger/messageQueue/messageQueue.go +++ b/mqtrigger/messageQueue/messageQueue.go @@ -21,9 +21,10 @@ import ( "time" log "github.com/sirupsen/logrus" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" - controllerClient "github.com/fission/fission/controller/client" + //"github.com/fission/fission" + "github.com/fission/fission/tpr" ) const ( @@ -47,21 +48,20 @@ type ( } MessageQueue interface { - subscribe(trigger fission.MessageQueueTrigger) (messageQueueSubscription, error) + subscribe(trigger *tpr.Messagequeuetrigger) (messageQueueSubscription, error) unsubscribe(triggerSub messageQueueSubscription) error } MessageQueueTriggerManager struct { - reqChan chan request - mqCfg MessageQueueConfig - triggers map[string]*triggerSubscription - controller *controllerClient.Client - messageQueue MessageQueue + reqChan chan request + mqCfg MessageQueueConfig + triggers map[string]*triggerSubscription + fissionClient *tpr.FissionClient + messageQueue MessageQueue } triggerSubscription struct { - fission.Metadata - funcMeta fission.Metadata + trigger tpr.Messagequeuetrigger subscription messageQueueSubscription } @@ -72,20 +72,18 @@ type ( } response struct { err error - triggers *map[string]messageQueueSubscription + triggers *map[string]*triggerSubscription } ) -func MakeMessageQueueTriggerManager(ctrlClient *controllerClient.Client, - routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { - +func MakeMessageQueueTriggerManager(fissionClient *tpr.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { var messageQueue MessageQueue var err error mqTriggerMgr := MessageQueueTriggerManager{ - reqChan: make(chan request), - triggers: make(map[string]*triggerSubscription), - controller: ctrlClient, + reqChan: make(chan request), + triggers: make(map[string]*triggerSubscription), + fissionClient: fissionClient, } switch mqConfig.MQType { case NATS: @@ -108,22 +106,21 @@ func (mqt *MessageQueueTriggerManager) service() { switch req.requestType { case ADD_TRIGGER: var err error - triggerUid := req.triggerSub.Uid - if _, ok := mqt.triggers[triggerUid]; ok { + k := tpr.CacheKey(&req.triggerSub.trigger.Metadata) + if _, ok := mqt.triggers[k]; ok { err = errors.New("Trigger already exists") } else { - mqt.triggers[triggerUid] = req.triggerSub + mqt.triggers[k] = req.triggerSub } req.respChan <- response{err: err} case GET_ALL_TRIGGERS: - copyTriggers := make(map[string]messageQueueSubscription) + copyTriggers := make(map[string]*triggerSubscription) for key, val := range mqt.triggers { copyTriggers[key] = val } req.respChan <- response{triggers: ©Triggers} case DELETE_TRIGGER: - triggerUid := req.triggerSub.Uid - delete(mqt.triggers, triggerUid) + delete(mqt.triggers, tpr.CacheKey(&req.triggerSub.trigger.Metadata)) } } } @@ -139,7 +136,7 @@ func (mqt *MessageQueueTriggerManager) addTrigger(triggerSub *triggerSubscriptio return r.err } -func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]messageQueueSubscription { +func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]*triggerSubscription { respChan := make(chan response) mqt.reqChan <- request{ requestType: GET_ALL_TRIGGERS, @@ -149,12 +146,12 @@ func (mqt *MessageQueueTriggerManager) getAllTriggers() *map[string]messageQueue return r.triggers } -func (mqt *MessageQueueTriggerManager) delTrigger(triggerUid string) { +func (mqt *MessageQueueTriggerManager) delTrigger(m *api.ObjectMeta) { mqt.reqChan <- request{ requestType: DELETE_TRIGGER, triggerSub: &triggerSubscription{ - Metadata: fission.Metadata{ - Uid: triggerUid, + trigger: tpr.Messagequeuetrigger{ + Metadata: *m, }, }, } @@ -162,57 +159,61 @@ func (mqt *MessageQueueTriggerManager) delTrigger(triggerUid string) { func (mqt *MessageQueueTriggerManager) syncTriggers() { for { - // TODO: handle error - newTriggers, err := mqt.controller.MessageQueueTriggerList(mqt.mqCfg.MQType) + // get new set of triggers + newTriggers, err := mqt.fissionClient.Messagequeuetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - log.Warnf("Failed to sync message queue trigger from controller: %v", err) + log.Fatalf("Failed to read message queue trigger list: %v", err) } - // sync trigger from controller - newTriggerMap := map[string]fission.MessageQueueTrigger{} - for _, trigger := range newTriggers { - newTriggerMap[trigger.Uid] = trigger + newTriggerMap := make(map[string]*tpr.Messagequeuetrigger) + for _, trigger := range newTriggers.Items { + newTriggerMap[tpr.CacheKey(&trigger.Metadata)] = &trigger } - currentTriggerSubMap := mqt.getAllTriggers() + + // get current set of triggers + currentTriggers := mqt.getAllTriggers() // register new triggers for key, trigger := range newTriggerMap { - if _, ok := (*currentTriggerSubMap)[key]; ok { + if _, ok := (*currentTriggers)[key]; ok { continue } + + // actually subscribe using the message queue client impl sub, err := mqt.messageQueue.subscribe(trigger) if err != nil { - log.Warnf("Message queue trigger %s created failed: %v", trigger.Name, err) + log.Warnf("Failed to subscribe to message queue trigger %s: %v", trigger.Metadata.Name, err) continue } + triggerSub := triggerSubscription{ - Metadata: fission.Metadata{ - Name: trigger.Name, - Uid: trigger.Uid, - }, - funcMeta: trigger.Function, + trigger: *trigger, subscription: sub, } + + // add to our list err = mqt.addTrigger(&triggerSub) if err != nil { - log.Warnf("Message queue trigger %s created failed: %v", trigger.Name, err) - continue + log.Fatalf("Message queue trigger %s addition failed: %v", trigger.Metadata.Name, err) } - log.Infof("Message queue trigger %s created", trigger.Name) + + log.Infof("Message queue trigger %s created", trigger.Metadata.Name) } // remove old triggers - for _, ts := range *currentTriggerSubMap { - triggerSub := ts.(*triggerSubscription) - if _, ok := newTriggerMap[triggerSub.Uid]; ok { + for key, triggerSub := range *currentTriggers { + if _, ok := newTriggerMap[key]; ok { continue } - if err := mqt.messageQueue.unsubscribe(triggerSub.subscription); err != nil { - log.Warnf("Message queue trigger %s deleted failed: %v", triggerSub.Name, err) - } else { - mqt.delTrigger(triggerSub.Uid) - log.Infof("Message queue trigger %s deleted", triggerSub.Name) + err := mqt.messageQueue.unsubscribe(triggerSub.subscription) + if err != nil { + log.Warnf("Failed to unsubscribe to trigger %s: %v", triggerSub.trigger.Metadata.Name, err) + continue } + mqt.delTrigger(&triggerSub.trigger.Metadata) + log.Infof("Message queue trigger %s deleted", triggerSub.trigger.Metadata.Name) } + + // TODO replace with a watch time.Sleep(3 * time.Second) } } diff --git a/mqtrigger/messageQueue/nats.go b/mqtrigger/messageQueue/nats.go index 39f6185f..4373f4a0 100644 --- a/mqtrigger/messageQueue/nats.go +++ b/mqtrigger/messageQueue/nats.go @@ -29,6 +29,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/fission/fission" + "github.com/fission/fission/tpr" ) const ( @@ -57,17 +58,17 @@ func makeNatsMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQu return nats, nil } -func (nats Nats) subscribe(trigger fission.MessageQueueTrigger) (messageQueueSubscription, error) { - subj := trigger.Topic +func (nats Nats) subscribe(trigger *tpr.Messagequeuetrigger) (messageQueueSubscription, error) { + subj := trigger.Spec.Topic if !isTopicValidForNats(subj) { - return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Topic)) + return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Spec.Topic)) } opts := []ns.SubscriptionOption{ // Create a durable subscription to nats, so that triggers could retrieve last unack message. // https://github.com/nats-io/go-nats-streaming#durable-subscriptions - ns.DurableName(trigger.Uid), + ns.DurableName(string(trigger.Metadata.UID)), // Nats-streaming server is auto-ack mode by default. Since we want nats-streaming server to // resend a message if the trigger does not ack it, we need to enable the manual ack mode, so that @@ -90,14 +91,21 @@ func isTopicValidForNats(topic string) bool { return nsUtil.IsSubjectValid(topic) } -func msgHandler(nats *Nats, trigger fission.MessageQueueTrigger) func(*ns.Msg) { +func msgHandler(nats *Nats, trigger *tpr.Messagequeuetrigger) func(*ns.Msg) { return func(msg *ns.Msg) { - url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(&trigger.Function), "/") + + // Support other function ref types + if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { + log.Fatalf("Unsupported function reference type (%v) for trigger %v", + trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) + } + + url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name), "/") log.Printf("Making HTTP request to %v", url) headers := map[string]string{ - "X-Fission-MQTrigger-Topic": trigger.Topic, - "X-Fission-MQTrigger-RespTopic": trigger.ResponseTopic, + "X-Fission-MQTrigger-Topic": trigger.Spec.Topic, + "X-Fission-MQTrigger-RespTopic": trigger.Spec.ResponseTopic, } // Create request @@ -129,10 +137,10 @@ func msgHandler(nats *Nats, trigger fission.MessageQueueTrigger) func(*ns.Msg) { log.Warningf("Failed to ack message: %v", err) } - if len(trigger.ResponseTopic) > 0 { - err = nats.nsConn.Publish(trigger.ResponseTopic, body) + if len(trigger.Spec.ResponseTopic) > 0 { + err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body) if err != nil { - log.Warningf("Failed to publish message to topic %s: %v", trigger.ResponseTopic, err) + log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) } } } diff --git a/poolmgr/api.go b/poolmgr/api.go index 854a5ceb..e88af431 100644 --- a/poolmgr/api.go +++ b/poolmgr/api.go @@ -29,54 +29,46 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/cache" - controllerclient "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) -type funcSvc struct { - function *fission.Metadata // function this pod/service is for - environment *fission.Environment // env it was obtained from - address string // Host:Port or IP:Port that the service can be reached at. - podName string // pod name (within the function namespace) +type ( + createFuncServiceRequest struct { + funcMeta *api.ObjectMeta + respChan chan *createFuncServiceResponse + } - ctime time.Time - atime time.Time -} + createFuncServiceResponse struct { + address string + err error + } -type createFuncServiceRequest struct { - funcMeta *fission.Metadata - respChan chan *createFuncServiceResponse -} -type createFuncServiceResponse struct { - address string - err error -} + Poolmgr struct { + gpm *GenericPoolManager + functionEnv *cache.Cache // map[string]tpr.Environment + fsCache *functionServiceCache + fissionClient *tpr.FissionClient -type API struct { - poolMgr *GenericPoolManager - functionEnv *cache.Cache // map[fission.Metadata]fission.Environment - fsCache *functionServiceCache - controller *controllerclient.Client - fsCreateChannels map[fission.Metadata]*sync.WaitGroup - requestChan chan *createFuncServiceRequest + fsCreateChannels map[string]*sync.WaitGroup // xxx no channels here, rename this + requestChan chan *createFuncServiceRequest + } +) - //functionService *cache.Cache // map[fission.Metadata]*funcSvc - //urlFuncSvc *cache.Cache // map[string]*funcSvc -} - -func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCache *functionServiceCache) *API { - api := API{ - poolMgr: gpm, - functionEnv: cache.MakeCache(time.Minute, 0), +func MakePoolmgr(gpm *GenericPoolManager, fissionClient *tpr.FissionClient, fissionNs string, fsCache *functionServiceCache) *Poolmgr { + poolMgr := &Poolmgr{ + gpm: gpm, + functionEnv: cache.MakeCache(10*time.Second, 0), fsCache: fsCache, - controller: controller, - fsCreateChannels: make(map[fission.Metadata]*sync.WaitGroup), + fissionClient: fissionClient, + fsCreateChannels: make(map[string]*sync.WaitGroup), requestChan: make(chan *createFuncServiceRequest), } - go api.serveCreateFuncServices() - return &api + go poolMgr.serveCreateFuncServices() + return poolMgr } // All non-cached function service requests go through this goroutine @@ -85,29 +77,29 @@ func MakeAPI(gpm *GenericPoolManager, controller *controllerclient.Client, fsCac // get specialized. In other words, it ensures that when there's an // ongoing request for a certain function, all other requests wait for // that request to complete. -func (api *API) serveCreateFuncServices() { +func (poolMgr *Poolmgr) serveCreateFuncServices() { for { - req := <-api.requestChan + req := <-poolMgr.requestChan m := req.funcMeta // Cache miss -- is this first one to request the func? - wg, found := api.fsCreateChannels[*m] + wg, found := poolMgr.fsCreateChannels[tpr.CacheKey(m)] if !found { // create a waitgroup for other requests for // the same function to wait on wg := &sync.WaitGroup{} wg.Add(1) - api.fsCreateChannels[*m] = wg + poolMgr.fsCreateChannels[tpr.CacheKey(m)] = wg // launch a goroutine for each request, to parallelize // the specialization of different functions go func() { - address, err := api.createServiceForFunction(m) + address, err := poolMgr.createServiceForFunction(m) req.respChan <- &createFuncServiceResponse{ address: address, err: err, } - delete(api.fsCreateChannels, *m) + delete(poolMgr.fsCreateChannels, tpr.CacheKey(m)) wg.Done() }() } else { @@ -117,7 +109,7 @@ func (api *API) serveCreateFuncServices() { wg.Wait() // get the function service from the cache - fsvc, err := api.fsCache.GetByFunction(m) + fsvc, err := poolMgr.fsCache.GetByFunction(m) address := "" if err == nil { address = fsvc.address @@ -131,7 +123,7 @@ func (api *API) serveCreateFuncServices() { } } -func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) { +func (poolMgr *Poolmgr) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request", 500) @@ -139,14 +131,14 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) } // get function metadata - m := fission.Metadata{} + m := api.ObjectMeta{} err = json.Unmarshal(body, &m) if err != nil { http.Error(w, "Failed to parse request", 400) return } - serviceName, err := api.getServiceForFunction(&m) + serviceName, err := poolMgr.getServiceForFunction(&m) if err != nil { code, msg := fission.GetHTTPError(err) log.Printf("Error: %v: %v", code, msg) @@ -156,55 +148,46 @@ func (api *API) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) w.Write([]byte(serviceName)) } -func (api *API) getFunctionEnv(m *fission.Metadata) (*fission.Environment, error) { - var env *fission.Environment +func (poolMgr *Poolmgr) getFunctionEnv(m *api.ObjectMeta) (*tpr.Environment, error) { + var env *tpr.Environment // Cached ? - result, err := api.functionEnv.Get(*m) + result, err := poolMgr.functionEnv.Get(tpr.CacheKey(m)) if err == nil { - env = result.(*fission.Environment) + env = result.(*tpr.Environment) return env, nil } // Cache miss -- get func from controller - log.Printf("[%v] getting function from controller", m) - f, err := api.controller.FunctionGet(m) + f, err := poolMgr.fissionClient.Functions(m.Namespace).Get(m.Name) if err != nil { return nil, err } // Get env from metadata - log.Printf("[%v] getting env from controller", m) - env, err = api.controller.EnvironmentGet(&f.Environment) + log.Printf("[%v] getting env", m) + env, err = poolMgr.fissionClient.Environments(f.Metadata.Namespace).Get(f.Spec.EnvironmentName) if err != nil { return nil, err } - // cache for future - api.functionEnv.Set(*m, env) + // cache for future lookups + poolMgr.functionEnv.Set(tpr.CacheKey(m), env) return env, nil } -func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) { - // Make sure we have the full metadata. This ensures that - // poolmgr does not implicitly interpret empty-UID as latest - // version. - if len(m.Uid) == 0 { - return "", fission.MakeError(fission.ErrorInvalidArgument, - fmt.Sprintf("invalid metadata for function %v", m.Name)) - } - +func (poolMgr *Poolmgr) getServiceForFunction(m *api.ObjectMeta) (string, error) { // Check function -> svc cache log.Printf("[%v] Checking for cached function service", m.Name) - fsvc, err := api.fsCache.GetByFunction(m) + fsvc, err := poolMgr.fsCache.GetByFunction(m) if err == nil { // Cached, return svc address return fsvc.address, nil } respChan := make(chan *createFuncServiceResponse) - api.requestChan <- &createFuncServiceRequest{ + poolMgr.requestChan <- &createFuncServiceRequest{ funcMeta: m, respChan: respChan, } @@ -212,20 +195,20 @@ func (api *API) getServiceForFunction(m *fission.Metadata) (string, error) { return resp.address, resp.err } -func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) { +func (poolMgr *Poolmgr) createServiceForFunction(m *api.ObjectMeta) (string, error) { // None exists, so create a new funcSvc: log.Printf("[%v] No cached function service found, creating one", m.Name) // from Func -> get Env log.Printf("[%v] getting environment for function", m.Name) - env, err := api.getFunctionEnv(m) + env, err := poolMgr.getFunctionEnv(m) if err != nil { return "", err } // from Env -> get GenericPool log.Printf("[%v] getting generic pool for env", m.Name) - pool, err := api.poolMgr.GetPool(env) + pool, err := poolMgr.gpm.GetPool(env) if err != nil { return "", err } @@ -241,7 +224,7 @@ func (api *API) createServiceForFunction(m *fission.Metadata) (string, error) { } // find funcSvc and update its atime -func (api *API) tapService(w http.ResponseWriter, r *http.Request) { +func (poolMgr *Poolmgr) tapService(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read request", 500) @@ -250,7 +233,7 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) { svcName := string(body) svcHost := strings.TrimPrefix(svcName, "http://") - err = api.fsCache.TouchByAddress(svcHost) + err = poolMgr.fsCache.TouchByAddress(svcHost) if err != nil { log.Printf("funcSvc tap error: %v", err) http.Error(w, "Not found", 404) @@ -259,11 +242,10 @@ func (api *API) tapService(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (api *API) Serve(port int) { +func (poolMgr *Poolmgr) Serve(port int) { r := mux.NewRouter() - r.HandleFunc("/v1/getServiceForFunction", api.getServiceForFunctionApi).Methods("POST") - r.HandleFunc("/v1/tapService", api.tapService).Methods("POST") - + r.HandleFunc("/v2/getServiceForFunction", poolMgr.getServiceForFunctionApi).Methods("POST") + r.HandleFunc("/v2/tapService", poolMgr.tapService).Methods("POST") address := fmt.Sprintf(":%v", port) log.Printf("starting poolmgr at port %v", port) log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r))) diff --git a/poolmgr/client/client.go b/poolmgr/client/client.go index 166dd0a1..d3ee2a17 100644 --- a/poolmgr/client/client.go +++ b/poolmgr/client/client.go @@ -26,6 +26,8 @@ import ( "strings" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" ) @@ -45,8 +47,9 @@ func MakeClient(poolmgrUrl string) *Client { return c } -func (c *Client) GetServiceForFunction(metadata *fission.Metadata) (string, error) { - poolmgrUrl := c.poolmgrUrl + "/v1/getServiceForFunction" +func (c *Client) GetServiceForFunction(metadata *api.ObjectMeta) (string, error) { + poolmgrUrl := c.poolmgrUrl + "/v2/getServiceForFunction" + body, err := json.Marshal(metadata) if err != nil { return "", err @@ -97,7 +100,7 @@ func (c *Client) TapService(serviceUrl *url.URL) { } func (c *Client) _tapService(serviceUrlStr string) error { - poolmgrUrl := c.poolmgrUrl + "/v1/tapService" + poolmgrUrl := c.poolmgrUrl + "/v2/tapService" resp, err := http.Post(poolmgrUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr))) if err != nil { diff --git a/poolmgr/functionServiceCache.go b/poolmgr/functionServiceCache.go index f4fece70..25225298 100644 --- a/poolmgr/functionServiceCache.go +++ b/poolmgr/functionServiceCache.go @@ -20,8 +20,10 @@ import ( "log" "time" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission/cache" + "github.com/fission/fission/tpr" ) type fscRequestType int @@ -34,10 +36,20 @@ const ( ) type ( + funcSvc struct { + function *api.ObjectMeta // function this pod/service is for + environment *tpr.Environment // function's environment + address string // Host:Port or IP:Port that the function's service can be reached at. + podName string // pod name (within the function namespace) + + ctime time.Time + atime time.Time + } + functionServiceCache struct { - byFunction *cache.Cache // function -> funcSvc : map[fission.Metadata]*funcSvc - byAddress *cache.Cache // address -> function : map[string]fission.Metadata - byPod *cache.Cache // podname -> function : map[string]fission.Metadata + byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc + byAddress *cache.Cache // address -> function : map[string]api.ObjectMeta + byPod *cache.Cache // podname -> function : map[string]api.ObjectMeta requestChannel chan *fscRequest } @@ -79,8 +91,8 @@ func (fsc *functionServiceCache) service() { byPodCopy := fsc.byPod.Copy() pods := make([]string, 0) for podNameI, mI := range byPodCopy { - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { resp.error = err } else { @@ -95,10 +107,9 @@ func (fsc *functionServiceCache) service() { case LOG: funcCopy := fsc.byFunction.Copy() log.Printf("Cache has %v entries", len(funcCopy)) - for mI, fsvcI := range funcCopy { - m := mI.(fission.Metadata) + for key, fsvcI := range funcCopy { fsvc := fsvcI.(*funcSvc) - log.Printf("%v:%v\t%v", m.Name, m.Uid, fsvc.podName) + log.Printf("%v\t%v", key, fsvc.podName) } case DELETE_BY_POD: resp.deleted, resp.error = fsc._deleteByPod(req.podName, req.age) @@ -107,11 +118,14 @@ func (fsc *functionServiceCache) service() { } } -func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, error) { - fsvcI, err := fsc.byFunction.Get(*m) +func (fsc *functionServiceCache) GetByFunction(m *api.ObjectMeta) (*funcSvc, error) { + key := tpr.CacheKey(m) + + fsvcI, err := fsc.byFunction.Get(key) if err != nil { return nil, err } + // update atime fsvc := fsvcI.(*funcSvc) fsvc.atime = time.Now() @@ -121,7 +135,7 @@ func (fsc *functionServiceCache) GetByFunction(m *fission.Metadata) (*funcSvc, e } func (fsc *functionServiceCache) Add(fsvc funcSvc) (error, *funcSvc) { - err, existing := fsc.byFunction.Set(*fsvc.function, &fsvc) + err, existing := fsc.byFunction.Set(tpr.CacheKey(fsvc.function), &fsvc) if err != nil { if existing != nil { f := existing.(*funcSvc) @@ -167,8 +181,8 @@ func (fsc *functionServiceCache) _touchByAddress(address string) error { if err != nil { return err } - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { return err } @@ -196,8 +210,8 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati if err != nil { return false, err } - m := mI.(fission.Metadata) - fsvcI, err := fsc.byFunction.Get(m) + m := mI.(api.ObjectMeta) + fsvcI, err := fsc.byFunction.Get(tpr.CacheKey(&m)) if err != nil { return false, err } @@ -207,7 +221,7 @@ func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Durati return false, nil } - fsc.byFunction.Delete(m) + fsc.byFunction.Delete(tpr.CacheKey(&m)) fsc.byAddress.Delete(fsvc.address) fsc.byPod.Delete(podName) return true, nil diff --git a/poolmgr/functionServiceCache_test.go b/poolmgr/functionServiceCache_test.go index d33e1b9c..5a4003a0 100644 --- a/poolmgr/functionServiceCache_test.go +++ b/poolmgr/functionServiceCache_test.go @@ -5,7 +5,10 @@ import ( "testing" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func TestFunctionServiceCache(t *testing.T) { @@ -18,16 +21,22 @@ func TestFunctionServiceCache(t *testing.T) { now := time.Now() fsvc = &funcSvc{ - function: &fission.Metadata{ + function: &api.ObjectMeta{ Name: "foo", - Uid: "1212", + UID: "1212", }, - environment: &fission.Environment{ - Metadata: fission.Metadata{ + environment: &tpr.Environment{ + Metadata: api.ObjectMeta{ Name: "foo-env", - Uid: "2323", + UID: "2323", + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: "fission/foo-env", + }, + Builder: fission.Builder{}, }, - RunContainerImageUrl: "fission/foo-env", }, address: "xxx", podName: "yyy", diff --git a/poolmgr/gp.go b/poolmgr/gp.go index 3036bc1c..9a0bdf57 100644 --- a/poolmgr/gp.go +++ b/poolmgr/gp.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "net/url" + "os" "strings" "time" @@ -38,7 +39,10 @@ import ( "k8s.io/client-go/1.5/pkg/util/intstr" "github.com/fission/fission" + "github.com/fission/fission/environments/fetcher" + fetcherClient "github.com/fission/fission/environments/fetcher/client" "github.com/fission/fission/logger" + "github.com/fission/fission/tpr" ) const POOLMGR_INSTANCEID_LABEL string = "poolmgrInstanceId" @@ -46,16 +50,16 @@ const POD_PHASE_RUNNING string = "Running" type ( GenericPool struct { - env *fission.Environment - replicas int32 // num idle pods - deployment *v1beta1.Deployment // kubernetes deployment - namespace string // namespace to keep our resources - podReadyTimeout time.Duration // timeout for generic pods to become ready - controllerUrl string + env *tpr.Environment + replicas int32 // num idle pods + deployment *v1beta1.Deployment // kubernetes deployment + namespace string // namespace to keep our resources + podReadyTimeout time.Duration // timeout for generic pods to become ready idlePodReapTime time.Duration // pods unused for idlePodReapTime are deleted fsCache *functionServiceCache // cache funcSvc's by function, address and podname useSvc bool // create k8s service for specialized pods poolInstanceId string // small random string to uniquify pod names + fetcherImage string kubernetesClient *kubernetes.Clientset instanceId string // poolmgr instance id labelsForPool map[string]string @@ -74,15 +78,20 @@ type ( ) func MakeGenericPool( - controllerUrl string, kubernetesClient *kubernetes.Clientset, - env *fission.Environment, + env *tpr.Environment, initialReplicas int32, namespace string, fsCache *functionServiceCache, instanceId string) (*GenericPool, error) { log.Printf("Creating pool for environment %v", env.Metadata) + + fetcherImage := os.Getenv("FETCHER_IMAGE") + if len(fetcherImage) == 0 { + fetcherImage = "fission/fetcher" + } + // TODO: in general we need to provide the user a way to configure pools. Initial // replicas, autoscaling params, various timeouts, etc. gp := &GenericPool{ @@ -92,19 +101,18 @@ func MakeGenericPool( kubernetesClient: kubernetesClient, namespace: namespace, podReadyTimeout: 5 * time.Minute, // TODO make this an env param? - controllerUrl: controllerUrl, idlePodReapTime: 3 * time.Minute, // TODO make this configurable fsCache: fsCache, poolInstanceId: uniuri.NewLen(8), instanceId: instanceId, - - useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start + fetcherImage: fetcherImage, + useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start } // Labels for generic deployment/RS/pods. gp.labelsForPool = map[string]string{ "environmentName": gp.env.Metadata.Name, - "environmentUid": gp.env.Metadata.Uid, + "environmentUid": string(gp.env.Metadata.UID), POOLMGR_INSTANCEID_LABEL: gp.instanceId, } @@ -220,10 +228,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*v1.Pod, error) } } -func (gp *GenericPool) labelsForFunction(metadata *fission.Metadata) map[string]string { +func (gp *GenericPool) labelsForFunction(metadata *api.ObjectMeta) map[string]string { return map[string]string{ "functionName": metadata.Name, - "functionUid": metadata.Uid, + "functionUid": string(metadata.UID), "unmanaged": "true", // this allows us to easily find pods not managed by the deployment POOLMGR_INSTANCEID_LABEL: gp.instanceId, } @@ -240,10 +248,30 @@ func (gp *GenericPool) scheduleDeletePod(name string) { }() } +func (gp *GenericPool) getFetcherUrl(podIP string) string { + testUrl := os.Getenv("TEST_FETCHER_URL") + if len(testUrl) != 0 { + // it takes a second or so for the test service to + // become routable once a pod is relabeled. This is + // super hacky, but only runs in unit tests. + time.Sleep(5 * time.Second) + return testUrl + } + return fmt.Sprintf("http://%v:8000/", podIP) +} + +func (gp *GenericPool) getSpecializeUrl(podIP string) string { + u := os.Getenv("TEST_SPECIALIZE_URL") + if len(u) != 0 { + return u + } + return fmt.Sprintf("http://%v:8888/specialize", podIP) +} + // specializePod chooses a pod, copies the required user-defined function to that pod // (via fetcher), and calls the function-run container to load it, resulting in a // specialized pod. -func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) error { +func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *api.ObjectMeta) error { // for fetcher we don't need to create a service, just talk to the pod directly podIP := pod.Status.PodIP if len(podIP) == 0 { @@ -251,28 +279,23 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er } // tell fetcher to get the function. - fetcherUrl := fmt.Sprintf("http://%v:8000/", podIP) - functionUrl := fmt.Sprintf("%v/v1/functions/%v?uid=%v&raw=1", - gp.controllerUrl, metadata.Name, metadata.Uid) - fetcherRequest := fmt.Sprintf("{\"url\": \"%v\", \"filename\": \"user\"}", functionUrl) - - log.Printf("[%v] calling fetcher to copy function", metadata) - resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader([]byte(fetcherRequest))) + fetcherUrl := gp.getFetcherUrl(podIP) + log.Printf("[%v] calling fetcher to copy function", metadata.Name) + err := fetcherClient.DoFetchRequest(fetcherUrl, &fetcher.FetchRequest{ + FetchType: fetcher.FETCH_DEPLOYMENT, + Function: *metadata, + Filename: "user", // XXX use function id instead + }) if err != nil { - // TODO we should retry this call in case fetcher hasn't come up yet return err } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return errors.New(fmt.Sprintf("Error from fetcher: %v", resp.Status)) - } // Tell logging helper about this function invocation gp.setupLogging(pod, metadata) // get function run container to specialize - log.Printf("[%v] specializing pod", metadata) - specializeUrl := fmt.Sprintf("http://%v:8888/specialize", podIP) + log.Printf("[%v] specializing pod", metadata.Name) + specializeUrl := gp.getSpecializeUrl(podIP) // retry the specialize call a few times in case the env server hasn't come up yet maxRetries := 20 @@ -311,7 +334,7 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er // creates the pool but doesn't wait for any pods to be ready. func (gp *GenericPool) createPool() error { poolDeploymentName := fmt.Sprintf("%v-%v-%v", - gp.env.Metadata.Name, gp.env.Metadata.Uid, strings.ToLower(gp.poolInstanceId)) + gp.env.Metadata.Name, gp.env.Metadata.UID, strings.ToLower(gp.poolInstanceId)) sharedMountPath := "/userfunc" deployment := &v1beta1.Deployment{ @@ -340,7 +363,7 @@ func (gp *GenericPool) createPool() error { Containers: []v1.Container{ { Name: gp.env.Metadata.Name, - Image: gp.env.RunContainerImageUrl, + Image: gp.env.Spec.Runtime.Image, ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", VolumeMounts: []v1.VolumeMount{ @@ -352,7 +375,7 @@ func (gp *GenericPool) createPool() error { }, { Name: "fetcher", - Image: "fission/fetcher", + Image: gp.fetcherImage, ImagePullPolicy: v1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", VolumeMounts: []v1.VolumeMount{ @@ -418,9 +441,9 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*v1.Ser return svc, err } -func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) { +func (gp *GenericPool) GetFuncSvc(m *api.ObjectMeta) (*funcSvc, error) { - log.Printf("[%v] Choosing pod from pool", m) + log.Printf("[%v] Choosing pod from pool", m.Name) newLabels := gp.labelsForFunction(m) pod, err := gp.choosePod(newLabels) if err != nil { @@ -437,8 +460,8 @@ func (gp *GenericPool) GetFuncSvc(m *fission.Metadata) (*funcSvc, error) { var svcHost string if gp.useSvc { svcName := fmt.Sprintf("svc-%v", m.Name) - if len(m.Uid) > 0 { - svcName += "-" + m.Uid + if len(m.UID) > 0 { + svcName += ("-" + string(m.UID)) } labels := gp.labelsForFunction(m) @@ -580,13 +603,13 @@ func (gp *GenericPool) destroy() error { // Calls the logging daemonset pod on the node where the given pod is // running. -func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *fission.Metadata) { +func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *api.ObjectMeta) { logReq := logger.LogRequest{ Namespace: pod.Namespace, Pod: pod.Name, Container: gp.env.Metadata.Name, FuncName: metadata.Name, - FuncUid: metadata.Uid, + FuncUid: string(metadata.UID), } reqbody, err := json.Marshal(logReq) if err != nil { diff --git a/poolmgr/gpm.go b/poolmgr/gpm.go index 3214a261..f25ccc13 100644 --- a/poolmgr/gpm.go +++ b/poolmgr/gpm.go @@ -21,9 +21,9 @@ import ( "time" "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" - "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) type requestType int @@ -35,19 +35,18 @@ const ( type ( GenericPoolManager struct { - pools map[fission.Environment]*GenericPool + pools map[string]*GenericPool kubernetesClient *kubernetes.Clientset namespace string - controllerUrl string - controllerClient *client.Client + fissionClient *tpr.FissionClient fsCache *functionServiceCache instanceId string requestChannel chan *request } request struct { requestType - env *fission.Environment - envList []fission.Environment + env *tpr.Environment + envList []tpr.Environment responseChannel chan *response } response struct { @@ -57,18 +56,18 @@ type ( ) func MakeGenericPoolManager( - controllerUrl string, + fissionClient *tpr.FissionClient, kubernetesClient *kubernetes.Clientset, - namespace string, + fissionNamespace string, + functionNamespace string, fsCache *functionServiceCache, instanceId string) *GenericPoolManager { gpm := &GenericPoolManager{ - pools: make(map[fission.Environment]*GenericPool), + pools: make(map[string]*GenericPool), kubernetesClient: kubernetesClient, - namespace: namespace, - controllerUrl: controllerUrl, - controllerClient: client.MakeClient(controllerUrl), + namespace: functionNamespace, + fissionClient: fissionClient, fsCache: fsCache, instanceId: instanceId, requestChannel: make(chan *request), @@ -85,30 +84,30 @@ func (gpm *GenericPoolManager) service() { switch req.requestType { case GET_POOL: var err error - pool, ok := gpm.pools[*req.env] + pool, ok := gpm.pools[tpr.CacheKey(&req.env.Metadata)] if !ok { pool, err = MakeGenericPool( - gpm.controllerUrl, gpm.kubernetesClient, req.env, + gpm.kubernetesClient, req.env, 3, // TODO configurable/autoscalable gpm.namespace, gpm.fsCache, gpm.instanceId) if err != nil { req.responseChannel <- &response{error: err} continue } - gpm.pools[*req.env] = pool + gpm.pools[tpr.CacheKey(&req.env.Metadata)] = pool } req.responseChannel <- &response{pool: pool} case CLEANUP_POOLS: - uids := make(map[string]bool) + latestEnvSet := make(map[string]bool) for _, env := range req.envList { - uids[env.Metadata.Uid] = true + latestEnvSet[tpr.CacheKey(&env.Metadata)] = true } - for env, pool := range gpm.pools { - _, ok := uids[env.Metadata.Uid] + for key, pool := range gpm.pools { + _, ok := latestEnvSet[key] if !ok { // Env no longer exists -- remove our cache - log.Printf("Destroying generic pool for environment [%v]", env) - delete(gpm.pools, env) + log.Printf("Destroying generic pool for environment [%v]", key) + delete(gpm.pools, key) // and delete the pool asynchronously. go pool.destroy() @@ -119,7 +118,7 @@ func (gpm *GenericPoolManager) service() { } } -func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool, error) { +func (gpm *GenericPoolManager) GetPool(env *tpr.Environment) (*GenericPool, error) { c := make(chan *response) gpm.requestChannel <- &request{ requestType: GET_POOL, @@ -130,7 +129,7 @@ func (gpm *GenericPoolManager) GetPool(env *fission.Environment) (*GenericPool, return resp.pool, resp.error } -func (gpm *GenericPoolManager) CleanupPools(envs []fission.Environment) { +func (gpm *GenericPoolManager) CleanupPools(envs []tpr.Environment) { gpm.requestChannel <- &request{ requestType: CLEANUP_POOLS, envList: envs, @@ -145,11 +144,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { time.Sleep(pollSleep) // get list of envs from controller - envs, err := gpm.controllerClient.EnvironmentList() + envs, err := gpm.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{}) if err != nil { failureCount++ if failureCount >= maxFailures { - log.Fatalf("Failed to connect to controller %v times: %v", maxFailures, err) + log.Fatalf("Failed %v times: %v", maxFailures, err) } } @@ -157,14 +156,14 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { // creating pools for envs that are actually used by functions. Also we might want // to keep these eagerly created pools smaller than the ones created when there are // actual function calls. - for i := range envs { - _, err := gpm.GetPool(&envs[i]) + for i := range envs.Items { + _, err := gpm.GetPool(&envs.Items[i]) if err != nil { log.Printf("eager-create pool failed: %v", err) } } // Clean up pools whose env was deleted - gpm.CleanupPools(envs) + gpm.CleanupPools(envs.Items) } } diff --git a/poolmgr/poolmgr.go b/poolmgr/poolmgr.go index 896cac96..75c12ff9 100644 --- a/poolmgr/poolmgr.go +++ b/poolmgr/poolmgr.go @@ -18,52 +18,29 @@ package poolmgr import ( "log" - "strings" "github.com/dchest/uniuri" - "k8s.io/client-go/1.5/kubernetes" - "k8s.io/client-go/1.5/rest" - controllerclient "github.com/fission/fission/controller/client" + "github.com/fission/fission/tpr" ) -// Get a kubernetes client using the pod's service account. This only -// works when we're running inside a kubernetes cluster. -func getKubernetesClient() (*kubernetes.Clientset, error) { - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - log.Printf("Error getting kubernetes client config: %v", err) - return nil, err - } - - // creates the clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Printf("Error getting kubernetes client: %v", err) - return nil, err - } - - return clientset, nil -} - -func StartPoolmgr(controllerUrl string, namespace string, port int) error { - controllerUrl = strings.TrimSuffix(controllerUrl, "/") - controllerClient := controllerclient.MakeClient(controllerUrl) - - kubernetesClient, err := getKubernetesClient() +// Start the poolmgr service. +func StartPoolmgr(fissionNamespace string, functionNamespace string, port int) error { + fissionClient, kubernetesClient, err := tpr.MakeFissionClient() if err != nil { log.Printf("Failed to get kubernetes client: %v", err) return err } instanceId := uniuri.NewLen(8) - cleanupOldPoolmgrResources(kubernetesClient, namespace, instanceId) + cleanupOldPoolmgrResources(kubernetesClient, functionNamespace, instanceId) fsCache := MakeFunctionServiceCache() - gpm := MakeGenericPoolManager(controllerUrl, kubernetesClient, namespace, fsCache, instanceId) + gpm := MakeGenericPoolManager( + fissionClient, kubernetesClient, fissionNamespace, + functionNamespace, fsCache, instanceId) - api := MakeAPI(gpm, controllerClient, fsCache) + api := MakePoolmgr(gpm, fissionClient, fissionNamespace, fsCache) go api.Serve(port) return nil diff --git a/poolmgr/poolmgr_test.go b/poolmgr/poolmgr_test.go new file mode 100644 index 00000000..98e3427d --- /dev/null +++ b/poolmgr/poolmgr_test.go @@ -0,0 +1,234 @@ +// +// This test depends on several env vars: +// +// KUBECONFIG has to point at a kube config with a cluster. The test +// will use the default context from that config. Be careful, +// don't point this at your production environment. The test is +// skipped if KUBECONFIG is undefined. +// +// TEST_SPECIALIZE_URL +// TEST_FETCHER_URL +// These need to point at :30001 and :30002, +// where is the address of any node in the test +// cluster. +// +// FETCHER_IMAGE +// Optional. Set this to a fetcher image; otherwise uses the +// default. +// + +// Here's how I run this on my setup, with minikube: +// TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v . + +package poolmgr + +import ( + "fmt" + "log" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/v1" + "k8s.io/client-go/1.5/pkg/labels" + "k8s.io/client-go/1.5/pkg/util/intstr" + + "github.com/fission/fission" + "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" + "io/ioutil" +) + +// return the number of pods in the given namespace matching the given labels +func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int { + pods, err := kubeClient.Pods(ns).List(api.ListOptions{ + LabelSelector: labels.Set(labelz).AsSelector(), + }) + if err != nil { + log.Panicf("Failed to list pods: %v", err) + } + return len(pods.Items) +} + +func createTestNamespace(kubeClient *kubernetes.Clientset, ns string) { + _, err := kubeClient.Namespaces().Create(&v1.Namespace{ + ObjectMeta: v1.ObjectMeta{ + Name: ns, + }, + }) + if err != nil { + log.Panicf("failed to create ns %v: %v", ns, err) + } +} + +// create a nodeport service +func createSvc(kubeClient *kubernetes.Clientset, ns string, name string, targetPort int, nodePort int32, labels map[string]string) *v1.Service { + svc, err := kubeClient.Services(ns).Create(&v1.Service{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + { + Protocol: v1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(targetPort), + NodePort: nodePort, + }, + }, + Selector: labels, + }, + }) + if err != nil { + log.Panicf("Failed to create svc: %v", err) + } + return svc +} + +func httpGet(url string) string { + resp, err := http.Get(url) + if err != nil { + log.Panicf("HTTP Get failed: URL %v: %v", url, err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Panicf("HTTP Get failed to read body: URL %v: %v", url, err) + } + return string(body) +} + +func TestPoolmgr(t *testing.T) { + // run in a random namespace so we can have concurrent tests + // on a given cluster + rand.Seed(time.Now().UTC().UnixNano()) + testId := rand.Intn(999) + fissionNs := fmt.Sprintf("test-%v", testId) + functionNs := fmt.Sprintf("test-function-%v", testId) + + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + t.Skip("Skipping test, no kubernetes cluster") + return + } + + // connect to k8s + // and get TPR client + fissionClient, kubeClient, err := tpr.MakeFissionClient() + if err != nil { + log.Panicf("failed to connect: %v", err) + } + + // create the test's namespaces + createTestNamespace(kubeClient, fissionNs) + defer kubeClient.Namespaces().Delete(fissionNs, nil) + + createTestNamespace(kubeClient, functionNs) + defer kubeClient.Namespaces().Delete(functionNs, nil) + + // make sure TPR types exist on cluster + err = tpr.EnsureFissionTPRs(kubeClient) + if err != nil { + log.Panicf("failed to ensure tprs: %v", err) + } + fissionClient.WaitForTPRs() + + // create an env on the cluster + env, err := fissionClient.Environments(fissionNs).Create(&tpr.Environment{ + Metadata: api.ObjectMeta{ + Name: "nodejs", + Namespace: fissionNs, + }, + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: "fission/node-env", + }, + Builder: fission.Builder{}, + }, + }) + if err != nil { + log.Panicf("failed to create env: %v", err) + } + + // create poolmgr + port := 9999 + err = StartPoolmgr(fissionNs, functionNs, port) + if err != nil { + log.Panicf("failed to start poolmgr: %v", err) + } + + // connect poolmgr client + poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port)) + + // Wait for pool to be created (we don't actually need to do + // this, since the API should do the right thing in any case). + // waitForPool(functionNs, "nodejs") + time.Sleep(6 * time.Second) + + // create a function + f := &tpr.Function{ + Metadata: api.ObjectMeta{ + Name: "hello", + Namespace: fissionNs, + }, + Spec: fission.FunctionSpec{ + Source: fission.Package{}, + Deployment: fission.Package{ + Type: fission.PackageTypeLiteral, + Literal: []byte(`module.exports = async function(context) { return { status: 200, body: "Hello, world!\n" }; }`), + }, + EnvironmentName: env.Metadata.Name, + }, + } + _, err = fissionClient.Functions(fissionNs).Create(f) + if err != nil { + log.Panicf("failed to create function: %v", err) + } + + // create a service to call fetcher and the env container + labels := map[string]string{"functionName": f.Metadata.Name} + var fetcherPort int32 = 30001 + fetcherSvc := createSvc(kubeClient, functionNs, fmt.Sprintf("%v-%v", f.Metadata.Name, "fetcher"), 8000, fetcherPort, labels) + defer kubeClient.Services(functionNs).Delete(fetcherSvc.ObjectMeta.Name, nil) + + var funcSvcPort int32 = 30002 + functionSvc := createSvc(kubeClient, functionNs, f.Metadata.Name, 8888, funcSvcPort, labels) + defer kubeClient.Services(functionNs).Delete(functionSvc.ObjectMeta.Name, nil) + + // the main test: get a service for a given function + t1 := time.Now() + svc, err := poolmgrClient.GetServiceForFunction(&f.Metadata) + if err != nil { + log.Panicf("failed to get func svc: %v", err) + } + log.Printf("svc for function created at: %v (in %v)", svc, time.Now().Sub(t1)) + + // ensure that a pod with the label functionName=f.Metadata.Name exists + podCount := countPods(kubeClient, functionNs, map[string]string{"functionName": f.Metadata.Name}) + if podCount != 1 { + log.Panicf("expected 1 function pod, found %v", podCount) + } + + // call the service to ensure it works + + // wait for a bit + + // tap service to simulate calling it again + + // make sure the same pod is still there + + // wait for idleTimeout to ensure the pod is removed + + // remove env + + // wait for pool to be destroyed + + // that's it +} diff --git a/router/functionHandler.go b/router/functionHandler.go index 0b32a441..b899c3d5 100644 --- a/router/functionHandler.go +++ b/router/functionHandler.go @@ -26,20 +26,20 @@ import ( "time" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" - "github.com/fission/fission" poolmgrClient "github.com/fission/fission/poolmgr/client" ) type functionHandler struct { fmap *functionServiceMap poolmgr *poolmgrClient.Client - Function fission.Metadata + function *api.ObjectMeta } func (fh *functionHandler) getServiceForFunction() (*url.URL, error) { // call poolmgr, get a url for a function - svcName, err := fh.poolmgr.GetServiceForFunction(&fh.Function) + svcName, err := fh.poolmgr.GetServiceForFunction(fh.function) if err != nil { return nil, err } @@ -99,16 +99,15 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request * } // cache lookup - serviceUrl, err := fh.fmap.lookup(&fh.Function) + serviceUrl, err := fh.fmap.lookup(fh.function) if err != nil { // Cache miss: request the Pool Manager to make a new service. - log.Printf("Not cached, getting new service for %v", fh.Function) + log.Printf("Not cached, getting new service for %v", fh.function) var poolErr error serviceUrl, poolErr = fh.getServiceForFunction() if poolErr != nil { - log.Printf("Failed to get service for function (%v,%v): %v", - fh.Function.Name, fh.Function.Uid, poolErr) + log.Printf("Failed to get service for function %v: %v", fh.function.Name, poolErr) // We might want a specific error code or header for fission // failures as opposed to user function bugs. http.Error(responseWriter, "Internal server error (fission)", 500) @@ -116,7 +115,7 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request * } // add it to the map - fh.fmap.assign(&fh.Function, serviceUrl) + fh.fmap.assign(fh.function, serviceUrl) } else { // if we're using our cache, asynchronously tell // poolmgr we're using this service diff --git a/router/functionHandler_test.go b/router/functionHandler_test.go index 1cb9b3dd..76d11981 100644 --- a/router/functionHandler_test.go +++ b/router/functionHandler_test.go @@ -23,7 +23,7 @@ import ( "net/url" "testing" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" ) func createBackendService(testResponseString string) *url.URL { @@ -49,11 +49,12 @@ func TestFunctionProxying(t *testing.T) { backendURL := createBackendService(testResponseString) log.Printf("Created backend svc at %v", backendURL) - fn := &fission.Metadata{Name: "foo", Uid: "xxx"} + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} + fmap := makeFunctionServiceMap(0) fmap.assign(fn, backendURL) - fh := &functionHandler{fmap: fmap, Function: *fn} + fh := &functionHandler{fmap: fmap, function: fn} functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler)) fhURL := functionHandlerServer.URL diff --git a/router/functionReferenceResolver.go b/router/functionReferenceResolver.go new file mode 100644 index 00000000..ffb18660 --- /dev/null +++ b/router/functionReferenceResolver.go @@ -0,0 +1,118 @@ +/* +Copyright 2017 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package router + +import ( + "fmt" + "time" + + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/cache" + "github.com/fission/fission/tpr" +) + +type ( + // functionReferenceResolver provides a resolver to turn a function + // reference into a resolveResult + functionReferenceResolver struct { + fissionClient *tpr.FissionClient + + // FunctionReference -> function metadata + refCache *cache.Cache + } + + resolveResultType int + + // resolveResult is the result of resolving a function reference; for now + // it's just the metadata of one function, but in the future could support + // a distribution of requests across two functions. + resolveResult struct { + resolveResultType + functionMetadata *api.ObjectMeta + } + + // namespacedFunctionReference is just a function reference plus a + // namespace. Since a function reference works on names, it's only + // meaningful within a namespace. + namespacedFunctionReference struct { + namespace string + functionReference fission.FunctionReference + } +) + +const ( + resolveResultSingleFunction = iota +) + +func makeFunctionReferenceResolver(fissionClient *tpr.FissionClient) *functionReferenceResolver { + return &functionReferenceResolver{ + fissionClient: fissionClient, + refCache: cache.MakeCache(time.Minute, 0), + } +} + +// resolve translates a namespace and a function reference to resolveResult. +// The resolveResult for now is just a function's metadata. In the future, some +// function ref types may resolve to two functions rather than just one +// (e.g. for incremental deployment), which will make the resolveResult a bit +// more complex. +func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.FunctionReference) (*resolveResult, error) { + nfr := namespacedFunctionReference{ + namespace: namespace, + functionReference: *fr, + } + + // check cache + rrInt, err := frr.refCache.Get(nfr) + if err == nil { + result := rrInt.(resolveResult) + return &result, nil + } + + // resolve on cache miss + var rr *resolveResult + + switch fr.Type { + case fission.FunctionReferenceTypeFunctionName: + rr, err = frr.resolveByName(namespace, fr.Name) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Unrecognized function reference type %v", fr.Type) + } + + // cache resolve result + frr.refCache.Set(nfr, *rr) + + return rr, nil +} + +// resolveByName simply looks up function by name in a namespace. +func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) { + f, err := frr.fissionClient.Functions(namespace).Get(name) + if err != nil { + return nil, err + } + rr := resolveResult{ + resolveResultType: resolveResultSingleFunction, + functionMetadata: &f.Metadata, + } + return &rr, nil +} diff --git a/router/functionServiceMap.go b/router/functionServiceMap.go index 6b08adf7..c0548e53 100644 --- a/router/functionServiceMap.go +++ b/router/functionServiceMap.go @@ -21,13 +21,24 @@ import ( "net/url" "time" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission/cache" ) -type functionServiceMap struct { - cache *cache.Cache // map[fission.Metadata]*url.URL -} +type ( + functionServiceMap struct { + cache *cache.Cache // map[metadataKey]*url.URL + } + + // api.ObjectMeta is not hashable, so we make a hashable copy + // of the subset of its fields that are identifiable. + metadataKey struct { + Name string + Namespace string + ResourceVersion string + } +) func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { return &functionServiceMap{ @@ -35,8 +46,17 @@ func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { } } -func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) { - item, err := fmap.cache.Get(*f) +func keyFromMetadata(m *api.ObjectMeta) *metadataKey { + return &metadataKey{ + Name: m.Name, + Namespace: m.Namespace, + ResourceVersion: m.ResourceVersion, + } +} + +func (fmap *functionServiceMap) lookup(f *api.ObjectMeta) (*url.URL, error) { + mk := keyFromMetadata(f) + item, err := fmap.cache.Get(*mk) if err != nil { return nil, err } @@ -44,8 +64,9 @@ func (fmap *functionServiceMap) lookup(f *fission.Metadata) (*url.URL, error) { return u, nil } -func (fmap *functionServiceMap) assign(f *fission.Metadata, serviceUrl *url.URL) { - err, old := fmap.cache.Set(*f, serviceUrl) +func (fmap *functionServiceMap) assign(f *api.ObjectMeta, serviceUrl *url.URL) { + mk := keyFromMetadata(f) + err, old := fmap.cache.Set(*mk, serviceUrl) if err != nil { if *serviceUrl == *(old.(*url.URL)) { return diff --git a/router/functionServiceMap_test.go b/router/functionServiceMap_test.go index a48221f8..61b3d930 100644 --- a/router/functionServiceMap_test.go +++ b/router/functionServiceMap_test.go @@ -20,12 +20,12 @@ import ( "net/url" "testing" - "github.com/fission/fission" + "k8s.io/client-go/1.5/pkg/api" ) func TestFunctionServiceMap(t *testing.T) { m := makeFunctionServiceMap(0) - fn := &fission.Metadata{Name: "foo", Uid: "012"} + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} u, err := url.Parse("/foo012") if err != nil { t.Errorf("can't parse url") diff --git a/router/httpTriggers.go b/router/httpTriggers.go index e8b0744f..2b8eec89 100644 --- a/router/httpTriggers.go +++ b/router/httpTriggers.go @@ -22,34 +22,44 @@ import ( "time" "github.com/gorilla/mux" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" "github.com/fission/fission" - controllerClient "github.com/fission/fission/controller/client" poolmgrClient "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" ) type HTTPTriggerSet struct { *functionServiceMap *mutableRouter - controller *controllerClient.Client - poolmgr *poolmgrClient.Client - triggers []fission.HTTPTrigger - functions []fission.Function + fissionClient *tpr.FissionClient + poolmgr *poolmgrClient.Client + resolver *functionReferenceResolver + triggers []tpr.Httptrigger + functions []tpr.Function } -func makeHTTPTriggerSet(fmap *functionServiceMap, controller *controllerClient.Client, poolmgr *poolmgrClient.Client) *HTTPTriggerSet { - triggers := make([]fission.HTTPTrigger, 1) +func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *tpr.FissionClient, poolmgr *poolmgrClient.Client, resolver *functionReferenceResolver) *HTTPTriggerSet { + triggers := make([]tpr.Httptrigger, 1) return &HTTPTriggerSet{ functionServiceMap: fmap, triggers: triggers, - controller: controller, + fissionClient: fissionClient, poolmgr: poolmgr, + resolver: resolver, } } func (ts *HTTPTriggerSet) subscribeRouter(mr *mutableRouter) { ts.mutableRouter = mr mr.updateRouter(ts.getRouter()) + + if ts.fissionClient == nil { + // Used in tests only. + log.Printf("Skipping continuous trigger updates") + return + } go ts.watchTriggers() } @@ -60,27 +70,33 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) { func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter := mux.NewRouter() - // make a function name -> latest version map - latestVersions := make(map[string]string) - for _, f := range ts.functions { - latestVersions[f.Metadata.Name] = f.Metadata.Uid - } - // HTTP triggers setup by the user homeHandled := false for _, trigger := range ts.triggers { - m := trigger.Function - if len(m.Uid) == 0 { - // explicitly use the latest function version - m.Uid = latestVersions[m.Name] + + // resolve function reference + rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference) + if err != nil { + // Unresolvable function reference. Report the error via + // the trigger's status. + go ts.updateTriggerStatusFailed(&trigger, err) + + // Ignore this route and let it 404. + continue } + + if rr.resolveResultType != resolveResultSingleFunction { + // not implemented yet + log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType) + } + fh := &functionHandler{ fmap: ts.functionServiceMap, - Function: m, + function: rr.functionMetadata, poolmgr: ts.poolmgr, } - muxRouter.HandleFunc(trigger.UrlPattern, fh.handler).Methods(trigger.Method) - if trigger.UrlPattern == "/" && trigger.Method == "GET" { + muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method) + if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" { homeHandled = true } } @@ -95,53 +111,74 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET") } - // Internal triggers for (the latest version of) each function + // Internal triggers for each function by name. Non-http + // triggers route into these. for _, function := range ts.functions { - m := fission.Metadata{Name: function.Metadata.Name} fh := &functionHandler{ fmap: ts.functionServiceMap, - Function: function.Metadata, + function: &function.Metadata, poolmgr: ts.poolmgr, } - muxRouter.HandleFunc(fission.UrlForFunction(&m), fh.handler) + muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler) } return muxRouter } +func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *tpr.Httptrigger, err error) { + // TODO +} + func (ts *HTTPTriggerSet) watchTriggers() { - if ts.controller == nil { - return - } - - // the number of connection failures we'll accept before quitting - maxFailures := 5 - - // amount of time to sleep between polling calls - pollSleepDuration := 3 * time.Second + // sync all http triggers + ts.syncTriggers() // Watch controller for updates to triggers and update the router accordingly. - // TODO change this to use a watch API; or maybe even watch etcd directly. - failureCount := 0 + rv := "" for { - triggers, err := ts.controller.HTTPTriggerList() + wi, err := ts.fissionClient.Httptriggers(api.NamespaceAll).Watch(api.ListOptions{ + ResourceVersion: rv, + }) if err != nil { - failureCount += 1 - if failureCount >= maxFailures { - log.Fatalf("Failed to connect to controller after %v retries: %v", failureCount, err) + log.Fatalf("Failed to watch http trigger list: %v", err) + } + + for { + ev, more := <-wi.ResultChan() + if !more { + // restart watch from last rv + break } - time.Sleep(pollSleepDuration) - continue + if ev.Type == watch.Error { + // restart watch from the start + rv = "" + time.Sleep(time.Second) + break + } + ht := ev.Object.(*tpr.Httptrigger) + rv = ht.Metadata.ResourceVersion + ts.syncTriggers() } - ts.triggers = triggers - - functions, err := ts.controller.FunctionList() - if err != nil { - log.Fatalf("Failed to get function list") - } - ts.functions = functions - - ts.mutableRouter.updateRouter(ts.getRouter()) - time.Sleep(pollSleepDuration) } } + +func (ts *HTTPTriggerSet) syncTriggers() { + log.Printf("Syncing http triggers") + + // get triggers + triggers, err := ts.fissionClient.Httptriggers(api.NamespaceAll).List(api.ListOptions{}) + if err != nil { + log.Fatalf("Failed to get http trigger list: %v", err) + } + ts.triggers = triggers.Items + + // get functions + functions, err := ts.fissionClient.Functions(api.NamespaceAll).List(api.ListOptions{}) + if err != nil { + log.Fatalf("Failed to get function list: %v", err) + } + ts.functions = functions.Items + + // make a new router and use it + ts.mutableRouter.updateRouter(ts.getRouter()) +} diff --git a/router/router.go b/router/router.go index 01cccf56..bc2ef636 100644 --- a/router/router.go +++ b/router/router.go @@ -49,8 +49,8 @@ import ( "github.com/gorilla/handlers" "github.com/gorilla/mux" - controllerClient "github.com/fission/fission/controller/client" poolmgrClient "github.com/fission/fission/poolmgr/client" + "github.com/fission/fission/tpr" ) // request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url @@ -70,12 +70,16 @@ func serve(port int, httpTriggerSet *HTTPTriggerSet) { http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr)) } -func Start(port int, controllerUrl string, poolmgrUrl string) { +func Start(port int, poolmgrUrl string) { fmap := makeFunctionServiceMap(time.Minute) - controller := controllerClient.MakeClient(controllerUrl) - poolmgr := poolmgrClient.MakeClient(poolmgrUrl) - triggers := makeHTTPTriggerSet(fmap, controller, poolmgr) + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + log.Fatalf("Error connecting to kubernetes API: %v", err) + } + poolmgr := poolmgrClient.MakeClient(poolmgrUrl) + resolver := makeFunctionReferenceResolver(fissionClient) + triggers := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, resolver) log.Printf("Starting router at port %v\n", port) serve(port, triggers) } diff --git a/router/router_test.go b/router/router_test.go index 59b5c7db..5a848e66 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -21,26 +21,64 @@ import ( "testing" "time" + "k8s.io/client-go/1.5/pkg/api" + "github.com/fission/fission" + "github.com/fission/fission/tpr" ) func TestRouter(t *testing.T) { - fmap := makeFunctionServiceMap(0) - fn := &fission.Metadata{Name: "foo", Uid: "xxx"} + // metadata for a fake function + fn := &api.ObjectMeta{Name: "foo", Namespace: api.NamespaceDefault} + // and a reference to it + fr := fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: fn.Name, + } + + // start a fake service testResponseString := "hi" testServiceUrl := createBackendService(testResponseString) + // set up the cache with this fake service + fmap := makeFunctionServiceMap(0) fmap.assign(fn, testServiceUrl) - triggers := makeHTTPTriggerSet(fmap, nil, nil) - triggerUrl := "/foo" - triggers.triggers = append(triggers.triggers, fission.HTTPTrigger{UrlPattern: triggerUrl, Function: *fn, Method: "GET"}) + // set up the resolver's cache for this function + frr := makeFunctionReferenceResolver(nil) + nfr := namespacedFunctionReference{ + namespace: api.NamespaceDefault, + functionReference: fr, + } + rr := resolveResult{ + resolveResultType: resolveResultSingleFunction, + functionMetadata: fn, + } + frr.refCache.Set(nfr, rr) + // HTTP trigger set with a trigger for this function + triggers := makeHTTPTriggerSet(fmap, nil, nil, frr) + triggerUrl := "/foo" + triggers.triggers = append(triggers.triggers, + tpr.Httptrigger{ + Metadata: api.ObjectMeta{ + Name: "xxx", + Namespace: api.NamespaceDefault, + }, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: triggerUrl, + FunctionReference: fr, + Method: "GET", + }, + }) + + // run the router port := 4242 go serve(port, triggers) time.Sleep(100 * time.Millisecond) + // hit the router testUrl := fmt.Sprintf("http://localhost:%v%v", port, triggerUrl) testRequest(testUrl, testResponseString) } diff --git a/timer/main.go b/timer/main.go index aff393e8..301464d2 100644 --- a/timer/main.go +++ b/timer/main.go @@ -17,14 +17,18 @@ limitations under the License. package timer import ( - controllerClient "github.com/fission/fission/controller/client" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) -func Start(controllerUrl string, routerUrl string) error { - controller := controllerClient.MakeClient(controllerUrl) +func Start(routerUrl string) error { + fissionClient, _, err := tpr.MakeFissionClient() + if err != nil { + return err + } + poster := publisher.MakeWebhookPublisher(routerUrl) - MakeTimerSync(controller, MakeTimer(poster)) + MakeTimerSync(fissionClient, MakeTimer(poster)) return nil } diff --git a/timer/timer.go b/timer/timer.go index 1b61544d..3118cfc0 100644 --- a/timer/timer.go +++ b/timer/timer.go @@ -20,9 +20,11 @@ import ( "log" "github.com/robfig/cron" + // "k8s.io/client-go/1.5/pkg/api" "github.com/fission/fission" "github.com/fission/fission/publisher" + "github.com/fission/fission/tpr" ) type requestType int @@ -40,14 +42,14 @@ type ( timerRequest struct { requestType - triggers []fission.TimeTrigger + triggers []tpr.Timetrigger responseChannel chan *timerResponse } timerResponse struct { error } timerTriggerWithCron struct { - trigger fission.TimeTrigger + trigger tpr.Timetrigger cron *cron.Cron } ) @@ -62,7 +64,7 @@ func MakeTimer(publisher publisher.Publisher) *Timer { return timer } -func (timer *Timer) Sync(triggers []fission.TimeTrigger) error { +func (timer *Timer) Sync(triggers []tpr.Timetrigger) error { req := &timerRequest{ requestType: SYNC, triggers: triggers, @@ -84,16 +86,21 @@ func (timer *Timer) svc() { } } -func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { +func (timer *Timer) syncCron(triggers []tpr.Timetrigger) error { + // add new triggers or update existing ones for _, t := range triggers { - if item, ok := timer.triggers[t.Name]; ok { + if item, ok := timer.triggers[t.Metadata.Name]; ok { // the item exists, update the item if needed - if item.trigger.Uid == t.Uid { + + // if both UID and ResourceVersion match, the + // two triggers are identical + if item.trigger.Metadata.UID == t.Metadata.UID && + item.trigger.Metadata.ResourceVersion == t.Metadata.ResourceVersion { continue } // update cron if the cron spec changed - if item.trigger.Cron != t.Cron { + if item.trigger.Spec.Cron != t.Spec.Cron { // if there is an cron running, stop it if item.cron != nil { item.cron.Stop() @@ -103,17 +110,18 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { item.trigger = t } else { - timer.triggers[t.Name] = &timerTriggerWithCron{ + timer.triggers[t.Metadata.Name] = &timerTriggerWithCron{ trigger: t, cron: timer.newCron(t), } } } + // process removed triggers for k, v := range timer.triggers { found := false for _, t := range triggers { - if t.Name == k { + if t.Metadata.Name == k { found = true break } @@ -121,7 +129,7 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { if !found { if v.cron != nil { v.cron.Stop() - log.Printf("Cron for time trigger %s stopped", v.trigger.Name) + log.Printf("Cron for time trigger %s stopped", v.trigger.Metadata.Name) } delete(timer.triggers, k) } @@ -130,15 +138,15 @@ func (timer *Timer) syncCron(triggers []fission.TimeTrigger) error { return nil } -func (timer *Timer) newCron(t fission.TimeTrigger) *cron.Cron { +func (timer *Timer) newCron(t tpr.Timetrigger) *cron.Cron { c := cron.New() - c.AddFunc(t.Cron, func() { + c.AddFunc(t.Spec.Cron, func() { headers := map[string]string{ - "X-Fission-Timer-Name": t.Name, + "X-Fission-Timer-Name": t.Metadata.Name, } - (*timer.publisher).Publish("", headers, fission.UrlForFunction(&t.Function)) + (*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name)) }) c.Start() - log.Printf("Add new cron for time trigger %v", t.Name) + log.Printf("Add new cron for time trigger %v", t.Metadata.Name) return c } diff --git a/timer/timerSync.go b/timer/timerSync.go index 692d37b9..0aa018b4 100644 --- a/timer/timerSync.go +++ b/timer/timerSync.go @@ -20,39 +20,36 @@ import ( "log" "time" - controllerClient "github.com/fission/fission/controller/client" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission/tpr" ) type ( TimerSync struct { - controller *controllerClient.Client - timer *Timer + fissionClient *tpr.FissionClient + timer *Timer } ) -func MakeTimerSync(controller *controllerClient.Client, timer *Timer) *TimerSync { +func MakeTimerSync(fissionClient *tpr.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ - controller: controller, - timer: timer, + fissionClient: fissionClient, + timer: timer, } go ws.syncSvc() return ws } func (ws *TimerSync) syncSvc() { - failureCount := 0 - maxFailures := 6 for { - triggers, err := ws.controller.TimeTriggerList() + triggers, err := ws.fissionClient.Timetriggers(api.NamespaceAll).List(api.ListOptions{}) if err != nil { - failureCount++ - if failureCount > maxFailures { - log.Fatalf("Failed to connect to controller: %v", err) - } - time.Sleep(10 * time.Second) - continue + log.Fatalf("Failed get time trigger list: %v", err) } - ws.timer.Sync(triggers) + ws.timer.Sync(triggers.Items) + + // TODO switch to watches time.Sleep(3 * time.Second) } } diff --git a/tpr/client.go b/tpr/client.go new file mode 100644 index 00000000..f5f41066 --- /dev/null +++ b/tpr/client.go @@ -0,0 +1,198 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "errors" + "os" + "time" + + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/unversioned" + "k8s.io/client-go/1.5/pkg/runtime" + "k8s.io/client-go/1.5/pkg/runtime/serializer" + "k8s.io/client-go/1.5/rest" + "k8s.io/client-go/1.5/tools/clientcmd" +) + +type ( + FissionClient struct { + tprClient *rest.RESTClient + } +) + +// Get a kubernetes client using the kubeconfig file at the +// environment var $KUBECONFIG, or an in-cluster config if that's +// undefined. +func GetKubernetesClient() (*rest.Config, *kubernetes.Clientset, error) { + var config *rest.Config + var err error + + // get the config, either from kubeconfig or using our + // in-cluster service account + kubeConfig := os.Getenv("KUBECONFIG") + if len(kubeConfig) != 0 { + config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + return nil, nil, err + } + } else { + config, err = rest.InClusterConfig() + if err != nil { + return nil, nil, err + } + } + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, nil, err + } + + return config, clientset, nil +} + +// GetTprClient gets a TPR client config +func GetTprClient(config *rest.Config) (*rest.RESTClient, error) { + // mutate config to add our types + configureClient(config) + + // make a REST client with that config + return rest.RESTClientFor(config) +} + +// configureClient sets up a REST client for Fission TPR types. +// +// This is copied from the client-go TPR example. (I don't understand +// all of it completely.) It registers our types with the global API +// "scheme" (api.Scheme), which keeps a directory of types [I guess so +// it can use the string in the Kind field to make a Go object?]. It +// also puts the fission TPR types under a "group version" which we +// create for our TPRs types. +func configureClient(config *rest.Config) { + groupversion := unversioned.GroupVersion{ + Group: "fission.io", + Version: "v1", + } + config.GroupVersion = &groupversion + config.APIPath = "/apis" + config.ContentType = runtime.ContentTypeJSON + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs} + + schemeBuilder := runtime.NewSchemeBuilder( + func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes( + groupversion, + &Function{}, + &FunctionList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Environment{}, + &EnvironmentList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Httptrigger{}, + &HttptriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Kuberneteswatchtrigger{}, + &KuberneteswatchtriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Timetrigger{}, + &TimetriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + scheme.AddKnownTypes( + groupversion, + &Messagequeuetrigger{}, + &MessagequeuetriggerList{}, + &api.ListOptions{}, + &api.DeleteOptions{}, + ) + return nil + }) + schemeBuilder.AddToScheme(api.Scheme) +} + +func waitForTPRs(tprClient *rest.RESTClient) error { + start := time.Now() + for { + fi := MakeFunctionInterface(tprClient, api.NamespaceDefault) + _, err := fi.List(api.ListOptions{}) + if err != nil { + time.Sleep(100 * time.Millisecond) + } else { + return nil + } + + if time.Now().Sub(start) > 30*time.Second { + return errors.New("timeout waiting for TPRs") + } + } +} + +func MakeFissionClient() (*FissionClient, *kubernetes.Clientset, error) { + config, kubeClient, err := GetKubernetesClient() + if err != nil { + return nil, nil, err + } + tprClient, err := GetTprClient(config) + if err != nil { + return nil, nil, err + } + fc := &FissionClient{ + tprClient: tprClient, + } + return fc, kubeClient, nil +} + +func (fc *FissionClient) Functions(ns string) FunctionInterface { + return MakeFunctionInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Environments(ns string) EnvironmentInterface { + return MakeEnvironmentInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Httptriggers(ns string) HttptriggerInterface { + return MakeHttptriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Kuberneteswatchtriggers(ns string) KuberneteswatchtriggerInterface { + return MakeKuberneteswatchtriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Timetriggers(ns string) TimetriggerInterface { + return MakeTimetriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) Messagequeuetriggers(ns string) MessagequeuetriggerInterface { + return MakeMessagequeuetriggerInterface(fc.tprClient, ns) +} +func (fc *FissionClient) WaitForTPRs() { + waitForTPRs(fc.tprClient) +} diff --git a/tpr/environment.go b/tpr/environment.go new file mode 100644 index 00000000..4367693d --- /dev/null +++ b/tpr/environment.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + EnvironmentInterface interface { + Create(*Environment) (*Environment, error) + Get(name string) (*Environment, error) + Update(*Environment) (*Environment, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*EnvironmentList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + environmentClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeEnvironmentInterface(tprClient *rest.RESTClient, namespace string) EnvironmentInterface { + return &environmentClient{ + client: tprClient, + namespace: namespace, + } +} + +func (ec *environmentClient) Create(e *Environment) (*Environment, error) { + var result Environment + err := ec.client.Post(). + Resource("environments"). + Namespace(ec.namespace). + Body(e). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Get(name string) (*Environment, error) { + var result Environment + err := ec.client.Get(). + Resource("environments"). + Namespace(ec.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Update(e *Environment) (*Environment, error) { + var result Environment + err := ec.client.Put(). + Resource("environments"). + Namespace(ec.namespace). + Name(e.Metadata.Name). + Body(e). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Delete(name string, opts *api.DeleteOptions) error { + return ec.client.Delete(). + Namespace(ec.namespace). + Resource("environments"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (ec *environmentClient) List(opts api.ListOptions) (*EnvironmentList, error) { + var result EnvironmentList + err := ec.client.Get(). + Namespace(ec.namespace). + Resource("environments"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (ec *environmentClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return ec.client.Get(). + Prefix("watch"). + Namespace(ec.namespace). + Resource("environments"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/function.go b/tpr/function.go new file mode 100644 index 00000000..186301ac --- /dev/null +++ b/tpr/function.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + FunctionInterface interface { + Create(*Function) (*Function, error) + Get(name string) (*Function, error) + Update(*Function) (*Function, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*FunctionList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + functionClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeFunctionInterface(tprClient *rest.RESTClient, namespace string) FunctionInterface { + return &functionClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *functionClient) Create(f *Function) (*Function, error) { + var result Function + err := fc.client.Post(). + Resource("functions"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Get(name string) (*Function, error) { + var result Function + err := fc.client.Get(). + Resource("functions"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Update(f *Function) (*Function, error) { + var result Function + err := fc.client.Put(). + Resource("functions"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("functions"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *functionClient) List(opts api.ListOptions) (*FunctionList, error) { + var result FunctionList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("functions"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *functionClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("functions"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/httptrigger.go b/tpr/httptrigger.go new file mode 100644 index 00000000..c70cf36e --- /dev/null +++ b/tpr/httptrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + HttptriggerInterface interface { + Create(*Httptrigger) (*Httptrigger, error) + Get(name string) (*Httptrigger, error) + Update(*Httptrigger) (*Httptrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*HttptriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + httpTriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeHttptriggerInterface(tprClient *rest.RESTClient, namespace string) HttptriggerInterface { + return &httpTriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (c *httpTriggerClient) Create(obj *Httptrigger) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Post(). + Resource("httptriggers"). + Namespace(c.namespace). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Get(name string) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Get(). + Resource("httptriggers"). + Namespace(c.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Update(obj *Httptrigger) (*Httptrigger, error) { + var result Httptrigger + err := c.client.Put(). + Resource("httptriggers"). + Namespace(c.namespace). + Name(obj.Metadata.Name). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.namespace). + Resource("httptriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (c *httpTriggerClient) List(opts api.ListOptions) (*HttptriggerList, error) { + var result HttptriggerList + err := c.client.Get(). + Namespace(c.namespace). + Resource("httptriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *httpTriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return c.client.Get(). + Prefix("watch"). + Namespace(c.namespace). + Resource("httptriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/resource.go b/tpr/key.go similarity index 52% rename from resource.go rename to tpr/key.go index ccfbf35f..0cb64407 100644 --- a/resource.go +++ b/tpr/key.go @@ -14,28 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fission +package tpr -func (f Function) Key() string { - return f.Metadata.Name -} +import ( + "fmt" -func (e Environment) Key() string { - return e.Metadata.Name -} + "k8s.io/client-go/1.5/pkg/api" +) -func (ht HTTPTrigger) Key() string { - return ht.Metadata.Name -} - -func (mqt MessageQueueTrigger) Key() string { - return mqt.Metadata.Name -} - -func (tt TimeTrigger) Key() string { - return tt.Metadata.Name -} - -func (w Watch) Key() string { - return w.Metadata.Name +// Given metadata, create a key that uniquely identifies the contents +// of the object. Since resourceVersion changes on every update and +// UIDs are unique, uid+resourceVersion identifies the +// content. (ResourceVersion may also update on status updates, so +// this will result in some unnecessary cache misses. That should be +// ok.) +func CacheKey(metadata *api.ObjectMeta) string { + return fmt.Sprintf("%v_%v", metadata.UID, metadata.ResourceVersion) } diff --git a/tpr/kubernetesWatchTrigger.go b/tpr/kubernetesWatchTrigger.go new file mode 100644 index 00000000..3255fd21 --- /dev/null +++ b/tpr/kubernetesWatchTrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + KuberneteswatchtriggerInterface interface { + Create(*Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) + Get(name string) (*Kuberneteswatchtrigger, error) + Update(*Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*KuberneteswatchtriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + kubernetesWatchTriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeKuberneteswatchtriggerInterface(tprClient *rest.RESTClient, namespace string) KuberneteswatchtriggerInterface { + return &kubernetesWatchTriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (c *kubernetesWatchTriggerClient) Create(obj *Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Post(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Get(name string) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Get(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Update(obj *Kuberneteswatchtrigger) (*Kuberneteswatchtrigger, error) { + var result Kuberneteswatchtrigger + err := c.client.Put(). + Resource("kuberneteswatchtriggers"). + Namespace(c.namespace). + Name(obj.Metadata.Name). + Body(obj). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (c *kubernetesWatchTriggerClient) List(opts api.ListOptions) (*KuberneteswatchtriggerList, error) { + var result KuberneteswatchtriggerList + err := c.client.Get(). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (c *kubernetesWatchTriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return c.client.Get(). + Prefix("watch"). + Namespace(c.namespace). + Resource("kuberneteswatchtriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/messagequeuetrigger.go b/tpr/messagequeuetrigger.go new file mode 100644 index 00000000..61f08e8e --- /dev/null +++ b/tpr/messagequeuetrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + MessagequeuetriggerInterface interface { + Create(*Messagequeuetrigger) (*Messagequeuetrigger, error) + Get(name string) (*Messagequeuetrigger, error) + Update(*Messagequeuetrigger) (*Messagequeuetrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*MessagequeuetriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + messagequeuetriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeMessagequeuetriggerInterface(tprClient *rest.RESTClient, namespace string) MessagequeuetriggerInterface { + return &messagequeuetriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *messagequeuetriggerClient) Create(f *Messagequeuetrigger) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Post(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Get(name string) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Get(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Update(f *Messagequeuetrigger) (*Messagequeuetrigger, error) { + var result Messagequeuetrigger + err := fc.client.Put(). + Resource("messagequeuetriggers"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *messagequeuetriggerClient) List(opts api.ListOptions) (*MessagequeuetriggerList, error) { + var result MessagequeuetriggerList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *messagequeuetriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("messagequeuetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/timetrigger.go b/tpr/timetrigger.go new file mode 100644 index 00000000..600bfe56 --- /dev/null +++ b/tpr/timetrigger.go @@ -0,0 +1,119 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/watch" + "k8s.io/client-go/1.5/rest" +) + +type ( + TimetriggerInterface interface { + Create(*Timetrigger) (*Timetrigger, error) + Get(name string) (*Timetrigger, error) + Update(*Timetrigger) (*Timetrigger, error) + Delete(name string, options *api.DeleteOptions) error + List(opts api.ListOptions) (*TimetriggerList, error) + Watch(opts api.ListOptions) (watch.Interface, error) + } + + timetriggerClient struct { + client *rest.RESTClient + namespace string + } +) + +func MakeTimetriggerInterface(tprClient *rest.RESTClient, namespace string) TimetriggerInterface { + return &timetriggerClient{ + client: tprClient, + namespace: namespace, + } +} + +func (fc *timetriggerClient) Create(f *Timetrigger) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Post(). + Resource("timetriggers"). + Namespace(fc.namespace). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Get(name string) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Get(). + Resource("timetriggers"). + Namespace(fc.namespace). + Name(name). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Update(f *Timetrigger) (*Timetrigger, error) { + var result Timetrigger + err := fc.client.Put(). + Resource("timetriggers"). + Namespace(fc.namespace). + Name(f.Metadata.Name). + Body(f). + Do().Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Delete(name string, opts *api.DeleteOptions) error { + return fc.client.Delete(). + Namespace(fc.namespace). + Resource("timetriggers"). + Name(name). + Body(opts). + Do(). + Error() +} + +func (fc *timetriggerClient) List(opts api.ListOptions) (*TimetriggerList, error) { + var result TimetriggerList + err := fc.client.Get(). + Namespace(fc.namespace). + Resource("timetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Do(). + Into(&result) + if err != nil { + return nil, err + } + return &result, nil +} + +func (fc *timetriggerClient) Watch(opts api.ListOptions) (watch.Interface, error) { + return fc.client.Get(). + Prefix("watch"). + Namespace(fc.namespace). + Resource("timetriggers"). + VersionedParams(&opts, api.ParameterCodec). + Watch() +} diff --git a/tpr/tpr.go b/tpr/tpr.go new file mode 100644 index 00000000..8620a855 --- /dev/null +++ b/tpr/tpr.go @@ -0,0 +1,106 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/kubernetes" + "k8s.io/client-go/1.5/pkg/api/errors" + "k8s.io/client-go/1.5/pkg/api/v1" + "k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1" +) + +// ensureTPR checks if the given TPR type exists, and creates it if +// needed. (Note that this creates the TPR type; it doesn't create any +// _instances_ of that type.) +func ensureTPR(clientset *kubernetes.Clientset, tpr *v1beta1.ThirdPartyResource) error { + _, err := clientset.Extensions().ThirdPartyResources().Get(tpr.ObjectMeta.Name) + if err != nil { + if errors.IsNotFound(err) { + _, err := clientset.Extensions().ThirdPartyResources().Create(tpr) + if err != nil { + return err + } + } + } + return nil +} + +func EnsureFissionTPRs(clientset *kubernetes.Clientset) error { + tprs := []v1beta1.ThirdPartyResource{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "function.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "environment.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Environments (function containers)", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "httptrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "HTTP triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "kuberneteswatchtrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Kubernetes watch triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "timetrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Time-based triggers for functions", + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "messagequeuetrigger.fission.io", + }, + Versions: []v1beta1.APIVersion{ + {Name: "v1"}, + }, + Description: "Messqge queue triggers for functions", + }, + } + for _, tpr := range tprs { + err := ensureTPR(clientset, &tpr) + if err != nil { + return err + } + } + return nil +} diff --git a/tpr/tpr_test.go b/tpr/tpr_test.go new file mode 100644 index 00000000..cda5646b --- /dev/null +++ b/tpr/tpr_test.go @@ -0,0 +1,429 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "log" + "os" + "testing" + "time" + + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/unversioned" + "k8s.io/client-go/1.5/rest" + + "github.com/fission/fission" +) + +func panicIf(err error) { + if err != nil { + log.Panicf("err: %v", err) + } +} + +func functionTests(tprClient *rest.RESTClient) { + // sample function object + function := &Function{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Function", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.FunctionSpec{ + Source: fission.Package{}, + Deployment: fission.Package{ + Literal: []byte("hi"), + }, + EnvironmentName: "xxx", + }, + } + + // Test function CRUD + fi := MakeFunctionInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + fi.Delete(function.Metadata.Name, nil) + + // create + f, err := fi.Create(function) + panicIf(err) + if f.Metadata.Name != function.Metadata.Name { + log.Panicf("Bad result from create: %v", f) + } + + // read + f, err = fi.Get(function.Metadata.Name) + panicIf(err) + if len(f.Spec.Deployment.Literal) != len(function.Spec.Deployment.Literal) { + log.Panicf("Bad result from Get: %v", f) + } + + log.Printf("f.Metadata = %#v", f.Metadata) + + // update + function.Spec.EnvironmentName = "yyy" + f, err = fi.Update(function) + panicIf(err) + + log.Printf("f.Metadata = %#v", f.Metadata) + + // list + fl, err := fi.List(api.ListOptions{}) + panicIf(err) + if len(fl.Items) != 1 { + log.Panicf("wrong count from list: %v", fl) + } + if fl.Items[0].Spec.EnvironmentName != function.Spec.EnvironmentName { + log.Panicf("bad object from list: %v", fl.Items[0]) + } + + // delete + err = fi.Delete(f.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := fi.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + f, err = fi.Create(function) + panicIf(err) + defer fi.Delete(f.Metadata.Name, nil) + + // assert that we get a watch event for the new function + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + wf, ok := ev.Object.(*Function) + if !ok { + log.Panicf("Can't cast to Function") + } + if wf.Spec.EnvironmentName != function.Spec.EnvironmentName { + log.Panicf("Bad object from watch: %#v", wf) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func environmentTests(tprClient *rest.RESTClient) { + // sample environment object + environment := &Environment{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Environment", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.EnvironmentSpec{ + Runtime: fission.Runtime{ + Image: "xxx", + }, + Builder: fission.Builder{ + Image: "yyy", + Command: "zzz", + }, + }, + } + + // Test environment CRUD + ei := MakeEnvironmentInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(environment.Metadata.Name, nil) + + // create + e, err := ei.Create(environment) + panicIf(err) + if e.Metadata.Name != environment.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(environment.Metadata.Name) + panicIf(err) + if len(e.Spec.Runtime.Image) != len(environment.Spec.Runtime.Image) { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + environment.Spec.Runtime.Image = "www" + e, err = ei.Update(environment) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Runtime.Image != environment.Spec.Runtime.Image { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(environment) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new environment + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Environment) + if !ok { + log.Panicf("Can't cast to Environment") + } + if obj.Spec.Runtime.Image != environment.Spec.Runtime.Image { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func httpTriggerTests(tprClient *rest.RESTClient) { + // sample httpTrigger object + httpTrigger := &Httptrigger{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Httptrigger", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.HTTPTriggerSpec{ + RelativeURL: "/hi", + Method: "GET", + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "hello", + }, + }, + } + + // Test httpTrigger CRUD + ei := MakeHttptriggerInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(httpTrigger.Metadata.Name, nil) + + // create + e, err := ei.Create(httpTrigger) + panicIf(err) + if e.Metadata.Name != httpTrigger.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(httpTrigger.Metadata.Name) + panicIf(err) + if len(e.Spec.Method) != len(httpTrigger.Spec.Method) { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + httpTrigger.Spec.Method = "POST" + e, err = ei.Update(httpTrigger) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Method != httpTrigger.Spec.Method { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(httpTrigger) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new httpTrigger + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Httptrigger) + if !ok { + log.Panicf("Can't cast to Httptrigger") + } + if obj.Spec.Method != httpTrigger.Spec.Method { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func kubernetesWatchTriggerTests(tprClient *rest.RESTClient) { + // sample kubernetesWatchTrigger object + kubernetesWatchTrigger := &Kuberneteswatchtrigger{ + TypeMeta: unversioned.TypeMeta{ + Kind: "Kuberneteswatchtrigger", + APIVersion: "fission.io/v1", + }, + Metadata: api.ObjectMeta{ + Name: "hello", + }, + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: "foo", + Type: "pod", + LabelSelector: map[string]string{ + "x": "y", + }, + FunctionReference: fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: "foo", + }, + }, + } + + // Test kubernetesWatchTrigger CRUD + ei := MakeKuberneteswatchtriggerInterface(tprClient, api.NamespaceDefault) + + // cleanup from old crashed tests, ignore errors + ei.Delete(kubernetesWatchTrigger.Metadata.Name, nil) + + // create + e, err := ei.Create(kubernetesWatchTrigger) + panicIf(err) + if e.Metadata.Name != kubernetesWatchTrigger.Metadata.Name { + log.Panicf("Bad result from create: %v", e) + } + + // read + e, err = ei.Get(kubernetesWatchTrigger.Metadata.Name) + panicIf(err) + if e.Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("Bad result from Get: %#v", e) + } + + // update + kubernetesWatchTrigger.Spec.Type = "service" + e, err = ei.Update(kubernetesWatchTrigger) + panicIf(err) + + // list + el, err := ei.List(api.ListOptions{}) + panicIf(err) + if len(el.Items) != 1 { + log.Panicf("wrong count from list: %v", el) + } + if el.Items[0].Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("bad object from list: %v", el.Items[0]) + } + + // delete + err = ei.Delete(e.Metadata.Name, nil) + panicIf(err) + + // start a watch + wi, err := ei.Watch(api.ListOptions{}) + panicIf(err) + + start := time.Now() + e, err = ei.Create(kubernetesWatchTrigger) + panicIf(err) + defer ei.Delete(e.Metadata.Name, nil) + + // assert that we get a watch event for the new kubernetesWatchTrigger + recvd := false + select { + case <-time.NewTimer(1 * time.Second).C: + if !recvd { + log.Panicf("Didn't get watch event") + } + case ev := <-wi.ResultChan(): + obj, ok := ev.Object.(*Kuberneteswatchtrigger) + if !ok { + log.Panicf("Can't cast to Kuberneteswatchtrigger") + } + if obj.Spec.Type != kubernetesWatchTrigger.Spec.Type { + log.Panicf("Bad object from watch: %#v", obj) + } + log.Printf("watch event took %v", time.Now().Sub(start)) + recvd = true + } + +} + +func TestTpr(t *testing.T) { + // skip test if no cluster available for testing + kubeconfig := os.Getenv("KUBECONFIG") + if len(kubeconfig) == 0 { + log.Println("Skipping test, no kubernetes cluster") + return + } + + // Create the client config. Needs the KUBECONFIG env var to + // point at a valid kubeconfig. + config, clientset, err := GetKubernetesClient() + panicIf(err) + + // init our types + err = EnsureFissionTPRs(clientset) + panicIf(err) + + // rest client with knowledge about our tpr types + tprClient, err := GetTprClient(config) + panicIf(err) + + err = waitForTPRs(tprClient) + panicIf(err) + + functionTests(tprClient) + environmentTests(tprClient) + httpTriggerTests(tprClient) + kubernetesWatchTriggerTests(tprClient) +} diff --git a/tpr/types.go b/tpr/types.go new file mode 100644 index 00000000..84c04b2e --- /dev/null +++ b/tpr/types.go @@ -0,0 +1,208 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tpr + +import ( + "k8s.io/client-go/1.5/pkg/api" + "k8s.io/client-go/1.5/pkg/api/meta" + "k8s.io/client-go/1.5/pkg/api/unversioned" + + "github.com/fission/fission" +) + +// +// To add a Fission TPR type: +// 1. Create a "spec" type, for everything in the type except metadata +// 2. Create the type with metadata + the spec +// 3. Create a list type (for example see FunctionList and Function, below) +// 4. Add methods at the bottom of this file for satisfying Object and List interfaces +// 5. Add the type to configureClient in client.go +// 6. Add the type to EnsureFissionTPRs in tpr.go +// 7. Add tests to tpr_test.go +// + +type ( + // Functions. + Function struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.FunctionSpec `json:"spec"` + } + FunctionList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Function `json:"items"` + } + + // Environments. + Environment struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.EnvironmentSpec `json:"spec"` + } + EnvironmentList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Environment `json:"items"` + } + + // HTTP Triggers. (Something in the TPR reflection stuff wants + // it to be spelled "Httptrigger" not "HTTPTrigger" or even + // "HttpTrigger". Bleh.) + Httptrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.HTTPTriggerSpec `json:"spec"` + } + HttptriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Httptrigger `json:"items"` + } + + // Kubernetes Watches as triggers + Kuberneteswatchtrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.KubernetesWatchTriggerSpec `json:"spec"` + } + KuberneteswatchtriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Kuberneteswatchtrigger `json:"items"` + } + + // Time triggers + Timetrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.TimeTriggerSpec `json:"spec"` + } + TimetriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Timetrigger `json:"items"` + } + + // Message Queue triggers + Messagequeuetrigger struct { + unversioned.TypeMeta `json:",inline"` + Metadata api.ObjectMeta `json:"metadata"` + Spec fission.MessageQueueTriggerSpec `json:"spec"` + } + MessagequeuetriggerList struct { + unversioned.TypeMeta `json:",inline"` + Metadata unversioned.ListMeta `json:"metadata"` + + Items []Messagequeuetrigger `json:"items"` + } +) + +// Each TPR type needs: +// GetObjectKind (to satisfy the Object interface) +// +// In addition, each singular TPR type needs: +// GetObjectMeta (to satisfy the ObjectMetaAccessor interface) +// +// And each list TPR type needs: +// GetListMeta (to satisfy the ListMetaAccessor interface) + +func (f *Function) GetObjectKind() unversioned.ObjectKind { + return &f.TypeMeta +} +func (e *Environment) GetObjectKind() unversioned.ObjectKind { + return &e.TypeMeta +} +func (ht *Httptrigger) GetObjectKind() unversioned.ObjectKind { + return &ht.TypeMeta +} +func (w *Kuberneteswatchtrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} +func (w *Timetrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} +func (w *Messagequeuetrigger) GetObjectKind() unversioned.ObjectKind { + return &w.TypeMeta +} + +func (f *Function) GetObjectMeta() meta.Object { + return &f.Metadata +} +func (e *Environment) GetObjectMeta() meta.Object { + return &e.Metadata +} +func (ht *Httptrigger) GetObjectMeta() meta.Object { + return &ht.Metadata +} +func (w *Kuberneteswatchtrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} +func (w *Timetrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} +func (w *Messagequeuetrigger) GetObjectMeta() meta.Object { + return &w.Metadata +} + +func (fl *FunctionList) GetObjectKind() unversioned.ObjectKind { + return &fl.TypeMeta +} +func (el *EnvironmentList) GetObjectKind() unversioned.ObjectKind { + return &el.TypeMeta +} +func (hl *HttptriggerList) GetObjectKind() unversioned.ObjectKind { + return &hl.TypeMeta +} +func (wl *KuberneteswatchtriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} +func (wl *TimetriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} +func (wl *MessagequeuetriggerList) GetObjectKind() unversioned.ObjectKind { + return &wl.TypeMeta +} + +func (fl *FunctionList) GetListMeta() unversioned.List { + return &fl.Metadata +} +func (el *EnvironmentList) GetListMeta() unversioned.List { + return &el.Metadata +} +func (hl *HttptriggerList) GetListMeta() unversioned.List { + return &hl.Metadata +} +func (wl *KuberneteswatchtriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} +func (wl *TimetriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} +func (wl *MessagequeuetriggerList) GetListMeta() unversioned.List { + return &wl.Metadata +} + +// In the client-go TPR example, UnmarshalJSON is defined here for the +// singular and list types. That's supposed to be a workaround for +// some ugorji bug. But we don't seem to need it, and all our tests +// pass without it, so we don't define any UnmarshalJSON methods. diff --git a/types.go b/types.go index 91575c8f..5f70a455 100644 --- a/types.go +++ b/types.go @@ -17,72 +17,155 @@ limitations under the License. package fission type ( - // Metadata is used as the general identifier for all kinds of - // resources managed by the controller. - Metadata struct { + // + // Functions and packages + // + + // ChecksumType specifies the checksum algorithm, such as + // sha256, used for a checksum. + ChecksumType string + + // Checksum of package contents when the contents are stored + // outside the Package struct. Type is the checksum algorithm; + // "sha256" is the only currently supported one. Sum is hex + // encoded. + Checksum struct { + Type ChecksumType `json:"type"` + Sum string `json:"sum"` + } + + // PackageType is either literal or URL, indicating whether + // the package is specified in the Package struct or + // externally. + PackageType string + + // Package contains or references a collection of source or + // binary files. + Package struct { + // Type defines how the package is specified: literal or URL. + Type PackageType `json:"type"` + + // Literal contents of the package. Can be used for + // encoding packages below TODO (256KB?) size. + Literal []byte `json:"literal"` + + // URL references a package. + URL string `json:"url"` + + // Checksum ensures the integrity of packages + // refereced by URL. Ignored for literals. + Checksum Checksum `json:"checksum"` + + // EntryPoint optionally specifies an entry point in + // the package. Each environment defines a default + // entry point, but that can be overridden here. + EntryPoint string `json:"entrypoint"` + } + + // FunctionSpec describes the contents of the function. + FunctionSpec struct { + // EnvironmentName is the name of the environment that this function is associated + // with. An Environment with this name should exist, otherwise the function cannot + // be invoked. + EnvironmentName string `json:"environmentName"` + + // Source is an source package for this function; it's used for the build step if + // the environment defines a build container. + Source Package `json:"source"` + + // Deployment is a deployable package for this function. This is the package that's + // loaded into the environment's runtime container. + Deployment Package `json:"deployment"` + } + + FunctionReferenceType string + + FunctionReference struct { + // Type indicates whether this function reference is by name or selector. For now, + // the only supported reference type is by name. Future reference types: + // * Function by label or annotation + // * Branch or tag of a versioned function + // * A "rolling upgrade" from one version of a function to another + Type FunctionReferenceType `json:"type"` + + // Name of the function. Name string `json:"name"` - Uid string `json:"uid,omitempty"` } - // Function is a unit of executable code. Though it's called - // a function, the code may have more than one function; it's - // usually some sort of module or package. - Function struct { - Metadata `json:"metadata"` - Environment Metadata `json:"environment"` - Code string `json:"code"` + // + // Environments + // + + Runtime struct { + // Image for containing the language runtime. + Image string `json:"image"` + + // LoadEndpointPort defines the port on which the + // server listens for function load + // requests. Optional; default 8888. + LoadEndpointPort int32 `json:"loadendpointport"` + + // LoadEndpointPath defines the relative URL on which + // the server listens for function load + // requests. Optional; default "/specialize". + LoadEndpointPath string `json:"loadendpointpath"` + + // FunctionEndpointPort defines the port on which the + // server listens for function requests. Optional; + // default 8888. + FunctionEndpointPort int32 `json:"functionendpointport"` + } + Builder struct { + Image string `json:"image"` + Command string `json:"command"` + } + EnvironmentSpec struct { + // Environment API version + Version int `json:"version"` + + // Runtime container image etc.; required + Runtime Runtime `json:"runtime"` + + // Optional + Builder Builder `json:"builder"` + + // Optional, but strongly encouraged. Used to populate + // links from UI, CLI, etc. + DocumentationURL string `json:"documentationurl"` } - // Environment identifies the language and OS specific - // resources that a function depends on. For now this - // includes only the function run container image. Later, - // this will also include build containers, as well as support - // tools like debuggers, profilers, etc. - Environment struct { - Metadata `json:"metadata"` - RunContainerImageUrl string `json:"runContainerImageUrl"` + // + // Triggers + // + + HTTPTriggerSpec struct { + Host string `json:"host"` + RelativeURL string `json:"relativeurl"` + Method string `json:"method"` + FunctionReference FunctionReference `json:"functionref"` } - // HTTPTrigger maps URL patterns to functions. Function.UID - // is optional; if absent, the latest version of the function - // will automatically be selected. - HTTPTrigger struct { - Metadata `json:"metadata"` - UrlPattern string `json:"urlpattern"` - Method string `json:"method"` - Function Metadata `json:"function"` + KubernetesWatchTriggerSpec struct { + Namespace string `json:"namespace"` + Type string `json:"type"` + LabelSelector map[string]string `json:"labelselector"` + FunctionReference FunctionReference `json:"functionref"` } - MessageQueueTrigger struct { - Metadata `json:"metadata"` - Function Metadata `json:"function"` - MessageQueueType string `json:"messageQueueType"` - Topic string `json:"topic"` - ResponseTopic string `json:"respTopic,omitempty"` - } - - // Watch is a specification of Kubernetes watch along with a URL to post events to. - Watch struct { - Metadata `json:"metadata"` - - Namespace string `json:"namespace"` - ObjType string `json:"objtype"` - LabelSelector string `json:"labelselector"` - FieldSelector string `json:"fieldselector"` - - Function Metadata `json:"function"` - - Target string `json:"target"` // Watch publish target (URL, NATS stream, etc) + // MessageQueueTriggerSpec defines a binding from a topic in a + // message queue to a function. + MessageQueueTriggerSpec struct { + FunctionReference FunctionReference `json:"functionref"` + MessageQueueType string `json:"messageQueueType"` + Topic string `json:"topic"` + ResponseTopic string `json:"respTopic,omitempty"` } // TimeTrigger invokes the specific function at a time or // times specified by a cron string. - TimeTrigger struct { - Metadata `json:"metadata"` - - Cron string `json:"cron"` - - Function Metadata `json:"function"` + TimeTriggerSpec struct { + Cron string `json:"cron"` + FunctionReference `json:"functionref"` } // Errors returned by the Fission API. @@ -94,6 +177,54 @@ type ( errorCode int ) +// +// Fission-Environment interface. The following types are not +// exposed in the Fission API, but rather used by Fission to +// talk to environments. +// +type ( + FunctionLoadRequest struct { + // FilePath is an absolute filesystem path to the + // function. What exactly is stored here is + // env-specific. Optional. + FilePath string `json:"filepath"` + + // Entrypoint has an environment-specific meaning; + // usually, it defines a function within a module + // containing multiple functions. Optional; default is + // environment-specific. + EntryPoint string `json:"entrypoint"` + + // URL to expose this function at. Optional; defaults + // to "/". + URL string `json:"url"` + } +) + +const ( + ChecksumTypeSHA256 ChecksumType = "sha256" +) + +const ( + // PackageTypeLiteral means the package contents are specified in the Literal field of + // resource itself. + PackageTypeLiteral PackageType = "literal" + + // PackageTypeUrl means the package contents are at the specified URL. + PackageTypeUrl PackageType = "url" +) + +const ( + // FunctionReferenceFunctionName means that the function + // reference is simply by function name. + FunctionReferenceTypeFunctionName = "name" + + // Other function reference types we'd like to support: + // Versioned function, latest version + // Versioned function. by semver "latest compatible" + // Set of function references (recursively), by percentage of traffic +) + const ( ErrorInternal = iota @@ -103,6 +234,8 @@ const ( ErrorInvalidArgument ErrorNoSpace ErrorNotImplmented + ErrorChecksumFail + ErrorSizeLimitExceeded ) // must match order and len of the above const @@ -114,4 +247,10 @@ var errorDescriptions = []string{ "Invalid argument", "No space", "Not implemented", + "Checksum verification failed", + "Size limit exceeded", } + +const ( + PackageLiteralSizeLimit int64 = 256 * 1024 +)