From 6ea465c67cbcf0f249fe8acd4d12d42d48c00bd7 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Thu, 1 Sep 2016 23:56:40 -0700 Subject: [PATCH 01/13] Simple single-threaded file store service Intended for storing function source code. --- src/controller/fileStore.go | 98 ++++++++++++++++++++++++++++++++ src/controller/fileStore_test.go | 58 +++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 src/controller/fileStore.go create mode 100644 src/controller/fileStore_test.go diff --git a/src/controller/fileStore.go b/src/controller/fileStore.go new file mode 100644 index 00000000..5cade415 --- /dev/null +++ b/src/controller/fileStore.go @@ -0,0 +1,98 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "io/ioutil" + "log" + "path" +) + +type requestType int + +const ( + READ requestType = iota + WRITE +) + +type ( + fileStore struct { + root string // abs path of root of filestore + requestChannel chan fileStoreRequest + } + + fileStoreRequest struct { + requestType + fileName string // relative path + fileContents []byte + responseChannel chan fileStoreResponse + } + + fileStoreResponse struct { + error + fileContents []byte + } +) + +func makeFileStore(path string) *fileStore { + fileStore := &fileStore{ + root: path, + requestChannel: make(chan fileStoreRequest), + } + go fileStore.fileStoreService() + return fileStore +} + +func (fs *fileStore) fileStoreService() { + for { + req := <-fs.requestChannel + response := &fileStoreResponse{} + + switch req.requestType { + case READ: + response.fileContents, response.error = ioutil.ReadFile(path.Join(fs.root, req.fileName)) + case WRITE: + response.error = ioutil.WriteFile(path.Join(fs.root, req.fileName), req.fileContents, 0600) + default: + log.Panic("bad request") + } + req.responseChannel <- *response + } +} + +func (fs *fileStore) read(fileName string) ([]byte, error) { + req := &fileStoreRequest{ + requestType: READ, + fileName: fileName, + responseChannel: make(chan fileStoreResponse), + } + fs.requestChannel <- *req + response := <-req.responseChannel + return response.fileContents, response.error +} + +func (fs *fileStore) write(fileName string, contents []byte) error { + req := &fileStoreRequest{ + requestType: WRITE, + fileName: fileName, + fileContents: contents, + responseChannel: make(chan fileStoreResponse), + } + fs.requestChannel <- *req + response := <-req.responseChannel + return response.error +} diff --git a/src/controller/fileStore_test.go b/src/controller/fileStore_test.go new file mode 100644 index 00000000..d93393c2 --- /dev/null +++ b/src/controller/fileStore_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "io/ioutil" + "log" + "os" + "testing" +) + +func TestFileStore(t *testing.T) { + // tmp dir + dir, err := ioutil.TempDir("", "testFileStore") + if err != nil { + t.Fatalf("error creating temp dir: %v", err) + } + defer os.RemoveAll(dir) + log.Printf("temp dir at %v", dir) + + // file store + fs := makeFileStore(dir) + + _, err = fs.read("nonexistent") + if err == nil { + t.Fatalf("expected an error") + } + + path := "foo" + contents := []byte("bar") + err = fs.write(path, contents) + if err != nil { + t.Fatalf("error: %v", err) + } + + observedContents, err := fs.read(path) + if err != nil { + t.Fatalf("error: %v", err) + } + + if string(observedContents) != string(contents) { + t.Fatalf("contents don't match") + } +} From 60de12dfce2ff7e2e9c8dec2ae232ec9ac847c4d Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Fri, 2 Sep 2016 11:44:53 -0700 Subject: [PATCH 02/13] Add delete command to fileStore --- src/controller/fileStore.go | 23 +++++++++++++++++++---- src/controller/fileStore_test.go | 5 +++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/controller/fileStore.go b/src/controller/fileStore.go index 5cade415..79f8b26a 100644 --- a/src/controller/fileStore.go +++ b/src/controller/fileStore.go @@ -19,6 +19,7 @@ package controller import ( "io/ioutil" "log" + "os" "path" ) @@ -27,6 +28,7 @@ type requestType int const ( READ requestType = iota WRITE + DELETE ) type ( @@ -67,6 +69,8 @@ func (fs *fileStore) fileStoreService() { response.fileContents, response.error = ioutil.ReadFile(path.Join(fs.root, req.fileName)) case WRITE: response.error = ioutil.WriteFile(path.Join(fs.root, req.fileName), req.fileContents, 0600) + case DELETE: + response.error = os.Remove(path.Join(fs.root, req.fileName)) default: log.Panic("bad request") } @@ -75,24 +79,35 @@ func (fs *fileStore) fileStoreService() { } func (fs *fileStore) read(fileName string) ([]byte, error) { - req := &fileStoreRequest{ + req := fileStoreRequest{ requestType: READ, fileName: fileName, responseChannel: make(chan fileStoreResponse), } - fs.requestChannel <- *req + fs.requestChannel <- req response := <-req.responseChannel return response.fileContents, response.error } func (fs *fileStore) write(fileName string, contents []byte) error { - req := &fileStoreRequest{ + req := fileStoreRequest{ requestType: WRITE, fileName: fileName, fileContents: contents, responseChannel: make(chan fileStoreResponse), } - fs.requestChannel <- *req + fs.requestChannel <- req + response := <-req.responseChannel + return response.error +} + +func (fs *fileStore) delete(fileName string) error { + req := fileStoreRequest{ + requestType: DELETE, + fileName: fileName, + responseChannel: make(chan fileStoreResponse), + } + fs.requestChannel <- req response := <-req.responseChannel return response.error } diff --git a/src/controller/fileStore_test.go b/src/controller/fileStore_test.go index d93393c2..79fac437 100644 --- a/src/controller/fileStore_test.go +++ b/src/controller/fileStore_test.go @@ -55,4 +55,9 @@ func TestFileStore(t *testing.T) { if string(observedContents) != string(contents) { t.Fatalf("contents don't match") } + + err = fs.delete(path) + if err != nil { + t.Fatalf("error: %v", err) + } } From 5c336043db9d297248ed55a576350b3db09b8575 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Thu, 8 Sep 2016 12:12:15 -0700 Subject: [PATCH 03/13] Move /src/controller/ to /controller --- {src/controller => controller}/fileStore.go | 0 {src/controller => controller}/fileStore_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {src/controller => controller}/fileStore.go (100%) rename {src/controller => controller}/fileStore_test.go (100%) diff --git a/src/controller/fileStore.go b/controller/fileStore.go similarity index 100% rename from src/controller/fileStore.go rename to controller/fileStore.go diff --git a/src/controller/fileStore_test.go b/controller/fileStore_test.go similarity index 100% rename from src/controller/fileStore_test.go rename to controller/fileStore_test.go From 94899a88a7992482cef7efeb26f31c22d3611002 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Fri, 9 Sep 2016 13:00:01 -0700 Subject: [PATCH 04/13] Some sort of generic etcd-backed store - A generic Resource interface, which is serializable and has a key() method to get an identifying key. - A resourceStore, which has create, read, update, delete and list. This wraps etcd key operations and serialization/deserialization in a convenient service. Serialization is provided to the resourceStore as a 'serialization' interface. This can be JSON or anything else. - The resourceStore also has file operations -- this has read, write and delete operations. The general idea is to store metadata in etcd and data on the fileStore (which could also be an object store, we don't really use any file-specific properties). Files are also versioned -- each writeFile writes to a new UUID and that UUID is stored in an etcd in-order list. readFile can retrieve an arbitrary version by UUID, or just get the latest contents. --- controller/jsonSerializer.go | 32 ++++++ controller/resourceStore.go | 191 +++++++++++++++++++++++++++++++ controller/resourceStore_test.go | 136 ++++++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100644 controller/jsonSerializer.go create mode 100644 controller/resourceStore.go create mode 100644 controller/resourceStore_test.go diff --git a/controller/jsonSerializer.go b/controller/jsonSerializer.go new file mode 100644 index 00000000..008ba608 --- /dev/null +++ b/controller/jsonSerializer.go @@ -0,0 +1,32 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "encoding/json" +) + +type JsonSerializer struct { +} + +func (j JsonSerializer) serialize(r resource) ([]byte, error) { + return json.Marshal(r) +} + +func (j JsonSerializer) deserialize(buf []byte, r resource) error { + return json.Unmarshal(buf, r) +} diff --git a/controller/resourceStore.go b/controller/resourceStore.go new file mode 100644 index 00000000..10409311 --- /dev/null +++ b/controller/resourceStore.go @@ -0,0 +1,191 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "errors" + "github.com/coreos/etcd/client" + "github.com/satori/go.uuid" + "golang.org/x/net/context" + "reflect" +) + +type ( + resource interface { + key() string + } + + serializer interface { + serialize(r resource) ([]byte, error) + deserialize(buf []byte, r resource) error + } + + resourceStore struct { + *fileStore + client.KeysAPI + serializer + } +) + +func makeResourceStore(fs *fileStore, ks client.KeysAPI, s serializer) *resourceStore { + return &resourceStore{fileStore: fs, KeysAPI: ks, serializer: s} +} + +func getTypeName(r resource) (string, error) { + typ := reflect.TypeOf(r) + if typ.Kind().String() == "ptr" { + typ = typ.Elem() + } + typName := typ.Name() + if len(typName) == 0 { + return "", errors.New("Failed to get type") + } + return typName, nil +} + +func getKey(r resource) (string, error) { + typName, err := getTypeName(r) + if err != nil { + return "", err + } + rkey := r.key() + return (typName + "/" + rkey), nil +} + +func (rs *resourceStore) create(r resource) error { + key, err := getKey(r) + if err != nil { + return err + } + + serialized, err := rs.serializer.serialize(r) + if err != nil { + return err + } + + _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), + &client.SetOptions{PrevExist: client.PrevNoExist}) + return err +} + +func (rs *resourceStore) read(rkey string, res resource) error { + typName, err := getTypeName(res) + if err != nil { + return err + } + key := typName + "/" + rkey + + resp, err := rs.KeysAPI.Get(context.Background(), key, nil) + if err != nil { + return err + } + return rs.serializer.deserialize([]byte(resp.Node.Value), res) +} + +func (rs *resourceStore) update(r resource) error { + key, err := getKey(r) + if err != nil { + return err + } + + serialized, err := rs.serializer.serialize(r) + if err != nil { + return err + } + + _, err = rs.KeysAPI.Set(context.Background(), key, string(serialized), + &client.SetOptions{PrevExist: client.PrevExist}) + return err +} + +func (rs *resourceStore) delete(typename, rkey string) error { + key := typename + "/" + rkey + _, err := rs.KeysAPI.Delete(context.Background(), key, nil) // ignore response + return err +} + +func (rs *resourceStore) getAll(key string) ([]string, error) { + resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Recursive: true}) + if err != nil { + return nil, err + } + + res := make([]string, 0, len(resp.Node.Nodes)) + for _, n := range resp.Node.Nodes { + res = append(res, n.Value) + } + return res, nil +} + +func (rs *resourceStore) writeFile(parentKey string, contents []byte) (string, string, error) { + uid := uuid.NewV4().String() + + err := rs.fileStore.write(uid, contents) + if err != nil { + return "", "", err + } + + resp, err := rs.KeysAPI.CreateInOrder(context.Background(), parentKey, uid, nil) + if err != nil { + _ = rs.fileStore.delete(uid) + return "", "", err + } + + return resp.Node.Key, uid, nil +} + +func (rs *resourceStore) readFile(key string, uid *string) ([]byte, error) { + resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) + if err != nil { + return nil, err + } + + if uid == nil { + // get latest + n := resp.Node.Nodes + uid = &n[len(n)-1].Value + } else { + // validate uid is in the list + found := false + for _, u := range resp.Node.Nodes { + if *uid == u.Value { + found = true + break + } + } + if !found { + return nil, errors.New("Invalid UUID") + } + } + + contents, err := rs.fileStore.read(*uid) + return contents, err +} + +func (rs *resourceStore) deleteFile(key string, uid string) error { + resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) + if err != nil { + return err + } + for _, u := range resp.Node.Nodes { + if u.Value == uid { + err = rs.fileStore.delete(u.Value) + return err + } + } + return errors.New("won't delete unreferenced file") +} diff --git a/controller/resourceStore_test.go b/controller/resourceStore_test.go new file mode 100644 index 00000000..cb2a1b1b --- /dev/null +++ b/controller/resourceStore_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + // "github.com/coreos/etcd/client" + "golang.org/x/net/context" + "io/ioutil" + "log" + "os" + "testing" +) + +type TestResource struct { + A string + B int +} + +func (tr TestResource) key() string { + return tr.A +} + +func check(err error) { + if err != nil { + log.Panicf("err: %v", err) + } +} + +func assert(b bool, msg string) { + if !b { + log.Panic("assertion failed: " + msg) + } +} + +func TestResourceStore(t *testing.T) { + // make a tmp dir + dir, err := ioutil.TempDir("", "testFileStore") + check(err) + defer os.RemoveAll(dir) + + fs := makeFileStore(dir) + + // assume etcd is running, connect to it + ks := getEtcdKeyAPI([]string{"http://localhost:2379"}) + s := JsonSerializer{} + rs := makeResourceStore(fs, ks, s) + + tr := TestResource{A: "hello", B: 1} + + // Delete the key first, in case of a panic'd previous test run; ignore errors + _ = rs.delete("TestResource", tr.key()) + + // Create + err = rs.create(tr) + check(err) + defer rs.delete("TestResource", tr.key()) + + // Etcd key /TestResource/hello should exist + _, err = ks.Get(context.Background(), "TestResource/hello", nil) + check(err) + + // Read + tr1 := TestResource{} + err = rs.read(tr.key(), &tr1) + check(err) + assert(tr1 == tr, "retrieved value must equal created value") + + // Update and Read + tr.B += 1 + err = rs.update(tr) + check(err) + err = rs.read(tr.key(), &tr1) + check(err) + assert(tr1 == tr, "retrieved value must equal updated value") + + // Get list + results, err := rs.getAll("TestResource") + check(err) + res := make([]TestResource, 0, 0) + for _, r := range results { + tmp := TestResource{} + err = s.deserialize([]byte(r), &tmp) + check(err) + res = append(res, tmp) + } + assert(res[0] == tr, "value from retrieved list must equal updated value") + + // file tests + fileKey := "foo" + fileContents1 := []byte("hello") + fileContents2 := []byte("world") + key, uid1, err := rs.writeFile(fileKey, fileContents1) + check(err) + defer rs.deleteFile(fileKey, uid1) + log.Printf("key = %v, uid = %v", key, uid1) + + // read latest + contents, err := rs.readFile(fileKey, nil) + check(err) + assert(string(contents) == string(fileContents1), "retrieved file contents must match written value") + + // update-- same key new contents + _, uid2, err := rs.writeFile(fileKey, fileContents2) + check(err) + defer rs.deleteFile(fileKey, uid2) + + // read latest + contents, err = rs.readFile(fileKey, nil) + check(err) + assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") + + // read by uid + // 1 + contents, err = rs.readFile(fileKey, &uid1) + check(err) + assert(string(contents) == string(fileContents1), "retrieved file contents must match updated value") + + // 2 + contents, err = rs.readFile(fileKey, &uid2) + check(err) + assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") +} From 7963d7527cab900abdeff2d40b768cb11483e4aa Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:09:00 -0700 Subject: [PATCH 05/13] Wrap resourceStore for Functions, HTTPTriggers and Environments FunctionStore uses fileStore for function code and resourceStore for the Function's metadata. Environment- and HTTPTrigger- Store are just thin wrappers on resourceStore. There is probably a better way to organize this code, maybe with reflection. --- controller/environmentStore.go | 79 +++++++++++++++++++ controller/functionStore.go | 137 +++++++++++++++++++++++++++++++++ controller/httpTriggerStore.go | 83 ++++++++++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 controller/environmentStore.go create mode 100644 controller/functionStore.go create mode 100644 controller/httpTriggerStore.go diff --git a/controller/environmentStore.go b/controller/environmentStore.go new file mode 100644 index 00000000..49bd8efa --- /dev/null +++ b/controller/environmentStore.go @@ -0,0 +1,79 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/satori/go.uuid" + + "github.com/platform9/fission" +) + +type EnvironmentStore struct { + resourceStore +} + +func (es *EnvironmentStore) create(e *fission.Environment) error { + e.Metadata.Uid = uuid.NewV4().String() + return es.resourceStore.create(e) +} + +func (es *EnvironmentStore) read(m fission.Metadata) (*fission.Environment, error) { + var e fission.Environment + err := es.resourceStore.read(m.Name, &e) + if err != nil { + return nil, err + } + return &e, nil +} + +func (es *EnvironmentStore) update(e *fission.Environment) error { + e.Metadata.Uid = uuid.NewV4().String() + return es.resourceStore.update(e) +} + +func (es *EnvironmentStore) delete(m fission.Metadata) error { + typeName, err := getTypeName(fission.Environment{}) + if err != nil { + return err + } + return es.resourceStore.delete(typeName, m.Name) +} + +func (es *EnvironmentStore) list() ([]fission.Environment, error) { + typeName, err := getTypeName(fission.Environment{}) + if err != nil { + return nil, err + } + + bufs, err := es.resourceStore.getAll(typeName) + if err != nil { + return nil, err + } + + triggers := make([]fission.Environment, 0, len(bufs)) + js := JsonSerializer{} + for _, buf := range bufs { + var e fission.Environment + err = js.deserialize([]byte(buf), &e) + if err != nil { + return nil, err + } + triggers = append(triggers, e) + } + + return triggers, nil +} diff --git a/controller/functionStore.go b/controller/functionStore.go new file mode 100644 index 00000000..a5bc6406 --- /dev/null +++ b/controller/functionStore.go @@ -0,0 +1,137 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + log "github.com/Sirupsen/logrus" + + "github.com/platform9/fission" +) + +type FunctionStore struct { + resourceStore +} + +func (fs *FunctionStore) Create(f *fission.Function) (string, error) { + code := []byte(f.Code) + _, uid, err := fs.resourceStore.writeFile(f.Key(), code) + if err != nil { + return "", err + } + + f.Metadata.Uid = uid + f.Code = "" + + err = fs.resourceStore.create(f) + if err != nil { + fs.resourceStore.deleteFile(f.Key(), uid) // ignore errors + return "", err + } + return f.Metadata.Uid, nil +} + +func (fs *FunctionStore) Get(m *fission.Metadata) (*fission.Function, error) { + var f fission.Function + err := fs.resourceStore.read(m.Name, &f) + if err != nil { + return nil, err + } + + var code []byte + if len(m.Uid) > 0 { + log.WithFields(log.Fields{"Uid": m.Uid}).Info("fetching by uid") + code, err = fs.resourceStore.readFile(m.Name, &m.Uid) + f.Metadata = *m + } else { + code, err = fs.resourceStore.readFile(m.Name, nil) + } + if err != nil { + return nil, err + } + + f.Code = string(code) + return &f, nil +} + +func (fs *FunctionStore) Update(f *fission.Function) (string, error) { + code := []byte(f.Code) + _, uid, err := fs.resourceStore.writeFile(f.Key(), code) + if err != nil { + return "", err + } + + var fnew fission.Function + err = fs.resourceStore.read(f.Metadata.Name, &fnew) + if err != nil { + fs.resourceStore.deleteFile(f.Key(), uid) // ignore err + return "", err + } + + fnew.Metadata.Uid = uid + fnew.Environment = f.Environment + + err = fs.resourceStore.update(fnew) + if err != nil { + fs.resourceStore.deleteFile(f.Key(), uid) // ignore err + return "", err + } + return uid, err +} + +func (fs *FunctionStore) Delete(m fission.Metadata) error { + if len(m.Uid) == 0 { + err := fs.resourceStore.deleteAllFiles(m.Name) + if err != nil { + return err + } + } else { + err := fs.resourceStore.deleteFile(m.Name, m.Uid) + if err != nil { + return err + } + } + typeName, err := getTypeName(fission.Function{}) + if err != nil { + return err + } + return fs.resourceStore.delete(typeName, m.Name) +} + +func (fs *FunctionStore) List() ([]fission.Function, error) { + typeName, err := getTypeName(fission.Function{}) + if err != nil { + return nil, err + } + + bufs, err := fs.resourceStore.getAll(typeName) + if err != nil { + return nil, err + } + + js := JsonSerializer{} + functions := make([]fission.Function, 0, len(bufs)) + for _, buf := range bufs { + var f fission.Function + err = js.deserialize([]byte(buf), &f) + if err != nil { + return nil, err + } + functions = append(functions, f) + } + + return functions, nil +} diff --git a/controller/httpTriggerStore.go b/controller/httpTriggerStore.go new file mode 100644 index 00000000..e2d42e5a --- /dev/null +++ b/controller/httpTriggerStore.go @@ -0,0 +1,83 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/satori/go.uuid" + + "github.com/platform9/fission" +) + +type HTTPTriggerStore struct { + resourceStore +} + +func (hts *HTTPTriggerStore) create(ht *fission.HTTPTrigger) error { + ht.Metadata.Uid = uuid.NewV4().String() + return hts.resourceStore.create(ht) +} + +func (hts *HTTPTriggerStore) read(m fission.Metadata) (*fission.HTTPTrigger, error) { + var ht fission.HTTPTrigger + err := hts.resourceStore.read(m.Name, &ht) + if err != nil { + return nil, err + } + return &ht, nil +} + +func (hts *HTTPTriggerStore) update(ht *fission.HTTPTrigger) error { + err := validateHTTPTrigger(ht) + if err != nil { + return err + } + ht.Metadata.Uid = uuid.NewV4().String() + return hts.resourceStore.update(ht) +} + +func (hts *HTTPTriggerStore) delete(m fission.Metadata) error { + typeName, err := getTypeName(fission.HTTPTrigger{}) + if err != nil { + return err + } + return hts.resourceStore.delete(typeName, m.Name) +} + +func (hts *HTTPTriggerStore) list() ([]fission.HTTPTrigger, error) { + typeName, err := getTypeName(fission.HTTPTrigger{}) + if err != nil { + return nil, err + } + + bufs, err := hts.resourceStore.getAll(typeName) + if err != nil { + return nil, err + } + + triggers := make([]fission.HTTPTrigger, 0, len(bufs)) + js := JsonSerializer{} + for _, buf := range bufs { + var ht fission.HTTPTrigger + err = js.deserialize([]byte(buf), &ht) + if err != nil { + return nil, err + } + triggers = append(triggers, ht) + } + + return triggers, nil +} From 66854ff2869c9ca556cf6e9d67e736fa88b7a93c Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:16:15 -0700 Subject: [PATCH 06/13] Add fission.Error type; add JSON annotations --- types.go | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/types.go b/types.go index 01b08d2d..9515478c 100644 --- a/types.go +++ b/types.go @@ -18,22 +18,19 @@ package fission type ( // Metadata is used as the general identifier for all kinds of - // resources managed by the controller. In general, when a - // resource is updated, the Name remains the same, but the UID - // changes. In other words, the UID identifies a particular - // value of a resource. + // resources managed by the controller. Metadata struct { - Name string - Uid string + Name string `json:"name"` + Uid string `json:"uid,omitempty"` } // Function is a unit of executable code. Though it's called // a function, the code may have more than one function; it's // usually some sort of module or package. Function struct { - Metadata - Environment Metadata - Code string + Metadata `json:"metadata"` + Environment Metadata `json:"environment"` + Code string `json:"code"` } // Environment identifies the language and OS specific @@ -42,16 +39,33 @@ type ( // this will also include build containers, as well as support // tools like debuggers, profilers, etc. Environment struct { - Metadata - RunContainerImageUrl string + Metadata `json:"metadata"` + RunContainerImageUrl string `json:"runContainerImageUrl"` } // HTTPTrigger maps URL patterns to functions. Function.UID // is optional; if absent, the latest version of the function // will automatically be selected. HTTPTrigger struct { - Metadata - UrlPattern string - Function Metadata + Metadata `json:"metadata"` + UrlPattern string `json:"urlpattern"` + Function Metadata `json:"function"` } + + // Errors returned by the Fission API. + Error struct { + Code errorCode `json:"code"` + Message string `json:"message"` + } + errorCode int +) + +const ( + ErrorInternal = iota + + ErrorNotAuthorized + ErrorNotFound + ErrorNameExists + ErrorInvalidArgument + ErrorNoSpace ) From a0cfadbd73b321588e5f02dee8b85f3ed0f85225 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:25:50 -0700 Subject: [PATCH 07/13] Fix deleteFile (missing deletes for etcd metadata) Also move resource and serializer interfaces to types.go --- controller/resourceStore.go | 67 +++++++++++++++++++++++++------- controller/resourceStore_test.go | 65 ++++++++++++++++++------------- controller/types.go | 28 +++++++++++++ 3 files changed, 118 insertions(+), 42 deletions(-) create mode 100644 controller/types.go diff --git a/controller/resourceStore.go b/controller/resourceStore.go index 10409311..4529747e 100644 --- a/controller/resourceStore.go +++ b/controller/resourceStore.go @@ -18,22 +18,15 @@ package controller import ( "errors" + "reflect" + + log "github.com/Sirupsen/logrus" "github.com/coreos/etcd/client" "github.com/satori/go.uuid" "golang.org/x/net/context" - "reflect" ) type ( - resource interface { - key() string - } - - serializer interface { - serialize(r resource) ([]byte, error) - deserialize(buf []byte, r resource) error - } - resourceStore struct { *fileStore client.KeysAPI @@ -62,7 +55,7 @@ func getKey(r resource) (string, error) { if err != nil { return "", err } - rkey := r.key() + rkey := r.Key() return (typName + "/" + rkey), nil } @@ -139,6 +132,7 @@ func (rs *resourceStore) writeFile(parentKey string, contents []byte) (string, s return "", "", err } + parentKey = "file/" + parentKey resp, err := rs.KeysAPI.CreateInOrder(context.Background(), parentKey, uid, nil) if err != nil { _ = rs.fileStore.delete(uid) @@ -149,6 +143,7 @@ func (rs *resourceStore) writeFile(parentKey string, contents []byte) (string, s } func (rs *resourceStore) readFile(key string, uid *string) ([]byte, error) { + key = "file/" + key resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) if err != nil { return nil, err @@ -168,7 +163,7 @@ func (rs *resourceStore) readFile(key string, uid *string) ([]byte, error) { } } if !found { - return nil, errors.New("Invalid UUID") + return nil, errors.New("Invalid UID " + *uid) } } @@ -177,15 +172,57 @@ func (rs *resourceStore) readFile(key string, uid *string) ([]byte, error) { } func (rs *resourceStore) deleteFile(key string, uid string) error { + key = "file/" + key + resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) + if err != nil { + return err + } + + var node *client.Node + for _, u := range resp.Node.Nodes { + if u.Value == uid { + node = u + } + } + if node == nil { + log.WithFields(log.Fields{"key": key, "uid": uid}).Error("unreferenced file") + return errors.New("won't delete unreferenced file") + } + + err = rs.fileStore.delete(node.Value) + if err != nil { + return err + } + + _, err = rs.KeysAPI.Delete(context.Background(), node.Key, nil) + if err != nil { + return err + } + + if len(resp.Node.Nodes) == 1 { + _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) + return err + } + return nil +} + +func (rs *resourceStore) deleteAllFiles(key string) error { + key = "file/" + key resp, err := rs.KeysAPI.Get(context.Background(), key, &client.GetOptions{Sort: true}) if err != nil { return err } for _, u := range resp.Node.Nodes { - if u.Value == uid { - err = rs.fileStore.delete(u.Value) + err = rs.fileStore.delete(u.Value) + if err != nil { + return err + } + + _, err = rs.KeysAPI.Delete(context.Background(), u.Key, nil) + if err != nil { return err } } - return errors.New("won't delete unreferenced file") + _, err = rs.KeysAPI.Delete(context.Background(), key, &client.DeleteOptions{Dir: true}) + return err } diff --git a/controller/resourceStore_test.go b/controller/resourceStore_test.go index cb2a1b1b..5ff312c3 100644 --- a/controller/resourceStore_test.go +++ b/controller/resourceStore_test.go @@ -17,12 +17,13 @@ limitations under the License. package controller import ( - // "github.com/coreos/etcd/client" - "golang.org/x/net/context" "io/ioutil" "log" "os" "testing" + + "github.com/coreos/etcd/client" + "golang.org/x/net/context" ) type TestResource struct { @@ -30,11 +31,11 @@ type TestResource struct { B int } -func (tr TestResource) key() string { +func (tr TestResource) Key() string { return tr.A } -func check(err error) { +func panicIf(err error) { if err != nil { log.Panicf("err: %v", err) } @@ -46,91 +47,101 @@ func assert(b bool, msg string) { } } -func TestResourceStore(t *testing.T) { +func getTestResourceStore() (*fileStore, client.KeysAPI, *resourceStore) { // make a tmp dir dir, err := ioutil.TempDir("", "testFileStore") - check(err) - defer os.RemoveAll(dir) - + panicIf(err) fs := makeFileStore(dir) // assume etcd is running, connect to it ks := getEtcdKeyAPI([]string{"http://localhost:2379"}) + s := JsonSerializer{} rs := makeResourceStore(fs, ks, s) + return fs, ks, rs +} + +func TestResourceStore(t *testing.T) { + fs, ks, rs := getTestResourceStore() + defer os.RemoveAll(fs.root) + + s := JsonSerializer{} + tr := TestResource{A: "hello", B: 1} // Delete the key first, in case of a panic'd previous test run; ignore errors - _ = rs.delete("TestResource", tr.key()) + _ = rs.delete("TestResource", tr.Key()) // Create - err = rs.create(tr) - check(err) - defer rs.delete("TestResource", tr.key()) + err := rs.create(tr) + panicIf(err) + defer ks.Delete(context.Background(), "/TestResource", &client.DeleteOptions{Dir: true}) + defer rs.delete("TestResource", tr.Key()) // Etcd key /TestResource/hello should exist _, err = ks.Get(context.Background(), "TestResource/hello", nil) - check(err) + panicIf(err) // Read tr1 := TestResource{} - err = rs.read(tr.key(), &tr1) - check(err) + err = rs.read(tr.Key(), &tr1) + panicIf(err) assert(tr1 == tr, "retrieved value must equal created value") // Update and Read tr.B += 1 err = rs.update(tr) - check(err) - err = rs.read(tr.key(), &tr1) - check(err) + panicIf(err) + err = rs.read(tr.Key(), &tr1) + panicIf(err) assert(tr1 == tr, "retrieved value must equal updated value") // Get list results, err := rs.getAll("TestResource") - check(err) + panicIf(err) res := make([]TestResource, 0, 0) for _, r := range results { tmp := TestResource{} err = s.deserialize([]byte(r), &tmp) - check(err) + panicIf(err) res = append(res, tmp) } assert(res[0] == tr, "value from retrieved list must equal updated value") // file tests - fileKey := "foo" + fileKey := "resourceStoreTest" fileContents1 := []byte("hello") fileContents2 := []byte("world") key, uid1, err := rs.writeFile(fileKey, fileContents1) - check(err) + panicIf(err) defer rs.deleteFile(fileKey, uid1) log.Printf("key = %v, uid = %v", key, uid1) // read latest contents, err := rs.readFile(fileKey, nil) - check(err) + panicIf(err) assert(string(contents) == string(fileContents1), "retrieved file contents must match written value") // update-- same key new contents _, uid2, err := rs.writeFile(fileKey, fileContents2) - check(err) + panicIf(err) defer rs.deleteFile(fileKey, uid2) // read latest contents, err = rs.readFile(fileKey, nil) - check(err) + panicIf(err) assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") // read by uid // 1 contents, err = rs.readFile(fileKey, &uid1) - check(err) + panicIf(err) assert(string(contents) == string(fileContents1), "retrieved file contents must match updated value") // 2 contents, err = rs.readFile(fileKey, &uid2) - check(err) + panicIf(err) assert(string(contents) == string(fileContents2), "retrieved file contents must match updated value") + } diff --git a/controller/types.go b/controller/types.go new file mode 100644 index 00000000..abc575ec --- /dev/null +++ b/controller/types.go @@ -0,0 +1,28 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +type ( + resource interface { + Key() string + } + + serializer interface { + serialize(r resource) ([]byte, error) + deserialize(buf []byte, r resource) error + } +) From be16c6a2aa774d4b3e2a9f18a461729484fc1922 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:44:19 -0700 Subject: [PATCH 08/13] Minor stuff: debug logs, unit test update --- controller/fileStore.go | 4 +++- controller/fileStore_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/controller/fileStore.go b/controller/fileStore.go index 79f8b26a..66337424 100644 --- a/controller/fileStore.go +++ b/controller/fileStore.go @@ -18,9 +18,10 @@ package controller import ( "io/ioutil" - "log" "os" "path" + + log "github.com/Sirupsen/logrus" ) type requestType int @@ -64,6 +65,7 @@ func (fs *fileStore) fileStoreService() { req := <-fs.requestChannel response := &fileStoreResponse{} + log.WithFields(log.Fields{"file": req.fileName, "type": req.requestType}).Debug("fileStore request") switch req.requestType { case READ: response.fileContents, response.error = ioutil.ReadFile(path.Join(fs.root, req.fileName)) diff --git a/controller/fileStore_test.go b/controller/fileStore_test.go index 79fac437..b150a26f 100644 --- a/controller/fileStore_test.go +++ b/controller/fileStore_test.go @@ -40,7 +40,7 @@ func TestFileStore(t *testing.T) { t.Fatalf("expected an error") } - path := "foo" + path := "fileStoreTest" contents := []byte("bar") err = fs.write(path, contents) if err != nil { From 14734f64354cde5043a9c30a54962129e8001088 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:45:13 -0700 Subject: [PATCH 09/13] HTTP API for Functions So far, just Function create/read/update/delete/list. FunctionApi glues HTTP request/response to functionStore. --- controller/api.go | 102 +++++++++++++++++++++++++ controller/functionApi.go | 151 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 controller/api.go create mode 100644 controller/functionApi.go diff --git a/controller/api.go b/controller/api.go new file mode 100644 index 00000000..533daa61 --- /dev/null +++ b/controller/api.go @@ -0,0 +1,102 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "net/http" + "os" + + log "github.com/Sirupsen/logrus" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + + "github.com/platform9/fission" +) + +type API struct { + FunctionStore + HTTPTriggerStore + EnvironmentStore +} + +func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) { + _, err := w.Write(resp) + if err != nil { + // this will probably fail too, but try anyway + api.respondWithError(w, err) + } +} + +func (api *API) respondWithError(w http.ResponseWriter, err error) { + var code int + var msg string + + fe, ok := err.(fission.Error) + if ok { + msg = fe.Message + switch fe.Code { + case fission.ErrorNotFound: + code = 404 + case fission.ErrorInvalidArgument: + code = 400 + case fission.ErrorNoSpace: + code = 500 + case fission.ErrorNotAuthorized: + code = 403 + default: + code = 500 + } + } else { + code = 500 + msg = err.Error() + } + log.Printf("Error: %v: %v", code, msg) + http.Error(w, msg, code) +} + +func (api *API) HomeHandler(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Fission API") +} + +func (api *API) serve(port int) { + r := mux.NewRouter() + r.HandleFunc("/", api.HomeHandler) + + r.HandleFunc("/functions", api.FunctionApiList).Methods("GET") + r.HandleFunc("/functions", api.FunctionApiCreate).Methods("POST") + r.HandleFunc("/functions/{function}", api.FunctionApiGet).Methods("GET") + r.HandleFunc("/functions/{function}", api.FunctionApiUpdate).Methods("PUT") + r.HandleFunc("/functions/{function}", api.FunctionApiDelete).Methods("DELETE") + + // r.HandleFunc("/triggers/http", api.HTTPTriggerApiList).Methods("GET") + // r.HandleFunc("/triggers/http", api.HTTPTriggerApiCreate).Methods("POST") + // r.HandleFunc("/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET") + // r.HandleFunc("/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT") + // r.HandleFunc("/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE") + + // r.HandleFunc("/environments", api.EnvironmentApiList).Methods("GET") + // r.HandleFunc("/environments", api.EnvironmentApiCreate).Methods("POST") + // r.HandleFunc("/environments/{environment}", api.EnvironmentApiGet).Methods("GET") + // r.HandleFunc("/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT") + // r.HandleFunc("/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE") + + address := fmt.Sprintf(":%v", port) + + log.WithFields(log.Fields{"port": port}).Info("Server started") + log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r))) +} diff --git a/controller/functionApi.go b/controller/functionApi.go new file mode 100644 index 00000000..5319cd35 --- /dev/null +++ b/controller/functionApi.go @@ -0,0 +1,151 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "io/ioutil" + "net/http" + + "encoding/json" + log "github.com/Sirupsen/logrus" + "github.com/gorilla/mux" + + "github.com/platform9/fission" +) + +func (api *API) FunctionApiList(w http.ResponseWriter, r *http.Request) { + funcs, err := api.FunctionStore.List() + if err != nil { + api.respondWithError(w, err) + return + } + + resp, err := json.Marshal(funcs) + if err != nil { + api.respondWithError(w, err) + return + } + + api.respondWithSuccess(w, resp) +} + +func (api *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + api.respondWithError(w, err) + } + + var f fission.Function + err = json.Unmarshal(body, &f) + if err != nil { + api.respondWithError(w, err) + return + } + + uid, err := api.FunctionStore.Create(&f) + if err != nil { + api.respondWithError(w, err) + return + } + + m := &fission.Metadata{Name: f.Name, Uid: uid} + resp, err := json.Marshal(m) + if err != nil { + api.respondWithError(w, err) + return + } + + api.respondWithSuccess(w, resp) +} + +func (api *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) { + var m fission.Metadata + + vars := mux.Vars(r) + m.Name = vars["function"] + m.Uid = r.FormValue("uid") // empty if uid is absent + + f, err := api.FunctionStore.Get(&m) + if err != nil { + api.respondWithError(w, err) + return + } + + resp, err := json.Marshal(f) + if err != nil { + api.respondWithError(w, err) + return + } + + api.respondWithSuccess(w, resp) +} + +func (api *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funcName := vars["function"] + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + api.respondWithError(w, err) + } + + var f fission.Function + err = json.Unmarshal(body, &f) + if err != nil { + api.respondWithError(w, err) + return + } + + if funcName != f.Metadata.Name { + err = fission.MakeError(fission.ErrorInvalidArgument, "Function name doesn't match URL") + api.respondWithError(w, err) + return + } + + uid, err := api.FunctionStore.Update(&f) + if err != nil { + api.respondWithError(w, err) + return + } + + m := &fission.Metadata{Name: f.Name, Uid: uid} + resp, err := json.Marshal(m) + if err != nil { + api.respondWithError(w, err) + return + } + api.respondWithSuccess(w, resp) +} + +func (api *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + var m fission.Metadata + m.Name = vars["function"] + + m.Uid = r.FormValue("uid") // empty if uid is absent + if len(m.Uid) == 0 { + log.WithFields(log.Fields{"function": m.Name}).Info("Deleting all versions") + } + + err := api.FunctionStore.Delete(m) + if err != nil { + api.respondWithError(w, err) + return + } + + api.respondWithSuccess(w, []byte("")) +} From 72ff031624b77cefa4091514766399ab393645bb Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:53:57 -0700 Subject: [PATCH 10/13] Controller API client. Just Functions so far. --- controller/client/client.go | 214 ++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 controller/client/client.go diff --git a/controller/client/client.go b/controller/client/client.go new file mode 100644 index 00000000..b7725e9e --- /dev/null +++ b/controller/client/client.go @@ -0,0 +1,214 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + + log "github.com/Sirupsen/logrus" + + "github.com/platform9/fission" +) + +type ( + Client struct { + Url string + } +) + +func New(serverUrl string) *Client { + return &Client{Url: serverUrl} +} + +func (c *Client) delete(relativeUrl string) error { + req, err := http.NewRequest("DELETE", c.url(relativeUrl), nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.New("Delete failed") + } else { + return errors.New("Delete failed: " + string(body)) + } + } + + return nil +} + +func (c *Client) put(relativeUrl string, contentType string, body []byte) (*http.Response, error) { + req, err := http.NewRequest("PUT", c.url(relativeUrl), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-type", contentType) + return http.DefaultClient.Do(req) +} + +func (c *Client) url(relativeUrl string) string { + return c.Url + "/" + relativeUrl +} + +func (c *Client) FunctionCreate(f *fission.Function) (*fission.Metadata, error) { + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + + resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + log.WithFields(log.Fields{ + "function": f.Metadata.Name, + "status": resp.StatusCode, + }).Error("Failed to create function") + return nil, errors.New("failed to create function") + } + + var m fission.Metadata + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + + return &m, nil +} + +func (c *Client) FunctionGet(m *fission.Metadata) (*fission.Function, error) { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + if len(m.Uid) > 0 { + relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) + } + + resp, err := http.Get(c.url(relativeUrl)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var f fission.Function + err = json.Unmarshal(body, &f) + if err != nil { + return nil, err + } + + return &f, nil +} + +func (c *Client) FunctionUpdate(f *fission.Function) (*fission.Metadata, error) { + reqbody, err := json.Marshal(f) + if err != nil { + return nil, err + } + relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name) + + resp, err := c.put(relativeUrl, "application/json", reqbody) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var m fission.Metadata + err = json.Unmarshal(body, &m) + if err != nil { + return nil, err + } + return &m, nil +} + +func (c *Client) FunctionDelete(m *fission.Metadata) error { + relativeUrl := fmt.Sprintf("functions/%v", m.Name) + if len(m.Uid) > 0 { + relativeUrl += fmt.Sprintf("?uid=%v", m.Uid) + } + err := c.delete(relativeUrl) + return err +} + +func (c *Client) FunctionList() ([]fission.Function, error) { + resp, err := http.Get(c.url("functions")) + if err != nil { + return nil, err + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + funcs := make([]fission.Function, 0) + err = json.Unmarshal(body, &funcs) + if err != nil { + return nil, err + } + + return funcs, nil +} + +// func (c *Client) HTTPTriggerCreate(f *fission.Function) (string, error) { +// } +// func (c *Client) HTTPTriggerGet(m *fission.Metadata) (*fission.HTTPTrigger, error) { +// } +// func (c *Client) HTTPTriggerUpdate(f *fission.HTTPTrigger) (string, error) { +// } +// func (c *Client) HTTPTriggerDelete(m *fission.Metadata) error { +// } +// func (c *Client) HTTPTriggerList() ([]fission.HTTPTrigger, error) { +// } + +// func (c *Client) EnvironmentCreate(f *fission.Environment) (string, error) { +// } +// func (c *Client) EnvironmentGet(m *fission.Metadata) (*fission.Environment, error) { +// } +// func (c *Client) EnvironmentUpdate(f *fission.Environment) (string, error) { +// } +// func (c *Client) EnvironmentDelete(m *fission.Metadata) error { +// } +// func (c *Client) EnvironmentList() ([]fission.Environment, error) { +// } From f02f58cd121a32160795da0eb9989f3c8fe59218 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:54:26 -0700 Subject: [PATCH 11/13] Controller API unit tests --- controller/api_test.go | 124 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 controller/api_test.go diff --git a/controller/api_test.go b/controller/api_test.go new file mode 100644 index 00000000..988a1841 --- /dev/null +++ b/controller/api_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "io/ioutil" + "net/http" + "testing" + "time" + + log "github.com/Sirupsen/logrus" + etcdClient "github.com/coreos/etcd/client" + "golang.org/x/net/context" + + "github.com/platform9/fission" + "github.com/platform9/fission/controller/client" +) + +func TestFunctionApi(t *testing.T) { + log.SetFormatter(&log.TextFormatter{DisableColors: true}) + + _, ks, rs := getTestResourceStore() + fs := &FunctionStore{resourceStore: *rs} + hts := &HTTPTriggerStore{resourceStore: *rs} + es := &EnvironmentStore{resourceStore: *rs} + + api := &API{ + FunctionStore: *fs, + HTTPTriggerStore: *hts, + EnvironmentStore: *es, + } + + testFunc := &fission.Function{ + Metadata: fission.Metadata{ + Name: "foo", + Uid: "", + }, + Environment: fission.Metadata{ + Name: "nodejs", + Uid: "xxx", + }, + Code: "code1", + } + + go api.serve(8888) + time.Sleep(500 * time.Millisecond) + + resp, err := http.Get("http://localhost:8888/") + panicIf(err) + _, err = ioutil.ReadAll(resp.Body) + panicIf(err) + + client := client.New("http://localhost:8888") + + _, err = ks.Delete(context.Background(), "Function", &etcdClient.DeleteOptions{Recursive: true}) + if err != nil { + log.Printf("failed to delete: %v", err) + } + + m, err := client.FunctionCreate(testFunc) + panicIf(err) + uid1 := m.Uid + log.Printf("Created function %v: %v", m.Name, m.Uid) + + testFunc.Code = "code2" + m, err = client.FunctionUpdate(testFunc) + panicIf(err) + uid2 := m.Uid + log.Printf("Updated function %v: %v", m.Name, m.Uid) + + m.Uid = uid1 + testFunc.Code = "code1" + f, err := client.FunctionGet(m) + panicIf(err) + + testFunc.Metadata.Uid = m.Uid + log.Printf("f = %#v", f) + log.Printf("testFunc = %#v", testFunc) + assert(*f == *testFunc, "first version should match when read by uid") + + m.Uid = uid2 + testFunc.Metadata.Uid = m.Uid + testFunc.Code = "code2" + f, err = client.FunctionGet(m) + panicIf(err) + + assert(*f == *testFunc, "second version should match when read by uid") + + m.Uid = "" + testFunc.Metadata.Uid = uid2 + testFunc.Code = "code2" + f, err = client.FunctionGet(m) + panicIf(err) + + assert(*f == *testFunc, "second version should match when read as latest") + + testFunc.Metadata.Name = "bar" + m, err = client.FunctionCreate(testFunc) + panicIf(err) + + funcs, err := client.FunctionList() + panicIf(err) + assert(len(funcs) == 2, + "created two functions, but didn't find them") + + err = client.FunctionDelete(&fission.Metadata{Name: "foo"}) + panicIf(err) + err = client.FunctionDelete(&fission.Metadata{Name: "bar"}) + panicIf(err) +} From 39ef1da4156ad539ba065628645485a08c06b831 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:55:13 -0700 Subject: [PATCH 12/13] Key methods for Resource types --- resource.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 resource.go diff --git a/resource.go b/resource.go new file mode 100644 index 00000000..c5cb9dab --- /dev/null +++ b/resource.go @@ -0,0 +1,13 @@ +package fission + +func (f Function) Key() string { + return f.Metadata.Name +} + +func (e Environment) Key() string { + return e.Metadata.Name +} + +func (ht HTTPTrigger) Key() string { + return ht.Metadata.Name +} From 641cae0206f83be394b0747ead8b73b901644cbc Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Tue, 20 Sep 2016 00:55:31 -0700 Subject: [PATCH 13/13] Fission error type --- error.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 error.go diff --git a/error.go b/error.go new file mode 100644 index 00000000..77d34a70 --- /dev/null +++ b/error.go @@ -0,0 +1,29 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fission + +import ( + "fmt" +) + +func (e Error) Error() string { + return fmt.Sprintf("(Error %v) %v", e.Code, e.Message) +} + +func MakeError(code int, msg string) Error { + return Error{Code: errorCode(code), Message: msg} +}