Add message queue trigger support (#218)

Adds support for message queue triggers based on the NATS-Streaming message queue. This lets the user map a queue topic to a function, and optionally send the function's response into another queue topic. The message queue trigger manager is designed to be message queue independent, so we should be able add support for more queues by adding implementations for the relatively simple mqtrigger.MessageQueue interface.
2017-07-05 17:19:58 -07:00
parent 0fd60eab3b
commit e4fe007838
24 changed files with 1417 additions and 10 deletions
@@ -36,6 +36,7 @@ type (
 		FunctionStore
 		HTTPTriggerStore
 		TimeTriggerStore
+		MessageQueueTriggerStore
 		EnvironmentStore
 		WatchStore
 	}
@@ -49,11 +50,12 @@ type (

 func MakeAPI(rs *ResourceStore) *API {
 	api := &API{
-		FunctionStore:    FunctionStore{ResourceStore: *rs},
-		HTTPTriggerStore: HTTPTriggerStore{ResourceStore: *rs},
-		TimeTriggerStore: TimeTriggerStore{ResourceStore: *rs},
-		EnvironmentStore: EnvironmentStore{ResourceStore: *rs},
-		WatchStore:       WatchStore{ResourceStore: *rs},
+		FunctionStore:            FunctionStore{ResourceStore: *rs},
+		HTTPTriggerStore:         HTTPTriggerStore{ResourceStore: *rs},
+		TimeTriggerStore:         TimeTriggerStore{ResourceStore: *rs},
+		MessageQueueTriggerStore: MessageQueueTriggerStore{ResourceStore: *rs},
+		EnvironmentStore:         EnvironmentStore{ResourceStore: *rs},
+		WatchStore:               WatchStore{ResourceStore: *rs},
 	}
 	return api
 }
@@ -130,6 +132,12 @@ func (api *API) Serve(port int) {
 	r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT")
 	r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE")

+	r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET")
+	r.HandleFunc("/v1/triggers/messagequeue", api.MessageQueueApiCreate).Methods("POST")
+	r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiGet).Methods("GET")
+	r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiUpdate).Methods("PUT")
+	r.HandleFunc("/v1/triggers/messagequeue/{mqTrigger}", api.MessageQueueApiDelete).Methods("DELETE")
+
 	r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST")

 	address := fmt.Sprintf(":%v", port)
@@ -640,3 +640,116 @@ func (c *Client) TimeTriggerList() ([]fission.TimeTrigger, error) {

 	return triggers, nil
 }
+
+func (c *Client) MessageQueueTriggerCreate(t *fission.MessageQueueTrigger) (*fission.Metadata, error) {
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m fission.Metadata
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) MessageQueueTriggerGet(m *fission.Metadata) (*fission.MessageQueueTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name)
+	if len(m.Uid) > 0 {
+		relativeUrl += fmt.Sprintf("?uid=%v", m.Uid)
+	}
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var t fission.MessageQueueTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &t, nil
+}
+
+func (c *Client) MessageQueueTriggerUpdate(mqTrigger *fission.MessageQueueTrigger) (*fission.Metadata, error) {
+	reqbody, err := json.Marshal(mqTrigger)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m fission.Metadata
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) MessageQueueTriggerDelete(m *fission.Metadata) error {
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name)
+	if len(m.Uid) > 0 {
+		relativeUrl += fmt.Sprintf("?uid=%v", m.Uid)
+	}
+	err := c.delete(relativeUrl)
+	return err
+}
+
+func (c *Client) MessageQueueTriggerList(mqType string) ([]fission.MessageQueueTrigger, error) {
+	relativeUrl := "triggers/messagequeue"
+	if len(mqType) > 0 {
+		relativeUrl += fmt.Sprintf("?mqtype=%v", mqType)
+	}
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	triggers := make([]fission.MessageQueueTrigger, 0)
+	err = json.Unmarshal(body, &triggers)
+	if err != nil {
+		return nil, err
+	}
+
+	return triggers, nil
+}
@@ -0,0 +1,166 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/fission/fission"
+)
+
+func (api *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) {
+	mqType := r.FormValue("mqtype")
+	triggers, err := api.MessageQueueTriggerStore.List(mqType)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(triggers)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	api.respondWithSuccess(w, resp)
+}
+
+func (api *API) MessageQueueApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	var mqTrigger fission.MessageQueueTrigger
+	err = json.Unmarshal(body, &mqTrigger)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	// trigger name must not conflict with any other trigger
+	// even they are different message queue type
+	triggers, err := api.MessageQueueTriggerStore.List("")
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	for _, trigger := range triggers {
+		if trigger.Name == mqTrigger.Name {
+			err = fission.MakeError(fission.ErrorNameExists,
+				"Message queue trigger with same name already exists")
+			api.respondWithError(w, err)
+			return
+		}
+	}
+
+	// save trigger info
+	uid, err := api.MessageQueueTriggerStore.Create(&mqTrigger)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid}
+	resp, err := json.Marshal(mqTriggerMeta)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+	api.respondWithSuccess(w, resp)
+}
+
+func (api *API) MessageQueueApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	mqTriggerMeta := fission.Metadata{
+		Name: vars["mqTrigger"],
+		Uid:  r.FormValue("uid"), // empty if uid is absent
+	}
+	mqTrigger, err := api.MessageQueueTriggerStore.Get(&mqTriggerMeta)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(mqTrigger)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	api.respondWithSuccess(w, resp)
+}
+
+func (api *API) MessageQueueApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	mqtName := vars["mqTrigger"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	var mqTrigger fission.MessageQueueTrigger
+	err = json.Unmarshal(body, &mqTrigger)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	if mqtName != mqTrigger.Metadata.Name {
+		err = fission.MakeError(fission.ErrorInvalidArgument, "Message queue trigger name doesn't match URL")
+		api.respondWithError(w, err)
+		return
+	}
+
+	uid, err := api.MessageQueueTriggerStore.Update(&mqTrigger)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+
+	mqTriggerMeta := fission.Metadata{Name: mqTrigger.Metadata.Name, Uid: uid}
+	resp, err := json.Marshal(mqTriggerMeta)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	api.respondWithSuccess(w, resp)
+}
+
+func (api *API) MessageQueueApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	mqTriggerMeta := fission.Metadata{
+		Name: vars["mqTrigger"],
+		Uid:  r.FormValue("uid"), // empty if uid is absent
+	}
+
+	if len(mqTriggerMeta.Uid) == 0 {
+		log.WithFields(log.Fields{"mqTrigger": mqTriggerMeta.Name}).Info("Deleting all versions")
+	}
+
+	err := api.MessageQueueTriggerStore.Delete(mqTriggerMeta)
+	if err != nil {
+		api.respondWithError(w, err)
+		return
+	}
+	api.respondWithSuccess(w, []byte(""))
+}
@@ -0,0 +1,80 @@
+/*
+CopyrigmqTrigger 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    mqTriggertp://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"github.com/fission/fission"
+	uuid "github.com/satori/go.uuid"
+)
+
+type MessageQueueTriggerStore struct {
+	ResourceStore
+}
+
+func (mqs *MessageQueueTriggerStore) Create(mqTrigger *fission.MessageQueueTrigger) (string, error) {
+	mqTrigger.Metadata.Uid = uuid.NewV4().String()
+	return mqTrigger.Metadata.Uid, mqs.ResourceStore.create(mqTrigger)
+}
+
+func (mqs *MessageQueueTriggerStore) Get(m *fission.Metadata) (*fission.MessageQueueTrigger, error) {
+	var mqTrigger fission.MessageQueueTrigger
+	err := mqs.ResourceStore.read(m.Name, &mqTrigger)
+	if err != nil {
+		return nil, err
+	}
+	return &mqTrigger, nil
+}
+
+func (mqs *MessageQueueTriggerStore) Update(mqTrigger *fission.MessageQueueTrigger) (string, error) {
+	mqTrigger.Metadata.Uid = uuid.NewV4().String()
+	return mqTrigger.Metadata.Uid, mqs.ResourceStore.update(mqTrigger)
+}
+
+func (mqs *MessageQueueTriggerStore) Delete(m fission.Metadata) error {
+	typeName, err := getTypeName(fission.MessageQueueTrigger{})
+	if err != nil {
+		return err
+	}
+	return mqs.ResourceStore.delete(typeName, m.Name)
+}
+
+func (mqs *MessageQueueTriggerStore) List(mqType string) ([]fission.MessageQueueTrigger, error) {
+	typeName, err := getTypeName(fission.MessageQueueTrigger{})
+	if err != nil {
+		return nil, err
+	}
+	bufs, err := mqs.ResourceStore.getAll(typeName)
+	if err != nil {
+		return nil, err
+	}
+
+	triggers := make([]fission.MessageQueueTrigger, 0, len(bufs))
+	js := JsonSerializer{}
+	for _, buf := range bufs {
+		var mqTrigger fission.MessageQueueTrigger
+		err = js.deserialize([]byte(buf), &mqTrigger)
+		if err != nil {
+			return nil, err
+		}
+		if len(mqType) > 0 && mqType != mqTrigger.MessageQueueType {
+			continue
+		}
+		triggers = append(triggers, mqTrigger)
+	}
+
+	return triggers, nil
+}