Add message queue trigger support (#218)

Adds support for message queue triggers based on the NATS-Streaming message queue. This lets the user map a queue topic to a function, and optionally send the function's response into another queue topic. The message queue trigger manager is designed to be message queue independent, so we should be able add support for more queues by adding implementations for the relatively simple mqtrigger.MessageQueue interface.
2017-07-05 17:19:58 -07:00
parent 0fd60eab3b
commit e4fe007838
24 changed files with 1417 additions and 10 deletions
@@ -0,0 +1,6 @@
+module.exports = async function(context) {
+    return {
+        status: 200,
+        body: "Hello, World!"
+    };
+}
@@ -0,0 +1,112 @@
+// This file originally came from official Nats.io GitHub repository.
+// You can reach original file with the following link:
+// https://github.com/nats-io/go-nats-streaming/tree/master/examples
+
+// Copyright 2012-2016 Apcera Inc. All rights reserved.
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/nats-io/go-nats-streaming"
+)
+
+var usageStr = `
+Usage: stan-pub [options] <subject> <message>
+
+Options:
+	-s, --server   <url>            NATS Streaming server URL(s)
+	-c, --cluster  <cluster name>   NATS Streaming cluster name
+	-id,--clientid <client ID>      NATS Streaming client ID
+	-a, --async                     Asynchronous publish mode
+`
+
+// NOTE: Use tls scheme for TLS, e.g. stan-pub -s tls://demo.nats.io:4443 foo hello
+func usage() {
+	fmt.Printf("%s\n", usageStr)
+	os.Exit(0)
+}
+
+func main() {
+	var clusterID string
+	var clientID string
+	var async bool
+	var URL string
+
+	flag.StringVar(&URL, "s", stan.DefaultNatsURL, "The nats server URLs (separated by comma)")
+	flag.StringVar(&URL, "server", stan.DefaultNatsURL, "The nats server URLs (separated by comma)")
+	flag.StringVar(&clusterID, "c", "test-cluster", "The NATS Streaming cluster ID")
+	flag.StringVar(&clusterID, "cluster", "test-cluster", "The NATS Streaming cluster ID")
+	flag.StringVar(&clientID, "id", "stan-pub", "The NATS Streaming client ID to connect with")
+	flag.StringVar(&clientID, "clientid", "stan-pub", "The NATS Streaming client ID to connect with")
+	flag.BoolVar(&async, "a", false, "Publish asynchronously")
+	flag.BoolVar(&async, "async", false, "Publish asynchronously")
+
+	log.SetFlags(0)
+	flag.Usage = usage
+	flag.Parse()
+
+	args := flag.Args()
+
+	if len(args) < 1 {
+		usage()
+	}
+
+	sc, err := stan.Connect(clusterID, clientID, stan.NatsURL(URL))
+	if err != nil {
+		log.Fatalf("Can't connect: %v.\nMake sure a NATS Streaming Server is running at: %s", err, URL)
+	}
+	defer sc.Close()
+
+	subj, msg := args[0], []byte(args[1])
+
+	ch := make(chan bool)
+	var glock sync.Mutex
+	var guid string
+	acb := func(lguid string, err error) {
+		glock.Lock()
+		log.Printf("Received ACK for guid %s\n", lguid)
+		defer glock.Unlock()
+		if err != nil {
+			log.Fatalf("Error in server ack for guid %s: %v\n", lguid, err)
+		}
+		if lguid != guid {
+			log.Fatalf("Expected a matching guid in ack callback, got %s vs %s\n", lguid, guid)
+		}
+		ch <- true
+	}
+
+	if async != true {
+		err = sc.Publish(subj, msg)
+		if err != nil {
+			log.Fatalf("Error during publish: %v\n", err)
+		}
+		log.Printf("Published [%s] : '%s'\n", subj, msg)
+	} else {
+		glock.Lock()
+		guid, err = sc.PublishAsync(subj, msg, acb)
+		if err != nil {
+			log.Fatalf("Error during async publish: %v\n", err)
+		}
+		glock.Unlock()
+		if guid == "" {
+			log.Fatal("Expected non-empty guid to be returned.")
+		}
+		log.Printf("Published [%s] : '%s' [guid: %s]\n", subj, msg, guid)
+
+		select {
+		case <-ch:
+			break
+		case <-time.After(5 * time.Second):
+			log.Fatal("timeout")
+		}
+
+	}
+}
@@ -0,0 +1,133 @@
+// This file originally came from official Nats.io GitHub repository.
+// You can reach original file with the following link:
+// https://github.com/nats-io/go-nats-streaming/tree/master/examples
+
+// Copyright 2012-2016 Apcera Inc. All rights reserved.
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"log"
+	"time"
+
+	"github.com/nats-io/go-nats-streaming"
+	"github.com/nats-io/go-nats-streaming/pb"
+)
+
+var usageStr = `
+Usage: stan-sub [options] <subject>
+
+Options:
+	-s, --server   <url>            NATS Streaming server URL(s)
+	-c, --cluster  <cluster name>   NATS Streaming cluster name
+	-id,--clientid <client ID>      NATS Streaming client ID
+
+Subscription Options:
+	--qgroup <name>                 Queue group
+	--seq <seqno>                   Start at seqno
+	--all                           Deliver all available messages
+	--last                          Deliver starting with last published message
+	--since <duration>              Deliver messages in last interval (e.g. 1s, 1hr)
+	         (for more information: https://golang.org/pkg/time/#ParseDuration)
+	--durable <name>                Durable subscriber name
+	--unsubscribe                   Unsubscribe the durable on exit
+`
+
+// NOTE: Use tls scheme for TLS, e.g. stan-sub -s tls://demo.nats.io:4443 foo
+func usage() {
+	log.Fatalf(usageStr)
+}
+
+func printMsg(m *stan.Msg) {
+	log.Printf("[%s]: '%s'", m.Subject, m.Data)
+}
+
+func main() {
+	var clusterID string
+	var clientID string
+	var showTime bool
+	var startSeq uint64
+	var startDelta string
+	var deliverAll bool
+	var deliverLast bool
+	var durable string
+	var qgroup string
+	var unsubscribe bool
+	var URL string
+
+	//	defaultID := fmt.Sprintf("client.%s", nuid.Next())
+
+	flag.StringVar(&URL, "s", stan.DefaultNatsURL, "The nats server URLs (separated by comma)")
+	flag.StringVar(&URL, "server", stan.DefaultNatsURL, "The nats server URLs (separated by comma)")
+	flag.StringVar(&clusterID, "c", "test-cluster", "The NATS Streaming cluster ID")
+	flag.StringVar(&clusterID, "cluster", "test-cluster", "The NATS Streaming cluster ID")
+	flag.StringVar(&clientID, "id", "", "The NATS Streaming client ID to connect with")
+	flag.StringVar(&clientID, "clientid", "", "The NATS Streaming client ID to connect with")
+	flag.BoolVar(&showTime, "t", false, "Display timestamps")
+	// Subscription options
+	flag.Uint64Var(&startSeq, "seq", 0, "Start at sequence no.")
+	flag.BoolVar(&deliverAll, "all", false, "Deliver all")
+	flag.BoolVar(&deliverLast, "last", false, "Start with last value")
+	flag.StringVar(&startDelta, "since", "", "Deliver messages since specified time offset")
+	flag.StringVar(&durable, "durable", "", "Durable subscriber name")
+	flag.StringVar(&qgroup, "qgroup", "", "Queue group name")
+	flag.BoolVar(&unsubscribe, "unsubscribe", false, "Unsubscribe the durable on exit")
+
+	log.SetFlags(0)
+	flag.Usage = usage
+	flag.Parse()
+
+	args := flag.Args()
+
+	if clientID == "" {
+		log.Printf("Error: A unique client ID must be specified.")
+		usage()
+	}
+	if len(args) < 1 {
+		log.Printf("Error: A subject must be specified.")
+		usage()
+	}
+
+	sc, err := stan.Connect(clusterID, clientID, stan.NatsURL(URL))
+	if err != nil {
+		log.Fatalf("Can't connect: %v.\nMake sure a NATS Streaming Server is running at: %s", err, URL)
+	}
+	// log.Printf("Connected to %s clusterID: [%s] clientID: [%s]\n", URL, clusterID, clientID)
+
+	subj := args[0]
+
+	exit := make(chan struct{})
+	mcb := func(msg *stan.Msg) {
+		printMsg(msg)
+		exit <- struct{}{}
+	}
+
+	startOpt := stan.StartAt(pb.StartPosition_NewOnly)
+
+	if startSeq != 0 {
+		startOpt = stan.StartAtSequence(startSeq)
+	} else if deliverLast == true {
+		startOpt = stan.StartWithLastReceived()
+	} else if deliverAll == true {
+		log.Print("subscribing with DeliverAllAvailable")
+		startOpt = stan.DeliverAllAvailable()
+	} else if startDelta != "" {
+		ago, err := time.ParseDuration(startDelta)
+		if err != nil {
+			sc.Close()
+			log.Fatal(err)
+		}
+		startOpt = stan.StartAtTimeDelta(ago)
+	}
+
+	sub, err := sc.QueueSubscribe(subj, qgroup, mcb, startOpt, stan.DurableName(durable))
+	if err != nil {
+		sc.Close()
+		log.Fatal(err)
+	}
+
+	<-exit
+	sub.Unsubscribe()
+}
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+clusterID="fissionMQTrigger"
+topic="foo.bar"
+resptopic="foo.foo"
+expectedRespOutput="[foo.foo]: 'Hello, World!'"
+FISSIONDIR=$GOPATH"/src/github.com/fission/fission"
+
+if [[ -z $NATS_STREAMING_URL ]]; then
+    echo "'NATS_STREAMING_URL' must not be empty. For example: export NATS_STREAMING_URL=nats://192.168.0.1:4222"
+    exit 1
+fi
+
+if [[ -z $FISSION_URL ]]; then
+    echo "'FISSION_URL' must not be empty. For example: export FISSION_URL=http://10.10.10.10"
+    exit 1
+fi
+
+cd $FISSIONDIR"/fission/"
+go build
+mv fission $FISSIONDIR"/test/mqtrigger"
+cd $FISSIONDIR"/test/mqtrigger"
+
+./fission env create --name nodejs --image fission/node-env
+./fission fn create --name hello1 --env nodejs --code main.js --method GET
+./fission route create --method GET --url /h1 --function hello1
+./fission mqtrigger create --name h1 --function hello1 --mqtype "nats-streaming" --topic "foo.bar" --resptopic "foo.foo"
+
+# wait until nats trigger is created
+sleep 5
+
+go run ./stan-pub.go -s $NATS_STREAMING_URL -c $clusterID -id clientPub $topic "" || exit 1
+
+response=$(go run ./stan-sub.go --last -s $NATS_STREAMING_URL -c $clusterID -id clientSub $resptopic 2>&1)
+
+if [[ "$response" != "$expectedRespOutput" ]]; then
+    echo "$response is not equal to $expectedRespOutput"
+    exit 1
+fi
+
+echo "Subscriber received expected response: $response"
+
+exit 0