Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add message queue service factory (#1537)

Browse Source
This commit is contained in:
Ta-Ching Chen
2020-02-20 23:30:39 +08:00
committed by GitHub
parent e0dc7f2f81
commit cc14044b6b
12 changed files with 218 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/mqtrigger/messageQueue/azurequeuestorage/asq.go
+16 -3
View File
@@ -30,14 +30,21 @@ import (
"time"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/fission/fission/pkg/utils"
"github.com/pkg/errors"
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeASQ, &Factory{})
validator.Register(fv1.MessageQueueTypeASQ, IsTopicValid)
}
// TODO: some of these constants should probably be environment variables
const (
// AzureQueuePollingInterval is the polling interval (default is 1 minute).
@@ -105,6 +112,12 @@ type AzureHTTPClient interface {
Do(req *http.Request) (*http.Response, error)
}
type Factory struct{}
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
type azureQueueService struct {
service storage.QueueServiceClient
}
@@ -179,7 +192,7 @@ func newAzureQueueService(client storage.Client) AzureQueueService {
}
}
func New(logger *zap.Logger, routerURL string, config messageQueue.Config) (messageQueue.MessageQueue, error) {
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME")
if len(account) == 0 {
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
@@ -198,7 +211,7 @@ func New(logger *zap.Logger, routerURL string, config messageQueue.Config) (mess
}
return &AzureStorageConnection{
logger: logger.Named("azue_storage"),
routerURL: routerURL,
routerURL: routerUrl,
service: newAzureQueueService(client),
httpClient: &http.Client{
Timeout: AzureFunctionInvocationTimeout,
pkg/mqtrigger/messageQueue/azurequeuestorage/asq_test.go
+6 -6
View File
@@ -113,10 +113,10 @@ func TestNewStorageConnectionMissingAccountName(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: fv1.MessageQueueTypeASQ,
Url: "",
})
}, DummyRouterURL)
require.Nil(t, connection)
require.Error(t, err, "Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
}
@@ -126,10 +126,10 @@ func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: fv1.MessageQueueTypeASQ,
Url: "",
})
}, DummyRouterURL)
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_NAME")
require.Nil(t, connection)
require.Error(t, err, "Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set")
@@ -141,10 +141,10 @@ func TestNewStorageConnection(t *testing.T) {
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5")
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: "azure-storage-queue",
Url: "",
})
}, DummyRouterURL)
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_NAME")
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_KEY")
require.NoError(t, err)
pkg/mqtrigger/messageQueue/kafka/kafka.go
+16 -2
View File
@@ -33,10 +33,17 @@ import (
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeKafka, &Factory{})
validator.Register(fv1.MessageQueueTypeKafka, IsTopicValid)
}
var (
// Need to use raw string to support escape sequence for - & . chars
validKafkaTopicName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-\._]*[a-zA-Z0-9]$`)
@@ -51,9 +58,15 @@ type (
authKeys map[string][]byte
tls bool
}
Factory struct{}
)
func New(logger *zap.Logger, routerUrl string, mqCfg messageQueue.Config) (messageQueue.MessageQueue, error) {
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
if len(routerUrl) == 0 || len(mqCfg.Url) == 0 {
return nil, errors.New("the router URL or MQ URL is empty")
}
@@ -340,7 +353,8 @@ func errorHandler(logger *zap.Logger, trigger *fv1.MessageQueueTrigger, producer
}
}
// The validation is based on Kafka's internal implementation: https://github.com/apache/kafka/blob/cde6d18983b5d58199f8857d8d61d7efcbe6e54a/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L36-L47
// The validation is based on Kafka's internal implementation:
// https://github.com/apache/kafka/blob/cde6d18983b5d58199f8857d8d61d7efcbe6e54a/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L36-L47
func IsTopicValid(topic string) bool {
if len(topic) == 0 {
return false
pkg/mqtrigger/messageQueue/nats/nats.go
+14 -1
View File
@@ -28,10 +28,17 @@ import (
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeNats, &Factory{})
validator.Register(fv1.MessageQueueTypeNats, IsTopicValid)
}
const (
natsClusterID = "fissionMQTrigger"
natsProtocol = "nats://"
@@ -45,9 +52,15 @@ type (
nsConn ns.Conn
routerUrl string
}
Factory struct{}
)
func New(logger *zap.Logger, routerUrl string, mqCfg messageQueue.Config) (messageQueue.MessageQueue, error) {
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url),
ns.SetConnectionLostHandler(func(conn ns.Conn, reason error) {
// TODO: Better way to handle connection lost problem.