Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add message queue service factory (#1537)

Browse Source
This commit is contained in:
Ta-Ching Chen
2020-02-20 23:30:39 +08:00
committed by GitHub
parent e0dc7f2f81
commit cc14044b6b
12 changed files with 218 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/mqtrigger/factory/factory.go
+62
View File
@@ -0,0 +1,62 @@
/*
Copyright 2020 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package factory
import (
"sync"
"github.com/pkg/errors"
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
)
var (
messageQueueFactories = make(map[fv1.MessageQueueType]MessageQueueFactory)
lock = sync.Mutex{}
)
type (
MessageQueueFactory interface {
Create(logger *zap.Logger, config messageQueue.Config, routerURL string) (messageQueue.MessageQueue, error)
}
)
func Register(mqType fv1.MessageQueueType, factory MessageQueueFactory) {
lock.Lock()
defer lock.Unlock()
if factory == nil {
panic("Nil message queue factory")
}
_, registered := messageQueueFactories[mqType]
if registered {
panic("Message queue factory already register")
}
messageQueueFactories[mqType] = factory
}
func Create(logger *zap.Logger, mqType fv1.MessageQueueType, mqConfig messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
factory, registered := messageQueueFactories[mqType]
if !registered {
return nil, errors.Errorf("no supported message queue type found for %q", mqType)
}
return factory.Create(logger, mqConfig, routerUrl)
}
pkg/mqtrigger/messageQueue/azurequeuestorage/asq.go
+16 -3
View File
@@ -30,14 +30,21 @@ import (
"time"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/fission/fission/pkg/utils"
"github.com/pkg/errors"
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeASQ, &Factory{})
validator.Register(fv1.MessageQueueTypeASQ, IsTopicValid)
}
// TODO: some of these constants should probably be environment variables
const (
// AzureQueuePollingInterval is the polling interval (default is 1 minute).
@@ -105,6 +112,12 @@ type AzureHTTPClient interface {
Do(req *http.Request) (*http.Response, error)
}
type Factory struct{}
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
type azureQueueService struct {
service storage.QueueServiceClient
}
@@ -179,7 +192,7 @@ func newAzureQueueService(client storage.Client) AzureQueueService {
}
}
func New(logger *zap.Logger, routerURL string, config messageQueue.Config) (messageQueue.MessageQueue, error) {
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME")
if len(account) == 0 {
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
@@ -198,7 +211,7 @@ func New(logger *zap.Logger, routerURL string, config messageQueue.Config) (mess
}
return &AzureStorageConnection{
logger: logger.Named("azue_storage"),
routerURL: routerURL,
routerURL: routerUrl,
service: newAzureQueueService(client),
httpClient: &http.Client{
Timeout: AzureFunctionInvocationTimeout,
pkg/mqtrigger/messageQueue/azurequeuestorage/asq_test.go
+6 -6
View File
@@ -113,10 +113,10 @@ func TestNewStorageConnectionMissingAccountName(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: fv1.MessageQueueTypeASQ,
Url: "",
})
}, DummyRouterURL)
require.Nil(t, connection)
require.Error(t, err, "Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
}
@@ -126,10 +126,10 @@ func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: fv1.MessageQueueTypeASQ,
Url: "",
})
}, DummyRouterURL)
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_NAME")
require.Nil(t, connection)
require.Error(t, err, "Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set")
@@ -141,10 +141,10 @@ func TestNewStorageConnection(t *testing.T) {
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5")
connection, err := New(logger, DummyRouterURL, messageQueue.Config{
connection, err := New(logger, messageQueue.Config{
MQType: "azure-storage-queue",
Url: "",
})
}, DummyRouterURL)
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_NAME")
_ = os.Unsetenv("AZURE_STORAGE_ACCOUNT_KEY")
require.NoError(t, err)
pkg/mqtrigger/messageQueue/kafka/kafka.go
+16 -2
View File
@@ -33,10 +33,17 @@ import (
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeKafka, &Factory{})
validator.Register(fv1.MessageQueueTypeKafka, IsTopicValid)
}
var (
// Need to use raw string to support escape sequence for - & . chars
validKafkaTopicName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-\._]*[a-zA-Z0-9]$`)
@@ -51,9 +58,15 @@ type (
authKeys map[string][]byte
tls bool
}
Factory struct{}
)
func New(logger *zap.Logger, routerUrl string, mqCfg messageQueue.Config) (messageQueue.MessageQueue, error) {
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
if len(routerUrl) == 0 || len(mqCfg.Url) == 0 {
return nil, errors.New("the router URL or MQ URL is empty")
}
@@ -340,7 +353,8 @@ func errorHandler(logger *zap.Logger, trigger *fv1.MessageQueueTrigger, producer
}
}
// The validation is based on Kafka's internal implementation: https://github.com/apache/kafka/blob/cde6d18983b5d58199f8857d8d61d7efcbe6e54a/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L36-L47
// The validation is based on Kafka's internal implementation:
// https://github.com/apache/kafka/blob/cde6d18983b5d58199f8857d8d61d7efcbe6e54a/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L36-L47
func IsTopicValid(topic string) bool {
if len(topic) == 0 {
return false
pkg/mqtrigger/messageQueue/nats/nats.go
+14 -1
View File
@@ -28,10 +28,17 @@ import (
"go.uber.org/zap"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/mqtrigger/factory"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/validator"
"github.com/fission/fission/pkg/utils"
)
func init() {
factory.Register(fv1.MessageQueueTypeNats, &Factory{})
validator.Register(fv1.MessageQueueTypeNats, IsTopicValid)
}
const (
natsClusterID = "fissionMQTrigger"
natsProtocol = "nats://"
@@ -45,9 +52,15 @@ type (
nsConn ns.Conn
routerUrl string
}
Factory struct{}
)
func New(logger *zap.Logger, routerUrl string, mqCfg messageQueue.Config) (messageQueue.MessageQueue, error) {
func (factory *Factory) Create(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
return New(logger, mqCfg, routerUrl)
}
func New(logger *zap.Logger, mqCfg messageQueue.Config, routerUrl string) (messageQueue.MessageQueue, error) {
conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url),
ns.SetConnectionLostHandler(func(conn ns.Conn, reason error) {
// TODO: Better way to handle connection lost problem.
pkg/mqtrigger/mqtmanager.go
+16 -27
View File
@@ -26,9 +26,6 @@ import (
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/crd"
"github.com/fission/fission/pkg/mqtrigger/messageQueue"
"github.com/fission/fission/pkg/mqtrigger/messageQueue/azurequeuestorage"
"github.com/fission/fission/pkg/mqtrigger/messageQueue/kafka"
"github.com/fission/fission/pkg/mqtrigger/messageQueue/nats"
"github.com/fission/fission/pkg/utils"
)
@@ -42,11 +39,12 @@ type (
requestType int
MessageQueueTriggerManager struct {
logger *zap.Logger
reqChan chan request
triggers map[string]*triggerSubscription
fissionClient *crd.FissionClient
messageQueue messageQueue.MessageQueue
logger *zap.Logger
reqChan chan request
triggers map[string]*triggerSubscription
fissionClient *crd.FissionClient
messageQueueType fv1.MessageQueueType
messageQueue messageQueue.MessageQueue
}
triggerSubscription struct {
@@ -66,13 +64,14 @@ type (
)
func MakeMessageQueueTriggerManager(logger *zap.Logger,
fissionClient *crd.FissionClient, messageQueue messageQueue.MessageQueue) *MessageQueueTriggerManager {
fissionClient *crd.FissionClient, mqType fv1.MessageQueueType, messageQueue messageQueue.MessageQueue) *MessageQueueTriggerManager {
mqTriggerMgr := MessageQueueTriggerManager{
logger: logger.Named("message_queue_trigger_manager"),
reqChan: make(chan request),
triggers: make(map[string]*triggerSubscription),
fissionClient: fissionClient,
messageQueue: messageQueue,
logger: logger.Named("message_queue_trigger_manager"),
reqChan: make(chan request),
triggers: make(map[string]*triggerSubscription),
fissionClient: fissionClient,
messageQueueType: mqType,
messageQueue: messageQueue,
}
return &mqTriggerMgr
}
@@ -154,7 +153,9 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
newTriggerMap := make(map[string]*fv1.MessageQueueTrigger)
for index := range newTriggers.Items {
newTrigger := &newTriggers.Items[index]
newTriggerMap[crd.CacheKey(&newTrigger.ObjectMeta)] = newTrigger
if newTrigger.Spec.MessageQueueType == mqt.messageQueueType {
newTriggerMap[crd.CacheKey(&newTrigger.ObjectMeta)] = newTrigger
}
}
// get current set of triggers
@@ -205,15 +206,3 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
time.Sleep(3 * time.Second)
}
}
func IsTopicValid(mqType fv1.MessageQueueType, topic string) bool {
switch mqType {
case fv1.MessageQueueTypeNats:
return nats.IsTopicValid(topic)
case fv1.MessageQueueTypeASQ:
return azurequeuestorage.IsTopicValid(topic)
case fv1.MessageQueueTypeKafka:
return kafka.IsTopicValid(topic)
}
return false
}
pkg/mqtrigger/validator/validator.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2020 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validator
import (
"sync"
)
var (
topicValidators = make(map[string]TopicValidator)
lock = sync.Mutex{}
)
type (
TopicValidator func(topic string) bool
)
func Register(mqType string, validator TopicValidator) {
lock.Lock()
defer lock.Unlock()
if validator == nil {
panic("Nil message queue topic validator")
}
_, registered := topicValidators[mqType]
if registered {
panic("Message queue topic validator already register")
}
topicValidators[mqType] = validator
}
func IsValidTopic(mqType string, topic string) bool {
validator, registered := topicValidators[mqType]
if !registered {
return false
}
return validator(topic)
}
func IsValidMessageQueue(mqType string) bool {
_, registered := topicValidators[mqType]
return registered
}