Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fission MQT integration with keda (#1657)

Browse Source 
Fission MQT integration based on Keda with Kafka connector
This commit is contained in:
Rahul Bhati
2020-07-14 21:35:48 +05:30
committed by GitHub
parent d6fc9f71d0
commit 8e0571c7c9
25 changed files with 2203 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/fission-cli/cmd/mqtrigger/command.go
+6 -2
View File
@@ -33,7 +33,9 @@ func Commands() *cobra.Command {
Required: []flag.Flag{flag.MqtFnName, flag.MqtTopic},
Optional: []flag.Flag{flag.MqtName, flag.MqtMQType, flag.MqtRespTopic,
flag.MqtErrorTopic, flag.MqtMaxRetries, flag.MqtMsgContentType,
flag.NamespaceFunction, flag.SpecSave, flag.SpecDry},
flag.NamespaceFunction, flag.SpecSave, flag.SpecDry, flag.MqtPollingInterval,
flag.MqtCooldownPeriod, flag.MqtMinReplicaCount, flag.MqtMaxReplicaCount, flag.MqtSecret,
flag.MqtMetadata, flag.MqtKind},
})
updateCmd := &cobra.Command{
@@ -45,7 +47,9 @@ func Commands() *cobra.Command {
wrapper.SetFlags(updateCmd, flag.FlagSet{
Required: []flag.Flag{flag.MqtName},
Optional: []flag.Flag{flag.MqtFnName, flag.MqtTopic, flag.MqtRespTopic, flag.MqtErrorTopic,
flag.MqtMaxRetries, flag.MqtMsgContentType, flag.NamespaceTrigger},
flag.MqtMaxRetries, flag.MqtMsgContentType, flag.NamespaceTrigger, flag.MqtPollingInterval,
flag.MqtCooldownPeriod, flag.MqtMinReplicaCount, flag.MqtMaxReplicaCount, flag.MqtMetadata,
flag.MqtSecret, flag.MqtKind},
})
deleteCmd := &cobra.Command{
pkg/fission-cli/cmd/mqtrigger/create.go
+35
View File
@@ -93,6 +93,34 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
return err
}
pollingInterval := int32(input.Int(flagkey.MqtPollingInterval))
if pollingInterval < 0 {
return errors.New("Polling interval must be greater than or equal to 0")
}
cooldownPeriod := int32(input.Int(flagkey.MqtCooldownPeriod))
if cooldownPeriod < 0 {
return errors.New("CooldownPeriod interval is the period to wait after the last trigger reported active before scaling the deployment back to 0, it must be greater than or equal to 0")
}
minReplicaCount := int32(input.Int(flagkey.MqtMinReplicaCount))
if minReplicaCount < 0 {
return errors.New("MinReplicaCount must be greater than or equal to 0")
}
maxReplicaCount := int32(input.Int(flagkey.MqtMaxReplicaCount))
if maxReplicaCount < 0 {
return errors.New("MaxReplicaCount must be greater than or equal to 0")
}
metadata := make(map[string]string)
metadataParams := input.StringSlice(flagkey.MqtMetadata)
_ = util.UpdateMapFromStringSlice(&metadata, metadataParams)
secret := input.String(flagkey.MqtSecret)
mqtKind := input.String(flagkey.MqtKind)
if input.Bool(flagkey.SpecSave) {
specDir := util.GetSpecDir(input)
fr, err := spec.ReadSpecs(specDir)
@@ -131,6 +159,13 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
ErrorTopic: errorTopic,
MaxRetries: maxRetries,
ContentType: contentType,
PollingInterval: &pollingInterval,
CooldownPeriod: &cooldownPeriod,
MinReplicaCount: &minReplicaCount,
MaxReplicaCount: &maxReplicaCount,
Metadata: metadata,
Secret: secret,
MqtKind: mqtKind,
},
}
pkg/fission-cli/cmd/mqtrigger/update.go
+39 -3
View File
@@ -26,6 +26,7 @@ import (
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
"github.com/fission/fission/pkg/fission-cli/util"
)
type UpdateSubCommand struct {
@@ -60,7 +61,13 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error {
maxRetries := input.Int(flagkey.MqtMaxRetries)
fnName := input.String(flagkey.MqtFnName)
contentType := input.String(flagkey.MqtMsgContentType)
pollingInterval := int32(input.Int(flagkey.MqtPollingInterval))
cooldownPeriod := int32(input.Int(flagkey.MqtCooldownPeriod))
minReplicaCount := int32(input.Int(flagkey.MqtMinReplicaCount))
maxReplicaCount := int32(input.Int(flagkey.MqtMaxReplicaCount))
metadataParams := input.StringSlice(flagkey.MqtMetadata)
secret := input.String(flagkey.MqtSecret)
mqtKind := input.String(flagkey.MqtKind)
// TODO : Find out if we can make a call to checkIfFunctionExists, in the same ns more importantly.
err = checkMQTopicAvailability(mqt.Spec.MessageQueueType, topic, respTopic)
@@ -81,7 +88,7 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error {
mqt.Spec.ErrorTopic = errorTopic
updated = true
}
if maxRetries > -1 {
if input.IsSet(flagkey.MqtMaxRetries) {
mqt.Spec.MaxRetries = maxRetries
updated = true
}
@@ -89,10 +96,39 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error {
mqt.Spec.FunctionReference.Name = fnName
updated = true
}
if len(contentType) > 0 {
if input.IsSet(flagkey.MqtMsgContentType) {
mqt.Spec.ContentType = contentType
updated = true
}
if input.IsSet(flagkey.MqtPollingInterval) {
mqt.Spec.PollingInterval = &pollingInterval
updated = true
}
if input.IsSet(flagkey.MqtCooldownPeriod) {
mqt.Spec.CooldownPeriod = &cooldownPeriod
updated = true
}
if input.IsSet(flagkey.MqtMinReplicaCount) {
mqt.Spec.MinReplicaCount = &minReplicaCount
updated = true
}
if input.IsSet(flagkey.MqtMaxReplicaCount) {
mqt.Spec.MaxReplicaCount = &maxReplicaCount
updated = true
}
if input.IsSet(flagkey.MqtMetadata) {
updated = updated || util.UpdateMapFromStringSlice(&mqt.Spec.Metadata, metadataParams)
}
if input.IsSet(flagkey.MqtSecret) {
mqt.Spec.Secret = secret
updated = true
}
if input.IsSet(flagkey.MqtKind) {
mqt.Spec.MqtKind = mqtKind
updated = true
}
if !updated {
return errors.New("Nothing changed, see 'help' for more details")
pkg/fission-cli/flag/flag.go
+15 -8
View File
@@ -128,14 +128,21 @@ var (
TtFnName = Flag{Type: String, Name: flagkey.TtFnName, Usage: "Function name"}
TtRound = Flag{Type: Int, Name: flagkey.TtRound, Usage: "Get next N rounds of invocation time", DefaultValue: 1}
MqtName = Flag{Type: String, Name: flagkey.MqtName, Usage: "Message queue trigger name"}
MqtFnName = Flag{Type: String, Name: flagkey.MqtFnName, Usage: "Function name"}
MqtMQType = Flag{Type: String, Name: flagkey.MqtMQType, Usage: "Message queue type, e.g. nats-streaming, azure-storage-queue, kafka", DefaultValue: "nats-streaming"}
MqtTopic = Flag{Type: String, Name: flagkey.MqtTopic, Usage: "Message queue Topic the trigger listens on"}
MqtRespTopic = Flag{Type: String, Name: flagkey.MqtRespTopic, Usage: "Topic that the function response is sent on (response discarded if unspecified)"}
MqtErrorTopic = Flag{Type: String, Name: flagkey.MqtErrorTopic, Usage: "Topic that the function error messages are sent to (errors discarded if unspecified"}
MqtMaxRetries = Flag{Type: Int, Name: flagkey.MqtMaxRetries, Usage: "Maximum number of times the function will be retried upon failure", DefaultValue: 0}
MqtMsgContentType = Flag{Type: String, Name: flagkey.MqtMsgContentType, Short: "c", Usage: "Content type of messages that publish to the topic", DefaultValue: "application/json"}
MqtName = Flag{Type: String, Name: flagkey.MqtName, Usage: "Message queue trigger name"}
MqtFnName = Flag{Type: String, Name: flagkey.MqtFnName, Usage: "Function name"}
MqtMQType = Flag{Type: String, Name: flagkey.MqtMQType, Usage: "Message queue type, e.g. nats-streaming, azure-storage-queue, kafka", DefaultValue: "nats-streaming"}
MqtTopic = Flag{Type: String, Name: flagkey.MqtTopic, Usage: "Message queue Topic the trigger listens on"}
MqtRespTopic = Flag{Type: String, Name: flagkey.MqtRespTopic, Usage: "Topic that the function response is sent on (response discarded if unspecified)"}
MqtErrorTopic = Flag{Type: String, Name: flagkey.MqtErrorTopic, Usage: "Topic that the function error messages are sent to (errors discarded if unspecified"}
MqtMaxRetries = Flag{Type: Int, Name: flagkey.MqtMaxRetries, Usage: "Maximum number of times the function will be retried upon failure", DefaultValue: 0}
MqtMsgContentType = Flag{Type: String, Name: flagkey.MqtMsgContentType, Short: "c", Usage: "Content type of messages that publish to the topic", DefaultValue: "application/json"}
MqtPollingInterval = Flag{Type: Int, Name: flagkey.MqtPollingInterval, Usage: "Interval to check the message source for up/down scaling operation of consumers", DefaultValue: 30}
MqtCooldownPeriod = Flag{Type: Int, Name: flagkey.MqtCooldownPeriod, Usage: "The period to wait after the last trigger reported active before scaling the consumer back to 0", DefaultValue: 300}
MqtMinReplicaCount = Flag{Type: Int, Name: flagkey.MqtMinReplicaCount, Usage: "Minimum number of replicas of consumers to scale down to", DefaultValue: 0}
MqtMaxReplicaCount = Flag{Type: Int, Name: flagkey.MqtMaxReplicaCount, Usage: "Maximum number of replicas of consumers to scale up to", DefaultValue: 100}
MqtMetadata = Flag{Type: StringSlice, Name: flagkey.MqtMetadata, Usage: "Metadata needed for connecting to source system in format: --metadata key1=value1 --metadata key2=value2"}
MqtSecret = Flag{Type: String, Name: flagkey.MqtSecret, Usage: "Name of secret object", DefaultValue: ""}
MqtKind = Flag{Type: String, Name: flagkey.MqtKind, Usage: "Kind of Message Queue Trigger, e.g. fission, keda", DefaultValue: "fission"}
EnvName = Flag{Type: String, Name: flagkey.EnvName, Usage: "Environment name"}
EnvPoolsize = Flag{Type: Int, Name: flagkey.EnvPoolsize, Usage: "Size of the pool", DefaultValue: 3}
pkg/fission-cli/flag/key/key.go
+15 -8
View File
@@ -81,14 +81,21 @@ const (
TtFnName = "function"
TtRound = "round"
MqtName = resourceName
MqtFnName = "function"
MqtMQType = "mqtype"
MqtTopic = "topic"
MqtRespTopic = "resptopic"
MqtErrorTopic = "errortopic"
MqtMaxRetries = "maxretries"
MqtMsgContentType = "contenttype"
MqtName = resourceName
MqtFnName = "function"
MqtMQType = "mqtype"
MqtTopic = "topic"
MqtRespTopic = "resptopic"
MqtErrorTopic = "errortopic"
MqtMaxRetries = "maxretries"
MqtMsgContentType = "contenttype"
MqtPollingInterval = "pollinginterval"
MqtCooldownPeriod = "cooldownperiod"
MqtMinReplicaCount = "minreplicacount"
MqtMaxReplicaCount = "maxreplicacount"
MqtMetadata = "metadata"
MqtSecret = "secret"
MqtKind = "mqtkind"
EnvName = resourceName
EnvPoolsize = "poolsize"
pkg/fission-cli/util/util.go
+15
View File
@@ -315,3 +315,18 @@ func GetSpecDir(input cli.Input) string {
}
return specDir
}
// UpdateMapFromStringSlice parses key, val from "key=val" string array and updates passed map
func UpdateMapFromStringSlice(dataMap *map[string]string, params []string) bool {
updated := false
for _, m := range params {
keyValue := strings.SplitN(m, "=", 2)
if len(keyValue) == 2 {
key := keyValue[0]
value := keyValue[1]
(*dataMap)[key] = value
updated = true
}
}
return updated
}