Fission MQT integration with keda (#1657)
Fission MQT integration based on Keda with Kafka connector
This commit is contained in:
Rahul Bhati
@@ -0,0 +1,15 @@
|
||||
module.exports = async function (context) {
|
||||
console.log(context.request.body);
|
||||
console.log("z-custom-name: " + context.request.headers["z-custom-name"]);
|
||||
console.log(
|
||||
"x-fission-function-name: " +
|
||||
context.request.headers["x-fission-function-name"]
|
||||
);
|
||||
let obj = context.request.body;
|
||||
let headers = context.request.headers;
|
||||
return {
|
||||
status: 200,
|
||||
headers: headers,
|
||||
body: obj,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,8 @@
|
||||
module kafka_pub
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/Shopify/sarama v1.23.1
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 h1:2T/jmrHeTezcCM58lvEQXs0UpQJCo5SoGAcg+mbSTIg=
|
||||
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs=
|
||||
github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM=
|
||||
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU=
|
||||
github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE=
|
||||
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
|
||||
gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=
|
||||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010=
|
||||
gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
|
||||
gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
sarama "github.com/Shopify/sarama"
|
||||
)
|
||||
|
||||
// Handler posts a message to Kafka Topic
|
||||
func Handler(w http.ResponseWriter, r *http.Request) {
|
||||
brokers := []string{"my-cluster-kafka-brokers.my-kafka-project.svc:9092"}
|
||||
producerConfig := sarama.NewConfig()
|
||||
producerConfig.Producer.RequiredAcks = sarama.WaitForAll
|
||||
producerConfig.Producer.Retry.Max = 100
|
||||
producerConfig.Producer.Retry.Backoff = 100
|
||||
producerConfig.Producer.Return.Successes = true
|
||||
producerConfig.Version = sarama.V1_0_0_0
|
||||
producer, err := sarama.NewSyncProducer(brokers, producerConfig)
|
||||
fmt.Println("Created a new producer ", producer)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for i := 0; i < 1000; i++ {
|
||||
headers := []sarama.RecordHeader{{Key: []byte("Z-Custom-Name"), Value: []byte("Kafka-Header-test")}}
|
||||
_, _, err = producer.SendMessage(&sarama.ProducerMessage{
|
||||
Topic: "topic2",
|
||||
Value: sarama.StringEncoder("{\"name\": \"testvalue\"}"),
|
||||
Headers: headers,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
w.Write([]byte(fmt.Sprintf("Failed to publish message to topic %s: %v", "testtopic", err)))
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Write([]byte("Successfully sent to testtopic"))
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
#!/bin/bash
|
||||
#test:disabled
|
||||
|
||||
# Create a function and trigger it using Kafka
|
||||
# This test requires KEDA and Kafka in the cluster
|
||||
# This test assumes that there are three topics are already created with 3 Partitions and 3 Replicas in the cluster
|
||||
|
||||
set -euo pipefail
|
||||
source $(dirname $0)/../../../utils.sh
|
||||
set +x
|
||||
|
||||
TEST_ID=$(generate_test_id)
|
||||
echo "TEST_ID = $TEST_ID"
|
||||
|
||||
tmp_dir="/tmp/test-$TEST_ID"
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
nodeenv="node-kafka-$TEST_ID"
|
||||
goenv="go-kafka-$TEST_ID"
|
||||
producerfunc="producer-func-$TEST_ID"
|
||||
consumerfunc="consumer-func-$TEST_ID"
|
||||
mqt="kafkatest-$TEST_ID"
|
||||
no_of_topic_partition=3 # Change this if number of partitions are different
|
||||
topic="topic2"
|
||||
resptopic="response-topic"
|
||||
errortopic="error-topic"
|
||||
bootstrap_server="my-cluster-kafka-brokers.my-kafka-project.svc:9092"
|
||||
consumer_group="my-group"
|
||||
cooldownperiod=30
|
||||
pollinginterval=30
|
||||
|
||||
export FISSION_ROUTER=127.0.0.1:11009
|
||||
|
||||
test_totalpods(){
|
||||
echo "Checking total number of scaledpods"
|
||||
|
||||
set +e
|
||||
while true; do
|
||||
|
||||
cnt=$(kubectl get deployment ${mqt} -o jsonpath='{.status.readyReplicas}')
|
||||
if [$cnt -ge $no_of_topic_partition]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
set -e
|
||||
}
|
||||
|
||||
export -f test_totalpods
|
||||
|
||||
cleanup() {
|
||||
log "Cleaning up..."
|
||||
clean_resource_by_id $TEST_ID
|
||||
rm -rf $tmp_dir
|
||||
}
|
||||
|
||||
if [ -z "${TEST_NOCLEANUP:-}" ]; then
|
||||
trap cleanup EXIT
|
||||
else
|
||||
log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
|
||||
fi
|
||||
|
||||
DIR=$(dirname $0)
|
||||
|
||||
log "Creating ${nodeenv} environment"
|
||||
fission env create --name ${nodeenv} --image ${NODE_RUNTIME_IMAGE}
|
||||
|
||||
log "Creating ${goenv} environment"
|
||||
fission env create --name ${goenv} --image ${GO_RUNTIME_IMAGE} --builder ${GO_BUILDER_IMAGE}
|
||||
|
||||
log "Creating package for Kafka producer"
|
||||
cp -r $DIR/kafka_pub $tmp_dir/
|
||||
pushd $tmp_dir/kafka_pub
|
||||
go mod vendor
|
||||
zip -qr kafka.zip *
|
||||
pkgName=$(fission package create --name kafka-${TEST_ID} --env ${goenv} --src kafka.zip|cut -f2 -d' '| tr -d \')
|
||||
|
||||
log "pkgName=${pkgName}"
|
||||
popd
|
||||
|
||||
timeout 300s bash -c "waitBuild $pkgName"
|
||||
log "Package ${pkgName} created"
|
||||
log "Creating function ${producerfunc}"
|
||||
fission fn create --name ${producerfunc} --env ${goenv} --pkg ${pkgName} --entrypoint Handler
|
||||
log "Creating function ${consumerfunc}"
|
||||
fission fn create --name ${consumerfunc} --env ${nodeenv} --code $DIR/hellokafka.js
|
||||
|
||||
log "Creating trigger $mqt"
|
||||
fission mqt create --name ${mqt} --function ${consumerfunc} --mqtype kafka --topic $topic --resptopic $resptopic --errortopic $errortopic --version2=true --metadata bootstrapServers=$bootstrap_server --metadata consumerGroup=$consumer_group --metadata topic=$topic --cooldownperiod=$cooldownperiod --pollinginterval=$pollinginterval
|
||||
|
||||
log "Create Route to producer function"
|
||||
fission route create --function ${producerfunc} --url /${producerfunc} --method GET
|
||||
|
||||
log "Run Producer function"
|
||||
funcURL=$FISSION_ROUTER/${producerfunc}
|
||||
success="Successfully sent to testtopic"
|
||||
echo $funcURL
|
||||
|
||||
timeout 100s bash -c "test_response $funcURL $success"
|
||||
|
||||
timeout 100s bash -c "test_totalpods"
|
||||
|
||||
log "Test PASSED"
|
||||
Reference in New Issue
Block a user