Kafka integration (#831)

Kafka integration with Fission enables invoking a function when a message arrives in a Kafka topic

charts/README.md

@@ -56,11 +56,16 @@ The following table lists the configurable parameters of the Fission chart and t
 | `logger.influxdbAdmin`          | Log database admin username | `admin`                                                    |
 | `logger.fluentdImage`           | Logger fluentd image        | `fission/fluentd`                                          |
 | `fissionUiImage`                | Fission ui image            | `fission/fission-ui:0.1.0`                                 |
-| `messageQueue`                  | Message queue type          | `nats-streaming`                                           |
-| `nats.authToken`                | Nats streaming auth token   | `defaultFissionAuthToken`                                  |
-| `nats.clusterID`                | Nats streaming clusterID    | `fissionMQTrigger`                                         |
-| `azureStorageQueue.accountName` | Azure storage account name  | None (required if `messageQueue` is `azure-storage-queue`) |
-| `azureStorageQueue.key`         | Azure storage access key    | None (required if `messageQueue` is `azure-storage-queue`) |
+| `nats.enabled`                | Nats streaming enabled   | `true`                                  |
+| `nats.authToken`                | Nats streaming auth token   | `defaultFissionAuthToken`(required if `nats.enabled` is `true`) |
+| `nats.clusterID`                | Nats streaming clusterID    | `fissionMQTrigger`(required if `nats.enabled` is `true`) |
+| `azureStorageQueue.enabled` * | Azure storage account name  | false |
+| `azureStorageQueue.accountName` | Azure storage account name  | None (required if `azureStorageQueue.enabled` is `true`) |
+| `azureStorageQueue.key`         | Azure storage access key    | None (required if `azureStorageQueue.enabled` is `true`) |
+| `kafka.enabled` *  | Kafka trigger enabled           | `false`                    |
+| `kafka.brokers`  | Kafka brokers uri               | `broker.kafka:9092`  (required if `kafka.enabled` is `true`)          |
+
+* - Please note that deploying of Azure Storage Queue or Kafka is not done by Fission chart and you will have to explicitly deploy them.
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
 

charts/fission-all/templates/deployment.yaml

@@ -342,7 +342,7 @@ metadata:
     svc: influxdb
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
-  type: ClusterIP 
+  type: ClusterIP
   ports:
   - port: 8086
     targetPort: 8086
@@ -531,7 +531,7 @@ spec:
 #      serviceAccount: fission-svc
 
 ---
-{{- if eq .Values.messageQueue.type "nats-streaming" }}
+{{- if .Values.nats.enabled }}
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -553,17 +553,16 @@ spec:
           "--auth", "{{ .Values.nats.authToken }}",
           "--max_channels", "0"
         ]
+
         ports:
         - containerPort: 4222
           hostPort: 4222
           protocol: TCP
-{{- end }}
-
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: mqtrigger
+  name: mqtrigger-nats-streaming
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
@@ -572,6 +571,7 @@ spec:
     metadata:
       labels:
         svc: mqtrigger
+        messagequeue: nats-streaming
     spec:
       containers:
       - name: mqtrigger
@@ -581,11 +581,65 @@ spec:
         args: ["--mqt", "--routerUrl", "http://router.{{ .Release.Namespace }}"]
         env:
         - name: MESSAGE_QUEUE_TYPE
-          value: {{ .Values.messageQueue.type }}
-        {{- if eq .Values.messageQueue.type "nats-streaming" }}
+          value: nats-streaming
         - name: MESSAGE_QUEUE_URL
           value: nats://{{ .Values.nats.authToken }}@nats-streaming:4222
-        {{- else if eq .Values.messageQueue.type "azure-storage-queue" }}
+      serviceAccount: fission-svc
+{{- end }}
+---
+{{- if .Values.kafka.enabled }}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: mqtrigger-kafka
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        svc: mqtrigger
+        messagequeue: kafka
+    spec:
+      containers:
+      - name: mqtrigger
+        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command: ["/fission-bundle"]
+        args: ["--mqt"]
+        env:
+        - name: MESSAGE_QUEUE_TYPE
+          value: kafka
+        - name: MESSAGE_QUEUE_URL
+          value: "{{.Values.kafka.brokers}}"
+      serviceAccount: fission-svc
+{{- end }}
+---
+{{- if .Values.azureStorageQueue.enabled }}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: mqtrigger-azure-storage-queue
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        svc: mqtrigger
+        messagequeue: azure-storage-queue
+    spec:
+      containers:
+      - name: mqtrigger
+        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command: ["/fission-bundle"]
+        args: ["--mqt", "--routerUrl", "http://router.{{ .Release.Namespace }}"]
+        env:
+        - name: MESSAGE_QUEUE_TYPE
+          value: azure-storage-queue
         - name: AZURE_STORAGE_ACCOUNT_NAME
           value: {{ required "An Azure storage account name is required." .Values.azureStorageQueue.accountName }}
         - name: AZURE_STORAGE_ACCOUNT_KEY
@@ -593,9 +647,8 @@ spec:
             secretKeyRef:
               name: azure-storage-account-key
               key: key
-        {{- end }}
       serviceAccount: fission-svc
-
+{{- end }}
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment

charts/fission-all/templates/secrets.yaml

@@ -10,7 +10,7 @@ data:
   password: {{ randAlphaNum 20 | b64enc | quote }}
 
 ---
-{{- if eq .Values.messageQueue.type "azure-storage-queue" }}
+{{- if .Values.azureStorageQueue.enabled }}
 apiVersion: v1
 kind: Secret
 metadata:

charts/fission-all/templates/svc.yaml

@@ -38,7 +38,7 @@ spec:
     svc: controller
 
 ---
-{{- if eq .Values.messageQueue.type "nats-streaming" }}
+{{- if .Values.nats.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -57,7 +57,6 @@ spec:
   selector:
     svc: nats-streaming
 {{- end }}
-
 ---
 apiVersion: v1
 kind: Service
@@ -73,4 +72,4 @@ spec:
   - port: 80
     targetPort: 8000
   selector:
-    svc: storagesvc
+    svc: storagesvc

charts/fission-all/values.yaml

@@ -53,21 +53,23 @@ logger:
   fluentdImage: fission/fluentd
   fluentdImageTag: 0.10.0
 
-## Type of Queue you would like to use
-## currently supports nats-streaming, azure-storage-queue
-messageQueue:
-  type: nats-streaming
-
 ## Message queue trigger config
-### NATS Streaming
+### NATS Streaming, enabled by default
 nats:
+  enabled: true
   authToken: "defaultFissionAuthToken"
   clusterID: "fissionMQTrigger"
 
-## Required if messageQueue type is azure-storage-queue
+## Azure-storage-queue: enable and configure the details
 azureStorageQueue:
+  enabled: false
   key: ""
   accountName: ""
+  
+## Kafka: enable and configure the details
+kafka:
+  enabled: false
+  brokers: 'broker.kafka:9092'
 
 ## Persist data to a persistent volume.
 persistence: