Add fluentd.conf as a configmap (#792)

Logger's - FluentD config can be passed as a configmap so that user can customize the configuration to their needs

.gitignore

@@ -11,6 +11,7 @@ environments/php7/vendor/
 *.bak
 *.tmp
 *~
+*.tgz
 
 .DS_Store
 vendor/

charts/fission-all/config/fluentd.conf

@@ -0,0 +1,59 @@
+# Hide all fluent-related logs
+<match fluent.**>
+  type null
+</match>
+
+# Collect all logs from the containers in the current namespace
+<source>
+  type tail
+  format json
+  time_key time
+  path "#{ENV['FLUENTD_PATH']}"
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+  tag fission.*
+  read_from_head true
+  refresh_interval 5
+</source>
+
+# Augment logs with Kubernetes metadata
+<filter fission.**>
+   type kubernetes_metadata
+</filter>
+
+# Simplify the nested objects to XX_YY_ZZ names
+<filter fission.**>
+   type flatten_hash
+   separator _
+</filter>
+
+# Add `funcuid` to the record (using the functionUid label)
+<match fission.**>
+   type record_reformer
+   enable_ruby false
+   tag log
+   <record>
+     funcuid ${kubernetes_labels_functionUid}
+   </record>
+</match>
+
+# Push logs into influxdb
+<match **>
+  @type influxdb
+    host  "#{ENV['INFLUXDB_ADDRESS']}"
+    port  "#{ENV['INFLUXDB_PORT']}"
+    dbname "#{ENV['INFLUXDB_DBNAME']}"
+    user  "#{ENV['INFLUXDB_USERNAME']}"
+    password  "#{ENV['INFLUXDB_PASSWD']}"
+    use_ssl false
+    time_precision ns
+    tag_keys ["funcuid"]
+    sequence_tag _seq
+    buffer_type file
+    buffer_path /var/log/fission/fluentd.buffer
+    buffer_chunk_limit 128m
+    buffer_queue_limit 256
+    flush_interval 5
+    retry_limit 10
+    retry_wait 1.0
+    num_threads 2
+</match>

charts/fission-all/requirements.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: prometheus
+  repository: https://kubernetes-charts.storage.googleapis.com
+  version: 7.1.0
+digest: sha256:867c8666d4dee6ae36b0e418f132f3187fd189bcbb36875f2c06e3fbe2402cfe
+generated: 2019-01-03T16:29:53.158684+01:00

charts/fission-all/templates/deployment.yaml

@@ -448,65 +448,6 @@ spec:
 ---
 {{- end }}
 apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: logger
-  labels:
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-spec:
-  template:
-    metadata:
-      labels:
-        svc: logger
-    spec:
-      containers:
-        - name: fluentd
-          image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}"
-          imagePullPolicy: {{ .Values.pullPolicy }}
-          env:
-            - name: INFLUXDB_ADDRESS
-              value: influxdb
-            - name: INFLUXDB_PORT
-              value: "8086"
-            - name: INFLUXDB_DBNAME
-              value: "fissionFunctionLog"
-            - name: INFLUXDB_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: influxdb
-                  key: username
-            - name: INFLUXDB_PASSWD
-              valueFrom:
-                secretKeyRef:
-                  name: influxdb
-                  key: password
-            - name: FLUENTD_PATH
-              value: /var/log/containers/*{{.Values.functionNamespace}}*.log
-          volumeMounts:
-            - name: container-log
-              mountPath: /var/log/
-              readOnly: true
-            - name: docker-log
-              mountPath: /var/lib/docker/containers
-              readOnly: true
-            - name: fission-log
-              mountPath: /var/log/fission
-              readOnly: false
-      serviceAccount: fission-svc
-      volumes:
-        - name: container-log
-          hostPath:
-              path: /var/log/
-        - name: docker-log
-          hostPath:
-              path: /var/lib/docker/containers
-        - name: fission-log
-          hostPath:
-              path: /var/log/fission
-  updateStrategy:
-    type: RollingUpdate
----
-apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: timer

charts/fission-all/templates/fluentd.yaml

@@ -0,0 +1,81 @@
+# Fluentd deployment for Fission
+#
+# Requires:
+# - service account: fission-svc
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-fission-fluentd
+data:
+{{ if .Files.Get "config/fluentd.conf" }}
+  td-agent.conf: |
+{{ .Files.Get "config/fluentd.conf" | indent 3 }}
+{{ else }}
+{{ fail "invalid chart" }}
+{{ end }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: logger
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  template:
+    metadata:
+      labels:
+        svc: logger
+    spec:
+      containers:
+        - name: fluentd
+          image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}"
+          imagePullPolicy: {{ .Values.pullPolicy }}
+          env:
+            - name: INFLUXDB_ADDRESS
+              value: influxdb
+            - name: INFLUXDB_PORT
+              value: "8086"
+            - name: INFLUXDB_DBNAME
+              value: "fissionFunctionLog"
+            - name: INFLUXDB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: influxdb
+                  key: username
+            - name: INFLUXDB_PASSWD
+              valueFrom:
+                secretKeyRef:
+                  name: influxdb
+                  key: password
+            - name: FLUENTD_PATH
+              value: /var/log/containers/*{{.Values.functionNamespace}}*.log
+          volumeMounts:
+            - name: container-log
+              mountPath: /var/log/
+              readOnly: true
+            - name: docker-log
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: fission-log
+              mountPath: /var/log/fission
+              readOnly: false
+            - name: fluentd-config
+              mountPath: /etc/td-agent/
+              readOnly: true
+      serviceAccount: fission-svc
+      volumes:
+        - name: container-log
+          hostPath:
+            path: /var/log/
+        - name: docker-log
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: fission-log
+          hostPath:
+            path: /var/log/fission
+        # Fluentd config location: /etc/td-agent/td-agent.conf
+        - name: fluentd-config
+          configMap:
+            name: {{ .Release.Name }}-fission-fluentd
+  updateStrategy:
+    type: RollingUpdate

logger/fluentd/fluent.conf

@@ -1,3 +1,4 @@
+# Default logger configfile - which is generally replaced by a more specific logger config at runtime (see charts/)
 <match fluent.**>
   type null
 </match>