From 16efdaa25c61e1c18f4ce15053953b6e74cf25ce Mon Sep 17 00:00:00 2001 From: Erwin van Eyk Date: Thu, 10 Jan 2019 13:09:22 +0100 Subject: [PATCH] Add fluentd.conf as a configmap (#792) Logger's - FluentD config can be passed as a configmap so that user can customize the configuration to their needs --- .gitignore | 1 + charts/fission-all/config/fluentd.conf | 59 ++++++++++++++ charts/fission-all/requirements.lock | 6 ++ charts/fission-all/templates/deployment.yaml | 59 -------------- charts/fission-all/templates/fluentd.yaml | 81 ++++++++++++++++++++ logger/fluentd/fluent.conf | 1 + 6 files changed, 148 insertions(+), 59 deletions(-) create mode 100644 charts/fission-all/config/fluentd.conf create mode 100644 charts/fission-all/requirements.lock create mode 100644 charts/fission-all/templates/fluentd.yaml diff --git a/.gitignore b/.gitignore index d946bac2..0b4af9a6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ environments/php7/vendor/ *.bak *.tmp *~ +*.tgz .DS_Store vendor/ diff --git a/charts/fission-all/config/fluentd.conf b/charts/fission-all/config/fluentd.conf new file mode 100644 index 00000000..49ab6bf1 --- /dev/null +++ b/charts/fission-all/config/fluentd.conf @@ -0,0 +1,59 @@ +# Hide all fluent-related logs + + type null + + +# Collect all logs from the containers in the current namespace + + type tail + format json + time_key time + path "#{ENV['FLUENTD_PATH']}" + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag fission.* + read_from_head true + refresh_interval 5 + + +# Augment logs with Kubernetes metadata + + type kubernetes_metadata + + +# Simplify the nested objects to XX_YY_ZZ names + + type flatten_hash + separator _ + + +# Add `funcuid` to the record (using the functionUid label) + + type record_reformer + enable_ruby false + tag log + + funcuid ${kubernetes_labels_functionUid} + + + +# Push logs into influxdb + + @type influxdb + host "#{ENV['INFLUXDB_ADDRESS']}" + port "#{ENV['INFLUXDB_PORT']}" + dbname "#{ENV['INFLUXDB_DBNAME']}" + user "#{ENV['INFLUXDB_USERNAME']}" + password "#{ENV['INFLUXDB_PASSWD']}" + use_ssl false + time_precision ns + tag_keys ["funcuid"] + sequence_tag _seq + buffer_type file + buffer_path /var/log/fission/fluentd.buffer + buffer_chunk_limit 128m + buffer_queue_limit 256 + flush_interval 5 + retry_limit 10 + retry_wait 1.0 + num_threads 2 + diff --git a/charts/fission-all/requirements.lock b/charts/fission-all/requirements.lock new file mode 100644 index 00000000..7d1a8b06 --- /dev/null +++ b/charts/fission-all/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: prometheus + repository: https://kubernetes-charts.storage.googleapis.com + version: 7.1.0 +digest: sha256:867c8666d4dee6ae36b0e418f132f3187fd189bcbb36875f2c06e3fbe2402cfe +generated: 2019-01-03T16:29:53.158684+01:00 diff --git a/charts/fission-all/templates/deployment.yaml b/charts/fission-all/templates/deployment.yaml index 1f7845e1..f18417f8 100644 --- a/charts/fission-all/templates/deployment.yaml +++ b/charts/fission-all/templates/deployment.yaml @@ -448,65 +448,6 @@ spec: --- {{- end }} apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: logger - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" -spec: - template: - metadata: - labels: - svc: logger - spec: - containers: - - name: fluentd - image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}" - imagePullPolicy: {{ .Values.pullPolicy }} - env: - - name: INFLUXDB_ADDRESS - value: influxdb - - name: INFLUXDB_PORT - value: "8086" - - name: INFLUXDB_DBNAME - value: "fissionFunctionLog" - - name: INFLUXDB_USERNAME - valueFrom: - secretKeyRef: - name: influxdb - key: username - - name: INFLUXDB_PASSWD - valueFrom: - secretKeyRef: - name: influxdb - key: password - - name: FLUENTD_PATH - value: /var/log/containers/*{{.Values.functionNamespace}}*.log - volumeMounts: - - name: container-log - mountPath: /var/log/ - readOnly: true - - name: docker-log - mountPath: /var/lib/docker/containers - readOnly: true - - name: fission-log - mountPath: /var/log/fission - readOnly: false - serviceAccount: fission-svc - volumes: - - name: container-log - hostPath: - path: /var/log/ - - name: docker-log - hostPath: - path: /var/lib/docker/containers - - name: fission-log - hostPath: - path: /var/log/fission - updateStrategy: - type: RollingUpdate ---- -apiVersion: extensions/v1beta1 kind: Deployment metadata: name: timer diff --git a/charts/fission-all/templates/fluentd.yaml b/charts/fission-all/templates/fluentd.yaml new file mode 100644 index 00000000..bd9d4ece --- /dev/null +++ b/charts/fission-all/templates/fluentd.yaml @@ -0,0 +1,81 @@ +# Fluentd deployment for Fission +# +# Requires: +# - service account: fission-svc +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-fission-fluentd +data: +{{ if .Files.Get "config/fluentd.conf" }} + td-agent.conf: | +{{ .Files.Get "config/fluentd.conf" | indent 3 }} +{{ else }} +{{ fail "invalid chart" }} +{{ end }} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: logger + labels: + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +spec: + template: + metadata: + labels: + svc: logger + spec: + containers: + - name: fluentd + image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}" + imagePullPolicy: {{ .Values.pullPolicy }} + env: + - name: INFLUXDB_ADDRESS + value: influxdb + - name: INFLUXDB_PORT + value: "8086" + - name: INFLUXDB_DBNAME + value: "fissionFunctionLog" + - name: INFLUXDB_USERNAME + valueFrom: + secretKeyRef: + name: influxdb + key: username + - name: INFLUXDB_PASSWD + valueFrom: + secretKeyRef: + name: influxdb + key: password + - name: FLUENTD_PATH + value: /var/log/containers/*{{.Values.functionNamespace}}*.log + volumeMounts: + - name: container-log + mountPath: /var/log/ + readOnly: true + - name: docker-log + mountPath: /var/lib/docker/containers + readOnly: true + - name: fission-log + mountPath: /var/log/fission + readOnly: false + - name: fluentd-config + mountPath: /etc/td-agent/ + readOnly: true + serviceAccount: fission-svc + volumes: + - name: container-log + hostPath: + path: /var/log/ + - name: docker-log + hostPath: + path: /var/lib/docker/containers + - name: fission-log + hostPath: + path: /var/log/fission + # Fluentd config location: /etc/td-agent/td-agent.conf + - name: fluentd-config + configMap: + name: {{ .Release.Name }}-fission-fluentd + updateStrategy: + type: RollingUpdate diff --git a/logger/fluentd/fluent.conf b/logger/fluentd/fluent.conf index e74c1d81..efa64a44 100644 --- a/logger/fluentd/fluent.conf +++ b/logger/fluentd/fluent.conf @@ -1,3 +1,4 @@ +# Default logger configfile - which is generally replaced by a more specific logger config at runtime (see charts/) type null