Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/logger/influxdb/Dockerfile

@@ -0,0 +1,38 @@
+# Based on tutumcloud/influxdb source repository: https://github.com/tutumcloud/influxdb
+
+FROM ubuntu:trusty
+MAINTAINER Ta-Ching Chen <contact@tachingchen.com>
+
+RUN apt-get update && apt-get install -y curl && apt-get clean && rm -rf /var/lib/apt/lists
+
+# Install InfluxDB
+ENV INFLUXDB_VERSION 1.6.4
+RUN curl -s -o /tmp/influxdb_latest_amd64.deb https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUXDB_VERSION}_amd64.deb && \
+  dpkg -i /tmp/influxdb_latest_amd64.deb && \
+  rm /tmp/influxdb_latest_amd64.deb && \
+  rm -rf /var/lib/apt/lists/*
+
+ADD types.db /usr/share/collectd/types.db
+ADD config.toml /config/config.toml
+ADD run.sh /run.sh
+RUN chmod +x /*.sh
+
+ENV PRE_CREATE_DB **None**
+ENV SSL_SUPPORT **False**
+ENV SSL_CERT **None**
+
+# Admin server WebUI
+EXPOSE 8083
+
+# HTTP API
+EXPOSE 8086
+
+# Raft port (for clustering, don't expose publicly!)
+#EXPOSE 8090
+
+# Protobuf port (for clustering, don't expose publicly!)
+#EXPOSE 8099
+
+VOLUME ["/data"]
+
+CMD ["/run.sh"]

pkg/logger/influxdb/README.md

@@ -0,0 +1,10 @@
+# InfluxDB Container Image
+
+Due to tutum/influxdb dockerhub repo is no longer exists and causes installation failure, we decide to fork the original 
+source repository to rebuild influxDB image so that users can change the image repo while keeping backward compatibility.
+
+The files and code changes are based on [tutumcloud/influxdb](https://github.com/tutumcloud/influxdb) source repository.
+All credits belongs to them and thanks for their hard work!
+
+## UPDATE 2018/10/30
+* Update InfluxDB version from 1.0.0 to 1.6.4  

pkg/logger/influxdb/config.toml

@@ -0,0 +1,294 @@
+### Welcome to the InfluxDB configuration file.
+
+# Once every 24 hours InfluxDB will report anonymous data to m.influxdb.com
+# The data includes raft id (random 8 bytes), os, arch, version, and metadata.
+# We don't track ip addresses of servers reporting. This is only used
+# to track the number of instances running and the versions, which
+# is very helpful for us.
+# Change this option to true to disable reporting.
+reporting-disabled = false
+
+# we'll try to get the hostname automatically, but if it the os returns something
+# that isn't resolvable by other servers in the cluster, use this option to
+# manually set the hostname
+# hostname = "localhost"
+
+###
+### [meta]
+###
+### Controls the parameters for the Raft consensus group that stores metadata
+### about the InfluxDB cluster.
+###
+
+[meta]
+  # Where the metadata/raft database is stored
+  dir = "/data/meta"
+
+  retention-autocreate = true
+
+  # If log messages are printed for the meta service
+  logging-enabled = true
+  pprof-enabled = false
+
+  # The default duration for leases.
+  lease-duration = "1m0s"
+
+###
+### [data]
+###
+### Controls where the actual shard data for InfluxDB lives and how it is
+### flushed from the WAL. "dir" may need to be changed to a suitable place
+### for your system, but the WAL settings are an advanced configuration. The
+### defaults should work for most systems.
+###
+
+[data]
+  # Controls if this node holds time series data shards in the cluster
+  enabled = true
+
+  dir = "/data/data"
+
+  # These are the WAL settings for the storage engine >= 0.9.3
+  wal-dir = "/data/wal"
+  wal-logging-enabled = true
+  data-logging-enabled = true
+
+  # Whether queries should be logged before execution. Very useful for troubleshooting, but will
+  # log any sensitive data contained within a query.
+  # query-log-enabled = true
+
+  # Settings for the TSM engine
+
+  # CacheMaxMemorySize is the maximum size a shard's cache can
+  # reach before it starts rejecting writes.
+  # cache-max-memory-size = 524288000
+
+  # CacheSnapshotMemorySize is the size at which the engine will
+  # snapshot the cache and write it to a TSM file, freeing up memory
+  # cache-snapshot-memory-size = 26214400
+
+  # CacheSnapshotWriteColdDuration is the length of time at
+  # which the engine will snapshot the cache and write it to
+  # a new TSM file if the shard hasn't received writes or deletes
+  # cache-snapshot-write-cold-duration = "1h"
+
+  # MinCompactionFileCount is the minimum number of TSM files
+  # that need to exist before a compaction cycle will run
+  # compact-min-file-count = 3
+
+  # CompactFullWriteColdDuration is the duration at which the engine
+  # will compact all TSM files in a shard if it hasn't received a
+  # write or delete
+  # compact-full-write-cold-duration = "24h"
+
+  # MaxPointsPerBlock is the maximum number of points in an encoded
+  # block in a TSM file. Larger numbers may yield better compression
+  # but could incur a performance penalty when querying
+  # max-points-per-block = 1000
+
+###
+### [cluster]
+###
+### Controls non-Raft cluster behavior, which generally includes how data is
+### shared across shards.
+###
+
+[cluster]
+  shard-writer-timeout = "5s" # The time within which a remote shard must respond to a write request.
+  write-timeout = "10s" # The time within which a write request must complete on the cluster.
+  max-concurrent-queries = 0 # The maximum number of concurrent queries that can run. 0 to disable.
+  query-timeout = "0s" # The time within a query must complete before being killed automatically. 0s to disable.
+  max-select-point = 0 # The maximum number of points to scan in a query. 0 to disable.
+  max-select-series = 0 # The maximum number of series to select in a query. 0 to disable.
+  max-select-buckets = 0 # The maximum number of buckets to select in an aggregate query. 0 to disable.
+
+###
+### [retention]
+###
+### Controls the enforcement of retention policies for evicting old data.
+###
+
+[retention]
+  enabled = true
+  check-interval = "30m"
+
+###
+### [shard-precreation]
+###
+### Controls the precreation of shards, so they are available before data arrives.
+### Only shards that, after creation, will have both a start- and end-time in the
+### future, will ever be created. Shards are never precreated that would be wholly
+### or partially in the past.
+
+[shard-precreation]
+  enabled = true
+  check-interval = "10m"
+  advance-period = "30m"
+
+###
+### Controls the system self-monitoring, statistics and diagnostics.
+###
+### The internal database for monitoring data is created automatically if
+### if it does not already exist. The target retention within this database
+### is called 'monitor' and is also created with a retention period of 7 days
+### and a replication factor of 1, if it does not exist. In all cases the
+### this retention policy is configured as the default for the database.
+
+[monitor]
+  store-enabled = true # Whether to record statistics internally.
+  store-database = "_internal" # The destination database for recorded statistics
+  store-interval = "10s" # The interval at which to record statistics
+
+###
+### [admin]
+###
+### Controls the availability of the built-in, web-based admin interface. If HTTPS is
+### enabled for the admin interface, HTTPS must also be enabled on the [http] service.
+###
+
+[admin]
+  enabled = true
+  bind-address = ":8083"
+  https-enabled = false
+  https-certificate = "/etc/ssl/influxdb.pem"
+
+###
+### [http]
+###
+### Controls how the HTTP endpoints are configured. These are the primary
+### mechanism for getting data into and out of InfluxDB.
+###
+
+[http]
+  enabled = true
+  bind-address = ":8086"
+  auth-enabled = false
+  log-enabled = true
+  write-tracing = false
+  pprof-enabled = false
+  https-enabled = false
+  https-certificate = "/etc/ssl/influxdb.pem"
+  max-row-limit = 10000
+
+###
+### [[graphite]]
+###
+### Controls one or many listeners for Graphite data.
+###
+
+[[graphite]]
+  enabled = false
+  database = "graphitedb"
+  bind-address = ":2003"
+  protocol = "tcp"
+  # consistency-level = "one"
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 5000 # will flush if this many points get buffered
+  # batch-pending = 10 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+  # udp-read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max.
+
+  ### This string joins multiple matching 'measurement' values providing more control over the final measurement name.
+  # separator = "."
+
+  ### Default tags that will be added to all metrics.  These can be overridden at the template level
+  ### or by tags extracted from metric
+  # tags = ["region=us-east", "zone=1c"]
+
+  ### Each template line requires a template pattern.  It can have an optional
+  ### filter before the template and separated by spaces.  It can also have optional extra
+  ### tags following the template.  Multiple tags should be separated by commas and no spaces
+  ### similar to the line protocol format.  There can be only one default template.
+  templates = [
+     # filter + template
+     #"*.app env.service.resource.measurement",
+     # filter + template + extra tag
+     #"stats.* .host.measurement* region=us-west,agent=sensu",
+     # default template. Ignore the first graphite component "servers"
+     "instance.profile.measurement*"
+ ]
+
+###
+### [collectd]
+###
+### Controls one or many listeners for collectd data.
+###
+
+[[collectd]]
+  enabled = false
+  # bind-address = ":25826"
+  # database = "collectd"
+  # typesdb = "/usr/share/collectd/types.db"
+  # retention-policy = ""
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+  # read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max.
+
+###
+### [opentsdb]
+###
+### Controls one or many listeners for OpenTSDB data.
+###
+
+[[opentsdb]]
+  enabled = false
+  # bind-address = ":4242"
+  # database = "opentsdb"
+  # retention-policy = ""
+  # consistency-level = "one"
+  # tls-enabled = false
+  # certificate= ""
+  # log-point-errors = true # Log an error for every malformed point.
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Only points
+  # metrics received over the telnet protocol undergo batching.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+
+###
+### [[udp]]
+###
+### Controls the listeners for InfluxDB line protocol data via UDP.
+###
+
+[[udp]]
+  enabled = false
+  bind-address = ":4444"
+  database = "udpdb"
+  # retention-policy = ""
+
+  # These next lines control how batching works. You should have this enabled
+  # otherwise you could get dropped metrics or poor performance. Batching
+  # will buffer points in memory if you have many coming in.
+
+  # batch-size = 1000 # will flush if this many points get buffered
+  # batch-pending = 5 # number of batches that may be pending in memory
+  # batch-timeout = "1s" # will flush at least this often even if we haven't hit buffer limit
+  # read-buffer = 0 # UDP Read buffer size, 0 means OS default. UDP listener will fail if set above OS max.
+
+  # set the expected UDP payload size; lower values tend to yield better performance, default is max UDP size 65536
+  # udp-payload-size = 65536
+
+###
+### [continuous_queries]
+###
+### Controls how continuous queries are run within InfluxDB.
+###
+
+[continuous_queries]
+  log-enabled = true
+  enabled = true
+  # run-interval = "1s" # interval for how often continuous queries will be checked if they need to run

pkg/logger/influxdb/run.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+set -m
+CONFIG_FILE="/config/config.toml"
+INFLUX_HOST="localhost"
+INFLUX_API_PORT="8086"
+API_URL="http://${INFLUX_HOST}:${INFLUX_API_PORT}"
+
+# Dynamically change the value of 'max-open-shards' to what 'ulimit -n' returns
+sed -i "s/^max-open-shards.*/max-open-shards = $(ulimit -n)/" ${CONFIG_FILE}
+
+# Configure InfluxDB Cluster
+if [ -n "${FORCE_HOSTNAME}" ]; then
+    if [ "${FORCE_HOSTNAME}" == "auto" ]; then
+        #set hostname with IPv4 eth0
+        HOSTIPNAME=$(ip a show dev eth0 | grep inet | grep eth0 | sed -e 's/^.*inet.//g' -e 's/\/.*$//g')
+        /usr/bin/perl -p -i -e "s/hostname = \"localhost\"/hostname = \"${HOSTIPNAME}\"/g" ${CONFIG_FILE}
+    else
+        /usr/bin/perl -p -i -e "s/hostname = \"localhost\"/hostname = \"${FORCE_HOSTNAME}\"/g" ${CONFIG_FILE}
+    fi
+fi
+
+# NOTE: 'seed-servers.' is nowhere to be found in config.toml, this cannot work anymore! NEED FOR REVIEW!
+# if [ -n "${SEEDS}" ]; then
+#     SEEDS=$(eval SEEDS=$SEEDS ; echo $SEEDS | grep '^\".*\"$' || echo "\""$SEEDS"\"" | sed -e 's/, */", "/g')
+#     /usr/bin/perl -p -i -e "s/^# seed-servers.*$/seed-servers = [${SEEDS}]/g" ${CONFIG_FILE}
+# fi
+
+if [ -n "${REPLI_FACTOR}" ]; then
+    /usr/bin/perl -p -i -e "s/replication-factor = 1/replication-factor = ${REPLI_FACTOR}/g" ${CONFIG_FILE}
+fi
+
+if [ "${PRE_CREATE_DB}" == "**None**" ]; then
+    unset PRE_CREATE_DB
+fi
+
+# NOTE: It seems this is not used anymore...
+#
+# if [ "${SSL_CERT}" == "**None**" ]; then
+#     unset SSL_CERT
+# fi
+#
+# if [ "${SSL_SUPPORT}" == "**False**" ]; then
+#     unset SSL_SUPPORT
+# fi
+
+# Add Graphite support
+if [ -n "${GRAPHITE_DB}" ]; then
+    echo "GRAPHITE_DB: ${GRAPHITE_DB}"
+    sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/false/true/; s/\"graphitedb\"/\"${GRAPHITE_DB}\"/g; }" ${CONFIG_FILE}
+fi
+
+if [ -n "${GRAPHITE_BINDING}" ]; then
+    echo "GRAPHITE_BINDING: ${GRAPHITE_BINDING}"
+    sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/\:2003/${GRAPHITE_BINDING}/; }" ${CONFIG_FILE}
+fi
+
+if [ -n "${GRAPHITE_PROTOCOL}" ]; then
+    echo "GRAPHITE_PROTOCOL: ${GRAPHITE_PROTOCOL}"
+    sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/tcp/${GRAPHITE_PROTOCOL}/; }" ${CONFIG_FILE}
+fi
+
+if [ -n "${GRAPHITE_TEMPLATE}" ]; then
+    echo "GRAPHITE_TEMPLATE: ${GRAPHITE_TEMPLATE}"
+    sed -i -r -e "/^\[\[graphite\]\]/, /^$/ { s/instance\.profile\.measurement\*/${GRAPHITE_TEMPLATE}/; }" ${CONFIG_FILE}
+fi
+
+# Add Collectd support
+if [ -n "${COLLECTD_DB}" ]; then
+    echo "COLLECTD_DB: ${COLLECTD_DB}"
+    sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/false/true/; s/( *)# *(.*)\"collectd\"/\1\2\"${COLLECTD_DB}\"/g;}" ${CONFIG_FILE}
+fi
+if [ -n "${COLLECTD_BINDING}" ]; then
+    echo "COLLECTD_BINDING: ${COLLECTD_BINDING}"
+    sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/( *)# *(.*)\":25826\"/\1\2\"${COLLECTD_BINDING}\"/g;}" ${CONFIG_FILE}
+fi
+if [ -n "${COLLECTD_RETENTION_POLICY}" ]; then
+    echo "COLLECTD_RETENTION_POLICY: ${COLLECTD_RETENTION_POLICY}"
+    sed -i -r -e "/^\[\[collectd\]\]/, /^$/ { s/( *)# *(retention-policy.*)\"\"/\1\2\"${COLLECTD_RETENTION_POLICY}\"/g;}" ${CONFIG_FILE}
+fi
+
+# Add UDP support
+if [ -n "${UDP_DB}" ]; then
+    sed -i -r -e "/^\[\[udp\]\]/, /^$/ { s/false/true/; s/#//g; s/\"udpdb\"/\"${UDP_DB}\"/g; }" ${CONFIG_FILE}
+fi
+if [ -n "${UDP_PORT}" ]; then
+    sed -i -r -e "/^\[\[udp\]\]/, /^$/ { s/4444/${UDP_PORT}/; }" ${CONFIG_FILE}
+fi
+
+
+echo "influxdb configuration: "
+cat ${CONFIG_FILE}
+echo "=> Starting InfluxDB ..."
+if [ -n "${JOIN}" ]; then
+  exec influxd -config=${CONFIG_FILE} -join ${JOIN} &
+else
+  exec influxd -config=${CONFIG_FILE} &
+fi
+
+# Pre create database on the initiation of the container
+if [ -n "${PRE_CREATE_DB}" ]; then
+    echo "=> About to create the following database: ${PRE_CREATE_DB}"
+    if [ -f "/data/.pre_db_created" ]; then
+        echo "=> Database had been created before, skipping ..."
+    else
+        arr=$(echo ${PRE_CREATE_DB} | tr ";" "\n")
+
+        #wait for the startup of influxdb
+        RET=1
+        while [[ RET -ne 0 ]]; do
+            echo "=> Waiting for confirmation of InfluxDB service startup ..."
+            sleep 3
+            curl -k ${API_URL}/ping 2> /dev/null
+            RET=$?
+        done
+        echo ""
+
+        PASS=${INFLUXDB_INIT_PWD:-root}
+        if [ -n "${ADMIN_USER}" ]; then
+          echo "=> Creating admin user"
+          influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -execute="CREATE USER ${ADMIN_USER} WITH PASSWORD '${PASS}' WITH ALL PRIVILEGES"
+          for x in $arr
+          do
+              echo "=> Creating database: ${x}"
+              influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -username=${ADMIN_USER} -password="${PASS}" -execute="create database ${x}"
+              influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -username=${ADMIN_USER} -password="${PASS}" -execute="grant all PRIVILEGES on ${x} to ${ADMIN_USER}"
+          done
+          echo ""
+        else
+          for x in $arr
+          do
+              echo "=> Creating database: ${x}"
+              influx -host=${INFLUX_HOST} -port=${INFLUX_API_PORT} -execute="create database \"${x}\""
+          done
+        fi
+
+        touch "/data/.pre_db_created"
+    fi
+else
+    echo "=> No database need to be pre-created"
+fi
+
+fg

pkg/logger/influxdb/types.db

@@ -0,0 +1,241 @@
+absolute		value:ABSOLUTE:0:U
+apache_bytes		value:DERIVE:0:U
+apache_connections	value:GAUGE:0:65535
+apache_idle_workers	value:GAUGE:0:65535
+apache_requests		value:DERIVE:0:U
+apache_scoreboard	value:GAUGE:0:65535
+ath_nodes		value:GAUGE:0:65535
+ath_stat		value:DERIVE:0:U
+backends		value:GAUGE:0:65535
+bitrate			value:GAUGE:0:4294967295
+blocked_clients value:GAUGE:0:U
+bytes			value:GAUGE:0:U
+cache_eviction		value:DERIVE:0:U
+cache_operation		value:DERIVE:0:U
+cache_ratio		value:GAUGE:0:100
+cache_result		value:DERIVE:0:U
+cache_size		value:GAUGE:0:U
+ceph_bytes		value:GAUGE:U:U
+ceph_latency	value:GAUGE:U:U
+ceph_rate			value:DERIVE:0:U
+changes_since_last_save   value:GAUGE:0:U
+charge			value:GAUGE:0:U
+compression_ratio	value:GAUGE:0:2
+compression		uncompressed:DERIVE:0:U, compressed:DERIVE:0:U
+connections		value:DERIVE:0:U
+conntrack		value:GAUGE:0:4294967295
+contextswitch		value:DERIVE:0:U
+count			value:GAUGE:0:U
+counter			value:COUNTER:U:U
+cpufreq			value:GAUGE:0:U
+cpu			value:DERIVE:0:U
+current_connections	value:GAUGE:0:U
+current_sessions	value:GAUGE:0:U
+current			value:GAUGE:U:U
+delay			value:GAUGE:-1000000:1000000
+derive			value:DERIVE:0:U
+df_complex		value:GAUGE:0:U
+df_inodes		value:GAUGE:0:U
+df			used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
+disk_latency		read:GAUGE:0:U, write:GAUGE:0:U
+disk_merged		read:DERIVE:0:U, write:DERIVE:0:U
+disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
+disk_ops_complex	value:DERIVE:0:U
+disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
+disk_time		read:DERIVE:0:U, write:DERIVE:0:U
+disk_io_time		io_time:DERIVE:0:U, weighted_io_time:DERIVE:0:U
+dns_answer		value:DERIVE:0:U
+dns_notify		value:DERIVE:0:U
+dns_octets		queries:DERIVE:0:U, responses:DERIVE:0:U
+dns_opcode		value:DERIVE:0:U
+dns_qtype_cached	value:GAUGE:0:4294967295
+dns_qtype		value:DERIVE:0:U
+dns_query		value:DERIVE:0:U
+dns_question		value:DERIVE:0:U
+dns_rcode		value:DERIVE:0:U
+dns_reject		value:DERIVE:0:U
+dns_request		value:DERIVE:0:U
+dns_resolver		value:DERIVE:0:U
+dns_response		value:DERIVE:0:U
+dns_transfer		value:DERIVE:0:U
+dns_update		value:DERIVE:0:U
+dns_zops		value:DERIVE:0:U
+drbd_resource	value:DERIVE:0:U
+duration		seconds:GAUGE:0:U
+email_check		value:GAUGE:0:U
+email_count		value:GAUGE:0:U
+email_size		value:GAUGE:0:U
+entropy			value:GAUGE:0:4294967295
+expired_keys    value:GAUGE:0:U
+fanspeed		value:GAUGE:0:U
+file_handles		value:GAUGE:0:U
+file_size		value:GAUGE:0:U
+files			value:GAUGE:0:U
+flow			value:GAUGE:0:U
+fork_rate		value:DERIVE:0:U
+frequency_offset	value:GAUGE:-1000000:1000000
+frequency		value:GAUGE:0:U
+fscache_stat		value:DERIVE:0:U
+gauge			value:GAUGE:U:U
+hash_collisions		value:DERIVE:0:U
+http_request_methods	value:DERIVE:0:U
+http_requests		value:DERIVE:0:U
+http_response_codes	value:DERIVE:0:U
+humidity		value:GAUGE:0:100
+if_collisions		value:DERIVE:0:U
+if_dropped		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_errors		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_multicast		value:DERIVE:0:U
+if_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
+if_rx_errors		value:DERIVE:0:U
+if_rx_octets		value:DERIVE:0:U
+if_tx_errors		value:DERIVE:0:U
+if_tx_octets		value:DERIVE:0:U
+invocations		value:DERIVE:0:U
+io_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+io_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
+ipt_bytes		value:DERIVE:0:U
+ipt_packets		value:DERIVE:0:U
+irq			value:DERIVE:0:U
+latency			value:GAUGE:0:U
+links			value:GAUGE:0:U
+load			shortterm:GAUGE:0:5000, midterm:GAUGE:0:5000, longterm:GAUGE:0:5000
+md_disks		value:GAUGE:0:U
+memcached_command	value:DERIVE:0:U
+memcached_connections	value:GAUGE:0:U
+memcached_items		value:GAUGE:0:U
+memcached_octets	rx:DERIVE:0:U, tx:DERIVE:0:U
+memcached_ops		value:DERIVE:0:U
+memory			value:GAUGE:0:281474976710656
+memory_lua		value:GAUGE:0:281474976710656
+multimeter		value:GAUGE:U:U
+mutex_operations	value:DERIVE:0:U
+mysql_commands		value:DERIVE:0:U
+mysql_handler		value:DERIVE:0:U
+mysql_locks		value:DERIVE:0:U
+mysql_log_position	value:DERIVE:0:U
+mysql_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+mysql_bpool_pages	value:GAUGE:0:U
+mysql_bpool_bytes	value:GAUGE:0:U
+mysql_bpool_counters	value:DERIVE:0:U
+mysql_innodb_data	value:DERIVE:0:U
+mysql_innodb_dblwr	value:DERIVE:0:U
+mysql_innodb_log	value:DERIVE:0:U
+mysql_innodb_pages	value:DERIVE:0:U
+mysql_innodb_row_lock	value:DERIVE:0:U
+mysql_innodb_rows	value:DERIVE:0:U
+mysql_select		value:DERIVE:0:U
+mysql_sort		value:DERIVE:0:U
+nfs_procedure		value:DERIVE:0:U
+nginx_connections	value:GAUGE:0:U
+nginx_requests		value:DERIVE:0:U
+node_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+node_rssi		value:GAUGE:0:255
+node_stat		value:DERIVE:0:U
+node_tx_rate		value:GAUGE:0:127
+objects			value:GAUGE:0:U
+operations		value:DERIVE:0:U
+packets			value:DERIVE:0:U
+pending_operations	value:GAUGE:0:U
+percent			value:GAUGE:0:100.1
+percent_bytes		value:GAUGE:0:100.1
+percent_inodes		value:GAUGE:0:100.1
+pf_counters		value:DERIVE:0:U
+pf_limits		value:DERIVE:0:U
+pf_source		value:DERIVE:0:U
+pf_states		value:GAUGE:0:U
+pf_state		value:DERIVE:0:U
+pg_blks			value:DERIVE:0:U
+pg_db_size		value:GAUGE:0:U
+pg_n_tup_c		value:DERIVE:0:U
+pg_n_tup_g		value:GAUGE:0:U
+pg_numbackends		value:GAUGE:0:U
+pg_scan			value:DERIVE:0:U
+pg_xact			value:DERIVE:0:U
+ping_droprate		value:GAUGE:0:100
+ping_stddev		value:GAUGE:0:65535
+ping			value:GAUGE:0:65535
+players			value:GAUGE:0:1000000
+power			value:GAUGE:0:U
+pressure			value:GAUGE:0:U
+protocol_counter	value:DERIVE:0:U
+ps_code			value:GAUGE:0:9223372036854775807
+ps_count		processes:GAUGE:0:1000000, threads:GAUGE:0:1000000
+ps_cputime		user:DERIVE:0:U, syst:DERIVE:0:U
+ps_data			value:GAUGE:0:9223372036854775807
+ps_disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
+ps_disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
+ps_pagefaults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
+ps_rss			value:GAUGE:0:9223372036854775807
+ps_stacksize		value:GAUGE:0:9223372036854775807
+ps_state		value:GAUGE:0:65535
+ps_vm			value:GAUGE:0:9223372036854775807
+pubsub        value:GAUGE:0:U
+queue_length		value:GAUGE:0:U
+records			value:GAUGE:0:U
+requests		value:GAUGE:0:U
+response_time		value:GAUGE:0:U
+response_code		value:GAUGE:0:U
+route_etx		value:GAUGE:0:U
+route_metric		value:GAUGE:0:U
+routes			value:GAUGE:0:U
+segments 		value:GAUGE:0:65535
+serial_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
+signal_noise		value:GAUGE:U:0
+signal_power		value:GAUGE:U:0
+signal_quality		value:GAUGE:0:U
+smart_poweron		value:GAUGE:0:U
+smart_powercycles	value:GAUGE:0:U
+smart_badsectors	value:GAUGE:0:U
+smart_temperature	value:GAUGE:-300:300
+smart_attribute         current:GAUGE:0:255, worst:GAUGE:0:255, threshold:GAUGE:0:255, pretty:GAUGE:0:U
+snr			value:GAUGE:0:U
+spam_check		value:GAUGE:0:U
+spam_score		value:GAUGE:U:U
+spl			value:GAUGE:U:U
+swap_io			value:DERIVE:0:U
+swap			value:GAUGE:0:1099511627776
+tcp_connections		value:GAUGE:0:4294967295
+temperature		value:GAUGE:U:U
+threads			value:GAUGE:0:U
+time_dispersion		value:GAUGE:-1000000:1000000
+timeleft		value:GAUGE:0:U
+time_offset		value:GAUGE:-1000000:1000000
+total_bytes		value:DERIVE:0:U
+total_connections	value:DERIVE:0:U
+total_objects		value:DERIVE:0:U
+total_operations	value:DERIVE:0:U
+total_requests		value:DERIVE:0:U
+total_sessions		value:DERIVE:0:U
+total_threads		value:DERIVE:0:U
+total_time_in_ms	value:DERIVE:0:U
+total_values		value:DERIVE:0:U
+uptime			value:GAUGE:0:4294967295
+users			value:GAUGE:0:65535
+vcl			value:GAUGE:0:65535
+vcpu			value:GAUGE:0:U
+virt_cpu_total		value:DERIVE:0:U
+virt_vcpu		value:DERIVE:0:U
+vmpage_action		value:DERIVE:0:U
+vmpage_faults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
+vmpage_io		in:DERIVE:0:U, out:DERIVE:0:U
+vmpage_number		value:GAUGE:0:4294967295
+volatile_changes	value:GAUGE:0:U
+voltage_threshold	value:GAUGE:U:U, threshold:GAUGE:U:U
+voltage			value:GAUGE:U:U
+vs_memory		value:GAUGE:0:9223372036854775807
+vs_processes		value:GAUGE:0:65535
+vs_threads		value:GAUGE:0:65535
+
+#
+# Legacy types
+# (required for the v5 upgrade target)
+#
+arc_counts		demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U
+arc_l2_bytes		read:COUNTER:0:U, write:COUNTER:0:U
+arc_l2_size		value:GAUGE:0:U
+arc_ratio		value:GAUGE:0:U
+arc_size		current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U
+mysql_qcache		hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U
+mysql_threads		running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U

pkg/logger/logger.go

@@ -0,0 +1,190 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logger
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	log "github.com/sirupsen/logrus"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	"github.com/fission/fission/pkg/crd"
+)
+
+var nodeName = os.Getenv("NODE_NAME")
+
+const (
+	originalContainerLogPath = "/var/log/containers"
+	fissionSymlinkPath       = "/var/log/fission"
+)
+
+func makePodLoggerController(zapLogger *zap.Logger, k8sClientSet *kubernetes.Clientset) k8sCache.Controller {
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
+	_, controller := k8sCache.NewInformer(lw, &corev1.Pod{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				pod := obj.(*corev1.Pod)
+				if !isValidFunctionPodOnNode(pod) || !utils.IsReadyPod(pod) {
+					return
+				}
+				err := createLogSymlinks(zapLogger, pod)
+				if err != nil {
+					funcName := pod.Labels[types.FUNCTION_NAME]
+					zapLogger.Error("error creating symlink",
+						zap.String("function", funcName), zap.Error(err))
+				}
+			},
+			UpdateFunc: func(_, obj interface{}) {
+				pod := obj.(*corev1.Pod)
+				if !isValidFunctionPodOnNode(pod) || !utils.IsReadyPod(pod) {
+					return
+				}
+				err := createLogSymlinks(zapLogger, pod)
+				if err != nil {
+					funcName := pod.Labels[types.FUNCTION_NAME]
+					zapLogger.Error("error creating symlink",
+						zap.String("function", funcName), zap.Error(err))
+				}
+			},
+			DeleteFunc: func(obj interface{}) {
+				// Do nothing here, let symlink reaper to recycle orphan symlink file
+			},
+		})
+	return controller
+}
+
+func createLogSymlinks(zapLogger *zap.Logger, pod *corev1.Pod) error {
+	for _, container := range pod.Status.ContainerStatuses {
+		containerUID, err := parseContainerString(container.ContainerID)
+		if err != nil {
+			zapLogger.Error("error parsing container uid",
+				zap.String("container", container.Name),
+				zap.String("pod", pod.Name),
+				zap.String("namespace", pod.Namespace),
+				zap.Error(err))
+			continue
+		}
+		containerLogPath := getLogPath(originalContainerLogPath, pod.Name, pod.Namespace, container.Name, containerUID)
+		symlinkLogPath := getLogPath(fissionSymlinkPath, pod.Name, pod.Namespace, container.Name, containerUID)
+
+		// check whether a symlink exists, if yes then ignore it
+		if _, err := os.Stat(symlinkLogPath); os.IsNotExist(err) {
+			err := os.Symlink(containerLogPath, symlinkLogPath)
+			if err != nil {
+				zapLogger.Error("error creating symlink",
+					zap.String("container", container.Name),
+					zap.String("pod", pod.Name),
+					zap.String("namespace", pod.Namespace),
+					zap.Error(err))
+			}
+		}
+	}
+
+	return nil
+}
+
+// isValidFunctionPodOnNode checks whether a pod is scheduled to the node the logger runs on
+// and examines it's metadata labels to ensure it's a qualified function pod.
+func isValidFunctionPodOnNode(pod *corev1.Pod) bool {
+	if pod.Spec.NodeName != nodeName {
+		return false
+	}
+	labels := []string{types.ENVIRONMENT_NAMESPACE, types.ENVIRONMENT_NAME, types.ENVIRONMENT_UID,
+		types.FUNCTION_NAMESPACE, types.FUNCTION_NAME, types.FUNCTION_UID, types.EXECUTOR_TYPE}
+	for _, l := range labels {
+		if len(pod.Labels[l]) == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// The ContainerID is consist of container engine type (docker://) and uuid of container.
+// (e.g., docker://f4ca66baaa715030e20273aaf5232635a144165f1cd8e34ca5175064c245b679)
+// This function tries to extract container uuid from ContainerID.
+func parseContainerString(containerID string) (string, error) {
+	// Trim the quotes and split the type and ID.
+	parts := strings.Split(strings.Trim(containerID, "\""), "://")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("invalid container ID: %q", containerID)
+	}
+	_, ID := parts[0], parts[1]
+	return ID, nil
+}
+
+func getLogPath(pathPrefix, podName, podNamespace, containerName, containerID string) string {
+	logName := fmt.Sprintf("%s_%s_%s-%s.log", podName, podNamespace, containerName, containerID)
+	return filepath.Join(pathPrefix, logName)
+}
+
+// symlinkReaper periodically checks and removes symlink file if it's target container log file is no longer exists.
+func symlinkReaper(zapLogger *zap.Logger) {
+	for {
+		select {
+		case <-time.After(5 * time.Minute):
+			err := filepath.Walk(fissionSymlinkPath, func(path string, info os.FileInfo, err error) error {
+				if target, e := os.Readlink(path); e == nil {
+					if _, pathErr := os.Stat(target); os.IsNotExist(pathErr) {
+						zapLogger.Debug("remove symlink file", zap.String("filepath", path))
+						os.Remove(path)
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				zapLogger.Error("error reaping symlink", zap.Error(err))
+			}
+		}
+	}
+}
+
+func Start() {
+	zapLogger, err := zap.NewProduction()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer zapLogger.Sync()
+
+	if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) {
+		zapLogger.Info("symlink path not exist, create it",
+			zap.String("fissionSymlinkPath", fissionSymlinkPath))
+		err = os.Mkdir(fissionSymlinkPath, 0755)
+		if err != nil {
+			zapLogger.Fatal("error creating fissionSymlinkPath", zap.Error(err))
+		}
+	}
+	go symlinkReaper(zapLogger)
+	_, kubernetesClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		log.Fatalf("Error starting pod watcher: %v", err)
+	}
+	controller := makePodLoggerController(zapLogger, kubernetesClient)
+	controller.Run(make(chan struct{}))
+	zapLogger.Fatal("Stop watching pod changes")
+}