Add function logs support (#53) (#131)

Add function log aggregation and persistence using Fluentd and InfluxDB. Fluentd and a helper sidecar run as a daemonset. The poolmgr sets up logging for each function pod, using the helper sidecar. Fluentd forwards logs to InfluxDB, which is run as a deployment and service. The client CLI directly queries InfluxDB for logs. Fluentd supports many outputs besides InfluxDB, so we aren't very tied to InfluxDB. The setup is somewhat manual, which we should be able to improve by integrating this into the helm chart. Diagram of component interactions: https://cloud.githubusercontent.com/assets/202578/23100399/b0e3ea00-f6ba-11e6-8f2f-6588cfef2e84.png
2017-03-22 15:59:42 -07:00
parent 1665235c14
commit a13015e75a
20 changed files with 1029 additions and 14 deletions
@@ -0,0 +1,68 @@
+# This file originally came from official Kubernetes GitHub repository.
+# You can reach original file with the following link:
+# https://github.com/kubernetes/kubernetes/tree/42fbf93fb0bb48d0592e2aa08c5ce6d28ab6d4b0/cluster/addons/fluentd-gcp/fluentd-gcp-image
+
+# Modification:
+# 1. add plugin "fluent-plugin-influxdb" for influxdb support
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This Dockerfile will build an image that is configured
+# to use Fluentd to collect all Docker container log files
+# and then cause them to be ingested using the Google Cloud
+# Logging API. This configuration assumes that the host performning
+# the collection is a VM that has been created with a logging.write
+# scope and that the Logging API has been enabled for the project
+# in the Google Developer Console.
+
+FROM gcr.io/google_containers/ubuntu-slim:0.6
+
+
+# Disable prompts from apt
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install build tools
+RUN apt-get -qq update && \
+    apt-get install -y -qq curl ca-certificates gcc make bash sudo && \
+    apt-get install -y -qq --reinstall lsb-base lsb-release && \
+    # Install logging agent and required gems
+    /usr/bin/curl -sSL https://toolbelt.treasuredata.com/sh/install-ubuntu-xenial-td-agent2.sh | sh && \
+    sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent && \
+    td-agent-gem install --no-document fluent-plugin-record-reformer -v 0.8.2 && \
+    td-agent-gem install --no-document fluent-plugin-systemd -v 0.0.5 && \
+    td-agent-gem install --no-document fluent-plugin-google-cloud -v 0.5.2 && \
+    td-agent-gem install --no-document fluent-plugin-detect-exceptions -v 0.0.4 && \
+    td-agent-gem install --no-document fluent-plugin-influxdb && \
+    # Remove build tools
+    apt-get remove -y -qq gcc make && \
+    apt-get autoremove -y -qq && \
+    apt-get clean -qq && \
+    # Remove unnecessary files
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+        /opt/td-agent/embedded/share/doc \
+        /opt/td-agent/embedded/share/gtk-doc \
+        /opt/td-agent/embedded/lib/postgresql \
+        /opt/td-agent/embedded/bin/postgres \
+        /opt/td-agent/embedded/share/postgresql \
+        /etc/td-agent/td-agent.conf
+
+# Copy the Fluentd configuration file for logging Docker container logs.
+COPY fluent.conf /etc/td-agent/td-agent.conf
+
+# Copy the entrypoint for the container
+COPY run.sh /run.sh
+
+# Start Fluentd to pick up our config that watches Docker container logs.
+CMD /run.sh $FLUENTD_ARGS
@@ -0,0 +1,2 @@
+#!/bin/sh
+docker build -t fission-daemonset-fluentd:latest .
@@ -0,0 +1,48 @@
+<match fluent.**>
+  type null
+</match>
+
+<source>
+  type tail
+  format json
+  time_key time
+  path /var/log/fission/*.log
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+  tag fission.*
+  read_from_head true
+  refresh_interval 5
+</source>
+
+<match fission.**>
+  type record_reformer
+  enable_ruby false
+  tag log
+  <record>
+    namespace ${tag_parts[4]}
+    pod       ${tag_parts[5]}
+    container ${tag_parts[6]}
+    funcname  ${tag_parts[7]}
+    funcuid   ${tag_parts[8]}
+  </record>
+</match>
+
+<match **>
+  @type influxdb
+    host  "#{ENV['INFLUXDB_ADDRESS']}"
+    port  "#{ENV['INFLUXDB_PORT']}"
+    dbname "#{ENV['INFLUXDB_DBNAME']}"
+    user  "#{ENV['INFLUXDB_USERNAME']}"
+    password  "#{ENV['INFLUXDB_PASSWD']}"
+    use_ssl false
+    time_precision s
+    tag_keys ["funcuid", "pod"]
+    sequence_tag _seq
+    buffer_type file
+    buffer_path /var/log/fission/fluentd.buffer
+    buffer_chunk_limit 128m
+    buffer_queue_limit 256
+    flush_interval 5
+    retry_limit 10
+    retry_wait 1.0
+    num_threads 2
+</match>
@@ -0,0 +1,27 @@
+# This file originally came from official Kubernetes GitHub repository.
+# You can reach original file with the following link:
+# https://github.com/kubernetes/kubernetes/tree/42fbf93fb0bb48d0592e2aa08c5ce6d28ab6d4b0/cluster/addons/fluentd-gcp/fluentd-gcp-image
+
+#!/bin/sh
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For systems without journald
+mkdir -p /var/log/journal
+
+LD_PRELOAD=/opt/td-agent/embedded/lib/libjemalloc.so
+RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9
+
+/usr/sbin/td-agent $@
@@ -0,0 +1,215 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/fission/fission"
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/rest"
+)
+
+func makelogRequestTracker() logRequestTracker {
+	return logRequestTracker{
+		logMap: make(map[string]LogRequest),
+	}
+}
+
+func (l logRequestTracker) Add(logReq LogRequest) {
+	l.Lock()
+	l.logMap[logReq.Pod] = logReq
+	l.Unlock()
+}
+
+func (l logRequestTracker) Get(pod string) LogRequest {
+	l.RLock()
+	logReq, ok := l.logMap[pod]
+	l.RUnlock()
+	if ok {
+		return logReq
+	}
+	return LogRequest{}
+}
+
+func (l logRequestTracker) Remove(logReq LogRequest) {
+	l.Lock()
+	delete(l.logMap, logReq.Pod)
+	l.Unlock()
+}
+
+// Get a kubernetes client using the pod's service account.  This only
+// works when we're running inside a kubernetes cluster.
+func getKubernetesClient() (*kubernetes.Clientset, error) {
+	// creates the in-cluster config
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		log.Printf("Error getting kubernetes client config: %v", err)
+		return nil, err
+	}
+
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Printf("Error getting kubernetes client: %v", err)
+		return nil, err
+	}
+
+	return clientset, nil
+}
+
+// make sure that the targetPath is a legitimate path for security purpose
+func validateFilePath(targetPath string, expectedPathPrefix string) bool {
+	targetPath = filepath.Clean(targetPath)
+	return strings.HasPrefix(targetPath, expectedPathPrefix)
+}
+
+// The ContainerID is consist of container engine type (docker://) and uuid of container.
+// (e.g., docker://f4ca66baaa715030e20273aaf5232635a144165f1cd8e34ca5175064c245b679)
+// This function tries to extract container uuid from ContainerID.
+func parseContainerString(containerID string) (string, error) {
+	// Trim the quotes and split the type and ID.
+	parts := strings.Split(strings.Trim(containerID, "\""), "://")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("invalid container ID: %q", containerID)
+	}
+	_, ID := parts[0], parts[1]
+	return ID, nil
+}
+
+func getcontainerID(kubeClient *kubernetes.Clientset, namespace, pod, container string) (string, error) {
+	podInfo, err := kubeClient.Core().Pods(namespace).Get(pod)
+	if err != nil {
+		log.Printf("Failed to get pod info: %v", err)
+		return "", err
+	}
+	var containerID string
+	for _, c := range podInfo.Status.ContainerStatuses {
+		if c.Name == container {
+			containerID, err = parseContainerString(c.ContainerID)
+			if err != nil {
+				log.Printf("Failed to get container id: %v", err)
+				return "", err
+			}
+			return containerID, nil
+		}
+	}
+	return "", fission.MakeError(404, "no matching container is found")
+}
+
+func getContainerLogPath(logReq LogRequest) (string, bool) {
+	logPath := fmt.Sprintf("/var/lib/docker/containers/%s/%s-json.log", logReq.ContainerID, logReq.ContainerID)
+	if !validateFilePath(logPath, "/var/lib/docker/containers") {
+		return "", false
+	}
+	return logPath, true
+}
+
+func getFissionLogSymlinkPath(logReq LogRequest) (string, bool) {
+	// pass function related information through a symlink name
+	logSymLink := fmt.Sprintf("/var/log/fission/%s.%s.%s.%s.%s.log", logReq.Namespace, logReq.Pod, logReq.ContainerID, logReq.FuncName, logReq.FuncUid)
+	if !validateFilePath(logSymLink, "/var/log/fission") {
+		return "", false
+	}
+	return logSymLink, true
+}
+
+func createLogSymlink(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read request", 500)
+		return
+	}
+	logReq := LogRequest{}
+	if err = json.Unmarshal(body, &logReq); err != nil {
+		w.Write([]byte(fmt.Sprintf("%v", err)))
+		return
+	}
+
+	kubernetesClient, err := getKubernetesClient()
+	if err != nil {
+		log.Warningf("Failed to get kubernetes client: %v", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	containerID, err := getcontainerID(kubernetesClient, logReq.Namespace, logReq.Pod, logReq.Container)
+	if err != nil || containerID == "" {
+		log.Warningf("Failed to get container id: %v", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	logReq.ContainerID = containerID
+	containerLogFilePath, isValidLogPath := getContainerLogPath(logReq)
+	fissionLogSymlinkPath, isValidSymlinkPath := getFissionLogSymlinkPath(logReq)
+	if !isValidLogPath || !isValidSymlinkPath {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	err = os.Symlink(containerLogFilePath, fissionLogSymlinkPath)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	logInfo.Add(logReq)
+	w.WriteHeader(http.StatusOK)
+}
+
+func removeLogSymlink(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	pod := vars["pod"]
+	logReq := logInfo.Get(pod)
+	if logReq.Pod == "" {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	fissionLogSymlinkPath, isValidSymlinkPath := getFissionLogSymlinkPath(logReq)
+	if !isValidSymlinkPath {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	err := os.Remove(fissionLogSymlinkPath)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+var logInfo logRequestTracker
+
+func Start() {
+	logInfo = makelogRequestTracker()
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/log", createLogSymlink).Methods("POST")
+	r.HandleFunc("/v1/log/{pod}", removeLogSymlink).Methods("DELETE")
+	address := fmt.Sprintf(":%v", 1234)
+	log.Printf("starting poolmgr at port %s", address)
+	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
+}
@@ -0,0 +1,35 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logger
+
+import "sync"
+
+type (
+	LogRequest struct {
+		Namespace   string `json:"namespace"`
+		Pod         string `json:"pod"`
+		Container   string `json:"container"`
+		FuncName    string `json:"funcname"`
+		FuncUid     string `json:"funcuid"`
+		ContainerID string `json:"-"`
+	}
+
+	logRequestTracker struct {
+		sync.RWMutex
+		logMap map[string]LogRequest
+	}
+)