Refactor logging to remove logger, use fluentd with kubernetes filter (#380)
Remove the `logger` container from the logging daemonset. Remove the outgoing call from the poolmgr to the logger. Use Fluentd's Kubernetes filter to add function name and UID to influx metadata. This means fluentd now figures out when to start collecting function logs on its own, without being informed by poolmgr. This is great for other execution strategies, and for autoscaling, where fission isn't in direct control of function pod creation. Also adds an integration test to make sure logging keeps working.
This commit is contained in:
prithviramesh
committed by
Soam Vasani
Soam Vasani
parent
74e05ba3a1
commit
8b40ad0b33
@@ -35,7 +35,7 @@ ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
# Install build tools
|
||||
RUN apt-get -qq update && \
|
||||
apt-get install -y -qq curl ca-certificates gcc make bash sudo && \
|
||||
apt-get install -y -qq curl ca-certificates gcc g++ make bash sudo && \
|
||||
apt-get install -y -qq --reinstall lsb-base lsb-release && \
|
||||
# Install logging agent and required gems
|
||||
/usr/bin/curl -sSL https://toolbelt.treasuredata.com/sh/install-ubuntu-xenial-td-agent2.sh | sh && \
|
||||
@@ -45,6 +45,8 @@ RUN apt-get -qq update && \
|
||||
td-agent-gem install --no-document fluent-plugin-google-cloud -v 0.5.2 && \
|
||||
td-agent-gem install --no-document fluent-plugin-detect-exceptions -v 0.0.4 && \
|
||||
td-agent-gem install --no-document fluent-plugin-influxdb && \
|
||||
td-agent-gem install --no-document fluent-plugin-kubernetes_metadata_filter && \
|
||||
td-agent-gem install --no-document fluent-plugin-flatten-hash && \
|
||||
# Remove build tools
|
||||
apt-get remove -y -qq gcc make && \
|
||||
apt-get autoremove -y -qq && \
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
#!/bin/sh
|
||||
docker build -t fission-daemonset-fluentd:latest .
|
||||
docker build -t fission-daemonset-fluentd:latest .
|
||||
|
||||
+20
-14
@@ -4,26 +4,32 @@
|
||||
|
||||
<source>
|
||||
type tail
|
||||
format json
|
||||
format json
|
||||
time_key time
|
||||
path /var/log/fission/*.log
|
||||
path "#{ENV['FLUENTD_PATH']}"
|
||||
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
||||
tag fission.*
|
||||
read_from_head true
|
||||
refresh_interval 5
|
||||
</source>
|
||||
|
||||
<filter fission.**>
|
||||
type kubernetes_metadata
|
||||
</filter>
|
||||
|
||||
|
||||
<filter fission.**>
|
||||
type flatten_hash
|
||||
separator _
|
||||
</filter>
|
||||
|
||||
<match fission.**>
|
||||
type record_reformer
|
||||
enable_ruby false
|
||||
tag log
|
||||
<record>
|
||||
namespace ${tag_parts[4]}
|
||||
pod ${tag_parts[5]}
|
||||
container ${tag_parts[6]}
|
||||
funcname ${tag_parts[7]}
|
||||
funcuid ${tag_parts[8]}
|
||||
</record>
|
||||
type record_reformer
|
||||
enable_ruby false
|
||||
tag log
|
||||
<record>
|
||||
funcuid ${kubernetes_labels_functionUid}
|
||||
</record>
|
||||
</match>
|
||||
|
||||
<match **>
|
||||
@@ -35,7 +41,7 @@
|
||||
password "#{ENV['INFLUXDB_PASSWD']}"
|
||||
use_ssl false
|
||||
time_precision s
|
||||
tag_keys ["funcuid", "pod"]
|
||||
tag_keys ["funcuid"]
|
||||
sequence_tag _seq
|
||||
buffer_type file
|
||||
buffer_path /var/log/fission/fluentd.buffer
|
||||
@@ -45,4 +51,4 @@
|
||||
retry_limit 10
|
||||
retry_wait 1.0
|
||||
num_threads 2
|
||||
</match>
|
||||
</match>
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/fission/fission"
|
||||
"github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
func makelogRequestTracker() logRequestTracker {
|
||||
return logRequestTracker{
|
||||
logMap: make(map[string]LogRequest),
|
||||
}
|
||||
}
|
||||
|
||||
func (l logRequestTracker) Add(logReq LogRequest) {
|
||||
l.Lock()
|
||||
l.logMap[logReq.Pod] = logReq
|
||||
l.Unlock()
|
||||
}
|
||||
|
||||
func (l logRequestTracker) Get(pod string) LogRequest {
|
||||
l.RLock()
|
||||
logReq, ok := l.logMap[pod]
|
||||
l.RUnlock()
|
||||
if ok {
|
||||
return logReq
|
||||
}
|
||||
return LogRequest{}
|
||||
}
|
||||
|
||||
func (l logRequestTracker) Remove(logReq LogRequest) {
|
||||
l.Lock()
|
||||
delete(l.logMap, logReq.Pod)
|
||||
l.Unlock()
|
||||
}
|
||||
|
||||
// Get a kubernetes client using the pod's service account. This only
|
||||
// works when we're running inside a kubernetes cluster.
|
||||
func getKubernetesClient() (*kubernetes.Clientset, error) {
|
||||
// creates the in-cluster config
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
log.Printf("Error getting kubernetes client config: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// creates the clientset
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
log.Printf("Error getting kubernetes client: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return clientset, nil
|
||||
}
|
||||
|
||||
// make sure that the targetPath is a legitimate path for security purpose
|
||||
func validateFilePath(targetPath string, expectedPathPrefix string) bool {
|
||||
targetPath = filepath.Clean(targetPath)
|
||||
return strings.HasPrefix(targetPath, expectedPathPrefix)
|
||||
}
|
||||
|
||||
// The ContainerID is consist of container engine type (docker://) and uuid of container.
|
||||
// (e.g., docker://f4ca66baaa715030e20273aaf5232635a144165f1cd8e34ca5175064c245b679)
|
||||
// This function tries to extract container uuid from ContainerID.
|
||||
func parseContainerString(containerID string) (string, error) {
|
||||
// Trim the quotes and split the type and ID.
|
||||
parts := strings.Split(strings.Trim(containerID, "\""), "://")
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("invalid container ID: %q", containerID)
|
||||
}
|
||||
_, ID := parts[0], parts[1]
|
||||
return ID, nil
|
||||
}
|
||||
|
||||
func getcontainerID(kubeClient *kubernetes.Clientset, namespace, pod, container string) (string, error) {
|
||||
podInfo, err := kubeClient.CoreV1().Pods(namespace).Get(pod, v1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Printf("Failed to get pod info: %v", err)
|
||||
return "", err
|
||||
}
|
||||
var containerID string
|
||||
for _, c := range podInfo.Status.ContainerStatuses {
|
||||
if c.Name == container {
|
||||
containerID, err = parseContainerString(c.ContainerID)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get container id: %v", err)
|
||||
return "", err
|
||||
}
|
||||
return containerID, nil
|
||||
}
|
||||
}
|
||||
return "", fission.MakeError(404, "no matching container is found")
|
||||
}
|
||||
|
||||
func getContainerLogPath(logReq LogRequest) (string, bool) {
|
||||
logPath := fmt.Sprintf("/var/lib/docker/containers/%s/%s-json.log", logReq.ContainerID, logReq.ContainerID)
|
||||
if !validateFilePath(logPath, "/var/lib/docker/containers") {
|
||||
return "", false
|
||||
}
|
||||
return logPath, true
|
||||
}
|
||||
|
||||
func getFissionLogSymlinkPath(logReq LogRequest) (string, bool) {
|
||||
// pass function related information through a symlink name
|
||||
logSymLink := fmt.Sprintf("/var/log/fission/%s.%s.%s.%s.%s.log", logReq.Namespace, logReq.Pod, logReq.ContainerID, logReq.FuncName, logReq.FuncUid)
|
||||
if !validateFilePath(logSymLink, "/var/log/fission") {
|
||||
return "", false
|
||||
}
|
||||
return logSymLink, true
|
||||
}
|
||||
|
||||
func createLogSymlink(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request", 500)
|
||||
return
|
||||
}
|
||||
logReq := LogRequest{}
|
||||
if err = json.Unmarshal(body, &logReq); err != nil {
|
||||
w.Write([]byte(fmt.Sprintf("%v", err)))
|
||||
return
|
||||
}
|
||||
|
||||
kubernetesClient, err := getKubernetesClient()
|
||||
if err != nil {
|
||||
log.Warningf("Failed to get kubernetes client: %v", err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
containerID, err := getcontainerID(kubernetesClient, logReq.Namespace, logReq.Pod, logReq.Container)
|
||||
if err != nil || containerID == "" {
|
||||
log.Warningf("Failed to get container id: %v", err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
logReq.ContainerID = containerID
|
||||
containerLogFilePath, isValidLogPath := getContainerLogPath(logReq)
|
||||
fissionLogSymlinkPath, isValidSymlinkPath := getFissionLogSymlinkPath(logReq)
|
||||
if !isValidLogPath || !isValidSymlinkPath {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
err = os.Symlink(containerLogFilePath, fissionLogSymlinkPath)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
logInfo.Add(logReq)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func removeLogSymlink(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
pod := vars["pod"]
|
||||
logReq := logInfo.Get(pod)
|
||||
if logReq.Pod == "" {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fissionLogSymlinkPath, isValidSymlinkPath := getFissionLogSymlinkPath(logReq)
|
||||
if !isValidSymlinkPath {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err := os.Remove(fissionLogSymlinkPath)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
var logInfo logRequestTracker
|
||||
|
||||
func Start() {
|
||||
logInfo = makelogRequestTracker()
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc("/v1/log", createLogSymlink).Methods("POST")
|
||||
r.HandleFunc("/v1/log/{pod}", removeLogSymlink).Methods("DELETE")
|
||||
address := fmt.Sprintf(":%v", 1234)
|
||||
log.Printf("starting logger at port %s", address)
|
||||
log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
|
||||
}
|
||||
Reference in New Issue
Block a user