Switch from fluentd to fluentbit for log forwarding (#1086)
This removes fluentd in favor of using fluentbit, which is lighter (in memory usage) and seems to be more actively maintained. Fluentbit's config file format is different from fluentd's. It also doesn't support the same record modification stuff that fluentd supports, so we have to change the influxdb query slightly. This means that after an upgrade, the new CLI may won't work for querying older logs. Hopefully, this slight breakage is acceptable; if users really need older logs they can use the older CLI.
This commit is contained in:
Soam Vasani
committed by
Ta-Ching Chen
Ta-Ching Chen
parent
0fc864f230
commit
4e4c8aa14f
@@ -1,70 +0,0 @@
|
||||
# This file originally came from official Kubernetes GitHub repository.
|
||||
# You can reach original file with the following link:
|
||||
# https://github.com/kubernetes/kubernetes/tree/42fbf93fb0bb48d0592e2aa08c5ce6d28ab6d4b0/cluster/addons/fluentd-gcp/fluentd-gcp-image
|
||||
|
||||
# Modification:
|
||||
# 1. add plugin "fluent-plugin-influxdb" for influxdb support
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This Dockerfile will build an image that is configured
|
||||
# to use Fluentd to collect all Docker container log files
|
||||
# and then cause them to be ingested using the Google Cloud
|
||||
# Logging API. This configuration assumes that the host performning
|
||||
# the collection is a VM that has been created with a logging.write
|
||||
# scope and that the Logging API has been enabled for the project
|
||||
# in the Google Developer Console.
|
||||
|
||||
FROM gcr.io/google_containers/ubuntu-slim:0.6
|
||||
|
||||
|
||||
# Disable prompts from apt
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
# Install build tools
|
||||
RUN apt-get -qq update && \
|
||||
apt-get install -y -qq curl ca-certificates gcc g++ make bash sudo && \
|
||||
apt-get install -y -qq --reinstall lsb-base lsb-release && \
|
||||
# Install logging agent and required gems
|
||||
/usr/bin/curl -sSL https://toolbelt.treasuredata.com/sh/install-ubuntu-xenial-td-agent2.sh | sh && \
|
||||
sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent && \
|
||||
td-agent-gem install --no-document fluent-plugin-record-reformer -v 0.8.2 && \
|
||||
td-agent-gem install --no-document fluent-plugin-systemd -v 0.0.5 && \
|
||||
td-agent-gem install --no-document fluent-plugin-google-cloud -v 0.5.2 && \
|
||||
td-agent-gem install --no-document fluent-plugin-detect-exceptions -v 0.0.4 && \
|
||||
td-agent-gem install --no-document fluent-plugin-influxdb -v 1.1.0 && \
|
||||
td-agent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 2.1.5 && \
|
||||
td-agent-gem install --no-document fluent-plugin-flatten-hash -v 0.5.1 && \
|
||||
# Remove build tools
|
||||
apt-get remove -y -qq gcc make && \
|
||||
apt-get autoremove -y -qq && \
|
||||
apt-get clean -qq && \
|
||||
# Remove unnecessary files
|
||||
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
|
||||
/opt/td-agent/embedded/share/doc \
|
||||
/opt/td-agent/embedded/share/gtk-doc \
|
||||
/opt/td-agent/embedded/lib/postgresql \
|
||||
/opt/td-agent/embedded/bin/postgres \
|
||||
/opt/td-agent/embedded/share/postgresql \
|
||||
/etc/td-agent/td-agent.conf
|
||||
|
||||
# Copy the Fluentd configuration file for logging Docker container logs.
|
||||
COPY fluent.conf /etc/td-agent/td-agent.conf
|
||||
|
||||
# Copy the entrypoint for the container
|
||||
COPY run.sh /run.sh
|
||||
|
||||
# Start Fluentd to pick up our config that watches Docker container logs.
|
||||
CMD /run.sh $FLUENTD_ARGS
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/bin/sh
|
||||
docker build -t fission-daemonset-fluentd:latest .
|
||||
@@ -1,55 +0,0 @@
|
||||
# Default logger configfile - which is generally replaced by a more specific logger config at runtime (see charts/)
|
||||
<match fluent.**>
|
||||
type null
|
||||
</match>
|
||||
|
||||
<source>
|
||||
type tail
|
||||
format json
|
||||
time_key time
|
||||
path "#{ENV['FLUENTD_PATH']}"
|
||||
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
||||
tag fission.*
|
||||
read_from_head true
|
||||
refresh_interval 5
|
||||
</source>
|
||||
|
||||
<filter fission.**>
|
||||
type kubernetes_metadata
|
||||
</filter>
|
||||
|
||||
|
||||
<filter fission.**>
|
||||
type flatten_hash
|
||||
separator _
|
||||
</filter>
|
||||
|
||||
<match fission.**>
|
||||
type record_reformer
|
||||
enable_ruby false
|
||||
tag log
|
||||
<record>
|
||||
funcuid ${kubernetes_labels_functionUid}
|
||||
</record>
|
||||
</match>
|
||||
|
||||
<match **>
|
||||
@type influxdb
|
||||
host "#{ENV['INFLUXDB_ADDRESS']}"
|
||||
port "#{ENV['INFLUXDB_PORT']}"
|
||||
dbname "#{ENV['INFLUXDB_DBNAME']}"
|
||||
user "#{ENV['INFLUXDB_USERNAME']}"
|
||||
password "#{ENV['INFLUXDB_PASSWD']}"
|
||||
use_ssl false
|
||||
time_precision ns
|
||||
tag_keys ["funcuid"]
|
||||
sequence_tag _seq
|
||||
buffer_type file
|
||||
buffer_path /var/log/fission/fluentd.buffer
|
||||
buffer_chunk_limit 128m
|
||||
buffer_queue_limit 256
|
||||
flush_interval 5
|
||||
retry_limit 10
|
||||
retry_wait 1.0
|
||||
num_threads 2
|
||||
</match>
|
||||
@@ -1,27 +0,0 @@
|
||||
# This file originally came from official Kubernetes GitHub repository.
|
||||
# You can reach original file with the following link:
|
||||
# https://github.com/kubernetes/kubernetes/tree/42fbf93fb0bb48d0592e2aa08c5ce6d28ab6d4b0/cluster/addons/fluentd-gcp/fluentd-gcp-image
|
||||
|
||||
#!/bin/sh
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# For systems without journald
|
||||
mkdir -p /var/log/journal
|
||||
|
||||
LD_PRELOAD=/opt/td-agent/embedded/lib/libjemalloc.so
|
||||
RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9
|
||||
|
||||
/usr/sbin/td-agent $@
|
||||
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.uber.org/zap"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
"github.com/fission/fission"
|
||||
"github.com/fission/fission/crd"
|
||||
)
|
||||
|
||||
var nodeName = os.Getenv("NODE_NAME")
|
||||
|
||||
const (
|
||||
originalContainerLogPath = "/var/log/containers"
|
||||
fissionSymlinkPath = "/var/log/fission"
|
||||
)
|
||||
|
||||
func makePodLoggerController(zapLogger *zap.Logger, k8sClientSet *kubernetes.Clientset) k8sCache.Controller {
|
||||
resyncPeriod := 30 * time.Second
|
||||
lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
|
||||
_, controller := k8sCache.NewInformer(lw, &corev1.Pod{}, resyncPeriod,
|
||||
k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
pod := obj.(*corev1.Pod)
|
||||
if !isValidFunctionPodOnNode(pod) || !fission.IsReadyPod(pod) {
|
||||
return
|
||||
}
|
||||
err := createLogSymlinks(zapLogger, pod)
|
||||
if err != nil {
|
||||
funcName := pod.Labels[fission.FUNCTION_NAME]
|
||||
zapLogger.Error("error creating symlink",
|
||||
zap.String("function", funcName), zap.Error(err))
|
||||
}
|
||||
},
|
||||
UpdateFunc: func(_, obj interface{}) {
|
||||
pod := obj.(*corev1.Pod)
|
||||
if !isValidFunctionPodOnNode(pod) || !fission.IsReadyPod(pod) {
|
||||
return
|
||||
}
|
||||
err := createLogSymlinks(zapLogger, pod)
|
||||
if err != nil {
|
||||
funcName := pod.Labels[fission.FUNCTION_NAME]
|
||||
zapLogger.Error("error creating symlink",
|
||||
zap.String("function", funcName), zap.Error(err))
|
||||
}
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
// Do nothing here, let symlink reaper to recycle orphan symlink file
|
||||
},
|
||||
})
|
||||
return controller
|
||||
}
|
||||
|
||||
func createLogSymlinks(zapLogger *zap.Logger, pod *corev1.Pod) error {
|
||||
for _, container := range pod.Status.ContainerStatuses {
|
||||
containerUID, err := parseContainerString(container.ContainerID)
|
||||
if err != nil {
|
||||
zapLogger.Error("error parsing container uid",
|
||||
zap.String("container", container.Name),
|
||||
zap.String("pod", pod.Name),
|
||||
zap.String("namespace", pod.Namespace),
|
||||
zap.Error(err))
|
||||
continue
|
||||
}
|
||||
containerLogPath := getLogPath(originalContainerLogPath, pod.Name, pod.Namespace, container.Name, containerUID)
|
||||
symlinkLogPath := getLogPath(fissionSymlinkPath, pod.Name, pod.Namespace, container.Name, containerUID)
|
||||
|
||||
// check whether a symlink exists, if yes then ignore it
|
||||
if _, err := os.Stat(symlinkLogPath); os.IsNotExist(err) {
|
||||
err := os.Symlink(containerLogPath, symlinkLogPath)
|
||||
if err != nil {
|
||||
zapLogger.Error("error creating symlink",
|
||||
zap.String("container", container.Name),
|
||||
zap.String("pod", pod.Name),
|
||||
zap.String("namespace", pod.Namespace),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isValidFunctionPodOnNode checks whether a pod is scheduled to the node the logger runs on
|
||||
// and examines it's metadata labels to ensure it's a qualified function pod.
|
||||
func isValidFunctionPodOnNode(pod *corev1.Pod) bool {
|
||||
if pod.Spec.NodeName != nodeName {
|
||||
return false
|
||||
}
|
||||
labels := []string{fission.ENVIRONMENT_NAMESPACE, fission.ENVIRONMENT_NAME, fission.ENVIRONMENT_UID,
|
||||
fission.FUNCTION_NAMESPACE, fission.FUNCTION_NAME, fission.FUNCTION_UID, fission.EXECUTOR_TYPE}
|
||||
for _, l := range labels {
|
||||
if len(pod.Labels[l]) == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The ContainerID is consist of container engine type (docker://) and uuid of container.
|
||||
// (e.g., docker://f4ca66baaa715030e20273aaf5232635a144165f1cd8e34ca5175064c245b679)
|
||||
// This function tries to extract container uuid from ContainerID.
|
||||
func parseContainerString(containerID string) (string, error) {
|
||||
// Trim the quotes and split the type and ID.
|
||||
parts := strings.Split(strings.Trim(containerID, "\""), "://")
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("invalid container ID: %q", containerID)
|
||||
}
|
||||
_, ID := parts[0], parts[1]
|
||||
return ID, nil
|
||||
}
|
||||
|
||||
func getLogPath(pathPrefix, podName, podNamespace, containerName, containerID string) string {
|
||||
logName := fmt.Sprintf("%s_%s_%s-%s.log", podName, podNamespace, containerName, containerID)
|
||||
return filepath.Join(pathPrefix, logName)
|
||||
}
|
||||
|
||||
// symlinkReaper periodically checks and removes symlink file if it's target container log file is no longer exists.
|
||||
func symlinkReaper(zapLogger *zap.Logger) {
|
||||
for {
|
||||
select {
|
||||
case <-time.After(5 * time.Minute):
|
||||
err := filepath.Walk(fissionSymlinkPath, func(path string, info os.FileInfo, err error) error {
|
||||
if target, e := os.Readlink(path); e == nil {
|
||||
if _, pathErr := os.Stat(target); os.IsNotExist(pathErr) {
|
||||
zapLogger.Debug("remove symlink file", zap.String("filepath", path))
|
||||
os.Remove(path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
zapLogger.Error("error reaping symlink", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Start() {
|
||||
zapLogger, err := zap.NewProduction()
|
||||
if err != nil {
|
||||
log.Fatalf("can't initialize zap logger: %v", err)
|
||||
}
|
||||
defer zapLogger.Sync()
|
||||
|
||||
go symlinkReaper(zapLogger)
|
||||
_, kubernetesClient, _, err := crd.MakeFissionClient()
|
||||
if err != nil {
|
||||
log.Fatalf("Error starting pod watcher: %v", err)
|
||||
}
|
||||
controller := makePodLoggerController(zapLogger, kubernetesClient)
|
||||
controller.Run(make(chan struct{}))
|
||||
zapLogger.Fatal("Stop watching pod changes")
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import "sync"
|
||||
|
||||
type (
|
||||
LogRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Pod string `json:"pod"`
|
||||
Container string `json:"container"`
|
||||
FuncName string `json:"funcname"`
|
||||
FuncUid string `json:"funcuid"`
|
||||
ContainerID string `json:"-"`
|
||||
}
|
||||
|
||||
logRequestTracker struct {
|
||||
sync.RWMutex
|
||||
logMap map[string]LogRequest
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user