Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch from fluentd to fluentbit for log forwarding (#1086)

Browse Source 
This removes fluentd in favor of using fluentbit, which is lighter (in
memory usage) and seems to be more actively maintained.

Fluentbit's config file format is different from fluentd's.  It also
doesn't support the same record modification stuff that fluentd
supports, so we have to change the influxdb query slightly.  This
means that after an upgrade, the new CLI may won't work for querying older 
logs.  Hopefully, this slight breakage is acceptable; if users 
really need older logs they can use the older CLI.
This commit is contained in:
Soam Vasani
2019-03-18 15:42:37 +08:00
committed by Ta-Ching Chen
parent 0fc864f230
commit 4e4c8aa14f
25 changed files with 366 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fission/logdb/influxdb.go
+38 -13
View File
@@ -66,10 +66,12 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
//the parameters above are only for the where clause and do not work with LIMIT
if filter.Pod != "" {
queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"pod\" = $pod AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
// wait for bug fix for fluent-bit influxdb plugin
queryCmd = "select * from /^log*/ where (\"funcuid\" = $funcuid OR \"kubernetes_labels_functionUid\" = $funcuid) AND \"pod\" = $pod AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
parameters["pod"] = filter.Pod
} else {
queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
// wait for bug fix for fluent-bit influxdb plugin
queryCmd = "select * from /^log*/ where (\"funcuid\" = $funcuid OR \"kubernetes_labels_functionUid\" = $funcuid) AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
}
query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
@@ -84,9 +86,13 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
//create map of columns to row indeces
indexMap := makeIndexMap(series.Columns)
container := indexMap["docker_container_id"]
// TODO: Remove fallback indexes. Some of index's name changed in fluent-bit, here we add extra fallbackIndexes to address compatibility problem.
container := indexMap["kubernetes_docker_id"]
container_1 := indexMap["docker_container_id"] // for backward compatibility
functionName := indexMap["kubernetes_labels_functionName"]
funcuid := indexMap["kubernetes_labels_functionUid"]
funcuid_1 := indexMap["funcuid"] // for backward compatibility
funcuid_2 := indexMap["kubernetes_labels_functionUid_1"] // for backward compatibility
logMessage := indexMap["log"]
nameSpace := indexMap["kubernetes_namespace_name"]
podName := indexMap["kubernetes_pod_name"]
@@ -102,18 +108,19 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
if err != nil {
return logEntries, err
}
logEntries = append(logEntries, LogEntry{
entry := LogEntry{
//The attributes of the LogEntry are selected as relative to their position in InfluxDB's line protocol response
Timestamp: t,
Container: row[container].(string), //docker_container_id
FuncName: row[functionName].(string), //kubernetes_labels_functionName
FuncUid: row[funcuid].(string), //funcuid
Message: strings.TrimSuffix(row[logMessage].(string), "\n"), //log field
Namespace: row[nameSpace].(string), //kubernetes_namespace_name
Pod: row[podName].(string), //kubernetes_pod_name
Stream: row[stream].(string), //stream
Sequence: seqNum, //sequence tag
})
Container: getEntryValue(row, container, container_1),
FuncName: getEntryValue(row, functionName, -1),
FuncUid: getEntryValue(row, funcuid, funcuid_1, funcuid_2),
Message: strings.TrimSuffix(getEntryValue(row, logMessage, -1), "\n"), //log field
Namespace: getEntryValue(row, nameSpace, -1),
Pod: getEntryValue(row, podName, -1),
Stream: getEntryValue(row, stream, -1),
Sequence: seqNum, //sequence tag
}
logEntries = append(logEntries, entry)
}
}
}
@@ -175,3 +182,21 @@ func (influx InfluxDB) query(query influxdbClient.Query) (*influxdbClient.Respon
}
return &response, nil
}
// getEntryValue returns a field value in string type of log entry by providing index of log entry.
// Since we switch from fluentd to fluent-bit, there are some field names' changed which will break
// CLI due to empty value. For backward compatibility, getEntryValue also supports to get value from
// fallbackIndex if exists, otherwise an empty string returned instead.
func getEntryValue(list []interface{}, index int, fallbackIndex ...int) string {
if index < len(list) && list[index] != nil {
return list[index].(string)
}
for _, i := range fallbackIndex {
if i >= 0 && i < len(list) && list[i] != nil {
return list[i].(string)
}
}
return ""
}