Add function logs support (#53) (#131)

Add function log aggregation and persistence using Fluentd and InfluxDB. Fluentd and a helper sidecar run as a daemonset. The poolmgr sets up logging for each function pod, using the helper sidecar. Fluentd forwards logs to InfluxDB, which is run as a deployment and service. The client CLI directly queries InfluxDB for logs. Fluentd supports many outputs besides InfluxDB, so we aren't very tied to InfluxDB. The setup is somewhat manual, which we should be able to improve by integrating this into the helm chart. Diagram of component interactions: https://cloud.githubusercontent.com/assets/202578/23100399/b0e3ea00-f6ba-11e6-8f2f-6588cfef2e84.png
2017-03-22 15:59:42 -07:00
parent 1665235c14
commit a13015e75a
20 changed files with 1029 additions and 14 deletions
@@ -17,6 +17,7 @@ limitations under the License.
 package main

 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -24,11 +25,13 @@ import (
 	"os/exec"
 	"strings"
 	"text/tabwriter"
+	"time"

 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"

 	"github.com/fission/fission"
+	"github.com/fission/fission/fission/logdb"
 )

 func fnFetchCode(filePath string) []byte {
@@ -276,3 +279,134 @@ func fnEdit(c *cli.Context) error {
 	fmt.Printf("function %v updated, new uuid: %v\n", newfn.Name, newfn.Uid)
 	return nil
 }
+
+func fnLogs(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		fatal("Need name of function, use --name")
+	}
+
+	dbHost := c.String("dbhost")
+	if len(dbHost) == 0 {
+		fatal("Need host address of log database, use --dbhost")
+	}
+
+	dbType := c.String("dbtype")
+	if len(dbType) == 0 {
+		dbType = logdb.INFLUXDB
+	}
+
+	fnPod := c.String("pod")
+	m := &fission.Metadata{Name: fnName}
+
+	f, err := client.FunctionGet(m)
+	checkErr(err, "get function")
+
+	auth := logdb.DBConfig{
+		DBType:   dbType,
+		Endpoint: dbHost,
+		Username: c.String("username"),
+		Password: c.String("password"),
+	}
+	logDB, err := logdb.GetLogDB(auth)
+	if err != nil {
+		fatal("failed to connect log database")
+	}
+
+	requestChan := make(chan struct{})
+	responseChan := make(chan struct{})
+	ctx := context.Background()
+
+	go func(ctx context.Context, requestChan, responseChan chan struct{}) {
+		t := time.Unix(0, 0*int64(time.Millisecond))
+		for {
+			select {
+			case <-requestChan:
+				logFilter := logdb.LogFilter{
+					Pod:      fnPod,
+					Function: f.Metadata.Name,
+					FuncUid:  f.Metadata.Uid,
+					Since:    t,
+				}
+				logEntries, err := logDB.GetLogs(logFilter)
+				if err != nil {
+					fatal("failed to query logs")
+				}
+				for _, logEntry := range logEntries {
+					if c.Bool("d") {
+						fmt.Printf("Timestamp: %s\nNamespace: %s\nFunction Name: %s\nFunction ID: %s\nPod: %s\nContainer: %s\nStream: %s\nLog: %s\n---\n",
+							logEntry.Timestamp, logEntry.Namespace, logEntry.FuncName, logEntry.FuncUid, logEntry.Pod, logEntry.Container, logEntry.Stream, logEntry.Message)
+					} else {
+						fmt.Printf("[%s] %s\n", logEntry.Timestamp, logEntry.Message)
+					}
+					t = logEntry.Timestamp
+				}
+				responseChan <- struct{}{}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}(ctx, requestChan, responseChan)
+
+	for {
+		requestChan <- struct{}{}
+		<-responseChan
+		if !c.Bool("f") {
+			ctx.Done()
+			return nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func fnPods(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		fatal("Need name of function, use --name")
+	}
+
+	dbHost := c.String("dbhost")
+	if len(dbHost) == 0 {
+		fatal("Need host address of log database, use --dbhost")
+	}
+
+	dbType := c.String("dbtype")
+	if len(dbType) == 0 {
+		dbType = logdb.INFLUXDB
+	}
+
+	m := &fission.Metadata{Name: fnName}
+
+	f, err := client.FunctionGet(m)
+	checkErr(err, "get function")
+
+	auth := logdb.DBConfig{
+		DBType:   dbType,
+		Endpoint: dbHost,
+		Username: c.String("username"),
+		Password: c.String("password"),
+	}
+	logDB, err := logdb.GetLogDB(auth)
+	if err != nil {
+		fatal("failed to connect log database")
+	}
+
+	logFilter := logdb.LogFilter{
+		Function: f.Metadata.Name,
+		FuncUid:  f.Metadata.Uid,
+	}
+	pods, err := logDB.GetPods(logFilter)
+	if err != nil {
+		fatal("failed to get pods of function")
+		return err
+	}
+	for _, pod := range pods {
+		fmt.Println(pod)
+	}
+
+	return err
+}
@@ -0,0 +1,118 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logdb
+
+import (
+	"log"
+	"time"
+
+	"fmt"
+
+	"strings"
+
+	influxdbClient "github.com/influxdata/influxdb/client/v2"
+)
+
+const (
+	INFLUXDB_DATABASE = "fissionFunctionLog"
+)
+
+func NewInfluxDB(cnf DBConfig) (InfluxDB, error) {
+	dbClient, err := influxdbClient.NewHTTPClient(influxdbClient.HTTPConfig{
+		Addr:     cnf.Endpoint,
+		Username: cnf.Username,
+		Password: cnf.Password,
+	})
+	if err != nil {
+		log.Fatal(err)
+		return InfluxDB{}, err
+	}
+
+	return InfluxDB{
+		dbClient: dbClient,
+	}, nil
+}
+
+type InfluxDB struct {
+	dbClient influxdbClient.Client
+}
+
+func (influx InfluxDB) GetPods(filter LogFilter) ([]string, error) {
+	query := fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' group by \"pod\"", filter.FuncUid)
+	q := influxdbClient.Query{
+		Command:  query,
+		Database: INFLUXDB_DATABASE,
+	}
+	response, err := influx.dbClient.Query(q)
+	if err != nil /*|| response.Err != ""*/ {
+		return []string{}, err
+	}
+	pods := []string{}
+	for _, r := range response.Results {
+		for _, series := range r.Series {
+			for _, pod := range series.Tags {
+				pods = append(pods, pod)
+			}
+		}
+	}
+	return pods, nil
+}
+
+func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
+	timestamp := filter.Since.UnixNano()
+	var query string
+	if filter.Pod != "" {
+		query = fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' AND \"pod\" = '%s' AND \"time\" > %d ORDER BY time ASC", filter.FuncUid, filter.Pod, timestamp)
+	} else {
+		query = fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' AND \"time\" > %d ORDER BY time ASC", filter.FuncUid, timestamp)
+	}
+
+	logEntries := []LogEntry{}
+	response, err := influx.query(query)
+	if err != nil {
+		return logEntries, nil
+	}
+	for _, r := range response.Results {
+		for _, series := range r.Series {
+			for _, row := range series.Values {
+				t, err := time.Parse(time.RFC3339, row[0].(string))
+				if err != nil {
+					log.Fatal(err)
+				}
+				logEntries = append(logEntries, LogEntry{
+					Timestamp: t,
+					Container: row[2].(string),
+					FuncName:  row[3].(string),
+					FuncUid:   row[4].(string),
+					Message:   strings.TrimSuffix(row[5].(string), "\n"),
+					Namespace: row[6].(string),
+					Pod:       row[7].(string),
+					Stream:    row[8].(string),
+				})
+			}
+		}
+	}
+	return logEntries, nil
+}
+
+func (influx InfluxDB) query(queryCmd string) (*influxdbClient.Response, error) {
+	q := influxdbClient.Query{
+		Command:  queryCmd,
+		Database: INFLUXDB_DATABASE,
+	}
+	return influx.dbClient.Query(q)
+}
@@ -0,0 +1,68 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logdb
+
+import (
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+const (
+	INFLUXDB = "influxdb"
+)
+
+type LogDatabase interface {
+	GetPods(LogFilter) ([]string, error)
+	GetLogs(LogFilter) ([]LogEntry, error)
+}
+
+type LogFilter struct {
+	Pod      string
+	Function string
+	FuncUid  string
+	Since    time.Time
+}
+
+type LogEntry struct {
+	Timestamp time.Time
+	Message   string
+	Stream    string
+	Container string
+	Namespace string
+	FuncName  string
+	FuncUid   string
+	Pod       string
+}
+
+type DBConfig struct {
+	DBType   string
+	Endpoint string
+	Username string
+	Password string
+}
+
+func GetLogDB(cnf DBConfig) (LogDatabase, error) {
+	switch cnf.DBType {
+	case INFLUXDB:
+		return NewInfluxDB(cnf)
+	}
+	log.WithFields(log.Fields{
+		"FISSION_LOGDB_URL": cnf.Endpoint,
+	}).Fatalf("FISSION_LOGDB_URL is incorrect, now only support %s", INFLUXDB)
+	return nil, nil
+}
@@ -41,6 +41,13 @@ func main() {
 	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
 	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "local path or URL for binary package"}
 	fnUidFlag := cli.StringFlag{Name: "uid", Usage: "function uid, optional (use latest if unspecified)"}
+	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
+	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
+	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
+	fnLogDBHostFlag := cli.StringFlag{Name: "dbhost", Usage: "log database host to connect to", EnvVar: "FISSION_LOGDB"}
+	fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"}
+	fnUserNameFlag := cli.StringFlag{Name: "username, u", Usage: "username for connecting log database"}
+	fnPasswordFlag := cli.StringFlag{Name: "password, p", Usage: "password for connecting log database"}
 	fnSubcommands := []cli.Command{
 		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGet},
@@ -49,6 +56,8 @@ func main() {
 		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnDelete},
 		{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
+		{Name: "logs", Usage: "Display funtion logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBHostFlag, fnLogDBTypeFlag, fnUserNameFlag, fnPasswordFlag}, Action: fnLogs},
+		{Name: "pods", Usage: "Display funtion pods", Flags: []cli.Flag{fnNameFlag, fnLogDBHostFlag, fnLogDBTypeFlag, fnUserNameFlag, fnPasswordFlag}, Action: fnPods},
 	}

 	// httptriggers