Add function logs support (#53) (#131)

Add function log aggregation and persistence using Fluentd and InfluxDB.

Fluentd and a helper sidecar run as a daemonset.  The poolmgr sets up logging for each function pod, using the helper sidecar. Fluentd forwards logs to InfluxDB, which is run as a deployment and service. The client CLI directly queries InfluxDB for logs.

Fluentd supports many outputs besides InfluxDB, so we aren't very tied to InfluxDB.

The setup is somewhat manual, which we should be able to improve by integrating this into the helm chart.

Diagram of component interactions: https://cloud.githubusercontent.com/assets/202578/23100399/b0e3ea00-f6ba-11e6-8f2f-6588cfef2e84.png

fission/function.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -24,11 +25,13 @@ import (
 	"os/exec"
 	"strings"
 	"text/tabwriter"
+	"time"
 
 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/fission/logdb"
 )
 
 func fnFetchCode(filePath string) []byte {
@@ -276,3 +279,134 @@ func fnEdit(c *cli.Context) error {
 	fmt.Printf("function %v updated, new uuid: %v\n", newfn.Name, newfn.Uid)
 	return nil
 }
+
+func fnLogs(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		fatal("Need name of function, use --name")
+	}
+
+	dbHost := c.String("dbhost")
+	if len(dbHost) == 0 {
+		fatal("Need host address of log database, use --dbhost")
+	}
+
+	dbType := c.String("dbtype")
+	if len(dbType) == 0 {
+		dbType = logdb.INFLUXDB
+	}
+
+	fnPod := c.String("pod")
+	m := &fission.Metadata{Name: fnName}
+
+	f, err := client.FunctionGet(m)
+	checkErr(err, "get function")
+
+	auth := logdb.DBConfig{
+		DBType:   dbType,
+		Endpoint: dbHost,
+		Username: c.String("username"),
+		Password: c.String("password"),
+	}
+	logDB, err := logdb.GetLogDB(auth)
+	if err != nil {
+		fatal("failed to connect log database")
+	}
+
+	requestChan := make(chan struct{})
+	responseChan := make(chan struct{})
+	ctx := context.Background()
+
+	go func(ctx context.Context, requestChan, responseChan chan struct{}) {
+		t := time.Unix(0, 0*int64(time.Millisecond))
+		for {
+			select {
+			case <-requestChan:
+				logFilter := logdb.LogFilter{
+					Pod:      fnPod,
+					Function: f.Metadata.Name,
+					FuncUid:  f.Metadata.Uid,
+					Since:    t,
+				}
+				logEntries, err := logDB.GetLogs(logFilter)
+				if err != nil {
+					fatal("failed to query logs")
+				}
+				for _, logEntry := range logEntries {
+					if c.Bool("d") {
+						fmt.Printf("Timestamp: %s\nNamespace: %s\nFunction Name: %s\nFunction ID: %s\nPod: %s\nContainer: %s\nStream: %s\nLog: %s\n---\n",
+							logEntry.Timestamp, logEntry.Namespace, logEntry.FuncName, logEntry.FuncUid, logEntry.Pod, logEntry.Container, logEntry.Stream, logEntry.Message)
+					} else {
+						fmt.Printf("[%s] %s\n", logEntry.Timestamp, logEntry.Message)
+					}
+					t = logEntry.Timestamp
+				}
+				responseChan <- struct{}{}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}(ctx, requestChan, responseChan)
+
+	for {
+		requestChan <- struct{}{}
+		<-responseChan
+		if !c.Bool("f") {
+			ctx.Done()
+			return nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func fnPods(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		fatal("Need name of function, use --name")
+	}
+
+	dbHost := c.String("dbhost")
+	if len(dbHost) == 0 {
+		fatal("Need host address of log database, use --dbhost")
+	}
+
+	dbType := c.String("dbtype")
+	if len(dbType) == 0 {
+		dbType = logdb.INFLUXDB
+	}
+
+	m := &fission.Metadata{Name: fnName}
+
+	f, err := client.FunctionGet(m)
+	checkErr(err, "get function")
+
+	auth := logdb.DBConfig{
+		DBType:   dbType,
+		Endpoint: dbHost,
+		Username: c.String("username"),
+		Password: c.String("password"),
+	}
+	logDB, err := logdb.GetLogDB(auth)
+	if err != nil {
+		fatal("failed to connect log database")
+	}
+
+	logFilter := logdb.LogFilter{
+		Function: f.Metadata.Name,
+		FuncUid:  f.Metadata.Uid,
+	}
+	pods, err := logDB.GetPods(logFilter)
+	if err != nil {
+		fatal("failed to get pods of function")
+		return err
+	}
+	for _, pod := range pods {
+		fmt.Println(pod)
+	}
+
+	return err
+}