Add function logs support (#53) (#131)

Add function log aggregation and persistence using Fluentd and InfluxDB.

Fluentd and a helper sidecar run as a daemonset.  The poolmgr sets up logging for each function pod, using the helper sidecar. Fluentd forwards logs to InfluxDB, which is run as a deployment and service. The client CLI directly queries InfluxDB for logs.

Fluentd supports many outputs besides InfluxDB, so we aren't very tied to InfluxDB.

The setup is somewhat manual, which we should be able to improve by integrating this into the helm chart.

Diagram of component interactions: https://cloud.githubusercontent.com/assets/202578/23100399/b0e3ea00-f6ba-11e6-8f2f-6588cfef2e84.png

poolmgr/gp.go

@@ -18,6 +18,7 @@ package poolmgr
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log"
@@ -37,6 +38,7 @@ import (
 	"k8s.io/client-go/1.5/pkg/util/intstr"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/logger"
 )
 
 const POOLMGR_INSTANCEID_LABEL string = "poolmgrInstanceId"
@@ -265,6 +267,9 @@ func (gp *GenericPool) specializePod(pod *v1.Pod, metadata *fission.Metadata) er
 		return errors.New(fmt.Sprintf("Error from fetcher: %v", resp.Status))
 	}
 
+	// Tell logging helper about this function invocation
+	gp.setupLogging(pod, metadata)
+
 	// get function run container to specialize
 	log.Printf("[%v] specializing pod", metadata)
 	specializeUrl := fmt.Sprintf("http://%v:8888/specialize", podIP)
@@ -490,6 +495,23 @@ func (gp *GenericPool) CleanupFunctionService(podName string) error {
 		return nil
 	}
 
+	pod, err := gp.kubernetesClient.Core().Pods(gp.namespace).Get(podName)
+	if err != nil {
+		return err
+	}
+
+	loggerUrl := fmt.Sprintf("http://%s:1234/v1/log/%s", pod.Spec.NodeName, pod.Name)
+	req, err := http.NewRequest("DELETE", loggerUrl, nil)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Printf("Error from %s daemonset logger: %v", pod.Spec.NodeName, err)
+	} else {
+		if resp.StatusCode != 200 {
+			log.Printf("Received not http 200(OK) status from %s daemonset logger: %s", pod.Spec.NodeName, resp.Status)
+		}
+		resp.Body.Close()
+	}
+
 	// delete pod
 	err = gp.kubernetesClient.Core().Pods(gp.namespace).Delete(podName, nil)
 	if err != nil {
@@ -556,3 +578,32 @@ func (gp *GenericPool) destroy() error {
 
 	return nil
 }
+
+// Calls the logging daemonset pod on the node where the given pod is
+// running.
+func (gp *GenericPool) setupLogging(pod *v1.Pod, metadata *fission.Metadata) {
+	logReq := logger.LogRequest{
+		Namespace: pod.Namespace,
+		Pod:       pod.Name,
+		Container: gp.env.Metadata.Name,
+		FuncName:  metadata.Name,
+		FuncUid:   metadata.Uid,
+	}
+	reqbody, err := json.Marshal(logReq)
+	if err != nil {
+		log.Printf("Error creating log request")
+		return
+	}
+	go func() {
+		loggerUrl := fmt.Sprintf("http://%s:1234/v1/log", pod.Status.HostIP)
+		resp, err := http.Post(loggerUrl, "application/json", bytes.NewReader(reqbody))
+		if err != nil {
+			log.Printf("Error connecting to %s log daemonset pod: %v", pod.Spec.NodeName, err)
+		} else {
+			if resp.StatusCode != 200 {
+				log.Printf("Error from %s log daemonset pod: %s", pod.Spec.NodeName, resp.Status)
+			}
+			resp.Body.Close()
+		}
+	}()
+}