Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retrieve function logs from controller (#207)

Browse Source 
Establish a proxy server from the controller to the log database.

Redirect query commands send from client to database then proxy back the db response.

Use parameter binding instead of fmt.Sprintf to prevent SQL injection.
This commit is contained in:
Ta-Ching Chen
2017-05-30 14:19:22 -07:00
committed by Soam Vasani
parent 531f69a037
commit 990d943f21
7 changed files with 158 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
INSTALL.md
+2 -19
View File
@@ -129,30 +129,13 @@ Finally, you're ready to use Fission!
### Enable Persistent Function Logs (Optional)
Fission uses InfluxDB to store logs and fluentd to forward them from
function pods into InfluxDB. To setup both InfluxDB and fluentd:
Edit `fission-logger.yaml` to add a username and password for the
Influxdb deployment. Then:
function pods into InfluxDB.
```
$ kubectl create -f fission-logger.yaml
```
On the client side,
If you're using minikube or a local cluster:
```
$ export FISSION_LOGDB=http://$(minikube ip):31315
```
If you're using GKE or other cloud:
```
$ export FISSION_LOGDB=http://$(kubectl --namespace fission get svc influxdb -o=jsonpath='{..ip}'):8086
```
That's it for setup. You can now use this to view function logs:
That's it for the basic setup. You can now use following command to view function logs:
```
$ fission function logs --name hello
controller/api.go
+36 -7
View File
@@ -21,21 +21,31 @@ import (
"net/http"
"os"
"runtime/debug"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/fission/fission"
"github.com/fission/fission/fission/logdb"
)
type API struct {
FunctionStore
HTTPTriggerStore
TimeTriggerStore
EnvironmentStore
WatchStore
}
type (
API struct {
FunctionStore
HTTPTriggerStore
TimeTriggerStore
EnvironmentStore
WatchStore
}
logDBConfig struct {
httpURL string
username string
password string
}
)
func MakeAPI(rs *ResourceStore) *API {
api := &API{
@@ -64,6 +74,23 @@ func (api *API) respondWithError(w http.ResponseWriter, err error) {
http.Error(w, msg, code)
}
func (api *API) getLogDBConfig(dbType string) logDBConfig {
dbType = strings.ToUpper(dbType)
// retrieve db auth config from the env
url := os.Getenv(fmt.Sprintf("%s_URL", dbType))
if url == "" {
// set up default database url
url = logdb.INFLUXDB_URL
}
username := os.Getenv(fmt.Sprintf("%s_USERNAME", dbType))
password := os.Getenv(fmt.Sprintf("%s_PASSWORD", dbType))
return logDBConfig{
httpURL: url,
username: username,
password: password,
}
}
func (api *API) HomeHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\"message\": \"Fission API\", \"version\": \"0.1.0\"}\n")
@@ -103,6 +130,8 @@ func (api *API) Serve(port int) {
r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT")
r.HandleFunc("/v1/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE")
r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST")
address := fmt.Sprintf(":%v", port)
log.WithFields(log.Fields{"port": port}).Info("Server started")
controller/functionApi.go
+34 -2
View File
@@ -17,14 +17,16 @@ limitations under the License.
package controller
import (
"encoding/base64"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
"encoding/json"
log "github.com/Sirupsen/logrus"
"github.com/gorilla/mux"
"encoding/base64"
"github.com/fission/fission"
)
@@ -171,3 +173,33 @@ func (api *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) {
api.respondWithSuccess(w, []byte(""))
}
// FunctionLogsApiPost establishes a proxy server to log database, and redirect
// query command send from client to database then proxy back the db response.
func (api *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
// get dbType from url
dbType := vars["dbType"]
// find correspond db http url
dbCnf := api.getLogDBConfig(dbType)
svcUrl, err := url.Parse(dbCnf.httpURL)
if err != nil {
log.Printf("Failed to establish proxy server for function logs: %v", err)
}
// set up proxy server director
director := func(req *http.Request) {
// only replace url Scheme and Host to remote influxDB
// and leave query string intact
req.URL.Scheme = svcUrl.Scheme
req.URL.Host = svcUrl.Host
req.URL.Path = svcUrl.Path
// set up http basic auth for database authentication
req.SetBasicAuth(dbCnf.username, dbCnf.password)
}
proxy := &httputil.ReverseProxy{
Director: director,
}
proxy.ServeHTTP(w, r)
}
fission/function.go
+7 -24
View File
@@ -288,11 +288,6 @@ func fnLogs(c *cli.Context) error {
fatal("Need name of function, use --name")
}
dbHost := c.String("dbhost")
if len(dbHost) == 0 {
fatal("Need host address of log database, use --dbhost")
}
dbType := c.String("dbtype")
if len(dbType) == 0 {
dbType = logdb.INFLUXDB
@@ -304,13 +299,9 @@ func fnLogs(c *cli.Context) error {
f, err := client.FunctionGet(m)
checkErr(err, "get function")
auth := logdb.DBConfig{
DBType: dbType,
Endpoint: dbHost,
Username: c.String("username"),
Password: c.String("password"),
}
logDB, err := logdb.GetLogDB(auth)
// client first send db query to controller, then controller will
// establishe a proxy server that bridges the client and the database.
logDB, err := logdb.GetLogDB(dbType, c.GlobalString("server"))
if err != nil {
fatal("failed to connect log database")
}
@@ -369,11 +360,6 @@ func fnPods(c *cli.Context) error {
fatal("Need name of function, use --name")
}
dbHost := c.String("dbhost")
if len(dbHost) == 0 {
fatal("Need host address of log database, use --dbhost")
}
dbType := c.String("dbtype")
if len(dbType) == 0 {
dbType = logdb.INFLUXDB
@@ -384,13 +370,9 @@ func fnPods(c *cli.Context) error {
f, err := client.FunctionGet(m)
checkErr(err, "get function")
auth := logdb.DBConfig{
DBType: dbType,
Endpoint: dbHost,
Username: c.String("username"),
Password: c.String("password"),
}
logDB, err := logdb.GetLogDB(auth)
// client first sends db query to the controller, then the controller
// will establish a proxy server that bridges the client and the database.
logDB, err := logdb.GetLogDB(dbType, c.GlobalString("server"))
if err != nil {
fatal("failed to connect log database")
}
@@ -404,6 +386,7 @@ func fnPods(c *cli.Context) error {
fatal("failed to get pods of function")
return err
}
fmt.Printf("NAME\t\n")
for _, pod := range pods {
fmt.Println(pod)
}
fission/logdb/influxdb.go
+73 -33
View File
@@ -17,47 +17,40 @@ limitations under the License.
package logdb
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"strings"
"time"
"fmt"
"strings"
influxdbClient "github.com/influxdata/influxdb/client/v2"
"github.com/fission/fission"
)
const (
INFLUXDB_DATABASE = "fissionFunctionLog"
INFLUXDB_URL = "http://influxdb:8086/query"
)
func NewInfluxDB(cnf DBConfig) (InfluxDB, error) {
dbClient, err := influxdbClient.NewHTTPClient(influxdbClient.HTTPConfig{
Addr: cnf.Endpoint,
Username: cnf.Username,
Password: cnf.Password,
})
if err != nil {
log.Fatal(err)
return InfluxDB{}, err
}
return InfluxDB{
dbClient: dbClient,
}, nil
func NewInfluxDB(serverURL string) (InfluxDB, error) {
return InfluxDB{endpoint: serverURL}, nil
}
type InfluxDB struct {
dbClient influxdbClient.Client
endpoint string
}
func (influx InfluxDB) GetPods(filter LogFilter) ([]string, error) {
query := fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' group by \"pod\"", filter.FuncUid)
q := influxdbClient.Query{
Command: query,
Database: INFLUXDB_DATABASE,
}
response, err := influx.dbClient.Query(q)
parameters := make(map[string]interface{})
parameters["funcuid"] = filter.FuncUid
queryCmd := "select * from \"log\" where \"funcuid\" = $funcuid group by \"pod\""
query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
response, err := influx.query(query)
if err != nil /*|| response.Err != ""*/ {
return []string{}, err
}
@@ -74,13 +67,22 @@ func (influx InfluxDB) GetPods(filter LogFilter) ([]string, error) {
func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
timestamp := filter.Since.UnixNano()
var query string
var queryCmd string
// please check "Example 4: Bind a parameter in the WHERE clause to specific tag value"
// at https://docs.influxdata.com/influxdb/v1.2/tools/api/
parameters := make(map[string]interface{})
parameters["funcuid"] = filter.FuncUid
parameters["time"] = timestamp
if filter.Pod != "" {
query = fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' AND \"pod\" = '%s' AND \"time\" > %d ORDER BY time ASC", filter.FuncUid, filter.Pod, timestamp)
queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"pod\" = $pod AND \"time\" > $time ORDER BY time ASC"
parameters["pod"] = filter.Pod
} else {
query = fmt.Sprintf("select * from \"log\" where \"funcuid\" = '%s' AND \"time\" > %d ORDER BY time ASC", filter.FuncUid, timestamp)
queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"time\" > $time ORDER BY time ASC"
}
query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
logEntries := []LogEntry{}
response, err := influx.query(query)
if err != nil {
@@ -109,10 +111,48 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
return logEntries, nil
}
func (influx InfluxDB) query(queryCmd string) (*influxdbClient.Response, error) {
q := influxdbClient.Query{
Command: queryCmd,
Database: INFLUXDB_DATABASE,
func (influx InfluxDB) query(query influxdbClient.Query) (*influxdbClient.Response, error) {
queryURL, err := url.Parse(influx.endpoint)
if err != nil {
return nil, err
}
return influx.dbClient.Query(q)
// connect to controller first, then controller will redirect our query command
// to influxdb and proxy back the db response.
queryURL.Path = fmt.Sprintf("/proxy/%s", INFLUXDB)
req, err := http.NewRequest("POST", queryURL.String(), nil)
if err != nil {
return nil, err
}
parametersBytes, err := json.Marshal(query.Parameters)
if err != nil {
return nil, err
}
// set up http URL query string
params := req.URL.Query()
params.Set("q", query.Command)
params.Set("db", query.Database)
params.Set("params", string(parametersBytes))
req.URL.RawQuery = params.Encode()
httpClient := http.Client{Timeout: 5 * time.Second}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fission.MakeErrorFromHTTP(resp)
}
// decode influxdb response
response := influxdbClient.Response{}
decoder := json.NewDecoder(resp.Body)
decoder.UseNumber()
if decoder.Decode(&response) != nil {
return nil, fmt.Errorf("Failed to decode influxdb response: %v", err)
}
return &response, nil
}
fission/logdb/logdb.go
+4 -13
View File
@@ -49,20 +49,11 @@ type LogEntry struct {
Pod string
}
type DBConfig struct {
DBType string
Endpoint string
Username string
Password string
}
func GetLogDB(cnf DBConfig) (LogDatabase, error) {
switch cnf.DBType {
func GetLogDB(dbType string, serverURL string) (LogDatabase, error) {
switch dbType {
case INFLUXDB:
return NewInfluxDB(cnf)
return NewInfluxDB(serverURL)
}
log.WithFields(log.Fields{
"FISSION_LOGDB_URL": cnf.Endpoint,
}).Fatalf("FISSION_LOGDB_URL is incorrect, now only support %s", INFLUXDB)
log.Fatalf("Log database type is incorrect, now only support %s", INFLUXDB)
return nil, nil
}
fission/main.go
+2 -5
View File
@@ -44,10 +44,7 @@ func main() {
fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
fnLogDBHostFlag := cli.StringFlag{Name: "dbhost", Usage: "log database host to connect to", EnvVar: "FISSION_LOGDB"}
fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"}
fnUserNameFlag := cli.StringFlag{Name: "username, u", Usage: "username for connecting log database"}
fnPasswordFlag := cli.StringFlag{Name: "password, p", Usage: "password for connecting log database"}
fnSubcommands := []cli.Command{
{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnGet},
@@ -56,8 +53,8 @@ func main() {
{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag}, Action: fnUpdate},
{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnUidFlag}, Action: fnDelete},
{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
{Name: "logs", Usage: "Display funtion logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBHostFlag, fnLogDBTypeFlag, fnUserNameFlag, fnPasswordFlag}, Action: fnLogs},
{Name: "pods", Usage: "Display funtion pods", Flags: []cli.Flag{fnNameFlag, fnLogDBHostFlag, fnLogDBTypeFlag, fnUserNameFlag, fnPasswordFlag}, Action: fnPods},
{Name: "logs", Usage: "Display funtion logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs},
{Name: "pods", Usage: "Display funtion pods", Flags: []cli.Flag{fnNameFlag, fnLogDBTypeFlag}, Action: fnPods},
}
// httptriggers