Binary Environment (#256)

Adds a fission environment for arbitrary linux binaries.

.gitignore

@@ -10,3 +10,4 @@ environments/php7/vendor/
 *~
 
 vendor/
+local/

environments/binary/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.5
+
+RUN apk update
+RUN apk add coreutils binutils findutils grep
+
+COPY . /app
+WORKDIR /app
+
+EXPOSE 8888
+ENTRYPOINT ["./server"]

environments/binary/README.md

@@ -0,0 +1,50 @@
+# Binary Environment Examples
+
+The `binary` runtime is a go server that uses a subprocess to invoke executables or execute shell scripts.
+
+Use Cases
+- Execute bash scripts
+- Run executables of languages that have no dedicated environment yet.
+
+## Words of Caution
+The environment runs on an alpine image with some additional utilty commandline tools installed, such as 'grep'. 
+However, in case you want to make use of more ecsoteric commandline tools, you should add the relevant apk to the 
+Dockerfile and build a new binary environment. See 'Compiling' for instructions.
+
+When executing functions using binaries, **ensure that the executable is built for the right architecture**. 
+Using the default binary environment this means that the binary should be build for Linux.
+
+## Usage
+The interface to the executable used by this environment is somewhat similar to a [CGI interface](https://en.wikipedia.org/wiki/Common_Gateway_Interface).
+This means that any HTTP headers are converted to environment variables of the form "HTTP_<header-name>". For example these
+are some of frequently occurring headers:
+
+```
+# Request Metadata
+CONTENT_LENGTH
+REQUEST_URI
+REQUEST_METHOD
+
+# HTTP Headers
+HTTP_ACCEPT
+HTTP_USER-AGENT
+...
+```
+
+The body of HTTP piped over the STDIN to the executable. 
+All output that is provided to the server over the STDOUT will be transformed into the HTTP response.
+
+
+## Compiling
+In order to build the Dockerfile, the server needs to be compiled to the right architecture.
+
+```bash
+sh ./build.sh
+```
+
+Build the Dockerfile:
+```bash
+docker build --tag=${USER}/binary-env .
+```
+
+See the [README](../../examples/binary/README.md) in the binary examples directory for usage instructions.

environments/binary/build.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+GOOS=linux GOARCH=386 go build -o server .
+

environments/binary/env.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Utility functions for working with environment variables
+type Env struct {
+	Vars []*EnvVar
+}
+
+type EnvVar struct {
+	Key string
+	Val string
+}
+
+func FromString(rawEnvVar string) *EnvVar {
+	parts := strings.SplitN(rawEnvVar, "=", 2)
+	return &EnvVar{parts[0], parts[1]}
+}
+
+func (ev *EnvVar) ToString() string {
+	return fmt.Sprintf("%s=%s", ev.Key, ev.Val)
+}
+
+func (e *Env) SetEnv(envVar *EnvVar) {
+	e.Vars = append(e.Vars, envVar)
+}
+
+func (e *Env) ToStringEnv() []string {
+	var result []string
+	for _, envVar := range e.Vars {
+		result = append(result, envVar.ToString())
+	}
+	return result
+}
+
+func NewEnv(stringEnv []string) *Env {
+	env := &Env{}
+	if stringEnv != nil {
+		for _, rawEnvVar := range stringEnv {
+			env.SetEnv(FromString(rawEnvVar))
+		}
+	}
+	return env
+}

environments/binary/server.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	DEFAULT_CODE_PATH          = "/userfunc/user"
+	DEFAULT_INTERNAL_CODE_PATH = "/bin/userfunc"
+)
+
+var specialized bool
+
+type BinaryServer struct {
+	fetchedCodePath  string
+	internalCodePath string
+}
+
+func (bs *BinaryServer) SpecializeHandler(w http.ResponseWriter, r *http.Request) {
+	if specialized {
+		w.WriteHeader(400)
+		w.Write([]byte("Not a generic container"))
+		return
+	}
+
+	_, err := os.Stat(bs.fetchedCodePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(bs.fetchedCodePath + ": not found"))
+			return
+		} else {
+			panic(err)
+		}
+	}
+
+	// Future: Check if executable is correct architecture/executable.
+
+	// Copy the executable to ensure that file is executable and immutable.
+	userFunc, err := ioutil.ReadFile(bs.fetchedCodePath)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Failed to read executable."))
+		return
+	}
+	err = ioutil.WriteFile(bs.internalCodePath, userFunc, 0555)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Failed to write executable to target location."))
+		return
+	}
+
+	fmt.Println("Specializing ...")
+	specialized = true
+	fmt.Println("Done")
+}
+
+func (bs *BinaryServer) InvocationHandler(w http.ResponseWriter, r *http.Request) {
+	if !specialized {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Generic container: no requests supported"))
+		return
+	}
+
+	// CGI-like passing of environment variables
+	execEnv := NewEnv(nil)
+	execEnv.SetEnv(&EnvVar{"REQUEST_METHOD", r.Method})
+	execEnv.SetEnv(&EnvVar{"REQUEST_URI", r.RequestURI})
+	execEnv.SetEnv(&EnvVar{"CONTENT_LENGTH", fmt.Sprintf("%d", r.ContentLength)})
+
+	for header, val := range r.Header {
+		execEnv.SetEnv(&EnvVar{fmt.Sprintf("HTTP_%s", strings.ToUpper(header)), val[0]})
+	}
+
+	// Future: could be improved by keeping subprocess open while environment is specialized
+	cmd := exec.Command(bs.internalCodePath)
+	cmd.Env = execEnv.ToStringEnv()
+
+	if r.ContentLength != 0 {
+		fmt.Println(r.ContentLength)
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf("Failed to get STDIN pipe: %s", err)))
+			panic(err)
+		}
+		_, err = io.Copy(stdin, r.Body)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf("Failed to pipe input: %s", err)))
+		}
+		stdin.Close()
+	}
+
+	out, err := cmd.Output()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(fmt.Sprintf("Function error: %s", err)))
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	w.Write(out)
+}
+
+func main() {
+	codePath := flag.String("c", DEFAULT_CODE_PATH, "Path to expected fetched executable.")
+	internalCodePath := flag.String("i", DEFAULT_INTERNAL_CODE_PATH, "Path to specialized executable.")
+	flag.Parse()
+	absInternalCodePath, err := filepath.Abs(*internalCodePath)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("Using fetched code path: %s\n", *codePath)
+	fmt.Printf("Using internal code path: %s\n", absInternalCodePath)
+
+	server := &BinaryServer{*codePath, absInternalCodePath}
+	http.HandleFunc("/", server.InvocationHandler)
+	http.HandleFunc("/specialize", server.SpecializeHandler)
+
+	fmt.Println("Listening on 8888 ...")
+	err = http.ListenAndServe(":8888", nil)
+	if err != nil {
+		panic(err)
+	}
+}

examples/binary/README.md

@@ -0,0 +1,80 @@
+# Binary Environment Examples
+
+The `binary` runtime is a go server that uses a subprocess to invoke executables or execute shell scripts.
+
+For more info read the [environment README](../../environments/binary/README.md).
+
+## Requirements
+
+First, set up your fission deployment with the binary environment.
+
+```bash
+fission env create --name binary-env --image fission/binary-env
+```
+
+## Example Usage
+
+### hello.sh
+`hello.sh` is an very basic shell script that returns `"Hello, World!"`.
+
+```bash
+# Upload the function to fission
+fission function create --name hello --env binary-env --code hello.sh
+
+# Map /hello to the hello function
+fission route create --method GET --url /hello --function hello
+
+# Run the function
+curl http://$FISSION_ROUTER/hello
+```
+
+This should return a HTTP response with the body `Hello World!`
+
+### echo.sh
+`echo.sh` shows the the use of STDIN to read the request body, echoing the input back in the response.
+
+```bash
+# Upload the function to fission
+fission function create --name echo --env binary-env --code echo.sh
+
+# Map /hello to the hello function
+fission route create --method POST --url /echo --function echo
+
+# Run the function
+curl -XPOST -d 'Echoooooo!'  http://$FISSION_ROUTER/echo
+```
+This should return a HTTP response with the body `... Echoooooo!`.
+
+
+### headers.sh
+`headers.sh` shows the access to the environment variables that hold the HTTP headers, returning the set HTTP headers.
+
+```bash
+# Upload the function to fission
+fission function create --name headers --env binary-env --code headers.sh
+
+# Map /hello to the hello function
+fission route create --url /headers --function headers
+
+# Run the function
+curl -H 'X-FOO: BAR'  http://$FISSION_ROUTER/headers
+```
+This should return a HTTP response with the body `... Echoooooo!`.
+
+### hello..go
+This example shows the differences between using shell scripts and binaries. `hello.go` returns `Hello World!` + the 
+environment variables it received from the server.
+
+```bash
+# Build the function targeted at the right architecture
+GOOS=linux GOARCH=386 go build -o hello-go-func hello.go
+
+# Upload the function to fission
+fission function create --name hello-go --env binary-env --code hello-go-func
+
+# Map /hello to the hello function
+fission route create --url /hello-go --function hello-go
+
+# Run the function
+curl -H 'X-GO: AWESOME!'  http://$FISSION_ROUTER/hello-go
+```

examples/binary/echo.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+printf "... "
+cat -

examples/binary/headers.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+env | grep "^HTTP_"

examples/binary/hello.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+// See README.md in the examples/binary directory for instructions
+func main() {
+	fmt.Println("Hello World!")
+	fmt.Printf("Environment: %v", os.Environ())
+}

examples/binary/hello.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Hello World!"