Binary Environment (#256)

Adds a fission environment for arbitrary linux binaries.

environments/binary/Dockerfile

@@ -0,0 +1,10 @@
+FROM alpine:3.5
+
+RUN apk update
+RUN apk add coreutils binutils findutils grep
+
+COPY . /app
+WORKDIR /app
+
+EXPOSE 8888
+ENTRYPOINT ["./server"]

environments/binary/README.md

@@ -0,0 +1,50 @@
+# Binary Environment Examples
+
+The `binary` runtime is a go server that uses a subprocess to invoke executables or execute shell scripts.
+
+Use Cases
+- Execute bash scripts
+- Run executables of languages that have no dedicated environment yet.
+
+## Words of Caution
+The environment runs on an alpine image with some additional utilty commandline tools installed, such as 'grep'. 
+However, in case you want to make use of more ecsoteric commandline tools, you should add the relevant apk to the 
+Dockerfile and build a new binary environment. See 'Compiling' for instructions.
+
+When executing functions using binaries, **ensure that the executable is built for the right architecture**. 
+Using the default binary environment this means that the binary should be build for Linux.
+
+## Usage
+The interface to the executable used by this environment is somewhat similar to a [CGI interface](https://en.wikipedia.org/wiki/Common_Gateway_Interface).
+This means that any HTTP headers are converted to environment variables of the form "HTTP_<header-name>". For example these
+are some of frequently occurring headers:
+
+```
+# Request Metadata
+CONTENT_LENGTH
+REQUEST_URI
+REQUEST_METHOD
+
+# HTTP Headers
+HTTP_ACCEPT
+HTTP_USER-AGENT
+...
+```
+
+The body of HTTP piped over the STDIN to the executable. 
+All output that is provided to the server over the STDOUT will be transformed into the HTTP response.
+
+
+## Compiling
+In order to build the Dockerfile, the server needs to be compiled to the right architecture.
+
+```bash
+sh ./build.sh
+```
+
+Build the Dockerfile:
+```bash
+docker build --tag=${USER}/binary-env .
+```
+
+See the [README](../../examples/binary/README.md) in the binary examples directory for usage instructions.

environments/binary/build.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+GOOS=linux GOARCH=386 go build -o server .
+

environments/binary/env.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Utility functions for working with environment variables
+type Env struct {
+	Vars []*EnvVar
+}
+
+type EnvVar struct {
+	Key string
+	Val string
+}
+
+func FromString(rawEnvVar string) *EnvVar {
+	parts := strings.SplitN(rawEnvVar, "=", 2)
+	return &EnvVar{parts[0], parts[1]}
+}
+
+func (ev *EnvVar) ToString() string {
+	return fmt.Sprintf("%s=%s", ev.Key, ev.Val)
+}
+
+func (e *Env) SetEnv(envVar *EnvVar) {
+	e.Vars = append(e.Vars, envVar)
+}
+
+func (e *Env) ToStringEnv() []string {
+	var result []string
+	for _, envVar := range e.Vars {
+		result = append(result, envVar.ToString())
+	}
+	return result
+}
+
+func NewEnv(stringEnv []string) *Env {
+	env := &Env{}
+	if stringEnv != nil {
+		for _, rawEnvVar := range stringEnv {
+			env.SetEnv(FromString(rawEnvVar))
+		}
+	}
+	return env
+}

environments/binary/server.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	DEFAULT_CODE_PATH          = "/userfunc/user"
+	DEFAULT_INTERNAL_CODE_PATH = "/bin/userfunc"
+)
+
+var specialized bool
+
+type BinaryServer struct {
+	fetchedCodePath  string
+	internalCodePath string
+}
+
+func (bs *BinaryServer) SpecializeHandler(w http.ResponseWriter, r *http.Request) {
+	if specialized {
+		w.WriteHeader(400)
+		w.Write([]byte("Not a generic container"))
+		return
+	}
+
+	_, err := os.Stat(bs.fetchedCodePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(bs.fetchedCodePath + ": not found"))
+			return
+		} else {
+			panic(err)
+		}
+	}
+
+	// Future: Check if executable is correct architecture/executable.
+
+	// Copy the executable to ensure that file is executable and immutable.
+	userFunc, err := ioutil.ReadFile(bs.fetchedCodePath)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Failed to read executable."))
+		return
+	}
+	err = ioutil.WriteFile(bs.internalCodePath, userFunc, 0555)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Failed to write executable to target location."))
+		return
+	}
+
+	fmt.Println("Specializing ...")
+	specialized = true
+	fmt.Println("Done")
+}
+
+func (bs *BinaryServer) InvocationHandler(w http.ResponseWriter, r *http.Request) {
+	if !specialized {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Generic container: no requests supported"))
+		return
+	}
+
+	// CGI-like passing of environment variables
+	execEnv := NewEnv(nil)
+	execEnv.SetEnv(&EnvVar{"REQUEST_METHOD", r.Method})
+	execEnv.SetEnv(&EnvVar{"REQUEST_URI", r.RequestURI})
+	execEnv.SetEnv(&EnvVar{"CONTENT_LENGTH", fmt.Sprintf("%d", r.ContentLength)})
+
+	for header, val := range r.Header {
+		execEnv.SetEnv(&EnvVar{fmt.Sprintf("HTTP_%s", strings.ToUpper(header)), val[0]})
+	}
+
+	// Future: could be improved by keeping subprocess open while environment is specialized
+	cmd := exec.Command(bs.internalCodePath)
+	cmd.Env = execEnv.ToStringEnv()
+
+	if r.ContentLength != 0 {
+		fmt.Println(r.ContentLength)
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf("Failed to get STDIN pipe: %s", err)))
+			panic(err)
+		}
+		_, err = io.Copy(stdin, r.Body)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(fmt.Sprintf("Failed to pipe input: %s", err)))
+		}
+		stdin.Close()
+	}
+
+	out, err := cmd.Output()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(fmt.Sprintf("Function error: %s", err)))
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	w.Write(out)
+}
+
+func main() {
+	codePath := flag.String("c", DEFAULT_CODE_PATH, "Path to expected fetched executable.")
+	internalCodePath := flag.String("i", DEFAULT_INTERNAL_CODE_PATH, "Path to specialized executable.")
+	flag.Parse()
+	absInternalCodePath, err := filepath.Abs(*internalCodePath)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("Using fetched code path: %s\n", *codePath)
+	fmt.Printf("Using internal code path: %s\n", absInternalCodePath)
+
+	server := &BinaryServer{*codePath, absInternalCodePath}
+	http.HandleFunc("/", server.InvocationHandler)
+	http.HandleFunc("/specialize", server.SpecializeHandler)
+
+	fmt.Println("Listening on 8888 ...")
+	err = http.ListenAndServe(":8888", nil)
+	if err != nil {
+		panic(err)
+	}
+}