Add Environment v2 Builder (#298)

Build server and client for environments.

This build server will run inside environment builders and invoke environment-specific scripts for doing builds. It provides a uniform language-independent API for builds to the fission build manager.

The change also includes an implementation of a builder for Python -- this "builder" acts on requirements.txt and runs `pip install` to collect all function deps.

builder/builder.go

@@ -0,0 +1,166 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/dchest/uniuri"
+)
+
+const (
+	// supported environment variables
+	envSrcPkg    = "SRC_PKG"
+	envDeployPkg = "DEPLOY_PKG"
+)
+
+type (
+	PackageBuildRequest struct {
+		SrcPkgFilename string `json:"srcPkgFilename"`
+		// Command for builder to run with.
+		// A build command consists of commands, parameters and environment variables.
+		// For now, two environment variables are supported:
+		// 1. SRC_PKG: path to source package directory
+		// 2. DEPLOY_PKG: path to deployment package directory
+		BuildCommand string `json:"command"`
+	}
+
+	PackageBuildResponse struct {
+		ArtifactFilename string `json:"artifactFilename"`
+		BuildLogs        string `json:"buildLogs"`
+	}
+
+	Builder struct {
+		sharedVolumePath string
+	}
+)
+
+func MakeBuilder(sharedVolumePath string) *Builder {
+	return &Builder{
+		sharedVolumePath: sharedVolumePath,
+	}
+}
+
+func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "", 405)
+		return
+	}
+
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Now().Sub(startTime)
+		log.Printf("elapsed time in build request = %v", elapsed)
+	}()
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("Error reading request body")
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	var req PackageBuildRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		log.Printf("Error parsing json body: %v", err)
+		http.Error(w, err.Error(), 400)
+		return
+	}
+	log.Printf("Builder received request: %v", req)
+
+	log.Println("Starting build...")
+	srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename)
+	deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6)))
+	deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename)
+	buildLogs, err := builder.build(req.BuildCommand, srcPkgPath, deployPkgPath)
+	if err != nil {
+		e := errors.New(fmt.Sprintf("Error building source package: %v", err))
+		http.Error(w, e.Error(), 500)
+		return
+	}
+
+	resp := PackageBuildResponse{
+		ArtifactFilename: deployPkgFilename,
+		BuildLogs:        buildLogs,
+	}
+
+	rBody, err := json.Marshal(resp)
+	if err != nil {
+		e := errors.New(fmt.Sprintf("Error encoding response body: %v", err))
+		http.Error(w, e.Error(), 500)
+		return
+	}
+
+	w.Header().Add("Content-Type", "application/json")
+	w.Write(rBody)
+	w.WriteHeader(http.StatusOK)
+}
+
+func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath string) (string, error) {
+	cmd := exec.Command(command)
+	cmd.Dir = srcPkgPath
+	// set env variables for build command
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("%v=%v", envSrcPkg, srcPkgPath),
+		fmt.Sprintf("%v=%v", envDeployPkg, deployPkgPath),
+	)
+
+	cmdReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error()))
+	}
+
+	scanner := bufio.NewScanner(cmdReader)
+
+	err = cmd.Start()
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error()))
+	}
+
+	var buildLogs string
+
+	fmt.Println("\n=== Build Logs ===")
+	for scanner.Scan() {
+		output := scanner.Text()
+		fmt.Println(output)
+		buildLogs += fmt.Sprintf("%v\n", output)
+	}
+	fmt.Println("==================\n")
+
+	if err := scanner.Err(); err != nil {
+		return "", errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error()))
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("Error waiting for cmd: %v", err.Error()))
+	}
+
+	return buildLogs, nil
+}

builder/client/client.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/fission/fission"
+	builder "github.com/fission/fission/builder"
+)
+
+type (
+	Client struct {
+		url string
+	}
+)
+
+func MakeClient(serverUrl string) *Client {
+	return &Client{
+		url: strings.TrimSuffix(serverUrl, "/"),
+	}
+}
+
+func (c *Client) Build(req *builder.PackageBuildRequest) error {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return err
+	}
+	resp, err := http.Post(c.url, "application/json", bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return fission.MakeErrorFromHTTP(resp)
+	}
+	return nil
+}

builder/cmd/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.4
+
+ADD builder /
+
+EXPOSE 8000

builder/cmd/build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+GOOS=linux GOARCH=386 go build -o builder .

builder/cmd/main.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"os"
+
+	builder "github.com/fission/fission/builder"
+)
+
+// Usage: builder <shared volume path>
+func main() {
+	dir := os.Args[1]
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(dir, os.ModeDir|0700)
+			if err != nil {
+				log.Fatalf("Error creating directory: %v", err)
+			}
+		}
+	}
+	builder := builder.MakeBuilder(dir)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", builder.Handler)
+	http.ListenAndServe(":8000", mux)
+}

environments/fetcher/fetcher.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -17,7 +18,6 @@ import (
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/tpr"
-	"io"
 )
 
 type (
@@ -102,7 +102,7 @@ func verifyChecksum(path string, checksum *fission.Checksum) error {
 
 func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "POST" {
-		http.Error(w, "", 404)
+		http.Error(w, "", 405)
 		return
 	}
 
@@ -123,7 +123,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 	err = json.Unmarshal(body, &req)
 	if err != nil {
 		log.Printf("Error reading request body: %v", err)
-		http.Error(w, err.Error(), 500)
+		http.Error(w, err.Error(), 400)
 		return
 	}
 	log.Printf("fetcher received request: %v", req)

environments/python3/builder/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.5
+
+RUN apk update
+RUN apk add --no-cache python3 python3-dev build-base
+RUN pip3 install --upgrade pip
+RUN rm -r /root/.cache
+
+ADD defaultBuildCmd /usr/local/bin/build
+ADD builder /
+
+EXPOSE 8000

environments/python3/builder/build.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+builderDir=${GOPATH}/src/github.com/fission/fission/builder/cmd
+pushd ${builderDir}
+GOOS=linux GOARCH=386 go build -o builder .
+popd
+cp ${builderDir}/builder .
+docker build -t python-builder .
+docker tag python-builder fission/python-builder:$tag
+docker push fission/python-builder:$tag
+

environments/python3/builder/defaultBuildCmd

@@ -0,0 +1,2 @@
+#!/bin/sh
+pip3 install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG}