diff --git a/builder/builder.go b/builder/builder.go new file mode 100644 index 00000000..afd7e560 --- /dev/null +++ b/builder/builder.go @@ -0,0 +1,166 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/dchest/uniuri" +) + +const ( + // supported environment variables + envSrcPkg = "SRC_PKG" + envDeployPkg = "DEPLOY_PKG" +) + +type ( + PackageBuildRequest struct { + SrcPkgFilename string `json:"srcPkgFilename"` + // Command for builder to run with. + // A build command consists of commands, parameters and environment variables. + // For now, two environment variables are supported: + // 1. SRC_PKG: path to source package directory + // 2. DEPLOY_PKG: path to deployment package directory + BuildCommand string `json:"command"` + } + + PackageBuildResponse struct { + ArtifactFilename string `json:"artifactFilename"` + BuildLogs string `json:"buildLogs"` + } + + Builder struct { + sharedVolumePath string + } +) + +func MakeBuilder(sharedVolumePath string) *Builder { + return &Builder{ + sharedVolumePath: sharedVolumePath, + } +} + +func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + http.Error(w, "", 405) + return + } + + startTime := time.Now() + defer func() { + elapsed := time.Now().Sub(startTime) + log.Printf("elapsed time in build request = %v", elapsed) + }() + + // parse request + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("Error reading request body") + http.Error(w, err.Error(), 500) + return + } + var req PackageBuildRequest + err = json.Unmarshal(body, &req) + if err != nil { + log.Printf("Error parsing json body: %v", err) + http.Error(w, err.Error(), 400) + return + } + log.Printf("Builder received request: %v", req) + + log.Println("Starting build...") + srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename) + deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6))) + deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename) + buildLogs, err := builder.build(req.BuildCommand, srcPkgPath, deployPkgPath) + if err != nil { + e := errors.New(fmt.Sprintf("Error building source package: %v", err)) + http.Error(w, e.Error(), 500) + return + } + + resp := PackageBuildResponse{ + ArtifactFilename: deployPkgFilename, + BuildLogs: buildLogs, + } + + rBody, err := json.Marshal(resp) + if err != nil { + e := errors.New(fmt.Sprintf("Error encoding response body: %v", err)) + http.Error(w, e.Error(), 500) + return + } + + w.Header().Add("Content-Type", "application/json") + w.Write(rBody) + w.WriteHeader(http.StatusOK) +} + +func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath string) (string, error) { + cmd := exec.Command(command) + cmd.Dir = srcPkgPath + // set env variables for build command + cmd.Env = append(os.Environ(), + fmt.Sprintf("%v=%v", envSrcPkg, srcPkgPath), + fmt.Sprintf("%v=%v", envDeployPkg, deployPkgPath), + ) + + cmdReader, err := cmd.StdoutPipe() + if err != nil { + return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error())) + } + + scanner := bufio.NewScanner(cmdReader) + + err = cmd.Start() + if err != nil { + return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error())) + } + + var buildLogs string + + fmt.Println("\n=== Build Logs ===") + for scanner.Scan() { + output := scanner.Text() + fmt.Println(output) + buildLogs += fmt.Sprintf("%v\n", output) + } + fmt.Println("==================\n") + + if err := scanner.Err(); err != nil { + return "", errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error())) + } + + err = cmd.Wait() + if err != nil { + return "", errors.New(fmt.Sprintf("Error waiting for cmd: %v", err.Error())) + } + + return buildLogs, nil +} diff --git a/builder/client/client.go b/builder/client/client.go new file mode 100644 index 00000000..0fe806c4 --- /dev/null +++ b/builder/client/client.go @@ -0,0 +1,56 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "bytes" + "encoding/json" + "net/http" + "strings" + + "github.com/fission/fission" + builder "github.com/fission/fission/builder" +) + +type ( + Client struct { + url string + } +) + +func MakeClient(serverUrl string) *Client { + return &Client{ + url: strings.TrimSuffix(serverUrl, "/"), + } +} + +func (c *Client) Build(req *builder.PackageBuildRequest) error { + body, err := json.Marshal(req) + if err != nil { + return err + } + resp, err := http.Post(c.url, "application/json", bytes.NewReader(body)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fission.MakeErrorFromHTTP(resp) + } + return nil +} diff --git a/builder/cmd/Dockerfile b/builder/cmd/Dockerfile new file mode 100644 index 00000000..11afba20 --- /dev/null +++ b/builder/cmd/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.4 + +ADD builder / + +EXPOSE 8000 diff --git a/builder/cmd/build.sh b/builder/cmd/build.sh new file mode 100755 index 00000000..69ba9c97 --- /dev/null +++ b/builder/cmd/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +GOOS=linux GOARCH=386 go build -o builder . diff --git a/builder/cmd/main.go b/builder/cmd/main.go new file mode 100644 index 00000000..b1619f85 --- /dev/null +++ b/builder/cmd/main.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + "net/http" + "os" + + builder "github.com/fission/fission/builder" +) + +// Usage: builder +func main() { + dir := os.Args[1] + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + err = os.MkdirAll(dir, os.ModeDir|0700) + if err != nil { + log.Fatalf("Error creating directory: %v", err) + } + } + } + builder := builder.MakeBuilder(dir) + mux := http.NewServeMux() + mux.HandleFunc("/", builder.Handler) + http.ListenAndServe(":8000", mux) +} diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 23d5ed7d..b42de2b4 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "io/ioutil" "log" "net/http" @@ -17,7 +18,6 @@ import ( "github.com/fission/fission" "github.com/fission/fission/tpr" - "io" ) type ( @@ -102,7 +102,7 @@ func verifyChecksum(path string, checksum *fission.Checksum) error { func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { - http.Error(w, "", 404) + http.Error(w, "", 405) return } @@ -123,7 +123,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(body, &req) if err != nil { log.Printf("Error reading request body: %v", err) - http.Error(w, err.Error(), 500) + http.Error(w, err.Error(), 400) return } log.Printf("fetcher received request: %v", req) diff --git a/environments/python3/builder/Dockerfile b/environments/python3/builder/Dockerfile new file mode 100644 index 00000000..2cb3c9cc --- /dev/null +++ b/environments/python3/builder/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine:3.5 + +RUN apk update +RUN apk add --no-cache python3 python3-dev build-base +RUN pip3 install --upgrade pip +RUN rm -r /root/.cache + +ADD defaultBuildCmd /usr/local/bin/build +ADD builder / + +EXPOSE 8000 diff --git a/environments/python3/builder/build.sh b/environments/python3/builder/build.sh new file mode 100755 index 00000000..b70d3580 --- /dev/null +++ b/environments/python3/builder/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e +builderDir=${GOPATH}/src/github.com/fission/fission/builder/cmd +pushd ${builderDir} +GOOS=linux GOARCH=386 go build -o builder . +popd +cp ${builderDir}/builder . +docker build -t python-builder . +docker tag python-builder fission/python-builder:$tag +docker push fission/python-builder:$tag + diff --git a/environments/python3/builder/defaultBuildCmd b/environments/python3/builder/defaultBuildCmd new file mode 100755 index 00000000..e9215130 --- /dev/null +++ b/environments/python3/builder/defaultBuildCmd @@ -0,0 +1,2 @@ +#!/bin/sh +pip3 install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG} \ No newline at end of file