Add builder manager support (#308)

This change orchestrates function builds. Environments (in v2) define a builder image, just like they do a runtime image. The builder image contains a build script that's invoked with source and deployment paths (as env vars). The buildermgr watches for environments with build images defined, and creates build deployments and services. Functions can define source and deployment. Buildermgr watches for functions with source code (and build status == pending) and invokes the environment's builder when appropriate. It captures logs from the build and sets the build status (success/failure) and build lots into the PackageStatus.
2017-09-25 07:53:09 -07:00
parent a720377f3c
commit e587eca08f
40 changed files with 1827 additions and 176 deletions
@@ -3,6 +3,8 @@ package client
 import (
 	"bytes"
 	"encoding/json"
+	"io/ioutil"
+	"log"
 	"net/http"
 	//"time"

@@ -11,7 +13,19 @@ import (
 	//"github.com/fission/fission/router"
 )

-func DoFetchRequest(fetcherUrl string, fr *fetcher.FetchRequest) error {
+type (
+	Client struct {
+		url string
+	}
+)
+
+func MakeClient(fetcherUrl string) *Client {
+	return &Client{
+		url: fetcherUrl,
+	}
+}
+
+func (c *Client) Fetch(fr *fetcher.FetchRequest) error {
 	body, err := json.Marshal(fr)
 	if err != nil {
 		return err
@@ -21,7 +35,7 @@ func DoFetchRequest(fetcherUrl string, fr *fetcher.FetchRequest) error {
 	// 	Transport: router.MakeRetryingRoundTripper(10, 50*time.Millisecond),
 	// }

-	resp, err := http.Post(fetcherUrl, "application/json", bytes.NewReader(body))
+	resp, err := http.Post(c.url, "application/json", bytes.NewReader(body))
 	if err != nil {
 		return err
 	}
@@ -33,3 +47,34 @@ func DoFetchRequest(fetcherUrl string, fr *fetcher.FetchRequest) error {

 	return nil
 }
+
+func (c *Client) Upload(fr *fetcher.UploadRequest) (*fetcher.UploadResponse, error) {
+	body, err := json.Marshal(fr)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.Post(c.url+"/upload", "application/json", bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, fission.MakeErrorFromHTTP(resp)
+	}
+
+	rBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Printf("Received upload response: %v", string(rBody))
+
+	uploadReq := fetcher.UploadResponse{}
+	err = json.Unmarshal([]byte(rBody), &uploadReq)
+	if err != nil {
+		return nil, err
+	}
+
+	return &uploadReq, nil
+}
@@ -21,6 +21,7 @@ func main() {
 	}
 	fetcher := fetcher.MakeFetcher(dir)
 	mux := http.NewServeMux()
-	mux.HandleFunc("/", fetcher.Handler)
+	mux.HandleFunc("/", fetcher.FetchHandler)
+	mux.HandleFunc("/upload", fetcher.UploadHandler)
 	http.ListenAndServe(":8000", mux)
 }
@@ -20,6 +20,7 @@ import (
 	"k8s.io/client-go/1.5/pkg/api"

 	"github.com/fission/fission"
+	storageSvcClient "github.com/fission/fission/storagesvc/client"
 	"github.com/fission/fission/tpr"
 )

@@ -28,12 +29,26 @@ type (

 	FetchRequest struct {
 		FetchType     FetchRequestType `json:"fetchType"`
-		Function      api.ObjectMeta   `json:"function"`
+		Package       api.ObjectMeta   `json:"package"`
 		Url           string           `json:"url"`
 		StorageSvcUrl string           `json:"storagesvcurl"`
 		Filename      string           `json:"filename"`
 	}

+	// UploadRequest send from builder manager describes which
+	// deployment package should be upload to storage service.
+	UploadRequest struct {
+		Filename      string `json:"filename"`
+		StorageSvcUrl string `json:"storagesvcurl"`
+	}
+
+	// UploadResponse defines the download url of an archive and
+	// its checksum.
+	UploadResponse struct {
+		ArchiveDownloadUrl string           `json:"archiveDownloadUrl"`
+		Checksum           fission.Checksum `json:"checksum"`
+	}
+
 	Fetcher struct {
 		sharedVolumePath string
 		fissionClient    *tpr.FissionClient
@@ -79,33 +94,44 @@ func downloadUrl(url string, localPath string) error {
 	return nil
 }

-func verifyChecksum(path string, checksum *fission.Checksum) error {
-	if checksum.Type != fission.ChecksumTypeSHA256 {
-		return fission.MakeError(fission.ErrorInvalidArgument, "Unsupported checksum type")
-	}
-
+func getChecksum(path string) (*fission.Checksum, error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer f.Close()

 	hasher := sha256.New()
 	_, err = io.Copy(hasher, f)
 	if err != nil {
-		return err
+		return nil, err
 	}

 	c := hex.EncodeToString(hasher.Sum(nil))
-	if c != checksum.Sum {
+
+	return &fission.Checksum{
+		Type: fission.ChecksumTypeSHA256,
+		Sum:  c,
+	}, nil
+}
+
+func verifyChecksum(path string, checksum *fission.Checksum) error {
+	if checksum.Type != fission.ChecksumTypeSHA256 {
+		return fission.MakeError(fission.ErrorInvalidArgument, "Unsupported checksum type")
+	}
+	c, err := getChecksum(path)
+	if err != nil {
+		return err
+	}
+	if c.Sum != checksum.Sum {
 		return fission.MakeError(fission.ErrorChecksumFail, "Checksum validation failed")
 	}
 	return nil
 }

-func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
+func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "POST" {
-		http.Error(w, "", 405)
+		http.Error(w, "only POST is supported on this endpoint", 405)
 		return
 	}

@@ -129,11 +155,13 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), 400)
 		return
 	}
-	log.Printf("fetcher received request: %v", req)
+	log.Printf("fetcher received fetch request: %v", req)

 	tmpFile := req.Filename + ".tmp"
 	tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile)

+	log.Printf("Start downloading...")
+
 	if req.FetchType == FETCH_URL {
 		// fetch the file and save it to the tmp path
 		err := downloadUrl(req.Url, tmpPath)
@@ -144,19 +172,8 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	} else {
-		// get function object
-		fn, err := fetcher.fissionClient.Functions(req.Function.Namespace).Get(req.Function.Name)
-		if err != nil {
-			e := fmt.Sprintf("Failed to get function: %v", err)
-			log.Printf(e)
-			http.Error(w, e, 500)
-			return
-		}
-
 		// get pkg
-		var pkg *tpr.Package
-		pkg, err = fetcher.fissionClient.
-			Packages(fn.Spec.Package.PackageRef.Namespace).Get(fn.Spec.Package.PackageRef.Name)
+		pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name)
 		if err != nil {
 			e := fmt.Sprintf("Failed to get package: %v", err)
 			log.Printf(e)
@@ -200,7 +217,6 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		}
-
 	}

 	// check file type here, if the file is a zip file unarchive it.
@@ -229,6 +245,85 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }

+func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "only POST is supported on this endpoint", 405)
+		return
+	}
+
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Now().Sub(startTime)
+		log.Printf("elapsed time in upload request = %v", elapsed)
+	}()
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Printf("Error reading request body")
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	var req UploadRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		log.Printf("Error reading request body: %v", err)
+		http.Error(w, err.Error(), 400)
+		return
+	}
+	log.Printf("fetcher received upload request: %v", req)
+
+	zipFilename := req.Filename + ".zip"
+	srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
+	dstFilepath := filepath.Join(fetcher.sharedVolumePath, zipFilename)
+
+	err = fetcher.archive(srcFilepath, dstFilepath)
+	if err != nil {
+		e := fmt.Sprintf("Error archiving zip file: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	log.Println("Start uploading...")
+	ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl)
+
+	fileID, err := ssClient.Upload(dstFilepath, nil)
+	if err != nil {
+		e := fmt.Sprintf("Error uploading zip file: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	sum, err := getChecksum(dstFilepath)
+	if err != nil {
+		e := fmt.Sprintf("Error calculating checksum of zip file: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	resp := UploadResponse{
+		ArchiveDownloadUrl: ssClient.GetUrl(fileID),
+		Checksum:           *sum,
+	}
+
+	rBody, err := json.Marshal(resp)
+	if err != nil {
+		e := fmt.Sprintf("Error encoding upload response: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	log.Println("Completed upload request")
+	w.Header().Add("Content-Type", "application/json")
+	w.Write(rBody)
+	w.WriteHeader(http.StatusOK)
+}
+
 func (fetcher *Fetcher) rename(src string, dst string) error {
 	err := os.Rename(src, dst)
 	if err != nil {
@@ -239,7 +334,21 @@ func (fetcher *Fetcher) rename(src string, dst string) error {

 // archive is a function that zips directory into a zip file
 func (fetcher *Fetcher) archive(src string, dst string) error {
-	return archiver.Zip.Make(dst, []string{src})
+	var files []string
+	target, err := os.Stat(src)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Failed to zip file: %v", err))
+	}
+	if target.IsDir() {
+		// list all
+		fs, _ := ioutil.ReadDir(src)
+		for _, f := range fs {
+			files = append(files, filepath.Join(src, f.Name()))
+		}
+	} else {
+		files = append(files, src)
+	}
+	return archiver.Zip.Make(dst, files)
 }

 // unarchive is a function that unzips a zip file to destination