Move builds to package level (#297)

Introduce _Archives_ as a type to reference arbitrary blobs.

_Packages_ are a pair of Source and Deployment archives. Packages have an environment reference.

Functions reference a package. Multiple functions can reference the same package.

controller/api_test.go

@@ -81,8 +81,10 @@ func TestFunctionApi(t *testing.T) {
 			Namespace: api.NamespaceDefault,
 		},
 		Spec: fission.FunctionSpec{
-			EnvironmentName: "nodejs",
+			Environment: fission.EnvironmentReference{
-			Deployment: fission.FunctionPackageRef{
+				Name: "nodejs",
+			},
+			Package: fission.FunctionPackageRef{
 				FunctionName: "xxx",
 			},
 		},
@@ -99,7 +101,7 @@ func TestFunctionApi(t *testing.T) {
 	_, err = g.client.FunctionCreate(testFunc)
 	assertNameReuseFailure(err, "function")
 
-	testFunc.Spec.Deployment.FunctionName = "yyy"
+	testFunc.Spec.Package.FunctionName = "yyy"
 	_, err = g.client.FunctionUpdate(testFunc)
 	panicIf(err)
 

controller/packageApi.go

@@ -65,7 +65,12 @@ func (a *API) PackageApiCreate(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Ensure size limits
-	if len(f.Spec.Literal) > 256*1024 {
+	if len(f.Spec.Source.Literal) > 256*1024 {
+		err := fission.MakeError(fission.ErrorInvalidArgument, "Package literal larger than 256K")
+		a.respondWithError(w, err)
+		return
+	}
+	if len(f.Spec.Deployment.Literal) > 256*1024 {
 		err := fission.MakeError(fission.ErrorInvalidArgument, "Package literal larger than 256K")
 		a.respondWithError(w, err)
 		return
@@ -104,7 +109,7 @@ func (a *API) PackageApiGet(w http.ResponseWriter, r *http.Request) {
 
 	var resp []byte
 	if raw != "" {
-		resp = []byte(f.Spec.Literal)
+		resp = []byte(f.Spec.Deployment.Literal)
 	} else {
 		resp, err = json.Marshal(f)
 		if err != nil {

environments/fetcher/fetcher.go

@@ -155,11 +155,8 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 
 		// get pkg
 		var pkg *tpr.Package
-		if req.FetchType == FETCH_SOURCE {
+		pkg, err = fetcher.fissionClient.
-			pkg, err = fetcher.fissionClient.Packages(fn.Spec.Source.PackageRef.Namespace).Get(fn.Spec.Source.PackageRef.Name)
+			Packages(fn.Spec.Package.PackageRef.Namespace).Get(fn.Spec.Package.PackageRef.Name)
-		} else if req.FetchType == FETCH_DEPLOYMENT {
-			pkg, err = fetcher.fissionClient.Packages(fn.Spec.Deployment.PackageRef.Namespace).Get(fn.Spec.Deployment.PackageRef.Name)
-		}
 		if err != nil {
 			e := fmt.Sprintf("Failed to get package: %v", err)
 			log.Printf(e)
@@ -167,10 +164,17 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
+		var archive *fission.Archive
+		if req.FetchType == FETCH_SOURCE {
+			archive = &pkg.Spec.Source
+		} else if req.FetchType == FETCH_DEPLOYMENT {
+			archive = &pkg.Spec.Deployment
+		}
+
 		// get package data as literal or by url
-		if len(pkg.Spec.Literal) > 0 {
+		if len(archive.Literal) > 0 {
 			// write pkg.Literal into tmpPath
-			err = ioutil.WriteFile(tmpPath, pkg.Spec.Literal, 0600)
+			err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
 			if err != nil {
 				e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err)
 				log.Printf(e)
@@ -180,7 +184,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 		} else {
 			// download and verify
 
-			err = downloadUrl(pkg.Spec.URL, tmpPath)
+			err = downloadUrl(archive.URL, tmpPath)
 			if err != nil {
 				e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err)
 				log.Printf(e)
@@ -188,7 +192,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 
-			err = verifyChecksum(tmpPath, &pkg.Spec.Checksum)
+			err = verifyChecksum(tmpPath, &archive.Checksum)
 			if err != nil {
 				e := fmt.Sprintf("Failed to verify checksum: %v", err)
 				log.Printf(e)

fission/function.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -44,14 +43,13 @@ func fileSize(filePath string) int64 {
 	return info.Size()
 }
 
-// updatePackageSpecWithFile uploads or serializes the file into the
+// upload a file and return a fission.Archive
-// provided package object
+func createArchive(client *client.Client, fileName string) *fission.Archive {
-func updatePackageSpecWithFile(client *client.Client, pkgSpec *fission.PackageSpec, fileName string) {
+	var archive fission.Archive
-	if fileSize(fileName) < fission.PackageLiteralSizeLimit {
+	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
-		pkgContents := getPackageContents(fileName)
+		contents := getContents(fileName)
-
+		archive.Type = fission.ArchiveTypeLiteral
-		pkgSpec.Type = fission.PackageTypeLiteral
+		archive.Literal = contents
-		pkgSpec.Literal = pkgContents
 	} else {
 		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
 		ssClient := storageSvcClient.MakeClient(u)
@@ -62,54 +60,13 @@ func updatePackageSpecWithFile(client *client.Client, pkgSpec *fission.PackageSp
 
 		archiveUrl := ssClient.GetUrl(id)
 
-		pkgSpec.Type = fission.PackageTypeUrl
+		archive.Type = fission.ArchiveTypeUrl
-		pkgSpec.URL = archiveUrl
+		archive.URL = archiveUrl
 	}
+	return &archive
 }
 
-// createPackageFromFile is a function that helps to upload the content
+func getContents(filePath string) []byte {
-// of given file to controller to create a TPR package resource, and then
-// return a function package reference for further usage.
-func createPackageFromFile(client *client.Client, fnName string, fileName string) fission.FunctionPackageRef {
-	pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
-	pkg := &tpr.Package{
-		Metadata: api.ObjectMeta{
-			Name:      pkgName,
-			Namespace: api.NamespaceDefault,
-		},
-	}
-
-	updatePackageSpecWithFile(client, &pkg.Spec, fileName)
-
-	_, err := client.PackageCreate(pkg)
-	checkErr(err, "upload package")
-
-	return fission.FunctionPackageRef{
-		PackageRef: fission.PackageRef{
-			Name:      pkgName,
-			Namespace: pkg.Metadata.Namespace,
-		},
-	}
-}
-
-// updatePackageContents is a function that reads content from given file
-// and updates the package content of TPR package resource.
-func updatePackageContents(client *client.Client, pkgName string, fileName string) error {
-	pkg, err := client.PackageGet(&api.ObjectMeta{
-		Name:      pkgName,
-		Namespace: api.NamespaceDefault,
-	})
-	if err != nil {
-		return errors.New(fmt.Sprintf("read package '%v'", pkgName))
-	}
-
-	updatePackageSpecWithFile(client, &pkg.Spec, fileName)
-
-	_, err = client.PackageUpdate(pkg)
-	return err
-}
-
-func getPackageContents(filePath string) []byte {
 	var code []byte
 	var err error
 
@@ -148,18 +105,39 @@ func fnCreate(c *cli.Context) error {
 			Namespace: api.NamespaceDefault,
 		},
 		Spec: fission.FunctionSpec{
-			EnvironmentName: envName,
+			Environment: fission.EnvironmentReference{
+				Name:      envName,
+				Namespace: api.NamespaceDefault,
+			},
 		},
 	}
 
+	var pkgSpec fission.PackageSpec
 	if len(srcPkgName) > 0 {
-		function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
+		pkgSpec.Source = *createArchive(client, srcPkgName)
 	}
 	if len(deployPkgName) > 0 {
-		function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
+		pkgSpec.Deployment = *createArchive(client, deployPkgName)
 	}
+	pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
+	pkg := &tpr.Package{
+		Metadata: api.ObjectMeta{
+			Name:      pkgName,
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: pkgSpec,
+	}
+	newpkg, err := client.PackageCreate(pkg)
+	checkErr(err, "create package")
 
-	_, err := client.FunctionCreate(function)
+	function.Spec.Package = fission.FunctionPackageRef{
+		PackageRef: fission.PackageRef{
+			Name:            newpkg.Name,
+			Namespace:       newpkg.Namespace,
+			ResourceVersion: newpkg.ResourceVersion,
+		},
+	}
+	_, err = client.FunctionCreate(function)
 	checkErr(err, "create function")
 
 	fmt.Printf("function '%v' created\n", fnName)
@@ -210,12 +188,12 @@ func fnGet(c *cli.Context) error {
 	checkErr(err, "get function")
 
 	pkg, err := client.PackageGet(&api.ObjectMeta{
-		Name:      fn.Spec.Deployment.PackageRef.Name,
+		Name:      fn.Spec.Package.PackageRef.Name,
-		Namespace: fn.Spec.Deployment.PackageRef.Namespace,
+		Namespace: fn.Spec.Package.PackageRef.Namespace,
 	})
 	checkErr(err, "get package")
 
-	os.Stdout.Write(pkg.Spec.Literal)
+	os.Stdout.Write(pkg.Spec.Deployment.Literal)
 	return err
 }
 
@@ -238,7 +216,7 @@ func fnGetMeta(c *cli.Context) error {
 	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
 	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV")
 	fmt.Fprintf(w, "%v\t%v\t%v\n",
-		f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName)
+		f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name)
 	w.Flush()
 	return err
 }
@@ -268,31 +246,30 @@ func fnUpdate(c *cli.Context) error {
 		fatal("Need --env or --code or --package or --srcpkg argument.")
 	}
 
-	// Now builder manager only starts a build if a function has a source package
+	if len(deployPkgName) != 0 || len(srcPkgName) != 0 {
-	// but no deployment package. This behavior will be changed after we move builds
+		// get existing package
-	// to package level (https://github.com/fission/fission/pull/297).
+		pkg, err := client.PackageGet(&api.ObjectMeta{
-
+			Name:      function.Spec.Package.PackageRef.Name,
-	if len(srcPkgName) > 0 {
+			Namespace: function.Spec.Package.PackageRef.Namespace,
-		// Check the existence of the package, create it if not exist.
+		})
-		if len(function.Spec.Source.PackageRef.Name) > 0 {
+		// update package spec
-			err := updatePackageContents(client, function.Spec.Source.PackageRef.Name, srcPkgName)
+		if len(srcPkgName) > 0 {
-			checkErr(err, "update source package")
+			archive := createArchive(client, srcPkgName)
-		} else {
+			pkg.Spec.Source = *archive
-			function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
 		}
-	}
+		if len(deployPkgName) > 0 {
-
+			archive := createArchive(client, deployPkgName)
-	if len(deployPkgName) > 0 {
+			pkg.Spec.Deployment = *archive
-		if len(function.Spec.Deployment.PackageRef.Name) > 0 {
-			err := updatePackageContents(client, function.Spec.Deployment.PackageRef.Name, deployPkgName)
-			checkErr(err, "update source package")
-		} else {
-			function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
 		}
+		// updage package object
+		newpkg, err := client.PackageUpdate(pkg)
+		checkErr(err, "update package")
+		// update function spec with resource version
+		function.Spec.Package.PackageRef.ResourceVersion = newpkg.ResourceVersion
 	}
 
 	if len(envName) > 0 {
-		function.Spec.EnvironmentName = envName
+		function.Spec.Environment.Name = envName
 	}
 
 	_, err = client.FunctionUpdate(function)
@@ -333,7 +310,7 @@ func fnList(c *cli.Context) error {
 	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV")
 	for _, f := range fns {
 		fmt.Fprintf(w, "%v\t%v\t%v\n",
-			f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName)
+			f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name)
 	}
 	w.Flush()
 

poolmgr/api.go

@@ -167,7 +167,7 @@ func (poolMgr *Poolmgr) getFunctionEnv(m *api.ObjectMeta) (*tpr.Environment, err
 
 	// Get env from metadata
 	log.Printf("[%v] getting env", m)
-	env, err = poolMgr.fissionClient.Environments(f.Metadata.Namespace).Get(f.Spec.EnvironmentName)
+	env, err = poolMgr.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
 	if err != nil {
 		return nil, err
 	}

tpr/tpr_test.go

@@ -46,15 +46,16 @@ func functionTests(tprClient *rest.RESTClient) {
 			Name: "hello",
 		},
 		Spec: fission.FunctionSpec{
-			Source: fission.FunctionPackageRef{},
+			Package: fission.FunctionPackageRef{
-			Deployment: fission.FunctionPackageRef{
 				PackageRef: fission.PackageRef{
 					Name:      "foo",
 					Namespace: "bar",
 				},
 				FunctionName: "hello",
 			},
-			EnvironmentName: "xxx",
+			Environment: fission.EnvironmentReference{
+				Name: "xxx",
+			},
 		},
 	}
 
@@ -74,14 +75,14 @@ func functionTests(tprClient *rest.RESTClient) {
 	// read
 	f, err = fi.Get(function.Metadata.Name)
 	panicIf(err)
-	if f.Spec.Deployment.FunctionName != function.Spec.Deployment.FunctionName {
+	if f.Spec.Environment.Name != function.Spec.Environment.Name {
 		log.Panicf("Bad result from Get: %v", f)
 	}
 
 	log.Printf("f.Metadata = %#v", f.Metadata)
 
 	// update
-	function.Spec.EnvironmentName = "yyy"
+	function.Spec.Environment.Name = "yyy"
 	f, err = fi.Update(function)
 	panicIf(err)
 
@@ -93,7 +94,7 @@ func functionTests(tprClient *rest.RESTClient) {
 	if len(fl.Items) != 1 {
 		log.Panicf("wrong count from list: %v", fl)
 	}
-	if fl.Items[0].Spec.EnvironmentName != function.Spec.EnvironmentName {
+	if fl.Items[0].Spec.Environment.Name != function.Spec.Environment.Name {
 		log.Panicf("bad object from list: %v", fl.Items[0])
 	}
 
@@ -122,7 +123,7 @@ func functionTests(tprClient *rest.RESTClient) {
 		if !ok {
 			log.Panicf("Can't cast to Function")
 		}
-		if wf.Spec.EnvironmentName != function.Spec.EnvironmentName {
+		if wf.Spec.Environment.Name != function.Spec.Environment.Name {
 			log.Panicf("Bad object from watch: %#v", wf)
 		}
 		log.Printf("watch event took %v", time.Now().Sub(start))

types.go

@@ -34,16 +34,16 @@ type (
 		Sum  string       `json:"sum"`
 	}
 
-	// PackageType is either literal or URL, indicating whether
+	// ArchiveType is either literal or URL, indicating whether
-	// the package is specified in the Package struct or
+	// the package is specified in the Archive struct or
 	// externally.
-	PackageType string
+	ArchiveType string
 
 	// Package contains or references a collection of source or
 	// binary files.
-	PackageSpec struct {
+	Archive struct {
 		// Type defines how the package is specified: literal or URL.
-		Type PackageType `json:"type"`
+		Type ArchiveType `json:"type"`
 
 		// Literal contents of the package. Can be used for
 		// encoding packages below TODO (256KB?) size.
@@ -57,12 +57,34 @@ type (
 		Checksum Checksum `json:"checksum"`
 	}
 
+	EnvironmentReference struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	}
+
+	BuildStatus string
+
+	PackageSpec struct {
+		Environment EnvironmentReference `json:"environment"`
+		Source      Archive              `json:"source"`
+		Deployment  Archive              `json:"deployment"`
+		// In the future, we can have a debug build here too
+	}
+	PackageStatus struct {
+		BuildStatus BuildStatus `json:"buildstatus"`
+		BuildLog    string      `json:"buildlog"` // output of the build (errors etc)
+	}
+
 	PackageRef struct {
-		Name      string
+		Namespace string `json:"namespace"`
-		Namespace string
+		Name      string `json:"name"`
+
+		// Including resource version in the reference forces the function to be updated on
+		// package update, making it possible to cache the function based on its metadata.
+		ResourceVersion string `json:"resourceversion"`
 	}
 	FunctionPackageRef struct {
-		PackageRef PackageRef
+		PackageRef PackageRef `json:"packageref"`
 
 		// FunctionName specifies a specific function within the package. This allows
 		// functions to share packages, by having different functions within the same
@@ -77,18 +99,13 @@ type (
 
 	// FunctionSpec describes the contents of the function.
 	FunctionSpec struct {
-		// EnvironmentName is the name of the environment that this function is associated
+		// Environment is the build and runtime environment that this function is
-		// with. An Environment with this name should exist, otherwise the function cannot
+		// associated with. An Environment with this name should exist, otherwise the
-		// be invoked.
+		// function cannot be invoked.
-		EnvironmentName string `json:"environmentName"`
+		Environment EnvironmentReference `json:"environment"`
 
-		// Source is an source package for this function; it's used for the build step if
+		// Reference to a package containing deployment and optionally the source
-		// the environment defines a build container.
+		Package FunctionPackageRef `json:"package"`
-		Source FunctionPackageRef `json:"source"`
-
-		// Deployment is a deployable package for this function. This is the package that's
-		// loaded into the environment's runtime container.
-		Deployment FunctionPackageRef `json:"deployment"`
 	}
 
 	FunctionReferenceType string
@@ -220,12 +237,19 @@ const (
 )
 
 const (
-	// PackageTypeLiteral means the package contents are specified in the Literal field of
+	// ArchiveTypeLiteral means the package contents are specified in the Literal field of
 	// resource itself.
-	PackageTypeLiteral PackageType = "literal"
+	ArchiveTypeLiteral ArchiveType = "literal"
 
-	// PackageTypeUrl means the package contents are at the specified URL.
+	// ArchiveTypeUrl means the package contents are at the specified URL.
-	PackageTypeUrl PackageType = "url"
+	ArchiveTypeUrl ArchiveType = "url"
+)
+
+const (
+	BuildStatusPending   = "pending"
+	BuildStatusRunning   = "running"
+	BuildStatusSucceeded = "succeeded"
+	BuildStatusFailed    = "failed"
 )
 
 const (
@@ -266,5 +290,5 @@ var errorDescriptions = []string{
 }
 
 const (
-	PackageLiteralSizeLimit int64 = 256 * 1024
+	ArchiveLiteralSizeLimit int64 = 256 * 1024
 )