Add env builder & srcpkg through cli (#296)

CLI support for specifying an environment builder and source packages.

fission/environment.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"text/tabwriter"
 
@@ -41,6 +42,14 @@ func envCreate(c *cli.Context) error {
 		fatal("Need an image, use --image.")
 	}
 
+	envBuilderImg := c.String("builder")
+
+	envBuildCmd := c.String("buildcmd")
+	if len(envBuilderImg) > 0 && len(envBuildCmd) == 0 {
+		log.Printf("No build command is specified, use the default build command.")
+		envBuildCmd = "build"
+	}
+
 	env := &tpr.Environment{
 		Metadata: api.ObjectMeta{
 			Name:      envName,
@@ -51,6 +60,10 @@ func envCreate(c *cli.Context) error {
 			Runtime: fission.Runtime{
 				Image: envImg,
 			},
+			Builder: fission.Builder{
+				Image:   envBuilderImg,
+				Command: envBuildCmd,
+			},
 		},
 	}
 
@@ -91,10 +104,12 @@ func envUpdate(c *cli.Context) error {
 	if len(envName) == 0 {
 		fatal("Need a name, use --name.")
 	}
-
 	envImg := c.String("image")
-	if len(envImg) == 0 {
+	envBuilderImg := c.String("builder")
-		fatal("Need an image, use --image.")
+	envBuildCmd := c.String("buildcmd")
+
+	if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 {
+		fatal("Need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command.")
 	}
 
 	env, err := client.EnvironmentGet(&api.ObjectMeta{
@@ -103,7 +118,15 @@ func envUpdate(c *cli.Context) error {
 	})
 	checkErr(err, "find environment")
 
-	env.Spec.Runtime.Image = envImg
+	if len(envImg) > 0 {
+		env.Spec.Runtime.Image = envImg
+	}
+	if len(envBuilderImg) > 0 {
+		env.Spec.Builder.Image = envBuilderImg
+	}
+	if len(envBuildCmd) > 0 {
+		env.Spec.Builder.Command = envBuildCmd
+	}
 
 	_, err = client.EnvironmentUpdate(env)
 	checkErr(err, "update environment")

fission/function.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -31,6 +32,7 @@ import (
 	"k8s.io/client-go/1.5/pkg/api"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/fission/logdb"
 	"github.com/fission/fission/tpr"
 )
@@ -41,6 +43,60 @@ func fileSize(filePath string) int64 {
 	return info.Size()
 }
 
+// createPackageFromFile is a function that helps to upload the content
+// of given file to controller to create a TPR package resource, and then
+// return a function package reference for further usage.
+func createPackageFromFile(client *client.Client, fnName string, fileName string) fission.FunctionPackageRef {
+	// TODO fallback to uploading + setting a Package URL
+	checkFileSize(fileName)
+	pkgContents := getPackageContents(fileName)
+	pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
+	pkg := &tpr.Package{
+		Metadata: api.ObjectMeta{
+			Name:      pkgName,
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: fission.PackageSpec{
+			Type:    fission.PackageTypeLiteral,
+			Literal: pkgContents,
+		},
+	}
+	_, err := client.PackageCreate(pkg)
+	checkErr(err, "upload package")
+
+	return fission.FunctionPackageRef{
+		PackageRef: fission.PackageRef{
+			Name:      pkgName,
+			Namespace: pkg.Metadata.Namespace,
+		},
+	}
+}
+
+// updatePackageContents is a function that reads content from given file
+// and updates the package content of TPR package resource.
+func updatePackageContents(client *client.Client, pkgName string, fileName string) error {
+	// TODO fallback to uploading + setting a Package URL
+	checkFileSize(fileName)
+	pkg, err := client.PackageGet(&api.ObjectMeta{
+		Name:      pkgName,
+		Namespace: api.NamespaceDefault,
+	})
+	if err != nil {
+		return errors.New(fmt.Sprintf("read package '%v'", pkgName))
+	}
+	pkg.Spec.Literal = getPackageContents(fileName)
+	_, err = client.PackageUpdate(pkg)
+	return err
+}
+
+func checkFileSize(fileName string) {
+	if fileSize(fileName) > fission.PackageLiteralSizeLimit {
+		// TODO fallback to uploading + setting a Package URL
+		fmt.Printf("File size >256k not supported yet")
+		os.Exit(1)
+	}
+}
+
 func getPackageContents(filePath string) []byte {
 	var code []byte
 	var err error
@@ -63,35 +119,17 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --env argument.")
 	}
 
-	fileName := c.String("code")
+	srcPkgName := c.String("srcpkg")
-	if len(fileName) == 0 {
+
-		fileName = c.String("package")
+	deployPkgName := c.String("code")
-		if len(fileName) == 0 {
+	if len(deployPkgName) == 0 {
-			fatal("Need --code or --package argument.")
+		deployPkgName = c.String("package")
-		}
 	}
 
-	if fileSize(fileName) > fission.PackageLiteralSizeLimit {
+	if len(srcPkgName) == 0 && len(deployPkgName) == 0 {
-		// TODO fallback to uploading + setting a Package URL
+		fatal("Need --code or --package to specify deployment package, or use --srcpkg to specify source package.")
-		fmt.Printf("File size not supported yet")
-		os.Exit(1)
 	}
 
-	pkgContents := getPackageContents(fileName)
-	pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
-	pkg := &tpr.Package{
-		Metadata: api.ObjectMeta{
-			Name:      pkgName,
-			Namespace: api.NamespaceDefault,
-		},
-		Spec: fission.PackageSpec{
-			Type:    fission.PackageTypeLiteral,
-			Literal: pkgContents,
-		},
-	}
-	_, err := client.PackageCreate(pkg)
-	checkErr(err, "upload package")
-
 	function := &tpr.Function{
 		Metadata: api.ObjectMeta{
 			Name:      fnName,
@@ -99,16 +137,17 @@ func fnCreate(c *cli.Context) error {
 		},
 		Spec: fission.FunctionSpec{
 			EnvironmentName: envName,
-			Deployment: fission.FunctionPackageRef{
-				PackageRef: fission.PackageRef{
-					Name:      pkgName,
-					Namespace: api.NamespaceDefault,
-				},
-			},
 		},
 	}
 
-	_, err = client.FunctionCreate(function)
+	if len(srcPkgName) > 0 {
+		function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
+	}
+	if len(deployPkgName) > 0 {
+		function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
+	}
+
+	_, err := client.FunctionCreate(function)
 	checkErr(err, "create function")
 
 	fmt.Printf("function '%v' created\n", fnName)
@@ -206,43 +245,47 @@ func fnUpdate(c *cli.Context) error {
 	})
 	checkErr(err, fmt.Sprintf("read function '%v'", fnName))
 
-	pkgName := function.Spec.Deployment.PackageRef.Name
-	pkg, err := client.PackageGet(&api.ObjectMeta{
-		Name:      pkgName,
-		Namespace: api.NamespaceDefault,
-	})
-	checkErr(err, fmt.Sprintf("read package '%v'", pkgName))
-
 	envName := c.String("env")
-	fileName := c.String("code")
+	deployPkgName := c.String("code")
-	if len(fileName) == 0 {
+	if len(deployPkgName) == 0 {
-		fileName = c.String("package")
+		deployPkgName = c.String("package")
+	}
+	srcPkgName := c.String("srcpkg")
+
+	if len(envName) == 0 && len(deployPkgName) == 0 && len(srcPkgName) == 0 {
+		fatal("Need --env or --code or --package or --srcpkg argument.")
 	}
 
-	if len(envName) == 0 && len(fileName) == 0 {
+	// Now builder manager only starts a build if a function has a source package
-		fatal("Need --env or --code or --package argument.")
+	// but no deployment package. This behavior will be changed after we move builds
-	}
+	// to package level (https://github.com/fission/fission/pull/297).
 
-	if len(fileName) > 0 {
+	if len(srcPkgName) > 0 {
-		if fileSize(fileName) > fission.PackageLiteralSizeLimit {
+		// Check the existence of the package, create it if not exist.
-			// TODO fallback to uploading + setting a Package URL
+		if len(function.Spec.Source.PackageRef.Name) > 0 {
-			fmt.Printf("File size >256k not supported yet")
+			err := updatePackageContents(client, function.Spec.Source.PackageRef.Name, srcPkgName)
-			os.Exit(1)
+			checkErr(err, "update source package")
+		} else {
+			function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
 		}
+	}
 
-		pkg.Spec.Literal = getPackageContents(fileName)
+	if len(deployPkgName) > 0 {
-
+		if len(function.Spec.Deployment.PackageRef.Name) > 0 {
-		_, err = client.PackageUpdate(pkg)
+			err := updatePackageContents(client, function.Spec.Deployment.PackageRef.Name, deployPkgName)
-		checkErr(err, "update package")
+			checkErr(err, "update source package")
+		} else {
+			function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
+		}
 	}
 
 	if len(envName) > 0 {
 		function.Spec.EnvironmentName = envName
-
-		_, err = client.FunctionUpdate(function)
-		checkErr(err, "update function")
 	}
 
+	_, err = client.FunctionUpdate(function)
+	checkErr(err, "update function")
+
 	fmt.Printf("function '%v' updated\n", fnName)
 	return err
 }

fission/main.go

@@ -40,15 +40,16 @@ func main() {
 	fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"}
 	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
 	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "local path or URL for binary package"}
+	fnSrcPackageFlag := cli.StringFlag{Name: "srcpkg", Usage: "local path or URL for source package"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
 	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
 	fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"}
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
-		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag}, Action: fnUpdate},
+		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcPackageFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete},
 		{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
 		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs},
@@ -96,10 +97,12 @@ func main() {
 	// environments
 	envNameFlag := cli.StringFlag{Name: "name", Usage: "Environment name"}
 	envImageFlag := cli.StringFlag{Name: "image", Usage: "Environment image URL"}
+	envBuilderImageFlag := cli.StringFlag{Name: "builder", Usage: "Environment builder image URL (optional)"}
+	envBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for environment builder to build source package (optional)"}
 	envSubcommands := []cli.Command{
-		{Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envImageFlag}, Action: envCreate},
+		{Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag}, Action: envCreate},
 		{Name: "get", Usage: "Get environment details", Flags: []cli.Flag{envNameFlag}, Action: envGet},
-		{Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envImageFlag}, Action: envUpdate},
+		{Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag}, Action: envUpdate},
 		{Name: "delete", Usage: "Delete environment", Flags: []cli.Flag{envNameFlag}, Action: envDelete},
 		{Name: "list", Usage: "List all environments", Flags: []cli.Flag{}, Action: envList},
 	}