Add builder manager support (#308)

This change orchestrates function builds. Environments (in v2) define a builder image, just like they do a runtime image. The builder image contains a build script that's invoked with source and deployment paths (as env vars). The buildermgr watches for environments with build images defined, and creates build deployments and services. Functions can define source and deployment. Buildermgr watches for functions with source code (and build status == pending) and invokes the environment's builder when appropriate. It captures logs from the build and sets the build status (success/failure) and build lots into the PackageStatus.
2017-09-25 07:53:09 -07:00
parent a720377f3c
commit e587eca08f
40 changed files with 1827 additions and 176 deletions
@@ -18,7 +18,6 @@ package main

 import (
 	"fmt"
-	"log"
 	"os"
 	"text/tabwriter"

@@ -42,12 +41,22 @@ func envCreate(c *cli.Context) error {
 		fatal("Need an image, use --image.")
 	}

+	envVersion := c.Int("version")
 	envBuilderImg := c.String("builder")
-
 	envBuildCmd := c.String("buildcmd")
-	if len(envBuilderImg) > 0 && len(envBuildCmd) == 0 {
-		log.Printf("No build command is specified, use the default build command.")
-		envBuildCmd = "build"
+
+	if len(envBuilderImg) > 0 {
+		envVersion = 2
+		if len(envBuildCmd) == 0 {
+			envBuildCmd = "build"
+		}
+	}
+
+	// Environment API interface version is not specified and
+	// builder image is empty, set default interface version
+	if envVersion == 0 {
+		fmt.Println("Use default environment v1 API interface")
+		envVersion = 1
 	}

 	env := &tpr.Environment{
@@ -56,7 +65,7 @@ func envCreate(c *cli.Context) error {
 			Namespace: api.NamespaceDefault,
 		},
 		Spec: fission.EnvironmentSpec{
-			Version: 1,
+			Version: envVersion,
 			Runtime: fission.Runtime{
 				Image: envImg,
 			},
@@ -121,6 +130,11 @@ func envUpdate(c *cli.Context) error {
 	if len(envImg) > 0 {
 		env.Spec.Runtime.Image = envImg
 	}
+
+	if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) {
+		fatal("Environment v1 API interface doesn't supported environment builder.")
+	}
+
 	if len(envBuilderImg) > 0 {
 		env.Spec.Builder.Image = envBuilderImg
 	}
@@ -28,7 +28,6 @@ import (
 	"text/tabwriter"
 	"time"

-	"github.com/dchest/uniuri"
 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 	"k8s.io/client-go/1.5/pkg/api"
@@ -85,6 +84,47 @@ func createArchive(client *client.Client, fileName string) *fission.Archive {
 	return &archive
 }

+func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *api.ObjectMeta {
+	pkgSpec := fission.PackageSpec{
+		Environment: fission.EnvironmentReference{
+			Namespace: api.NamespaceDefault,
+			Name:      envName,
+		},
+	}
+	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
+
+	if len(deployArchiveName) > 0 {
+		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
+		if len(srcArchiveName) > 0 {
+			fmt.Println("Deployment may be overwritten by builder manager after source package compilation")
+		}
+	}
+	if len(srcArchiveName) > 0 {
+		pkgSpec.Source = *createArchive(client, srcArchiveName)
+		// set pending status to package
+		pkgStatus = fission.BuildStatusPending
+	}
+
+	if len(buildcmd) > 0 {
+		pkgSpec.BuildCommand = buildcmd
+	}
+
+	pkgName := strings.ToLower(uuid.NewV4().String())
+	pkg := &tpr.Package{
+		Metadata: api.ObjectMeta{
+			Name:      pkgName,
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: pkgSpec,
+		Status: fission.PackageStatus{
+			BuildStatus: pkgStatus,
+		},
+	}
+	pkgMetadata, err := client.PackageCreate(pkg)
+	checkErr(err, "create package")
+	return pkgMetadata
+}
+
 func getContents(filePath string) []byte {
 	var code []byte
 	var err error
@@ -97,6 +137,10 @@ func getContents(filePath string) []byte {
 func fnCreate(c *cli.Context) error {
 	client := getClient(c.GlobalString("server"))

+	if len(c.String("package")) > 0 {
+		fatal("--package is deprecated, please use --deploy instead.")
+	}
+
 	fnName := c.String("name")
 	if len(fnName) == 0 {
 		fatal("Need --name argument.")
@@ -107,17 +151,24 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --env argument.")
 	}

-	srcPkgName := c.String("srcpkg")
-
-	deployPkgName := c.String("code")
-	if len(deployPkgName) == 0 {
-		deployPkgName = c.String("package")
+	srcArchiveName := c.String("src")
+	deployArchiveName := c.String("code")
+	if len(deployArchiveName) == 0 {
+		deployArchiveName = c.String("deploy")
 	}

-	if len(srcPkgName) == 0 && len(deployPkgName) == 0 {
-		fatal("Need --code or --package to specify deployment package, or use --srcpkg to specify source package.")
+	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+		fatal("Need --code or --deploy to specify deployment archive, or use --src to specify source archive.")
 	}

+	entrypoint := c.String("entrypoint")
+	buildcmd := c.String("buildcmd")
+	if len(buildcmd) == 0 {
+		buildcmd = "/builder"
+	}
+
+	pkgMetadata := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+
 	function := &tpr.Function{
 		Metadata: api.ObjectMeta{
 			Name:      fnName,
@@ -128,35 +179,18 @@ func fnCreate(c *cli.Context) error {
 				Name:      envName,
 				Namespace: api.NamespaceDefault,
 			},
+			Package: fission.FunctionPackageRef{
+				FunctionName: entrypoint,
+				PackageRef: fission.PackageRef{
+					Namespace:       pkgMetadata.Namespace,
+					Name:            pkgMetadata.Name,
+					ResourceVersion: pkgMetadata.ResourceVersion,
+				},
+			},
 		},
 	}

-	var pkgSpec fission.PackageSpec
-	if len(srcPkgName) > 0 {
-		pkgSpec.Source = *createArchive(client, srcPkgName)
-	}
-	if len(deployPkgName) > 0 {
-		pkgSpec.Deployment = *createArchive(client, deployPkgName)
-	}
-	pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
-	pkg := &tpr.Package{
-		Metadata: api.ObjectMeta{
-			Name:      pkgName,
-			Namespace: api.NamespaceDefault,
-		},
-		Spec: pkgSpec,
-	}
-	newpkg, err := client.PackageCreate(pkg)
-	checkErr(err, "create package")
-
-	function.Spec.Package = fission.FunctionPackageRef{
-		PackageRef: fission.PackageRef{
-			Name:            newpkg.Name,
-			Namespace:       newpkg.Namespace,
-			ResourceVersion: newpkg.ResourceVersion,
-		},
-	}
-	_, err = client.FunctionCreate(function)
+	_, err := client.FunctionCreate(function)
 	checkErr(err, "create function")

 	fmt.Printf("function '%v' created\n", fnName)
@@ -248,6 +282,10 @@ func fnUpdate(c *cli.Context) error {
 		fatal("Need name of function, use --name")
 	}

+	if len(c.String("package")) > 0 {
+		fatal("--package is deprecated, please use --deploy instead.")
+	}
+
 	function, err := client.FunctionGet(&api.ObjectMeta{
 		Name:      fnName,
 		Namespace: api.NamespaceDefault,
@@ -255,42 +293,50 @@ func fnUpdate(c *cli.Context) error {
 	checkErr(err, fmt.Sprintf("read function '%v'", fnName))

 	envName := c.String("env")
-	deployPkgName := c.String("code")
-	if len(deployPkgName) == 0 {
-		deployPkgName = c.String("package")
+	deployArchiveName := c.String("code")
+	if len(deployArchiveName) == 0 {
+		deployArchiveName = c.String("deploy")
 	}
-	srcPkgName := c.String("srcpkg")
+	srcArchiveName := c.String("src")

-	if len(envName) == 0 && len(deployPkgName) == 0 && len(srcPkgName) == 0 {
-		fatal("Need --env or --code or --package or --srcpkg argument.")
-	}
-
-	if len(deployPkgName) != 0 || len(srcPkgName) != 0 {
-		// get existing package
-		pkg, err := client.PackageGet(&api.ObjectMeta{
-			Name:      function.Spec.Package.PackageRef.Name,
-			Namespace: function.Spec.Package.PackageRef.Namespace,
-		})
-		// update package spec
-		if len(srcPkgName) > 0 {
-			archive := createArchive(client, srcPkgName)
-			pkg.Spec.Source = *archive
-		}
-		if len(deployPkgName) > 0 {
-			archive := createArchive(client, deployPkgName)
-			pkg.Spec.Deployment = *archive
-		}
-		// updage package object
-		newpkg, err := client.PackageUpdate(pkg)
-		checkErr(err, "update package")
-		// update function spec with resource version
-		function.Spec.Package.PackageRef.ResourceVersion = newpkg.ResourceVersion
+	if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 {
+		fatal("Need --env or --code or --package or --deploy argument.")
 	}

 	if len(envName) > 0 {
 		function.Spec.Environment.Name = envName
 	}

+	entrypoint := c.String("entrypoint")
+	if len(entrypoint) > 0 {
+		function.Spec.Package.FunctionName = entrypoint
+	}
+
+	pkg, err := client.PackageGet(&api.ObjectMeta{
+		Name:      function.Spec.Package.PackageRef.Name,
+		Namespace: function.Spec.Package.PackageRef.Namespace,
+	})
+	checkErr(err, fmt.Sprintf("read package '%v'", function.Spec.Package.PackageRef.Name))
+
+	buildcmd := c.String("buildcmd")
+	if len(buildcmd) == 0 {
+		// use previous build command if not specified.
+		buildcmd = pkg.Spec.BuildCommand
+	}
+
+	if len(deployArchiveName) > 0 || len(srcArchiveName) > 0 {
+		// create a new package for function
+		pkgMetadata := createPackage(client,
+			function.Spec.Environment.Name, srcArchiveName, deployArchiveName, buildcmd)
+
+		// update function spec with resource version
+		function.Spec.Package.PackageRef = fission.PackageRef{
+			Namespace:       pkgMetadata.Namespace,
+			Name:            pkgMetadata.Name,
+			ResourceVersion: pkgMetadata.ResourceVersion,
+		}
+	}
+
 	_, err = client.FunctionUpdate(function)
 	checkErr(err, "update function")

@@ -40,17 +40,20 @@ func main() {
 	fnNameFlag := cli.StringFlag{Name: "name", Usage: "function name"}
 	fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"}
 	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
-	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "local path or URL for binary package"}
-	fnSrcPackageFlag := cli.StringFlag{Name: "srcpkg", Usage: "local path or URL for source package"}
+	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "(Deprecated) local path or URL for binary package"}
+	fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
+	fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
 	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
 	fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"}
+	fnEntryPointFlag := cli.StringFlag{Name: "entrypoint", Usage: "entry point for environment v2 to load with"}
+	fnBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "build command for builder to run with"}
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcPackageFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
-		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcPackageFlag}, Action: fnUpdate},
+		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete},
 		{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
 		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs},
@@ -100,8 +103,9 @@ func main() {
 	envImageFlag := cli.StringFlag{Name: "image", Usage: "Environment image URL"}
 	envBuilderImageFlag := cli.StringFlag{Name: "builder", Usage: "Environment builder image URL (optional)"}
 	envBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for environment builder to build source package (optional)"}
+	envVersionFlag := cli.IntFlag{Name: "version", Usage: "Environment API version: defaults to 1 (means v1 interface)"}
 	envSubcommands := []cli.Command{
-		{Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag}, Action: envCreate},
+		{Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envVersionFlag}, Action: envCreate},
 		{Name: "get", Usage: "Get environment details", Flags: []cli.Flag{envNameFlag}, Action: envGet},
 		{Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag}, Action: envUpdate},
 		{Name: "delete", Usage: "Delete environment", Flags: []cli.Flag{envNameFlag}, Action: envDelete},