Declarative application specifications for Fission (#422)

This change enables users to have declarative specifications for Fission resources. Users can specify their "app" in a set of spec files, and use a new fission CLI command to "apply" these specs to a running cluster.

The new "spec" CLI also includes archiving of local source files, and a file watcher that re-builds archives and uploads them on file changes, and a package build watcher that waits for package builds on the CLI. 

A "--spec" option is also added to "function create", and will be added to other resources in future changes.  This option causes a YAML to be outputted to the specs directory instead of the resource being created on the cluster.

The CLI "fission spec --help" outputs usage information.

fission/buildwatch.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+)
+
+type (
+	// packageBuildWatcher is used to watch a set of in-progress builds.
+	packageBuildWatcher struct {
+		// fission client
+		fclient *client.Client
+
+		// set of packages already printed, ensures we don't duplicate the notifications
+		finished map[string]bool
+
+		// set of metadata in the app spec.  packages outside this set should be ignored.
+		pkgMeta map[string]metav1.ObjectMeta
+	}
+)
+
+func makePackageBuildWatcher(fclient *client.Client) *packageBuildWatcher {
+	return &packageBuildWatcher{
+		fclient:  fclient,
+		finished: make(map[string]bool),
+		pkgMeta:  make(map[string]metav1.ObjectMeta),
+	}
+}
+
+func (w *packageBuildWatcher) addPackages(pkgMeta map[string]metav1.ObjectMeta) {
+	for k, v := range pkgMeta {
+		w.pkgMeta[k] = v
+	}
+}
+
+func (w *packageBuildWatcher) watch(ctx context.Context) {
+	for {
+		// non-blocking check if we're cancelled
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		// poll list of packages (TODO: convert to watch)
+		pkgs, err := w.fclient.PackageList()
+		checkErr(err, "Getting list of packages")
+
+		// find packages that (a) are in the app spec and (b) have an interesting
+		// build status (either succeeded or failed; not "none")
+		keepWaiting := false
+		buildpkgs := make([]crd.Package, 0)
+		for _, pkg := range pkgs {
+			_, ok := w.pkgMeta[mapKey(&pkg.Metadata)]
+			if !ok {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusNone {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusPending ||
+				pkg.Status.BuildStatus == fission.BuildStatusRunning {
+				keepWaiting = true
+			}
+			buildpkgs = append(buildpkgs, pkg)
+		}
+
+		// print package status, and error logs if any
+		for _, pkg := range buildpkgs {
+			k := pkgKey(&pkg)
+			if _, printed := w.finished[k]; printed {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusFailed {
+				w.finished[k] = true
+				fmt.Printf("--- Build FAILED: ---\n%v\n------\n", pkg.Status.BuildLog)
+			} else if pkg.Status.BuildStatus == fission.BuildStatusSucceeded {
+				w.finished[k] = true
+				fmt.Printf("--- Build SUCCEEDED ---\n")
+				if len(pkg.Status.BuildLog) > 0 {
+					fmt.Printf("%v\n------\n", pkg.Status.BuildLog)
+				}
+			}
+		}
+
+		// if there are no builds running, we can stop polling
+		if !keepWaiting {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func pkgKey(pkg *crd.Package) string {
+	// packages are mutable so we want to keep track of them by resource version
+	return fmt.Sprintf("%v:%v:%v", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Metadata.ResourceVersion)
+}

fission/common.go

@@ -25,9 +25,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	"github.com/dchest/uniuri"
 	uuid "github.com/satori/go.uuid"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -104,8 +107,27 @@ func fileSize(filePath string) int64 {
 	return info.Size()
 }
 
+func fileChecksum(fileName string) (*fission.Checksum, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %v: %v", fileName, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to calculate checksum for %v", fileName)
+	}
+
+	return &fission.Checksum{
+		Type: fission.ChecksumTypeSHA256,
+		Sum:  hex.EncodeToString(h.Sum(nil)),
+	}, nil
+}
+
 // upload a file and return a fission.Archive
-func createArchive(client *client.Client, fileName string) *fission.Archive {
+func createArchive(client *client.Client, fileName string, specFile string) *fission.Archive {
 	var archive fission.Archive
 
 	// fetch archive from arbitrary url if fileName is a url
@@ -113,6 +135,23 @@ func createArchive(client *client.Client, fileName string) *fission.Archive {
 		fileName = downloadToTempFile(fileName)
 	}
 
+	if len(specFile) > 0 {
+		// create an ArchiveUploadSpec and reference it from the archive
+		aus := &ArchiveUploadSpec{
+			Name:         kubifyName(path.Base(fileName)),
+			IncludeGlobs: []string{fileName},
+		}
+		// save the uploadspec
+		err := specSave(*aus, specFile)
+		checkErr(err, fmt.Sprintf("write spec file %v", specFile))
+		// create the archive
+		ar := &fission.Archive{
+			Type: fission.ArchiveTypeUrl,
+			URL:  fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name),
+		}
+		return ar
+	}
+
 	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
 		contents := getContents(fileName)
 		archive.Type = fission.ArchiveTypeLiteral
@@ -130,26 +169,15 @@ func createArchive(client *client.Client, fileName string) *fission.Archive {
 		archive.Type = fission.ArchiveTypeUrl
 		archive.URL = archiveUrl
 
-		f, err := os.Open(fileName)
-		if err != nil {
-			checkErr(err, fmt.Sprintf("find file %v", fileName))
-		}
-		defer f.Close()
+		csum, err := fileChecksum(fileName)
+		checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
 
-		h := sha256.New()
-		if _, err := io.Copy(h, f); err != nil {
-			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
-		}
-
-		archive.Checksum = fission.Checksum{
-			Type: fission.ChecksumTypeSHA256,
-			Sum:  hex.EncodeToString(h.Sum(nil)),
-		}
+		archive.Checksum = *csum
 	}
 	return &archive
 }
 
-func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string, specFile string) *metav1.ObjectMeta {
 	pkgSpec := fission.PackageSpec{
 		Environment: fission.EnvironmentReference{
 			Namespace: metav1.NamespaceDefault,
@@ -158,20 +186,28 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive
 	}
 	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
 
+	var pkgName string
 	if len(deployArchiveName) > 0 {
-		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
+		if len(specFile) > 0 { // we should do this in all cases, i think
+			pkgStatus = fission.BuildStatusNone
+		}
+		pkgSpec.Deployment = *createArchive(client, deployArchiveName, specFile)
+		pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveName), uniuri.NewLen(4)))
 	}
 	if len(srcArchiveName) > 0 {
-		pkgSpec.Source = *createArchive(client, srcArchiveName)
+		pkgSpec.Source = *createArchive(client, srcArchiveName, specFile)
 		// set pending status to package
 		pkgStatus = fission.BuildStatusPending
+		pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveName), uniuri.NewLen(4)))
 	}
 
 	if len(buildcmd) > 0 {
 		pkgSpec.BuildCommand = buildcmd
 	}
 
-	pkgName := strings.ToLower(uuid.NewV4().String())
+	if len(pkgName) == 0 {
+		pkgName = strings.ToLower(uuid.NewV4().String())
+	}
 	pkg := &crd.Package{
 		Metadata: metav1.ObjectMeta{
 			Name:      pkgName,
@@ -182,9 +218,16 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive
 			BuildStatus: pkgStatus,
 		},
 	}
-	pkgMetadata, err := client.PackageCreate(pkg)
-	checkErr(err, "create package")
-	return pkgMetadata
+
+	if len(specFile) > 0 {
+		err := specSave(*pkg, specFile)
+		checkErr(err, "save package spec")
+		return &pkg.Metadata
+	} else {
+		pkgMetadata, err := client.PackageCreate(pkg)
+		checkErr(err, "create package")
+		return pkgMetadata
+	}
 }
 
 func getContents(filePath string) []byte {
@@ -263,3 +306,39 @@ func downloadURL(fileUrl string) (io.ReadCloser, error) {
 	}
 	return resp.Body, nil
 }
+
+// make a kubernetes compliant name out of an arbitrary string
+func kubifyName(old string) string {
+	// Kubernetes maximum name length (for some names; others can be 253 chars)
+	maxLen := 63
+
+	newName := strings.ToLower(old)
+
+	// replace disallowed chars with '-'
+	inv, err := regexp.Compile("[^-a-z0-9]")
+	checkErr(err, "compile regexp")
+	newName = string(inv.ReplaceAll([]byte(newName), []byte("-")))
+
+	// trim leading non-alphabetic
+	leadingnonalpha, err := regexp.Compile("^[^a-z]+")
+	checkErr(err, "compile regexp")
+	newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{}))
+
+	// trim trailing
+	trailing, err := regexp.Compile("[^a-z0-9]+$")
+	checkErr(err, "compile regexp")
+	newName = string(trailing.ReplaceAll([]byte(newName), []byte{}))
+
+	// truncate to length
+	if len(newName) > maxLen {
+		newName = newName[0:maxLen]
+	}
+
+	// if we removed everything, call this thing "default". maybe
+	// we should generate a unique name...
+	if len(newName) == 0 {
+		newName = "default"
+	}
+
+	return newName
+}

fission/function.go

@@ -112,6 +112,14 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --name argument.")
 	}
 
+	// user wants a spec, create a yaml file with package and function
+	spec := false
+	specFile := ""
+	if c.Bool("spec") {
+		spec = true
+		specFile = fmt.Sprintf("function-%v.yaml", fnName)
+	}
+
 	fnList, err := client.FunctionList()
 	checkErr(err, "get function list")
 	// check function existence before creating package
@@ -182,7 +190,7 @@ func fnCreate(c *cli.Context) error {
 		buildcmd := c.String("buildcmd")
 
 		// create new package
-		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, specFile)
 	}
 
 	//TODO Warn user about resources at fn level overriding the env resources
@@ -241,6 +249,14 @@ func fnCreate(c *cli.Context) error {
 		function.Spec.ConfigMaps = append(function.Spec.ConfigMaps, newCfgMap)
 	}
 
+	// if we're writing a spec, don't create the function
+	if spec {
+		err = specSave(*function, specFile)
+		checkErr(err, "create function spec")
+		return nil
+
+	}
+
 	_, err = client.FunctionCreate(function)
 	checkErr(err, "create function")
 

fission/main.go

@@ -68,9 +68,10 @@ func main() {
 	fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"}
 	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
 	fnExecutorTypeFlag := cli.StringFlag{Name: "executortype", Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy' defaults to 'poolmgr'"}
+	fnSpecSaveFlag := cli.BoolFlag{Name: "spec", Usage: "Save function to the spec directory instead of creating it"}
 
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnSpecSaveFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
 		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu}, Action: fnUpdate},
@@ -167,6 +168,7 @@ func main() {
 		{Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete},
 	}
 
+	// upgrades, data migrations
 	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
 	upgradeSubCommands := []cli.Command{
 		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
@@ -180,6 +182,20 @@ func main() {
 		{Name: "restore", Usage: "Restore state dumped from a pre-0.4 Fission cluster. Requires Fission 0.4, which uses Kubernetes CustomResources.", Flags: []cli.Flag{migrateFileFlag}, Action: migrateRestoreCRD},
 	}
 
+	// specs
+	specDirFlag := cli.StringFlag{Name: "specdir", Usage: "Directory to store specs, defaults to ./specs"}
+	specNameFlag := cli.StringFlag{Name: "name", Usage: "(optional) Name for the app, applied to resources as a Kubernetes annotation"}
+	specWaitFlag := cli.BoolFlag{Name: "wait", Usage: "Wait for package builds"}
+	specWatchFlag := cli.BoolFlag{Name: "watch", Usage: "Watch local files for change, and re-apply specs as necessary"}
+	specDeleteFlag := cli.BoolFlag{Name: "delete", Usage: "Allow apply to delete resources that no longer exist in the specification"}
+	specSubCommands := []cli.Command{
+		{Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag}, Action: specInit},
+		{Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: specValidate},
+		{Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: specApply},
+		{Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: specDestroy},
+		{Name: "helm", Usage: "Create a helm chart from the app specification", Flags: []cli.Flag{specDirFlag}, Action: specHelm},
+	}
+
 	app.Commands = []cli.Command{
 		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
 		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -188,6 +204,7 @@ func main() {
 		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
 		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
 		{Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands},
+		{Name: "spec", Aliases: []string{"specs"}, Usage: "Manage a declarative app specification", Subcommands: specSubCommands},
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands},
 	}

fission/package.go

@@ -76,7 +76,7 @@ func pkgCreate(c *cli.Context) error {
 		fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
 	}
 
-	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, "")
 	fmt.Printf("Package '%v' created\n", meta.GetName())
 
 	return nil
@@ -146,13 +146,13 @@ func updatePackage(client *client.Client, pkg *crd.Package, envName,
 	}
 
 	if len(srcArchiveName) > 0 {
-		srcArchiveMetadata = createArchive(client, srcArchiveName)
+		srcArchiveMetadata = createArchive(client, srcArchiveName, "")
 		pkg.Spec.Source = *srcArchiveMetadata
 		needToBuild = true
 	}
 
 	if len(deployArchiveName) > 0 {
-		deployArchiveMetadata = createArchive(client, deployArchiveName)
+		deployArchiveMetadata = createArchive(client, deployArchiveName, "")
 		pkg.Spec.Deployment = *deployArchiveMetadata
 	}
 

fission/types.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+const (
+	FISSION_DEPLOYMENT_NAME_KEY = "fission-name"
+	FISSION_DEPLOYMENT_UID_KEY  = "fission-uid"
+)
+
+// CLI spec types
+type (
+	// DeploymentConfig is the global configuration for a set of Fission specs.
+	DeploymentConfig struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "DeploymentConfig".
+		TypeMeta `json:",inline"`
+
+		// Name is a user-friendly name for the deployment. It is also stored in
+		// all uploaded resources as an annotation.
+		Name string `json:"name"`
+
+		// UID uniquely identifies the deployment. It is stored as a label and
+		// used to find resources to clean up when local specs are changed.
+		UID string `json:"uid"`
+	}
+
+	// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
+	//
+	// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
+	// using the name specified in the archive.  The fission spec applier will
+	// replace the archive:// URL with a real HTTP URL after uploading the file.
+	ArchiveUploadSpec struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "ArchiveUploadSpec".
+		TypeMeta `json:",inline"`
+
+		// Name is a local name that can be used to reference this archive. It
+		// must be unique; duplicate names will cause an error while handling
+		// specs.
+		Name string `json:"name"`
+
+		// RootDir specifies the root that the globs below are relative to. It
+		// is optional and defaults to the parent directory of the spec
+		// directory: for example, if the deployment config is at
+		// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
+		RootDir string `json:"rootdir,omitempty"`
+
+		// IncludeGlobs is a list of Unix shell globs to include
+		IncludeGlobs []string `json:"include,omitempty"`
+
+		// ExcludeGlobs is a list of globs to exclude from the set specified by
+		// IncludeGlobs.
+		ExcludeGlobs []string `json:"exclude,omitempty"`
+	}
+
+	// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
+	// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
+	// Kubernetes does.
+	TypeMeta struct {
+		Kind       string `json:"kind,omitempty"`
+		APIVersion string `json:"apiVersion,omitempty"`
+	}
+)

fission/upgrade.go

@@ -291,7 +291,7 @@ func upgradeRestoreState(c *cli.Context) error {
 		tmpfile.Close()
 
 		// upload
-		archive := createArchive(client, tmpfile.Name())
+		archive := createArchive(client, tmpfile.Name(), "")
 		os.Remove(tmpfile.Name())
 
 		// create pkg