Declarative application specifications for Fission (#422)

This change enables users to have declarative specifications for Fission resources. Users can specify their "app" in a set of spec files, and use a new fission CLI command to "apply" these specs to a running cluster.

The new "spec" CLI also includes archiving of local source files, and a file watcher that re-builds archives and uploads them on file changes, and a package build watcher that waits for package builds on the CLI. 

A "--spec" option is also added to "function create", and will be added to other resources in future changes.  This option causes a YAML to be outputted to the specs directory instead of the resource being created on the cluster.

The CLI "fission spec --help" outputs usage information.

examples/go/README.md

@@ -1,40 +1,17 @@
-# Go examples
+# Hello World in Go on Fission
 
-The `go` runtime uses the [`plugin` package](https://golang.org/pkg/plugin/) to dynamically load an HTTP handler.
-
-## Requirements
-
-First, set up your fission deployment with the go environment.
-
-```
-fission env create --name go-env --image fission/go-env:1.8.1
-```
-
-To ensure that you build functions using the same version as the
-runtime, fission provides a docker image and helper script for
-building functions.
-
-## Example Usage
-
-### hello.go
-
-`hello.go` is an very basic HTTP handler returning `"Hello, World!"`.
+`hello.go` is an very simple fission function that says "Hello, World!".
 
 ```bash
-# Download the build helper script
-$ curl https://raw.githubusercontent.com/fission/fission/master/environments/go/builder/go-function-build > go-function-build
-$ chmod +x go-function-build
+# This command creates the environment and function, and waits for the
+# function build.  Look at the YAML files in specs/ for details about
+# how those are specified.
+$ fission spec apply --wait
+1 environment created
+1 package created
+1 function created
 
-# Build the function as a plugin. Outputs result to 'function.so'
-$ go-function-build hello.go
-
-# Upload the function to fission
-$ fission function create --name hello --env go-env --package function.so
-
-# Map /hello to the hello function
-$ fission route create --method GET --url /hello --function hello
-
-# Run the function
-$ curl http://$FISSION_ROUTER/hello
+# This run the function and prints its output
+$ fission function test --name hello
 Hello, World!
 ```

examples/go/hello.go

@@ -4,7 +4,8 @@ import (
 	"net/http"
 )
 
+// Handler is the entry point for this fission function
 func Handler(w http.ResponseWriter, r *http.Request) {
-	msg := "Hello, World!"
+	msg := "Hello, world!\n"
 	w.Write([]byte(msg))
 }

examples/go/specs/env.yaml

@@ -0,0 +1,12 @@
+apiVersion: fission.io/v1
+kind: Environment
+metadata:
+  name: go
+  namespace: default
+spec:
+  version: 2
+  builder:
+    command: build
+    image: fission/go-build-env:20171206-2
+  runtime:
+    image: fission/go-env:20171206

examples/go/specs/fission-deployment-config.yaml

@@ -0,0 +1,6 @@
+# This file is generated by the 'fission spec init' command.
+# See the README in this directory for background and usage information.
+# Do not edit the UID below: that will break 'fission spec apply'
+kind: DeploymentConfig
+name: hello-go
+uid: a8cdb63c-9be8-4a59-9427-89051afecd7b

examples/go/specs/function-hello.yaml

@@ -0,0 +1,38 @@
+kind: ArchiveUploadSpec
+name: hello-go
+include:
+- hello.go
+
+---
+apiVersion: fission.io/v1
+kind: Package
+metadata:
+  creationTimestamp: null
+  name: hello-go-pkg
+  namespace: default
+spec:
+  environment:
+    name: go
+    namespace: default
+  source:
+    type: url
+    url: archive://hello-go
+status:
+  buildstatus: pending
+
+---
+apiVersion: fission.io/v1
+kind: Function
+metadata:
+  creationTimestamp: null
+  name: hello-go
+  namespace: default
+spec:
+  environment:
+    name: go
+    namespace: default
+  package:
+    functionName: Handler
+    packageref:
+      name: hello-go-pkg
+      namespace: default

examples/spec-example/README.md

@@ -0,0 +1,11 @@
+This is the root directory of a declaratively specified fission "application".  The app
+contains source code for one function (a simple "hello world") in the hello/hello.py
+file.
+
+The `specs` directory contains YAML files that specify the Fission environment and
+function.
+
+You can create this app on your cluster by running `fission spec apply` from this
+directory.  See `fission spec --help` for other options.
+
+After applying the spec, you can test the function with `fission fn test --name hello`.

examples/spec-example/hello/build.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+# check syntax
+python -m compileall -l ${SRC_PKG}
+
+# install deps
+pip install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG}

examples/spec-example/hello/hello.py

@@ -0,0 +1,4 @@
+import yaml
+
+def main():
+    return "hello, world!!\n"

examples/spec-example/hello/requirements.txt

@@ -0,0 +1 @@
+pyyaml

examples/spec-example/specs/env.yaml

@@ -0,0 +1,12 @@
+apiVersion: fission.io/v1
+kind: Environment
+metadata:
+  name: python-27
+  namespace: default
+spec:
+  version: 2
+  builder:
+    command: build
+    image: fission/python-build-env-2.7:0.4.0rc
+  runtime:
+    image: fission/python-env-2.7:0.4.0rc

examples/spec-example/specs/fission-config.yaml

@@ -0,0 +1,3 @@
+kind: DeploymentConfig
+name: spec-example
+uid: 27438a48-4191-4b5e-95b2-7793624317b9

examples/spec-example/specs/hello.yaml

@@ -0,0 +1,38 @@
+apiVersion: fission.io/v1
+kind: Function
+metadata:
+  name: hello
+  namespace: default
+spec:
+  environment:
+    name: python-27
+    namespace: default
+  package:
+    packageref:
+      name: hello-pkg
+      namespace: default
+    functionName: hello.main
+
+---
+apiVersion: fission.io/v1
+kind: Package
+metadata:
+  name: hello-pkg
+  namespace: default
+spec:
+  source:
+    url: archive://hello-archive
+  buildcmd: "./build.sh"
+  environment:
+    name: python-27
+    namespace: default
+status:
+  buildstatus: pending
+
+---
+kind: ArchiveUploadSpec
+name: hello-archive
+include:
+  - "hello/*.py"
+  - "hello/*.sh"
+  - "hello/requirements.txt"

fission/buildwatch.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+)
+
+type (
+	// packageBuildWatcher is used to watch a set of in-progress builds.
+	packageBuildWatcher struct {
+		// fission client
+		fclient *client.Client
+
+		// set of packages already printed, ensures we don't duplicate the notifications
+		finished map[string]bool
+
+		// set of metadata in the app spec.  packages outside this set should be ignored.
+		pkgMeta map[string]metav1.ObjectMeta
+	}
+)
+
+func makePackageBuildWatcher(fclient *client.Client) *packageBuildWatcher {
+	return &packageBuildWatcher{
+		fclient:  fclient,
+		finished: make(map[string]bool),
+		pkgMeta:  make(map[string]metav1.ObjectMeta),
+	}
+}
+
+func (w *packageBuildWatcher) addPackages(pkgMeta map[string]metav1.ObjectMeta) {
+	for k, v := range pkgMeta {
+		w.pkgMeta[k] = v
+	}
+}
+
+func (w *packageBuildWatcher) watch(ctx context.Context) {
+	for {
+		// non-blocking check if we're cancelled
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		// poll list of packages (TODO: convert to watch)
+		pkgs, err := w.fclient.PackageList()
+		checkErr(err, "Getting list of packages")
+
+		// find packages that (a) are in the app spec and (b) have an interesting
+		// build status (either succeeded or failed; not "none")
+		keepWaiting := false
+		buildpkgs := make([]crd.Package, 0)
+		for _, pkg := range pkgs {
+			_, ok := w.pkgMeta[mapKey(&pkg.Metadata)]
+			if !ok {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusNone {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusPending ||
+				pkg.Status.BuildStatus == fission.BuildStatusRunning {
+				keepWaiting = true
+			}
+			buildpkgs = append(buildpkgs, pkg)
+		}
+
+		// print package status, and error logs if any
+		for _, pkg := range buildpkgs {
+			k := pkgKey(&pkg)
+			if _, printed := w.finished[k]; printed {
+				continue
+			}
+			if pkg.Status.BuildStatus == fission.BuildStatusFailed {
+				w.finished[k] = true
+				fmt.Printf("--- Build FAILED: ---\n%v\n------\n", pkg.Status.BuildLog)
+			} else if pkg.Status.BuildStatus == fission.BuildStatusSucceeded {
+				w.finished[k] = true
+				fmt.Printf("--- Build SUCCEEDED ---\n")
+				if len(pkg.Status.BuildLog) > 0 {
+					fmt.Printf("%v\n------\n", pkg.Status.BuildLog)
+				}
+			}
+		}
+
+		// if there are no builds running, we can stop polling
+		if !keepWaiting {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func pkgKey(pkg *crd.Package) string {
+	// packages are mutable so we want to keep track of them by resource version
+	return fmt.Sprintf("%v:%v:%v", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Metadata.ResourceVersion)
+}

fission/common.go

@@ -25,9 +25,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	"github.com/dchest/uniuri"
 	uuid "github.com/satori/go.uuid"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -104,8 +107,27 @@ func fileSize(filePath string) int64 {
 	return info.Size()
 }
 
+func fileChecksum(fileName string) (*fission.Checksum, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %v: %v", fileName, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to calculate checksum for %v", fileName)
+	}
+
+	return &fission.Checksum{
+		Type: fission.ChecksumTypeSHA256,
+		Sum:  hex.EncodeToString(h.Sum(nil)),
+	}, nil
+}
+
 // upload a file and return a fission.Archive
-func createArchive(client *client.Client, fileName string) *fission.Archive {
+func createArchive(client *client.Client, fileName string, specFile string) *fission.Archive {
 	var archive fission.Archive
 
 	// fetch archive from arbitrary url if fileName is a url
@@ -113,6 +135,23 @@ func createArchive(client *client.Client, fileName string) *fission.Archive {
 		fileName = downloadToTempFile(fileName)
 	}
 
+	if len(specFile) > 0 {
+		// create an ArchiveUploadSpec and reference it from the archive
+		aus := &ArchiveUploadSpec{
+			Name:         kubifyName(path.Base(fileName)),
+			IncludeGlobs: []string{fileName},
+		}
+		// save the uploadspec
+		err := specSave(*aus, specFile)
+		checkErr(err, fmt.Sprintf("write spec file %v", specFile))
+		// create the archive
+		ar := &fission.Archive{
+			Type: fission.ArchiveTypeUrl,
+			URL:  fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name),
+		}
+		return ar
+	}
+
 	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
 		contents := getContents(fileName)
 		archive.Type = fission.ArchiveTypeLiteral
@@ -130,26 +169,15 @@ func createArchive(client *client.Client, fileName string) *fission.Archive {
 		archive.Type = fission.ArchiveTypeUrl
 		archive.URL = archiveUrl
 
-		f, err := os.Open(fileName)
-		if err != nil {
-			checkErr(err, fmt.Sprintf("find file %v", fileName))
-		}
-		defer f.Close()
+		csum, err := fileChecksum(fileName)
+		checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
 
-		h := sha256.New()
-		if _, err := io.Copy(h, f); err != nil {
-			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
-		}
-
-		archive.Checksum = fission.Checksum{
-			Type: fission.ChecksumTypeSHA256,
-			Sum:  hex.EncodeToString(h.Sum(nil)),
-		}
+		archive.Checksum = *csum
 	}
 	return &archive
 }
 
-func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string, specFile string) *metav1.ObjectMeta {
 	pkgSpec := fission.PackageSpec{
 		Environment: fission.EnvironmentReference{
 			Namespace: metav1.NamespaceDefault,
@@ -158,20 +186,28 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive
 	}
 	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
 
+	var pkgName string
 	if len(deployArchiveName) > 0 {
-		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
+		if len(specFile) > 0 { // we should do this in all cases, i think
+			pkgStatus = fission.BuildStatusNone
+		}
+		pkgSpec.Deployment = *createArchive(client, deployArchiveName, specFile)
+		pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveName), uniuri.NewLen(4)))
 	}
 	if len(srcArchiveName) > 0 {
-		pkgSpec.Source = *createArchive(client, srcArchiveName)
+		pkgSpec.Source = *createArchive(client, srcArchiveName, specFile)
 		// set pending status to package
 		pkgStatus = fission.BuildStatusPending
+		pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveName), uniuri.NewLen(4)))
 	}
 
 	if len(buildcmd) > 0 {
 		pkgSpec.BuildCommand = buildcmd
 	}
 
-	pkgName := strings.ToLower(uuid.NewV4().String())
+	if len(pkgName) == 0 {
+		pkgName = strings.ToLower(uuid.NewV4().String())
+	}
 	pkg := &crd.Package{
 		Metadata: metav1.ObjectMeta{
 			Name:      pkgName,
@@ -182,9 +218,16 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive
 			BuildStatus: pkgStatus,
 		},
 	}
-	pkgMetadata, err := client.PackageCreate(pkg)
-	checkErr(err, "create package")
-	return pkgMetadata
+
+	if len(specFile) > 0 {
+		err := specSave(*pkg, specFile)
+		checkErr(err, "save package spec")
+		return &pkg.Metadata
+	} else {
+		pkgMetadata, err := client.PackageCreate(pkg)
+		checkErr(err, "create package")
+		return pkgMetadata
+	}
 }
 
 func getContents(filePath string) []byte {
@@ -263,3 +306,39 @@ func downloadURL(fileUrl string) (io.ReadCloser, error) {
 	}
 	return resp.Body, nil
 }
+
+// make a kubernetes compliant name out of an arbitrary string
+func kubifyName(old string) string {
+	// Kubernetes maximum name length (for some names; others can be 253 chars)
+	maxLen := 63
+
+	newName := strings.ToLower(old)
+
+	// replace disallowed chars with '-'
+	inv, err := regexp.Compile("[^-a-z0-9]")
+	checkErr(err, "compile regexp")
+	newName = string(inv.ReplaceAll([]byte(newName), []byte("-")))
+
+	// trim leading non-alphabetic
+	leadingnonalpha, err := regexp.Compile("^[^a-z]+")
+	checkErr(err, "compile regexp")
+	newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{}))
+
+	// trim trailing
+	trailing, err := regexp.Compile("[^a-z0-9]+$")
+	checkErr(err, "compile regexp")
+	newName = string(trailing.ReplaceAll([]byte(newName), []byte{}))
+
+	// truncate to length
+	if len(newName) > maxLen {
+		newName = newName[0:maxLen]
+	}
+
+	// if we removed everything, call this thing "default". maybe
+	// we should generate a unique name...
+	if len(newName) == 0 {
+		newName = "default"
+	}
+
+	return newName
+}

fission/function.go

@@ -112,6 +112,14 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --name argument.")
 	}
 
+	// user wants a spec, create a yaml file with package and function
+	spec := false
+	specFile := ""
+	if c.Bool("spec") {
+		spec = true
+		specFile = fmt.Sprintf("function-%v.yaml", fnName)
+	}
+
 	fnList, err := client.FunctionList()
 	checkErr(err, "get function list")
 	// check function existence before creating package
@@ -182,7 +190,7 @@ func fnCreate(c *cli.Context) error {
 		buildcmd := c.String("buildcmd")
 
 		// create new package
-		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, specFile)
 	}
 
 	//TODO Warn user about resources at fn level overriding the env resources
@@ -241,6 +249,14 @@ func fnCreate(c *cli.Context) error {
 		function.Spec.ConfigMaps = append(function.Spec.ConfigMaps, newCfgMap)
 	}
 
+	// if we're writing a spec, don't create the function
+	if spec {
+		err = specSave(*function, specFile)
+		checkErr(err, "create function spec")
+		return nil
+
+	}
+
 	_, err = client.FunctionCreate(function)
 	checkErr(err, "create function")
 

fission/main.go

@@ -68,9 +68,10 @@ func main() {
 	fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"}
 	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
 	fnExecutorTypeFlag := cli.StringFlag{Name: "executortype", Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy' defaults to 'poolmgr'"}
+	fnSpecSaveFlag := cli.BoolFlag{Name: "spec", Usage: "Save function to the spec directory instead of creating it"}
 
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnSpecSaveFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
 		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu}, Action: fnUpdate},
@@ -167,6 +168,7 @@ func main() {
 		{Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete},
 	}
 
+	// upgrades, data migrations
 	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
 	upgradeSubCommands := []cli.Command{
 		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
@@ -180,6 +182,20 @@ func main() {
 		{Name: "restore", Usage: "Restore state dumped from a pre-0.4 Fission cluster. Requires Fission 0.4, which uses Kubernetes CustomResources.", Flags: []cli.Flag{migrateFileFlag}, Action: migrateRestoreCRD},
 	}
 
+	// specs
+	specDirFlag := cli.StringFlag{Name: "specdir", Usage: "Directory to store specs, defaults to ./specs"}
+	specNameFlag := cli.StringFlag{Name: "name", Usage: "(optional) Name for the app, applied to resources as a Kubernetes annotation"}
+	specWaitFlag := cli.BoolFlag{Name: "wait", Usage: "Wait for package builds"}
+	specWatchFlag := cli.BoolFlag{Name: "watch", Usage: "Watch local files for change, and re-apply specs as necessary"}
+	specDeleteFlag := cli.BoolFlag{Name: "delete", Usage: "Allow apply to delete resources that no longer exist in the specification"}
+	specSubCommands := []cli.Command{
+		{Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag}, Action: specInit},
+		{Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: specValidate},
+		{Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: specApply},
+		{Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: specDestroy},
+		{Name: "helm", Usage: "Create a helm chart from the app specification", Flags: []cli.Flag{specDirFlag}, Action: specHelm},
+	}
+
 	app.Commands = []cli.Command{
 		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
 		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -188,6 +204,7 @@ func main() {
 		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
 		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
 		{Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands},
+		{Name: "spec", Aliases: []string{"specs"}, Usage: "Manage a declarative app specification", Subcommands: specSubCommands},
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands},
 	}

fission/package.go

@@ -76,7 +76,7 @@ func pkgCreate(c *cli.Context) error {
 		fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
 	}
 
-	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, "")
 	fmt.Printf("Package '%v' created\n", meta.GetName())
 
 	return nil
@@ -146,13 +146,13 @@ func updatePackage(client *client.Client, pkg *crd.Package, envName,
 	}
 
 	if len(srcArchiveName) > 0 {
-		srcArchiveMetadata = createArchive(client, srcArchiveName)
+		srcArchiveMetadata = createArchive(client, srcArchiveName, "")
 		pkg.Spec.Source = *srcArchiveMetadata
 		needToBuild = true
 	}
 
 	if len(deployArchiveName) > 0 {
-		deployArchiveMetadata = createArchive(client, deployArchiveName)
+		deployArchiveMetadata = createArchive(client, deployArchiveName, "")
 		pkg.Spec.Deployment = *deployArchiveMetadata
 	}
 

fission/types.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+const (
+	FISSION_DEPLOYMENT_NAME_KEY = "fission-name"
+	FISSION_DEPLOYMENT_UID_KEY  = "fission-uid"
+)
+
+// CLI spec types
+type (
+	// DeploymentConfig is the global configuration for a set of Fission specs.
+	DeploymentConfig struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "DeploymentConfig".
+		TypeMeta `json:",inline"`
+
+		// Name is a user-friendly name for the deployment. It is also stored in
+		// all uploaded resources as an annotation.
+		Name string `json:"name"`
+
+		// UID uniquely identifies the deployment. It is stored as a label and
+		// used to find resources to clean up when local specs are changed.
+		UID string `json:"uid"`
+	}
+
+	// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
+	//
+	// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
+	// using the name specified in the archive.  The fission spec applier will
+	// replace the archive:// URL with a real HTTP URL after uploading the file.
+	ArchiveUploadSpec struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "ArchiveUploadSpec".
+		TypeMeta `json:",inline"`
+
+		// Name is a local name that can be used to reference this archive. It
+		// must be unique; duplicate names will cause an error while handling
+		// specs.
+		Name string `json:"name"`
+
+		// RootDir specifies the root that the globs below are relative to. It
+		// is optional and defaults to the parent directory of the spec
+		// directory: for example, if the deployment config is at
+		// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
+		RootDir string `json:"rootdir,omitempty"`
+
+		// IncludeGlobs is a list of Unix shell globs to include
+		IncludeGlobs []string `json:"include,omitempty"`
+
+		// ExcludeGlobs is a list of globs to exclude from the set specified by
+		// IncludeGlobs.
+		ExcludeGlobs []string `json:"exclude,omitempty"`
+	}
+
+	// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
+	// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
+	// Kubernetes does.
+	TypeMeta struct {
+		Kind       string `json:"kind,omitempty"`
+		APIVersion string `json:"apiVersion,omitempty"`
+	}
+)

fission/upgrade.go

@@ -291,7 +291,7 @@ func upgradeRestoreState(c *cli.Context) error {
 		tmpfile.Close()
 
 		// upload
-		archive := createArchive(client, tmpfile.Name())
+		archive := createArchive(client, tmpfile.Name(), "")
 		os.Remove(tmpfile.Name())
 
 		// create pkg

glide.lock

@@ -1,5 +1,5 @@
-hash: 7c938e591b0602b9fd3412aeffd3f1438fc77eaea80c8d9dd03431adb0e1e1c9
-updated: 2017-12-13T16:13:45.935783+08:00
+hash: 93c80adbba10750a7fca4536f93880f152527a8f04eedf40a8ea8cbadeb88779
+updated: 2017-12-02T21:38:02.116016618-08:00
 imports:
 - name: cloud.google.com/go
   version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -46,6 +46,8 @@ imports:
   - log
 - name: github.com/emicklei/go-restful-swagger12
   version: dcef7f55730566d41eae5db10e7d6981829720f6
+- name: github.com/fsnotify/fsnotify
+  version: 4da3e2cfbabc9f751898f250b49f2439785783a1
 - name: github.com/ghodss/yaml
   version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee
 - name: github.com/go-openapi/analysis
@@ -134,6 +136,8 @@ imports:
   version: a0006b13c722f7f12368c00a3d3c2ae8a999a0c6
   subpackages:
   - xxHash32
+- name: github.com/pkg/errors
+  version: f15c970de5b76fac0b59abb32d62c17cc7bed265
 - name: github.com/PuerkitoBio/purell
   version: 8a290539e2e8629dbc4e6bad948158f790ec31f4
 - name: github.com/PuerkitoBio/urlesc

glide.yaml

@@ -48,3 +48,6 @@ import:
   version: ^v0.4.0
 - package: github.com/graymeta/stow
 - package: github.com/mholt/archiver
+- package: github.com/pkg/errors
+- package: github.com/fsnotify/fsnotify
+

test/tests/test_spec_create/hello.py

@@ -0,0 +1,2 @@
+def main():
+    return "hello\n"

test/tests/test_spec_create/test_spec.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+#test:disabled
+
+set -euo pipefail
+
+fn=spec-$(date +%N)
+env=python-$fn
+
+# init
+fission spec init
+
+# verify init
+[ -d specs ]
+[ -f specs/README ]
+[ -f specs/fission-deployment-config.yaml ]
+
+# TODO replace with `fission env create --spec`
+cat > specs/env.yaml <<EOF
+apiVersion: fission.io/v1
+kind: Environment
+metadata:
+  name: $env
+  namespace: default
+spec:
+  version: 1
+  runtime:
+    image: fission/python-env:0.4.0
+EOF
+
+# create env
+fission spec apply
+trap "fission env delete --name $env" EXIT
+
+# verify env created
+fission env list | grep python
+
+# generate function spec
+fission fn create --spec --name $fn --env $env --code $(dirname $0)/hello.py
+
+# verify that function spec exists and has ArchiveUploadSpec, Package and Function
+[ -f specs/function-$fn.yaml ]
+grep ArchiveUploadSpec specs/*.yaml
+grep Package specs/*.yaml
+grep Function specs/*.yaml
+
+# create function
+fission spec apply
+trap "fission fn delete --name $fn" EXIT
+
+# verify function
+fission fn list | grep $fn
+
+sleep 3
+
+fission fn test --name $fn | grep -i hello
+
+

types.go

@@ -35,8 +35,8 @@ type (
 	// "sha256" is the only currently supported one. Sum is hex
 	// encoded.
 	Checksum struct {
-		Type ChecksumType `json:"type"`
-		Sum  string       `json:"sum"`
+		Type ChecksumType `json:"type,omitempty"`
+		Sum  string       `json:"sum,omitempty"`
 	}
 
 	// ArchiveType is either literal or URL, indicating whether
@@ -48,18 +48,18 @@ type (
 	// binary files.
 	Archive struct {
 		// Type defines how the package is specified: literal or URL.
-		Type ArchiveType `json:"type"`
+		Type ArchiveType `json:"type,omitempty"`
 
 		// Literal contents of the package. Can be used for
 		// encoding packages below TODO (256KB?) size.
-		Literal []byte `json:"literal"`
+		Literal []byte `json:"literal,omitempty"`
 
 		// URL references a package.
-		URL string `json:"url"`
+		URL string `json:"url,omitempty"`
 
 		// Checksum ensures the integrity of packages
 		// refereced by URL. Ignored for literals.
-		Checksum Checksum `json:"checksum"`
+		Checksum Checksum `json:"checksum,omitempty"`
 	}
 
 	EnvironmentReference struct {
@@ -81,14 +81,14 @@ type (
 
 	PackageSpec struct {
 		Environment  EnvironmentReference `json:"environment"`
-		Source       Archive              `json:"source"`
-		Deployment   Archive              `json:"deployment"`
-		BuildCommand string               `json:"buildcmd"`
+		Source       Archive              `json:"source,omitempty"`
+		Deployment   Archive              `json:"deployment,omitempty"`
+		BuildCommand string               `json:"buildcmd,omitempty"`
 		// In the future, we can have a debug build here too
 	}
 	PackageStatus struct {
-		BuildStatus BuildStatus `json:"buildstatus"`
-		BuildLog    string      `json:"buildlog"` // output of the build (errors etc)
+		BuildStatus BuildStatus `json:"buildstatus,omitempty"`
+		BuildLog    string      `json:"buildlog,omitempty"` // output of the build (errors etc)
 	}
 
 	PackageRef struct {
@@ -97,7 +97,7 @@ type (
 
 		// Including resource version in the reference forces the function to be updated on
 		// package update, making it possible to cache the function based on its metadata.
-		ResourceVersion string `json:"resourceversion"`
+		ResourceVersion string `json:"resourceversion,omitempty"`
 	}
 	FunctionPackageRef struct {
 		PackageRef PackageRef `json:"packageref"`
@@ -110,7 +110,7 @@ type (
 		// build and runtime environments.
 		//
 		// This is optional: if unspecified, the environment has a default name.
-		FunctionName string `json:"functionName"`
+		FunctionName string `json:"functionName,omitempty"`
 	}
 
 	//ExecutorType is the primary executor for an environment
@@ -211,10 +211,10 @@ type (
 	}
 	Builder struct {
 		// Image for containing the language runtime.
-		Image string `json:"image"`
+		Image string `json:"image,omitempty"`
 
 		// (Optional) Default build command to run for this build environment.
-		Command string `json:"command"`
+		Command string `json:"command,omitempty"`
 	}
 	EnvironmentSpec struct {
 		// Environment API version
@@ -228,17 +228,16 @@ type (
 
 		// Optional, but strongly encouraged. Used to populate
 		// links from UI, CLI, etc.
-		DocumentationURL string `json:"documentationurl"`
+		DocumentationURL string `json:"documentationurl,omitempty"`
 
-		// Optional
-		// Defaults to 'Single'
-		AllowedFunctionsPerContainer AllowedFunctionsPerContainer `json:"allowedFunctionsPerContainer"`
+		// Optional, defaults to 'AllowedFunctionsPerContainerSingle'
+		AllowedFunctionsPerContainer AllowedFunctionsPerContainer `json:"allowedFunctionsPerContainer,omitempty"`
 
 		// Request and limit resources for the environment
 		Resources v1.ResourceRequirements `json:"resources"`
 
 		// The initial pool size for environment
-		Poolsize int `json:"poolsize"`
+		Poolsize int `json:"poolsize,omitempty"`
 	}
 
 	AllowedFunctionsPerContainer string