From 07c7b759d281e16b095132412ed04cec2ffd00e4 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Mon, 5 Feb 2018 21:58:44 -0800 Subject: [PATCH] Declarative application specifications for Fission (#422) This change enables users to have declarative specifications for Fission resources. Users can specify their "app" in a set of spec files, and use a new fission CLI command to "apply" these specs to a running cluster. The new "spec" CLI also includes archiving of local source files, and a file watcher that re-builds archives and uploads them on file changes, and a package build watcher that waits for package builds on the CLI. A "--spec" option is also added to "function create", and will be added to other resources in future changes. This option causes a YAML to be outputted to the specs directory instead of the resource being created on the cluster. The CLI "fission spec --help" outputs usage information. --- examples/go/README.md | 45 +- examples/go/hello.go | 3 +- examples/go/specs/env.yaml | 12 + .../go/specs/fission-deployment-config.yaml | 6 + examples/go/specs/function-hello.yaml | 38 + examples/spec-example/README.md | 11 + examples/spec-example/hello/__init__.py | 0 examples/spec-example/hello/build.sh | 9 + examples/spec-example/hello/hello.py | 4 + examples/spec-example/hello/requirements.txt | 1 + examples/spec-example/specs/env.yaml | 12 + .../spec-example/specs/fission-config.yaml | 3 + examples/spec-example/specs/hello.yaml | 38 + fission/buildwatch.go | 120 ++ fission/common.go | 123 +- fission/function.go | 18 +- fission/main.go | 19 +- fission/package.go | 6 +- fission/spec.go | 1441 +++++++++++++++++ fission/types.go | 77 + fission/upgrade.go | 2 +- glide.lock | 8 +- glide.yaml | 3 + test/tests/test_spec_create/hello.py | 2 + test/tests/test_spec_create/test_spec.sh | 58 + types.go | 39 +- 26 files changed, 2013 insertions(+), 85 deletions(-) create mode 100644 examples/go/specs/env.yaml create mode 100644 examples/go/specs/fission-deployment-config.yaml create mode 100644 examples/go/specs/function-hello.yaml create mode 100644 examples/spec-example/README.md create mode 100644 examples/spec-example/hello/__init__.py create mode 100755 examples/spec-example/hello/build.sh create mode 100644 examples/spec-example/hello/hello.py create mode 100644 examples/spec-example/hello/requirements.txt create mode 100644 examples/spec-example/specs/env.yaml create mode 100644 examples/spec-example/specs/fission-config.yaml create mode 100644 examples/spec-example/specs/hello.yaml create mode 100644 fission/buildwatch.go create mode 100644 fission/spec.go create mode 100644 fission/types.go create mode 100644 test/tests/test_spec_create/hello.py create mode 100755 test/tests/test_spec_create/test_spec.sh diff --git a/examples/go/README.md b/examples/go/README.md index afd91e31..17bb1634 100644 --- a/examples/go/README.md +++ b/examples/go/README.md @@ -1,40 +1,17 @@ -# Go examples +# Hello World in Go on Fission -The `go` runtime uses the [`plugin` package](https://golang.org/pkg/plugin/) to dynamically load an HTTP handler. - -## Requirements - -First, set up your fission deployment with the go environment. - -``` -fission env create --name go-env --image fission/go-env:1.8.1 -``` - -To ensure that you build functions using the same version as the -runtime, fission provides a docker image and helper script for -building functions. - -## Example Usage - -### hello.go - -`hello.go` is an very basic HTTP handler returning `"Hello, World!"`. +`hello.go` is an very simple fission function that says "Hello, World!". ```bash -# Download the build helper script -$ curl https://raw.githubusercontent.com/fission/fission/master/environments/go/builder/go-function-build > go-function-build -$ chmod +x go-function-build +# This command creates the environment and function, and waits for the +# function build. Look at the YAML files in specs/ for details about +# how those are specified. +$ fission spec apply --wait +1 environment created +1 package created +1 function created -# Build the function as a plugin. Outputs result to 'function.so' -$ go-function-build hello.go - -# Upload the function to fission -$ fission function create --name hello --env go-env --package function.so - -# Map /hello to the hello function -$ fission route create --method GET --url /hello --function hello - -# Run the function -$ curl http://$FISSION_ROUTER/hello +# This run the function and prints its output +$ fission function test --name hello Hello, World! ``` diff --git a/examples/go/hello.go b/examples/go/hello.go index c58ebf27..88c004fe 100644 --- a/examples/go/hello.go +++ b/examples/go/hello.go @@ -4,7 +4,8 @@ import ( "net/http" ) +// Handler is the entry point for this fission function func Handler(w http.ResponseWriter, r *http.Request) { - msg := "Hello, World!" + msg := "Hello, world!\n" w.Write([]byte(msg)) } diff --git a/examples/go/specs/env.yaml b/examples/go/specs/env.yaml new file mode 100644 index 00000000..c2716724 --- /dev/null +++ b/examples/go/specs/env.yaml @@ -0,0 +1,12 @@ +apiVersion: fission.io/v1 +kind: Environment +metadata: + name: go + namespace: default +spec: + version: 2 + builder: + command: build + image: fission/go-build-env:20171206-2 + runtime: + image: fission/go-env:20171206 diff --git a/examples/go/specs/fission-deployment-config.yaml b/examples/go/specs/fission-deployment-config.yaml new file mode 100644 index 00000000..0967a979 --- /dev/null +++ b/examples/go/specs/fission-deployment-config.yaml @@ -0,0 +1,6 @@ +# This file is generated by the 'fission spec init' command. +# See the README in this directory for background and usage information. +# Do not edit the UID below: that will break 'fission spec apply' +kind: DeploymentConfig +name: hello-go +uid: a8cdb63c-9be8-4a59-9427-89051afecd7b diff --git a/examples/go/specs/function-hello.yaml b/examples/go/specs/function-hello.yaml new file mode 100644 index 00000000..a6da6abb --- /dev/null +++ b/examples/go/specs/function-hello.yaml @@ -0,0 +1,38 @@ +kind: ArchiveUploadSpec +name: hello-go +include: +- hello.go + +--- +apiVersion: fission.io/v1 +kind: Package +metadata: + creationTimestamp: null + name: hello-go-pkg + namespace: default +spec: + environment: + name: go + namespace: default + source: + type: url + url: archive://hello-go +status: + buildstatus: pending + +--- +apiVersion: fission.io/v1 +kind: Function +metadata: + creationTimestamp: null + name: hello-go + namespace: default +spec: + environment: + name: go + namespace: default + package: + functionName: Handler + packageref: + name: hello-go-pkg + namespace: default diff --git a/examples/spec-example/README.md b/examples/spec-example/README.md new file mode 100644 index 00000000..5b3b7e74 --- /dev/null +++ b/examples/spec-example/README.md @@ -0,0 +1,11 @@ +This is the root directory of a declaratively specified fission "application". The app +contains source code for one function (a simple "hello world") in the hello/hello.py +file. + +The `specs` directory contains YAML files that specify the Fission environment and +function. + +You can create this app on your cluster by running `fission spec apply` from this +directory. See `fission spec --help` for other options. + +After applying the spec, you can test the function with `fission fn test --name hello`. diff --git a/examples/spec-example/hello/__init__.py b/examples/spec-example/hello/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/spec-example/hello/build.sh b/examples/spec-example/hello/build.sh new file mode 100755 index 00000000..3edecfcb --- /dev/null +++ b/examples/spec-example/hello/build.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +# check syntax +python -m compileall -l ${SRC_PKG} + +# install deps +pip install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG} diff --git a/examples/spec-example/hello/hello.py b/examples/spec-example/hello/hello.py new file mode 100644 index 00000000..95c7208b --- /dev/null +++ b/examples/spec-example/hello/hello.py @@ -0,0 +1,4 @@ +import yaml + +def main(): + return "hello, world!!\n" diff --git a/examples/spec-example/hello/requirements.txt b/examples/spec-example/hello/requirements.txt new file mode 100644 index 00000000..c3726e8b --- /dev/null +++ b/examples/spec-example/hello/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/examples/spec-example/specs/env.yaml b/examples/spec-example/specs/env.yaml new file mode 100644 index 00000000..6ede44ff --- /dev/null +++ b/examples/spec-example/specs/env.yaml @@ -0,0 +1,12 @@ +apiVersion: fission.io/v1 +kind: Environment +metadata: + name: python-27 + namespace: default +spec: + version: 2 + builder: + command: build + image: fission/python-build-env-2.7:0.4.0rc + runtime: + image: fission/python-env-2.7:0.4.0rc diff --git a/examples/spec-example/specs/fission-config.yaml b/examples/spec-example/specs/fission-config.yaml new file mode 100644 index 00000000..5b63e83f --- /dev/null +++ b/examples/spec-example/specs/fission-config.yaml @@ -0,0 +1,3 @@ +kind: DeploymentConfig +name: spec-example +uid: 27438a48-4191-4b5e-95b2-7793624317b9 diff --git a/examples/spec-example/specs/hello.yaml b/examples/spec-example/specs/hello.yaml new file mode 100644 index 00000000..b4c12e96 --- /dev/null +++ b/examples/spec-example/specs/hello.yaml @@ -0,0 +1,38 @@ +apiVersion: fission.io/v1 +kind: Function +metadata: + name: hello + namespace: default +spec: + environment: + name: python-27 + namespace: default + package: + packageref: + name: hello-pkg + namespace: default + functionName: hello.main + +--- +apiVersion: fission.io/v1 +kind: Package +metadata: + name: hello-pkg + namespace: default +spec: + source: + url: archive://hello-archive + buildcmd: "./build.sh" + environment: + name: python-27 + namespace: default +status: + buildstatus: pending + +--- +kind: ArchiveUploadSpec +name: hello-archive +include: + - "hello/*.py" + - "hello/*.sh" + - "hello/requirements.txt" diff --git a/fission/buildwatch.go b/fission/buildwatch.go new file mode 100644 index 00000000..4eb847c1 --- /dev/null +++ b/fission/buildwatch.go @@ -0,0 +1,120 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission" + "github.com/fission/fission/controller/client" + "github.com/fission/fission/crd" +) + +type ( + // packageBuildWatcher is used to watch a set of in-progress builds. + packageBuildWatcher struct { + // fission client + fclient *client.Client + + // set of packages already printed, ensures we don't duplicate the notifications + finished map[string]bool + + // set of metadata in the app spec. packages outside this set should be ignored. + pkgMeta map[string]metav1.ObjectMeta + } +) + +func makePackageBuildWatcher(fclient *client.Client) *packageBuildWatcher { + return &packageBuildWatcher{ + fclient: fclient, + finished: make(map[string]bool), + pkgMeta: make(map[string]metav1.ObjectMeta), + } +} + +func (w *packageBuildWatcher) addPackages(pkgMeta map[string]metav1.ObjectMeta) { + for k, v := range pkgMeta { + w.pkgMeta[k] = v + } +} + +func (w *packageBuildWatcher) watch(ctx context.Context) { + for { + // non-blocking check if we're cancelled + select { + case <-ctx.Done(): + return + default: + } + + // poll list of packages (TODO: convert to watch) + pkgs, err := w.fclient.PackageList() + checkErr(err, "Getting list of packages") + + // find packages that (a) are in the app spec and (b) have an interesting + // build status (either succeeded or failed; not "none") + keepWaiting := false + buildpkgs := make([]crd.Package, 0) + for _, pkg := range pkgs { + _, ok := w.pkgMeta[mapKey(&pkg.Metadata)] + if !ok { + continue + } + if pkg.Status.BuildStatus == fission.BuildStatusNone { + continue + } + if pkg.Status.BuildStatus == fission.BuildStatusPending || + pkg.Status.BuildStatus == fission.BuildStatusRunning { + keepWaiting = true + } + buildpkgs = append(buildpkgs, pkg) + } + + // print package status, and error logs if any + for _, pkg := range buildpkgs { + k := pkgKey(&pkg) + if _, printed := w.finished[k]; printed { + continue + } + if pkg.Status.BuildStatus == fission.BuildStatusFailed { + w.finished[k] = true + fmt.Printf("--- Build FAILED: ---\n%v\n------\n", pkg.Status.BuildLog) + } else if pkg.Status.BuildStatus == fission.BuildStatusSucceeded { + w.finished[k] = true + fmt.Printf("--- Build SUCCEEDED ---\n") + if len(pkg.Status.BuildLog) > 0 { + fmt.Printf("%v\n------\n", pkg.Status.BuildLog) + } + } + } + + // if there are no builds running, we can stop polling + if !keepWaiting { + return + } + time.Sleep(time.Second) + } +} + +func pkgKey(pkg *crd.Package) string { + // packages are mutable so we want to keep track of them by resource version + return fmt.Sprintf("%v:%v:%v", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Metadata.ResourceVersion) +} diff --git a/fission/common.go b/fission/common.go index b67ae12b..ef686a18 100644 --- a/fission/common.go +++ b/fission/common.go @@ -25,9 +25,12 @@ import ( "io/ioutil" "net/http" "os" + "path" "path/filepath" + "regexp" "strings" + "github.com/dchest/uniuri" uuid "github.com/satori/go.uuid" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,8 +107,27 @@ func fileSize(filePath string) int64 { return info.Size() } +func fileChecksum(fileName string) (*fission.Checksum, error) { + f, err := os.Open(fileName) + if err != nil { + return nil, fmt.Errorf("failed to open file %v: %v", fileName, err) + } + defer f.Close() + + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return nil, fmt.Errorf("failed to calculate checksum for %v", fileName) + } + + return &fission.Checksum{ + Type: fission.ChecksumTypeSHA256, + Sum: hex.EncodeToString(h.Sum(nil)), + }, nil +} + // upload a file and return a fission.Archive -func createArchive(client *client.Client, fileName string) *fission.Archive { +func createArchive(client *client.Client, fileName string, specFile string) *fission.Archive { var archive fission.Archive // fetch archive from arbitrary url if fileName is a url @@ -113,6 +135,23 @@ func createArchive(client *client.Client, fileName string) *fission.Archive { fileName = downloadToTempFile(fileName) } + if len(specFile) > 0 { + // create an ArchiveUploadSpec and reference it from the archive + aus := &ArchiveUploadSpec{ + Name: kubifyName(path.Base(fileName)), + IncludeGlobs: []string{fileName}, + } + // save the uploadspec + err := specSave(*aus, specFile) + checkErr(err, fmt.Sprintf("write spec file %v", specFile)) + // create the archive + ar := &fission.Archive{ + Type: fission.ArchiveTypeUrl, + URL: fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name), + } + return ar + } + if fileSize(fileName) < fission.ArchiveLiteralSizeLimit { contents := getContents(fileName) archive.Type = fission.ArchiveTypeLiteral @@ -130,26 +169,15 @@ func createArchive(client *client.Client, fileName string) *fission.Archive { archive.Type = fission.ArchiveTypeUrl archive.URL = archiveUrl - f, err := os.Open(fileName) - if err != nil { - checkErr(err, fmt.Sprintf("find file %v", fileName)) - } - defer f.Close() + csum, err := fileChecksum(fileName) + checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) - } - - archive.Checksum = fission.Checksum{ - Type: fission.ChecksumTypeSHA256, - Sum: hex.EncodeToString(h.Sum(nil)), - } + archive.Checksum = *csum } return &archive } -func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta { +func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string, specFile string) *metav1.ObjectMeta { pkgSpec := fission.PackageSpec{ Environment: fission.EnvironmentReference{ Namespace: metav1.NamespaceDefault, @@ -158,20 +186,28 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive } var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded + var pkgName string if len(deployArchiveName) > 0 { - pkgSpec.Deployment = *createArchive(client, deployArchiveName) + if len(specFile) > 0 { // we should do this in all cases, i think + pkgStatus = fission.BuildStatusNone + } + pkgSpec.Deployment = *createArchive(client, deployArchiveName, specFile) + pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveName), uniuri.NewLen(4))) } if len(srcArchiveName) > 0 { - pkgSpec.Source = *createArchive(client, srcArchiveName) + pkgSpec.Source = *createArchive(client, srcArchiveName, specFile) // set pending status to package pkgStatus = fission.BuildStatusPending + pkgName = kubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveName), uniuri.NewLen(4))) } if len(buildcmd) > 0 { pkgSpec.BuildCommand = buildcmd } - pkgName := strings.ToLower(uuid.NewV4().String()) + if len(pkgName) == 0 { + pkgName = strings.ToLower(uuid.NewV4().String()) + } pkg := &crd.Package{ Metadata: metav1.ObjectMeta{ Name: pkgName, @@ -182,9 +218,16 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive BuildStatus: pkgStatus, }, } - pkgMetadata, err := client.PackageCreate(pkg) - checkErr(err, "create package") - return pkgMetadata + + if len(specFile) > 0 { + err := specSave(*pkg, specFile) + checkErr(err, "save package spec") + return &pkg.Metadata + } else { + pkgMetadata, err := client.PackageCreate(pkg) + checkErr(err, "create package") + return pkgMetadata + } } func getContents(filePath string) []byte { @@ -263,3 +306,39 @@ func downloadURL(fileUrl string) (io.ReadCloser, error) { } return resp.Body, nil } + +// make a kubernetes compliant name out of an arbitrary string +func kubifyName(old string) string { + // Kubernetes maximum name length (for some names; others can be 253 chars) + maxLen := 63 + + newName := strings.ToLower(old) + + // replace disallowed chars with '-' + inv, err := regexp.Compile("[^-a-z0-9]") + checkErr(err, "compile regexp") + newName = string(inv.ReplaceAll([]byte(newName), []byte("-"))) + + // trim leading non-alphabetic + leadingnonalpha, err := regexp.Compile("^[^a-z]+") + checkErr(err, "compile regexp") + newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{})) + + // trim trailing + trailing, err := regexp.Compile("[^a-z0-9]+$") + checkErr(err, "compile regexp") + newName = string(trailing.ReplaceAll([]byte(newName), []byte{})) + + // truncate to length + if len(newName) > maxLen { + newName = newName[0:maxLen] + } + + // if we removed everything, call this thing "default". maybe + // we should generate a unique name... + if len(newName) == 0 { + newName = "default" + } + + return newName +} diff --git a/fission/function.go b/fission/function.go index 77bea125..4ddbdb59 100644 --- a/fission/function.go +++ b/fission/function.go @@ -112,6 +112,14 @@ func fnCreate(c *cli.Context) error { fatal("Need --name argument.") } + // user wants a spec, create a yaml file with package and function + spec := false + specFile := "" + if c.Bool("spec") { + spec = true + specFile = fmt.Sprintf("function-%v.yaml", fnName) + } + fnList, err := client.FunctionList() checkErr(err, "get function list") // check function existence before creating package @@ -182,7 +190,7 @@ func fnCreate(c *cli.Context) error { buildcmd := c.String("buildcmd") // create new package - pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd) + pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, specFile) } //TODO Warn user about resources at fn level overriding the env resources @@ -241,6 +249,14 @@ func fnCreate(c *cli.Context) error { function.Spec.ConfigMaps = append(function.Spec.ConfigMaps, newCfgMap) } + // if we're writing a spec, don't create the function + if spec { + err = specSave(*function, specFile) + checkErr(err, "create function spec") + return nil + + } + _, err = client.FunctionCreate(function) checkErr(err, "create function") diff --git a/fission/main.go b/fission/main.go index 2cd9e1e0..7bf1d74c 100644 --- a/fission/main.go +++ b/fission/main.go @@ -68,9 +68,10 @@ func main() { fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"} fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"} fnExecutorTypeFlag := cli.StringFlag{Name: "executortype", Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy' defaults to 'poolmgr'"} + fnSpecSaveFlag := cli.BoolFlag{Name: "spec", Usage: "Save function to the spec directory instead of creating it"} fnSubcommands := []cli.Command{ - {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate}, + {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnSpecSaveFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, fnSecretnsFlag, fnCfgMapnsFlag}, Action: fnCreate}, {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet}, {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta}, {Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu}, Action: fnUpdate}, @@ -167,6 +168,7 @@ func main() { {Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete}, } + // upgrades, data migrations upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"} upgradeSubCommands := []cli.Command{ {Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState}, @@ -180,6 +182,20 @@ func main() { {Name: "restore", Usage: "Restore state dumped from a pre-0.4 Fission cluster. Requires Fission 0.4, which uses Kubernetes CustomResources.", Flags: []cli.Flag{migrateFileFlag}, Action: migrateRestoreCRD}, } + // specs + specDirFlag := cli.StringFlag{Name: "specdir", Usage: "Directory to store specs, defaults to ./specs"} + specNameFlag := cli.StringFlag{Name: "name", Usage: "(optional) Name for the app, applied to resources as a Kubernetes annotation"} + specWaitFlag := cli.BoolFlag{Name: "wait", Usage: "Wait for package builds"} + specWatchFlag := cli.BoolFlag{Name: "watch", Usage: "Watch local files for change, and re-apply specs as necessary"} + specDeleteFlag := cli.BoolFlag{Name: "delete", Usage: "Allow apply to delete resources that no longer exist in the specification"} + specSubCommands := []cli.Command{ + {Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag}, Action: specInit}, + {Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: specValidate}, + {Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: specApply}, + {Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: specDestroy}, + {Name: "helm", Usage: "Create a helm chart from the app specification", Flags: []cli.Flag{specDirFlag}, Action: specHelm}, + } + app.Commands = []cli.Command{ {Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands}, {Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands}, @@ -188,6 +204,7 @@ func main() { {Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands}, {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, {Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands}, + {Name: "spec", Aliases: []string{"specs"}, Usage: "Manage a declarative app specification", Subcommands: specSubCommands}, {Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands}, {Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands}, } diff --git a/fission/package.go b/fission/package.go index cd11fe00..1d582383 100644 --- a/fission/package.go +++ b/fission/package.go @@ -76,7 +76,7 @@ func pkgCreate(c *cli.Context) error { fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") } - meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd) + meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd, "") fmt.Printf("Package '%v' created\n", meta.GetName()) return nil @@ -146,13 +146,13 @@ func updatePackage(client *client.Client, pkg *crd.Package, envName, } if len(srcArchiveName) > 0 { - srcArchiveMetadata = createArchive(client, srcArchiveName) + srcArchiveMetadata = createArchive(client, srcArchiveName, "") pkg.Spec.Source = *srcArchiveMetadata needToBuild = true } if len(deployArchiveName) > 0 { - deployArchiveMetadata = createArchive(client, deployArchiveName) + deployArchiveMetadata = createArchive(client, deployArchiveName, "") pkg.Spec.Deployment = *deployArchiveMetadata } diff --git a/fission/spec.go b/fission/spec.go new file mode 100644 index 00000000..f89fbbee --- /dev/null +++ b/fission/spec.go @@ -0,0 +1,1441 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/ghodss/yaml" + "github.com/mholt/archiver" + "github.com/pkg/errors" + "github.com/satori/go.uuid" + "github.com/urfave/cli" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission" + "github.com/fission/fission/controller/client" + "github.com/fission/fission/crd" + "io/ioutil" +) + +const SPEC_API_VERSION = "fission.io/v1" + +const ARCHIVE_URL_PREFIX string = "archive://" + +const SPEC_README = ` +Fission Specs +============= + +This is a set of specifications for a Fission app. This includes functions, +environments, and triggers; we collectively call these things "resources". + +How to use these specs +---------------------- + +These specs are handled with the 'fission spec' command. See 'fission spec --help'. + +'fission spec apply' will "apply" all resources specified in this directory to your +cluster. That means it checks what resources exist on your cluster, what resources are +specified in the specs directory, and reconciles the difference by creating, updating or +deleting resources on the cluster. + +'fission spec apply' will also package up your source code (or compiled binaries) and +upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in +this directory to figure out which files to archive. + +You can use 'fission spec apply --watch' to watch for file changes and continuously keep +the cluster updated. + +You can add YAMLs to this directory by writing them manually, but it's easier to generate +them. Use 'fission function create --spec' to generate a function spec, +'fission environment create --spec' to generate an environment spec, and so on. + +You can edit any of the files in this directory, except 'fission-deployment-config.yaml', +which contains a UID that you should never change. To apply your changes simply use +'fission spec apply'. + +fission-deployment-config.yaml +------------------------------ + +fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate +resources on the cluster to resources in this directory. + +All resources created by 'fission spec apply' are annotated with this UID. Resources on +the cluster that are _not_ annotated with this UID are never modified or deleted by +fission. + +` + +type ( + FissionResources struct { + deploymentConfig DeploymentConfig + packages []crd.Package + functions []crd.Function + environments []crd.Environment + httpTriggers []crd.HTTPTrigger + kubernetesWatchTriggers []crd.KubernetesWatchTrigger + timeTriggers []crd.TimeTrigger + messageQueueTriggers []crd.MessageQueueTrigger + archiveUploadSpecs []ArchiveUploadSpec + + sourceMap SourceMap + } + + resourceApplyStatus struct { + created []*metav1.ObjectMeta + updated []*metav1.ObjectMeta + deleted []*metav1.ObjectMeta + } + + SourceMap struct { + // xxx + } +) + +func getSpecDir(c *cli.Context) string { + specDir := c.String("specs") + if len(specDir) == 0 { + specDir = "specs" + } + return specDir +} + +// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new +// fission-config.yaml in specDir. +func writeDeploymentConfig(specDir string, dc *DeploymentConfig) error { + y, err := yaml.Marshal(dc) + if err != nil { + return err + } + + msg := []byte("# This file is generated by the 'fission spec init' command.\n" + + "# See the README in this directory for background and usage information.\n" + + "# Do not edit the UID below: that will break 'fission spec apply'\n") + + err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644) + if err != nil { + return err + } + return nil +} + +// specInit just initializes an empty spec directory and adds some +// sample YAMLs in there that might be useful. +func specInit(c *cli.Context) error { + // Figure out spec directory + specDir := getSpecDir(c) + + name := c.String("name") + if len(name) == 0 { + // come up with a name using the current dir + dir, err := filepath.Abs(".") + checkErr(err, "get current working directory") + basename := filepath.Base(dir) + name = kubifyName(basename) + } + + // Create spec dir + fmt.Printf("Creating fission spec directory '%v'\n", specDir) + err := os.MkdirAll(specDir, 0755) + checkErr(err, fmt.Sprintf("create spec directory '%v'", specDir)) + + // Add a bit of documentation to the spec dir here + err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644) + if err != nil { + return err + } + + // Write the deployment config + dc := DeploymentConfig{ + TypeMeta: TypeMeta{ + APIVersion: SPEC_API_VERSION, + Kind: "DeploymentConfig", + }, + Name: name, + + // All resources will be annotated with the UID when they're created. This allows + // us to be idempotent, as well as to delete resources when their specs are + // removed. + UID: uuid.NewV4().String(), + } + err = writeDeploymentConfig(specDir, &dc) + checkErr(err, "write deployment config") + + // Other possible things to do here: + // - add example specs to the dir to make it easy to manually + // add new ones + + return nil +} + +// specValidate parses a set of specs and checks for references to +// resources that don't exist. +func specValidate(c *cli.Context) error { + //specDir := getSpecDir(c) + + // parse all specs + // verify references: + // functions from triggers + // packages from functions + + // find unreferenced uploads + + return nil +} + +// parseYaml takes one yaml document, figures out its type, parses it, and puts it in +// the right list in the given fission resources set. +func parseYaml(path string, b []byte, fr *FissionResources) error { + + // Figure out the object type by unmarshaling into the TypeMeta struct; then + // unmarshal again into the "real" struct once we know the type. There's almost + // certainly a better way to do this... + var tm TypeMeta + err := yaml.Unmarshal(b, &tm) + switch tm.Kind { + case "Package": + var v crd.Package + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.packages = append(fr.packages, v) + case "Function": + var v crd.Function + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.functions = append(fr.functions, v) + case "Environment": + var v crd.Environment + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.environments = append(fr.environments, v) + case "HTTPTrigger": + var v crd.HTTPTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.httpTriggers = append(fr.httpTriggers, v) + case "KubernetesWatchTrigger": + var v crd.KubernetesWatchTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.kubernetesWatchTriggers = append(fr.kubernetesWatchTriggers, v) + case "TimeTrigger": + var v crd.TimeTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.timeTriggers = append(fr.timeTriggers, v) + case "MessageQueueTrigger": + var v crd.MessageQueueTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.messageQueueTriggers = append(fr.messageQueueTriggers, v) + + // The following are not CRDs + + case "DeploymentConfig": + var v DeploymentConfig + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.deploymentConfig = v + case "ArchiveUploadSpec": + var v ArchiveUploadSpec + err = yaml.Unmarshal(b, &v) + if err != nil { + warn(fmt.Sprintf("Failed to parse %v in %v: %v", tm.Kind, path, err)) + return err + } + fr.archiveUploadSpecs = append(fr.archiveUploadSpecs, v) + default: + // no need to error out just because there's some extra files around; + // also good for compatibility. + warn(fmt.Sprintf("Ignoring unknown type %v in %v", tm.Kind, path)) + } + + return nil +} + +// readSpecs reads all specs in the specified directory and returns a parsed set of +// fission resources. +func readSpecs(specDir string) (*FissionResources, error) { + fr := FissionResources{ + packages: make([]crd.Package, 0), + functions: make([]crd.Function, 0), + environments: make([]crd.Environment, 0), + httpTriggers: make([]crd.HTTPTrigger, 0), + kubernetesWatchTriggers: make([]crd.KubernetesWatchTrigger, 0), + timeTriggers: make([]crd.TimeTrigger, 0), + messageQueueTriggers: make([]crd.MessageQueueTrigger, 0), + } + + // Users can organize the specdir into subdirs if they want to. + err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error { + // For now just read YAML files. We'll add jsonnet at some point. Skip + // unsupported files. + if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + return nil + } + // read + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + // handle the case where there are multiple YAML docs per file. go-yaml + // doesn't support this directly, yet. + docs := bytes.Split(b, []byte("\n---")) + for _, doc := range docs { + d := []byte(strings.TrimSpace(string(doc))) + if len(d) != 0 { + // parse this document and add whatever is in it to fr + err = parseYaml(path, d, &fr) + if err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return nil, err + } + + return &fr, nil +} + +func ignoreFile(path string) bool { + return (strings.Contains(path, "/.#") || // editor autosave files + strings.HasSuffix(path, "~")) // editor backups, usually +} + +func waitForFileWatcherToSettleDown(watcher *fsnotify.Watcher) error { + // Wait a bit for things to settle down in case a bunch of + // files changed; also drain all events that queue up during + // the wait interval. + time.Sleep(500 * time.Millisecond) + for { + select { + case _ = <-watcher.Events: + time.Sleep(200 * time.Millisecond) + continue + case err := <-watcher.Errors: + return err + default: + return nil + } + } +} + +// specApply compares the specs in the spec/config/ directory to the +// deployed resources on the cluster, and reconciles the differences +// by creating, updating or deleting resources on the cluster. +// +// specApply is idempotent. +// +// specApply is *not* transactional -- if the user hits Ctrl-C, or their laptop dies +// etc, while doing an apply, they will get a partially applied deployment. However, +// they can retry their apply command once they're back online. +func specApply(c *cli.Context) error { + fclient := getClient(c.GlobalString("server")) + specDir := getSpecDir(c) + + deleteResources := c.Bool("delete") + watchResources := c.Bool("watch") + waitForBuild := c.Bool("wait") + + var watcher *fsnotify.Watcher + var pbw *packageBuildWatcher + + if watchResources || waitForBuild { + // init package build watcher + pbw = makePackageBuildWatcher(fclient) + } + + if watchResources { + var err error + watcher, err = fsnotify.NewWatcher() + checkErr(err, "create file watcher") + + // add watches + rootDir := filepath.Clean(specDir + "/..") + err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + checkErr(err, "scan project files") + + if ignoreFile(path) { + return nil + } + err = watcher.Add(path) + checkErr(err, fmt.Sprintf("watch path %v", path)) + return nil + }) + checkErr(err, "scan files to watch") + } + + for { + // read all specs + fr, err := readSpecs(specDir) + checkErr(err, "read specs") + + // make changes to the cluster based on the specs + pkgMeta, as, err := apply(fclient, specDir, fr, deleteResources) + checkErr(err, "apply specs") + printApplyStatus(as) + + var ctx context.Context + var pkgWatchCancel context.CancelFunc + if watchResources || waitForBuild { + // watch package builds + ctx, pkgWatchCancel = context.WithCancel(context.Background()) + pbw.addPackages(pkgMeta) + } + + if watchResources { + // if we're watching for files, we don't need to wait for builds to complete + go pbw.watch(ctx) + } else if waitForBuild { + // synchronously wait for build if --wait was specified + pbw.watch(ctx) + } + + if !watchResources { + break + } + + // listen for file watch events + fmt.Println("Watching files for changes...") + + waitloop: + for { + select { + case e := <-watcher.Events: + if ignoreFile(e.Name) { + continue waitloop + } + + fmt.Printf("Noticed a file change, reapplying specs...\n") + + // Builds that finish after this cancellation will be + // printed in the next watchPackageBuildStatus call. + pkgWatchCancel() + + err = waitForFileWatcherToSettleDown(watcher) + checkErr(err, "watching files") + + break waitloop + case err := <-watcher.Errors: + checkErr(err, "watching files") + } + } + } + return nil +} + +// printApplyStatus prints a summary of what changed on the cluster as the result of a spec apply +// operation. +func printApplyStatus(applyStatus map[string]resourceApplyStatus) { + changed := false + for typ, ras := range applyStatus { + n := len(ras.created) + if n > 0 { + changed = true + fmt.Printf("%v %v created: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.created), ", ")) + } + n = len(ras.updated) + if n > 0 { + changed = true + fmt.Printf("%v %v updated: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.updated), ", ")) + } + n = len(ras.deleted) + if n > 0 { + changed = true + fmt.Printf("%v %v deleted: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.deleted), ", ")) + } + } + + if !changed { + fmt.Println("Everything up to date.") + } +} + +// metadataNames extracts a slice of names from a slice of object metadata. +func metadataNames(ms []*metav1.ObjectMeta) []string { + s := make([]string, len(ms)) + for i, m := range ms { + s[i] = m.Name + } + return s +} + +// pluralize returns the plural of word if num is zero or more than one. +func pluralize(num int, word string) string { + if num == 1 { + return word + } + return word + "s" +} + +// specDestroy destroys everything in the spec. +func specDestroy(c *cli.Context) error { + fclient := getClient(c.GlobalString("server")) + + // get specdir + specDir := getSpecDir(c) + + // read everything + fr, err := readSpecs(specDir) + checkErr(err, "read specs") + + // set desired state to nothing, but keep the UID so "apply" can find it + emptyFr := FissionResources{} + emptyFr.deploymentConfig = fr.deploymentConfig + + // "apply" the empty state + _, _, err = apply(fclient, specDir, &emptyFr, true) + checkErr(err, "delete resources") + + return nil +} + +// applyArchives figures out the set of archives that need to be uploaded, and uploads them. +func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) error { + + // archive:// URL -> archive map. + archiveFiles := make(map[string]fission.Archive) + + // We'll first populate archiveFiles with references to local files, and then modify it to + // point at archive URLs. + + // create archives locally and calculate checksums + for _, aus := range fr.archiveUploadSpecs { + ar, err := localArchiveFromSpec(specDir, &aus) + if err != nil { + return err + } + archiveUrl := fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name) + archiveFiles[archiveUrl] = *ar + } + + // get list of packages, make content-indexed map of available archives + availableArchives := make(map[string]string) // (sha256 -> url) + pkgs, err := fclient.PackageList() + if err != nil { + return err + } + for _, pkg := range pkgs { + for _, ar := range []fission.Archive{pkg.Spec.Source, pkg.Spec.Deployment} { + if ar.Type == fission.ArchiveTypeUrl && len(ar.URL) > 0 { + availableArchives[ar.Checksum.Sum] = ar.URL + } + } + } + + // upload archives that we need to, updating the map + for name, ar := range archiveFiles { + if ar.Type == fission.ArchiveTypeLiteral { + continue + } + // does the archive exist already? + if url, ok := availableArchives[ar.Checksum.Sum]; ok { + fmt.Printf("archive %v exists, not uploading\n", name) + a := archiveFiles[name] + a.URL = url + } else { + // doesn't exist, upload + fmt.Printf("uploading archive %v\n", name) + uploadedAr := createArchive(fclient, ar.URL, "") + archiveFiles[name] = *uploadedAr + } + } + + // resolve references to urls in packages to be applied + for i := range fr.packages { + for _, ar := range []*fission.Archive{&fr.packages[i].Spec.Source, &fr.packages[i].Spec.Deployment} { + if strings.HasPrefix(ar.URL, ARCHIVE_URL_PREFIX) { + availableAr, ok := archiveFiles[ar.URL] + if !ok { + return fmt.Errorf("Unknown archive name %v", strings.TrimPrefix(ar.URL, ARCHIVE_URL_PREFIX)) + } + ar.Type = availableAr.Type + ar.Literal = availableAr.Literal + ar.URL = availableAr.URL + ar.Checksum = availableAr.Checksum + } + } + } + return nil +} + +// apply applies the given set of fission resources. +func apply(fclient *client.Client, specDir string, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, map[string]resourceApplyStatus, error) { + + applyStatus := make(map[string]resourceApplyStatus) + + // upload archives that need to be uploaded. Changes archive references in fr.packages. + err := applyArchives(fclient, specDir, fr) + if err != nil { + return nil, nil, err + } + + _, ras, err := applyEnvironments(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "environment apply failed") + } + applyStatus["environment"] = *ras + + pkgMeta, ras, err := applyPackages(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "package apply failed") + } + applyStatus["package"] = *ras + + // Each reference to a package from a function must contain the resource version + // of the package. This ensures that various caches can invalidate themselves + // when the package changes. + for i, f := range fr.functions { + k := mapKey(&metav1.ObjectMeta{ + Namespace: f.Spec.Package.PackageRef.Namespace, + Name: f.Spec.Package.PackageRef.Name, + }) + m, ok := pkgMeta[k] + if !ok { + // the function references a package that doesn't exist in the + // spec. It may exist outside the spec, but we're going to treat + // that as an error, so that we encourage self-contained specs. + // Is there a good use case for non-self contained specs? + return nil, nil, fmt.Errorf("Function %v/%v references package %v/%v, which doesn't exist in the specs", + f.Metadata.Namespace, f.Metadata.Name, f.Spec.Package.PackageRef.Namespace, f.Spec.Package.PackageRef.Name) + } + fr.functions[i].Spec.Package.PackageRef.ResourceVersion = m.ResourceVersion + } + + _, ras, err = applyFunctions(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "function apply failed") + } + applyStatus["function"] = *ras + + _, ras, err = applyHTTPTriggers(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "HTTPTrigger apply failed") + } + applyStatus["HTTPTrigger"] = *ras + + _, ras, err = applyKubernetesWatchTriggers(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "KubernetesWatchTrigger apply failed") + } + applyStatus["KubernetesWatchTrigger"] = *ras + + _, ras, err = applyTimeTriggers(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "TimeTrigger apply failed") + } + applyStatus["TimeTrigger"] = *ras + + _, ras, err = applyMessageQueueTriggers(fclient, fr, delete) + if err != nil { + return nil, nil, errors.Wrap(err, "MessageQueueTrigger apply failed") + } + applyStatus["MessageQueueTrigger"] = *ras + + return pkgMeta, applyStatus, nil +} + +// localArchiveFromSpec creates an archive on the local filesystem from the given spec, +// and returns its path and checksum. +func localArchiveFromSpec(specDir string, aus *ArchiveUploadSpec) (*fission.Archive, error) { + // get root dir + var rootDir string + if len(aus.RootDir) == 0 { + rootDir = filepath.Clean(specDir + "/..") + } else { + rootDir = aus.RootDir + } + + // get a list of files from the include/exclude globs. + // + // XXX if there are lots of globs it's probably more efficient + // to do a filepath.Walk and call path.Match on each path... + files := make([]string, 0) + for _, relativeGlob := range aus.IncludeGlobs { + absGlob := rootDir + "/" + relativeGlob + f, err := filepath.Glob(absGlob) + if err != nil { + warn(fmt.Sprintf("Invalid glob in archive %v: %v", aus.Name, relativeGlob)) + return nil, err + } + files = append(files, f...) + // xxx handle excludeGlobs here + } + + if len(files) == 0 { + return nil, fmt.Errorf("Archive '%v' is empty", aus.Name) + } + + // if it's just one file, use its path directly + var archiveFileName string + if len(files) == 1 { + archiveFileName = files[0] + } else { + // zip up the file list + archiveFile, err := ioutil.TempFile("", fmt.Sprintf("fission-archive-%v", aus.Name)) + if err != nil { + return nil, err + } + archiveFileName = archiveFile.Name() + err = archiver.Zip.Make(archiveFileName, files) + if err != nil { + return nil, err + } + } + + // figure out if we're making a literal or a URL-based archive + if fileSize(archiveFileName) < fission.ArchiveLiteralSizeLimit { + contents := getContents(archiveFileName) + return &fission.Archive{ + Type: fission.ArchiveTypeLiteral, + Literal: contents, + }, nil + } else { + // checksum + csum, err := fileChecksum(archiveFileName) + if err != nil { + return nil, fmt.Errorf("failed to calculate archive checksum for %v (%v): %v", aus.Name, archiveFileName, err) + } + + // archive object + return &fission.Archive{ + Type: fission.ArchiveTypeUrl, + // we should be actually be adding a "file://" prefix, but this archive is only an + // intermediate step, so just the path works fine. + URL: archiveFileName, + Checksum: *csum, + }, nil + + } +} + +// specHelm creates a helm chart from a spec directory and a +// deployment config. +func specHelm(c *cli.Context) error { + return nil +} + +func mapKey(m *metav1.ObjectMeta) string { + return fmt.Sprintf("%v:%v", m.Namespace, m.Name) +} + +func applyDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) { + if m.Annotations == nil { + m.Annotations = make(map[string]string) + } + m.Annotations[FISSION_DEPLOYMENT_NAME_KEY] = fr.deploymentConfig.Name + m.Annotations[FISSION_DEPLOYMENT_UID_KEY] = fr.deploymentConfig.UID +} + +func hasDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) bool { + if m.Annotations == nil { + return false + } + uid, ok := m.Annotations[FISSION_DEPLOYMENT_UID_KEY] + if ok && uid == fr.deploymentConfig.UID { + return true + } + return false +} + +func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.PackageList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.Package, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.Package) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.packages { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + keep := false + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + keep = true + } else if reflect.DeepEqual(existingObj.Spec.Environment, o.Spec.Environment) && + !reflect.DeepEqual(existingObj.Spec.Source, fission.Archive{}) && + reflect.DeepEqual(existingObj.Spec.Source, o.Spec.Source) && + existingObj.Spec.BuildCommand == o.Spec.BuildCommand { + + keep = true + } + + if keep { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.PackageUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.PackageCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.PackageDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.FunctionList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.Function, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.Function) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.functions { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.FunctionUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.FunctionCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.FunctionDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.EnvironmentList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.Environment, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.Environment) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.environments { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.EnvironmentUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.EnvironmentCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.EnvironmentDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.HTTPTriggerList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.HTTPTrigger, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.HTTPTrigger) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.httpTriggers { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.HTTPTriggerUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.HTTPTriggerCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.HTTPTriggerDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.WatchList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.KubernetesWatchTrigger, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.KubernetesWatchTrigger) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.kubernetesWatchTriggers { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.WatchUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.WatchCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.WatchDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.TimeTriggerList() + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.TimeTrigger, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.TimeTrigger) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.timeTriggers { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.TimeTriggerUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.TimeTriggerCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.TimeTriggerDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { + // get list + allObjs, err := fclient.MessageQueueTriggerList("") + if err != nil { + return nil, nil, err + } + + // filter + objs := make([]crd.MessageQueueTrigger, 0) + for _, o := range allObjs { + if hasDeploymentConfig(&o.Metadata, fr) { + objs = append(objs, o) + } + } + + // index + existent := make(map[string]crd.MessageQueueTrigger) + for _, obj := range objs { + existent[mapKey(&obj.Metadata)] = obj + } + metadataMap := make(map[string]metav1.ObjectMeta) + + // desired set. used to compute the set to delete. + desired := make(map[string]bool) + + var ras resourceApplyStatus + + // create or update desired state + for _, o := range fr.messageQueueTriggers { + // apply deploymentConfig so we can find our objects on future apply invocations + applyDeploymentConfig(&o.Metadata, fr) + + // index desired state + desired[mapKey(&o.Metadata)] = true + + // exists? + existingObj, ok := existent[mapKey(&o.Metadata)] + if ok { + // ok, a resource with the same name exists, is it the same? + if reflect.DeepEqual(existingObj.Spec, o.Spec) { + // nothing to do on the server + metadataMap[mapKey(&o.Metadata)] = existingObj.Metadata + } else { + // update + o.Metadata.ResourceVersion = existingObj.Metadata.ResourceVersion + newmeta, err := fclient.MessageQueueTriggerUpdate(&o) + if err != nil { + return nil, nil, err + } + ras.updated = append(ras.updated, newmeta) + // keep track of metadata in case we need to create a reference to it + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } else { + // create + newmeta, err := fclient.MessageQueueTriggerCreate(&o) + if err != nil { + return nil, nil, err + } + ras.created = append(ras.created, newmeta) + metadataMap[mapKey(&o.Metadata)] = *newmeta + } + } + + // deletes + if delete { + // objs is already filtered with our UID + for _, o := range objs { + _, wanted := desired[mapKey(&o.Metadata)] + if !wanted { + err := fclient.MessageQueueTriggerDelete(&o.Metadata) + if err != nil { + return nil, nil, err + } + ras.deleted = append(ras.deleted, &o.Metadata) + fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) + } + } + } + + return metadataMap, &ras, nil +} + +// called from `fission function create --spec` +func specSave(resource interface{}, specFile string) error { + specDir := "specs" + + // verify + if _, err := os.Stat(filepath.Join(specDir, "fission-deployment-config.yaml")); os.IsNotExist(err) { + return errors.Wrap(err, "Couldn't find specs, run `fission spec init` first") + } + + // make sure we're writing a known type + var data []byte + var err error + switch typedres := resource.(type) { + case ArchiveUploadSpec: + typedres.Kind = "ArchiveUploadSpec" + data, err = yaml.Marshal(typedres) + case crd.Package: + typedres.TypeMeta.APIVersion = SPEC_API_VERSION + typedres.TypeMeta.Kind = "Package" + data, err = yaml.Marshal(typedres) + case crd.Function: + typedres.TypeMeta.APIVersion = SPEC_API_VERSION + typedres.TypeMeta.Kind = "Function" + data, err = yaml.Marshal(typedres) + default: + return fmt.Errorf("can't save resource %#v", resource) + } + if err != nil { + return errors.Wrap(err, "Couldn't marshal YAML") + } + + filename := filepath.Join(specDir, specFile) + // check if the file is new + newFile := false + if _, err := os.Stat(filename); os.IsNotExist(err) { + newFile = true + } + + // open spec file to append or write + f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return errors.Wrap(err, "couldn't create spec file") + } + defer f.Close() + + // if we're appending, add a yaml document separator + if !newFile { + _, err = f.Write([]byte("\n---\n")) + if err != nil { + return errors.Wrap(err, "couldn't write to spec file") + } + } + + // write our resource + _, err = f.Write(data) + if err != nil { + return errors.Wrap(err, "couldn't write to spec file") + } + return nil +} diff --git a/fission/types.go b/fission/types.go new file mode 100644 index 00000000..75974394 --- /dev/null +++ b/fission/types.go @@ -0,0 +1,77 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +const ( + FISSION_DEPLOYMENT_NAME_KEY = "fission-name" + FISSION_DEPLOYMENT_UID_KEY = "fission-uid" +) + +// CLI spec types +type ( + // DeploymentConfig is the global configuration for a set of Fission specs. + DeploymentConfig struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "DeploymentConfig". + TypeMeta `json:",inline"` + + // Name is a user-friendly name for the deployment. It is also stored in + // all uploaded resources as an annotation. + Name string `json:"name"` + + // UID uniquely identifies the deployment. It is stored as a label and + // used to find resources to clean up when local specs are changed. + UID string `json:"uid"` + } + + // ArchiveUploadSpec specifies a set of files to be archived and uploaded. + // + // The resulting archive can be referenced as archive:// in PackageSpecs, + // using the name specified in the archive. The fission spec applier will + // replace the archive:// URL with a real HTTP URL after uploading the file. + ArchiveUploadSpec struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "ArchiveUploadSpec". + TypeMeta `json:",inline"` + + // Name is a local name that can be used to reference this archive. It + // must be unique; duplicate names will cause an error while handling + // specs. + Name string `json:"name"` + + // RootDir specifies the root that the globs below are relative to. It + // is optional and defaults to the parent directory of the spec + // directory: for example, if the deployment config is at + // /path/to/project/specs/config.yaml, the RootDir is /path/to/project. + RootDir string `json:"rootdir,omitempty"` + + // IncludeGlobs is a list of Unix shell globs to include + IncludeGlobs []string `json:"include,omitempty"` + + // ExcludeGlobs is a list of globs to exclude from the set specified by + // IncludeGlobs. + ExcludeGlobs []string `json:"exclude,omitempty"` + } + + // TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and + // unmarshal local-only objects (like ArchiveUploadSpec) the same way that + // Kubernetes does. + TypeMeta struct { + Kind string `json:"kind,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + } +) diff --git a/fission/upgrade.go b/fission/upgrade.go index 37c322b8..71f299f4 100644 --- a/fission/upgrade.go +++ b/fission/upgrade.go @@ -291,7 +291,7 @@ func upgradeRestoreState(c *cli.Context) error { tmpfile.Close() // upload - archive := createArchive(client, tmpfile.Name()) + archive := createArchive(client, tmpfile.Name(), "") os.Remove(tmpfile.Name()) // create pkg diff --git a/glide.lock b/glide.lock index 80b46af8..e7438546 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 7c938e591b0602b9fd3412aeffd3f1438fc77eaea80c8d9dd03431adb0e1e1c9 -updated: 2017-12-13T16:13:45.935783+08:00 +hash: 93c80adbba10750a7fca4536f93880f152527a8f04eedf40a8ea8cbadeb88779 +updated: 2017-12-02T21:38:02.116016618-08:00 imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -46,6 +46,8 @@ imports: - log - name: github.com/emicklei/go-restful-swagger12 version: dcef7f55730566d41eae5db10e7d6981829720f6 +- name: github.com/fsnotify/fsnotify + version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/ghodss/yaml version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee - name: github.com/go-openapi/analysis @@ -134,6 +136,8 @@ imports: version: a0006b13c722f7f12368c00a3d3c2ae8a999a0c6 subpackages: - xxHash32 +- name: github.com/pkg/errors + version: f15c970de5b76fac0b59abb32d62c17cc7bed265 - name: github.com/PuerkitoBio/purell version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 - name: github.com/PuerkitoBio/urlesc diff --git a/glide.yaml b/glide.yaml index 5ab39eef..eb108a9c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -48,3 +48,6 @@ import: version: ^v0.4.0 - package: github.com/graymeta/stow - package: github.com/mholt/archiver +- package: github.com/pkg/errors +- package: github.com/fsnotify/fsnotify + diff --git a/test/tests/test_spec_create/hello.py b/test/tests/test_spec_create/hello.py new file mode 100644 index 00000000..2e38204d --- /dev/null +++ b/test/tests/test_spec_create/hello.py @@ -0,0 +1,2 @@ +def main(): + return "hello\n" diff --git a/test/tests/test_spec_create/test_spec.sh b/test/tests/test_spec_create/test_spec.sh new file mode 100755 index 00000000..f45830db --- /dev/null +++ b/test/tests/test_spec_create/test_spec.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +#test:disabled + +set -euo pipefail + +fn=spec-$(date +%N) +env=python-$fn + +# init +fission spec init + +# verify init +[ -d specs ] +[ -f specs/README ] +[ -f specs/fission-deployment-config.yaml ] + +# TODO replace with `fission env create --spec` +cat > specs/env.yaml <