Improve archive package user experience (#927)

common.go

@@ -149,12 +149,11 @@ func MakeArchive(targetName string, globs ...string) (string, error) {
 	for _, glob := range globs {
 		f, err := filepath.Glob(glob)
 		if err != nil {
-			log.Info(fmt.Sprintf("Invalid glob %v: %v", glob, err))
+			log.Warn(fmt.Sprintf("Invalid glob %v: %v", glob, err))
 			return "", err
 		}
 		files = append(files, f...)
 	}
-
 	// zip up the file list
 	err := archiver.Zip.Make(targetName, files)
 	if err != nil {

fission/function.go

@@ -252,10 +252,15 @@ func fnCreate(c *cli.Context) error {
 			}
 		}
 
-		srcArchiveName := c.String("src")
-		deployArchiveName := c.String("code")
-		if len(deployArchiveName) == 0 {
-			deployArchiveName = c.String("deploy")
+		srcArchiveName := c.StringSlice("src")
+		var deployArchiveName []string
+		codeFlag := false
+		code := c.String("code")
+		if len(code) == 0 {
+			deployArchiveName = c.StringSlice("deploy")
+		} else {
+			deployArchiveName = append(deployArchiveName, c.String("code"))
+			codeFlag = true
 		}
 		// fatal when both src & deploy archive are empty
 		if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
@@ -265,7 +270,7 @@ func fnCreate(c *cli.Context) error {
 		buildcmd := c.String("buildcmd")
 
 		// create new package in the same namespace as the function.
-		pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, specFile)
+		pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, specFile, codeFlag)
 
 		fmt.Printf("package '%v' created\n", pkgMetadata.Name)
 	}
@@ -466,11 +471,17 @@ func fnUpdate(c *cli.Context) error {
 		envNamespace = ""
 	}
 
-	deployArchiveName := c.String("code")
-	if len(deployArchiveName) == 0 {
-		deployArchiveName = c.String("deploy")
+	var deployArchiveName []string
+	codeFlag := false
+	code := c.String("code")
+	if len(code) == 0 {
+		deployArchiveName = c.StringSlice("deploy")
+	} else {
+		deployArchiveName = append(deployArchiveName, c.String("code"))
+		codeFlag = true
 	}
-	srcArchiveName := c.String("src")
+
+	srcArchiveName := c.StringSlice("src")
 	pkgName := c.String("pkg")
 	entrypoint := c.String("entrypoint")
 	buildcmd := c.String("buildcmd")
@@ -563,7 +574,7 @@ func fnUpdate(c *cli.Context) error {
 			log.Fatal("Package is used by multiple functions, use --force to force update")
 		}
 
-		pkgMetadata, err = updatePackage(client, pkg, pkg.Spec.Environment.Name, pkg.Spec.Environment.Namespace, srcArchiveName, deployArchiveName, buildcmd, false)
+		pkgMetadata, err = updatePackage(client, pkg, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, false, codeFlag)
 		util.CheckErr(err, fmt.Sprintf("update package '%v'", pkgName))
 
 		fmt.Printf("package '%v' updated\n", pkgMetadata.GetName())

fission/main.go

@@ -90,8 +90,8 @@ func newCliApp() *cli.App {
 	fnNameFlag := cli.StringFlag{Name: "name", Usage: "function name"}
 	fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"}
 	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
-	fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
-	fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"}
+	fnDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
+	fnSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"}
 	fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored), should be in the same namespace as the function"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
@@ -234,8 +234,8 @@ func newCliApp() *cli.App {
 	pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"}
 	pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"}
 	pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"}
-	pkgSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
-	pkgDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
+	pkgSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
+	pkgDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
 	pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"}
 	pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"}
 	pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "orphan packages that are not referenced by any function"}

fission/package.go

@@ -30,10 +30,12 @@ import (
 	"path/filepath"
 	"strings"
 	"text/tabwriter"
+	"time"
 
 	"github.com/dchest/uniuri"
 	"github.com/fission/fission/fission/util"
 	storageSvcClient "github.com/fission/fission/storagesvc/client"
+	"github.com/mholt/archiver"
 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -82,15 +84,15 @@ func pkgCreate(c *cli.Context) error {
 		log.Fatal("Need --env argument.")
 	}
 	envNamespace := c.String("envNamespace")
-	srcArchiveName := c.String("src")
-	deployArchiveName := c.String("deploy")
+	srcArchive := c.StringSlice("src")
+	deployArchive := c.StringSlice("deploy")
 	buildcmd := c.String("buildcmd")
 
-	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+	if len(srcArchive) == 0 && len(deployArchive) == 0 {
 		log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
 	}
 
-	meta := createPackage(client, pkgNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, "")
+	meta := createPackage(client, pkgNamespace, envName, envNamespace, srcArchive, deployArchive, buildcmd, "", false)
 	fmt.Printf("Package '%v' created\n", meta.GetName())
 
 	return nil
@@ -108,15 +110,15 @@ func pkgUpdate(c *cli.Context) error {
 	force := c.Bool("f")
 	envName := c.String("env")
 	envNamespace := c.String("envNamespace")
-	srcArchiveName := c.String("src")
-	deployArchiveName := c.String("deploy")
+	srcArchive := c.StringSlice("src")
+	deployArchive := c.StringSlice("deploy")
 	buildcmd := c.String("buildcmd")
 
-	if len(srcArchiveName) > 0 && len(deployArchiveName) > 0 {
+	if len(srcArchive) > 0 && len(deployArchive) > 0 {
 		log.Fatal("Need either of --src or --deploy and not both arguments.")
 	}
 
-	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 &&
+	if len(srcArchive) == 0 && len(deployArchive) == 0 &&
 		len(envName) == 0 && len(buildcmd) == 0 {
 		log.Fatal("Need --env or --src or --deploy or --buildcmd argument.")
 	}
@@ -146,7 +148,7 @@ func pkgUpdate(c *cli.Context) error {
 	}
 
 	newPkgMeta, err := updatePackage(client, pkg,
-		envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, false)
+		envName, envNamespace, srcArchive, deployArchive, buildcmd, false, false)
 	if err != nil {
 		util.CheckErr(err, "update package")
 	}
@@ -163,8 +165,8 @@ func pkgUpdate(c *cli.Context) error {
 	return nil
 }
 
-func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespace,
-	srcArchiveName, deployArchiveName, buildcmd string, forceRebuild bool) (*metav1.ObjectMeta, error) {
+func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespace string,
+	srcArchive []string, deployArchive []string, buildcmd string, forceRebuild bool, codeFlag bool) (*metav1.ObjectMeta, error) {
 
 	var srcArchiveMetadata, deployArchiveMetadata *fission.Archive
 	needToBuild := false
@@ -184,13 +186,20 @@ func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespac
 		needToBuild = true
 	}
 
-	if len(srcArchiveName) > 0 {
+	if len(srcArchive) > 0 {
+		srcArchiveName := archiveParser(srcArchive, envName)
 		srcArchiveMetadata = createArchive(client, srcArchiveName, "")
 		pkg.Spec.Source = *srcArchiveMetadata
 		needToBuild = true
 	}
 
-	if len(deployArchiveName) > 0 {
+	if len(deployArchive) > 0 {
+		var deployArchiveName string
+		if codeFlag {
+			deployArchiveName = deployArchive[0]
+		} else {
+			deployArchiveName = archiveParser(deployArchive, envName)
+		}
 		deployArchiveMetadata = createArchive(client, deployArchiveName, "")
 		pkg.Spec.Deployment = *deployArchiveMetadata
 		// Users may update the env, envNS and deploy archive at the same time,
@@ -439,7 +448,7 @@ func pkgRebuild(c *cli.Context) error {
 			pkg.Metadata.Name, fission.BuildStatusFailed))
 	}
 
-	_, err = updatePackage(client, pkg, "", "", "", "", "", true)
+	_, err = updatePackage(client, pkg, "", "", nil, nil, "", true, false)
 	util.CheckErr(err, "update package")
 
 	fmt.Printf("Retrying build for pkg %v. Use \"fission pkg info --name %v\" to view status.\n", pkg.Metadata.Name, pkg.Metadata.Name)
@@ -530,7 +539,7 @@ func createArchive(client *client.Client, fileName string, specFile string) *fis
 	return &archive
 }
 
-func createPackage(client *client.Client, pkgNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd string, specFile string) *metav1.ObjectMeta {
+func createPackage(client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchive []string, deployArchive []string, buildcmd string, specFile string, codeFlag bool) *metav1.ObjectMeta {
 	pkgSpec := fission.PackageSpec{
 		Environment: fission.EnvironmentReference{
 			Namespace: envNamespace,
@@ -540,14 +549,21 @@ func createPackage(client *client.Client, pkgNamespace, envName, envNamespace, s
 	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
 
 	var pkgName string
-	if len(deployArchiveName) > 0 {
+	if len(deployArchive) > 0 {
 		if len(specFile) > 0 { // we should do this in all cases, i think
 			pkgStatus = fission.BuildStatusNone
 		}
+		var deployArchiveName string
+		if codeFlag {
+			deployArchiveName = deployArchive[0]
+		} else {
+			deployArchiveName = archiveParser(deployArchive, envName)
+		}
 		pkgSpec.Deployment = *createArchive(client, deployArchiveName, specFile)
 		pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveName), uniuri.NewLen(4)))
 	}
-	if len(srcArchiveName) > 0 {
+	if len(srcArchive) > 0 {
+		srcArchiveName := archiveParser(srcArchive, envName)
 		pkgSpec.Source = *createArchive(client, srcArchiveName, specFile)
 		// set pending status to package
 		pkgStatus = fission.BuildStatusPending
@@ -652,3 +668,23 @@ func downloadURL(fileUrl string) (io.ReadCloser, error) {
 	}
 	return resp.Body, nil
 }
+
+func archiveParser(archiveInput []string, envName string) string {
+	var archiveName = ""
+
+	if (len(archiveInput) == 1 && archiver.Zip.Match(archiveInput[0])) ||
+		(len(archiveInput) == 1 && (strings.HasPrefix(archiveInput[0], "http://") || strings.HasPrefix(archiveInput[0], "https://"))) {
+		return archiveInput[0]
+	}
+
+	tmpDir, err := fission.GetTempDir()
+	if err != nil {
+		util.CheckErr(err, "create archive file")
+	}
+	archiveName, err = fission.MakeArchive(filepath.Join(tmpDir, fmt.Sprintf("%v-%v", envName, time.Now().Unix())), archiveInput...)
+	if err != nil {
+		util.CheckErr(err, "create archive file")
+	}
+
+	return archiveName
+}

fission/spec.go

@@ -969,6 +969,9 @@ func localArchiveFromSpec(specDir string, aus *ArchiveUploadSpec) (*fission.Arch
 	// XXX if there are lots of globs it's probably more efficient
 	// to do a filepath.Walk and call path.Match on each path...
 	files := make([]string, 0)
+	if len(aus.IncludeGlobs) == 1 && archiver.Zip.Match(aus.IncludeGlobs[0]) {
+		files = append(files, aus.IncludeGlobs[0])
+	}
 	for _, relativeGlob := range aus.IncludeGlobs {
 		absGlob := rootDir + "/" + relativeGlob
 		f, err := filepath.Glob(absGlob)

test/tests/test_package_command.sh

@@ -2,16 +2,26 @@
 
 set -euo pipefail
 
-# Use package command to create two packages one with source
-# archive and the other with deploy archive. Also, create a 
-# function to test the packages created by package command are 
+# Use package command to create packages of type:
+#     1) Multiple source files from a directory
+#     2) Source archive file
+# TBD 3) Source file from a HTTP location
+#     4) Deployment files from a directory
+#     5) Deployment archive
+# TBD 6) Deployment archive from a HTTP location
+# TBD 7) Multiple files from  multiple directories
+# Then create a function to test the packages created by package command are 
 # able to work.
 
 ROOT=$(dirname $0)/../..
 PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test
 PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
 
-fn=python-srcbuild-$(date +%s)
+fn1=python-srcbuild1-$(date +%s)
+fn2=python-srcbuild2-$(date +%s)
+
+fn4=python-deploy4-$(date +%s)
+fn5=python-deploy5-$(date +%s)
 
 waitBuild() {
     log "Waiting for builder manager to finish the build"
@@ -55,7 +65,10 @@ export -f waitEnvBuilder
 cleanup() {
     log "Cleaning up..."
     fission env delete --name python || true
-    fission fn delete --name $fn || true
+    fission fn delete --name $fn1 || true
+    fission fn delete --name $fn2 || true
+    fission fn delete --name $fn4 || true
+    fission fn delete --name $fn5 || true
 }
 
 if [ -z "${TEST_NOCLEANUP:-}" ]; then
@@ -71,24 +84,62 @@ log "Creating python env"
 fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
 
 timeout 180s bash -c "waitEnvBuilder python"
-
-log "Creating pacakage with source archive"
-zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
-pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
-
+# 1) Multiple source files (multiple inputs, Using * expression, from a directory)
+# Currently only * expression implemented as a test
+pushd $ROOT/examples/python/
+pkg1=$(fission package create --src "sourcepkg/*" --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+popd
 # wait for build to finish at most 60s
-timeout 60s bash -c "waitBuild $pkgName"
-
-log "Creating function " $fn
-fission fn create --name $fn --pkg $pkgName --entrypoint "user.main"
+timeout 60s bash -c "waitBuild $pkg1"
+log "Creating function " $fn1
+fission fn create --name $fn1 --pkg $pkg1 --entrypoint "user.main"
 
 log "Creating route"
-fission route create --function $fn --url /$fn --method GET
+fission route create --function $fn1 --url /$fn1 --method GET
 
 log "Waiting for router to catch up"
 sleep 3
   
-checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}'
+checkFunctionResponse $fn1 'a: 1 b: {c: 3, d: 4}'
+
+# 2) Source archive file
+log "Creating pacakage with source archive"
+zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
+pkg2=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+
+# wait for build to finish at most 60s
+timeout 60s bash -c "waitBuild $pkg2"
+
+log "Creating function " $fn2
+fission fn create --name $fn2 --pkg $pkg2 --entrypoint "user.main"
+
+log "Creating route"
+fission route create --function $fn2 --url /$fn2 --method GET
+
+log "Waiting for router to catch up"
+sleep 3
+  
+checkFunctionResponse $fn2 'a: 1 b: {c: 3, d: 4}'
+
+# 3) Source file from a HTTP location
+# TBD
+
+# 4) Deployment files from a directory
+pushd $ROOT/examples/python/
+pkg4=$(fission package create --deploy "multifile/*" --env python| cut -f2 -d' '| tr -d \')
+popd
+log "Creating function " $fn4
+fission fn create --name $fn4 --pkg $pkg4 --entrypoint "main.main"
+
+log "Creating route"
+fission route create --function $fn4 --url /$fn4 --method GET
+
+log "Waiting for router to catch up"
+sleep 3
+  
+checkFunctionResponse $fn4 'Hello, world!'
+
+# 5) Deployment archive
 
 log "Creating package with deploy archive"
 mkdir testDir
@@ -97,13 +148,20 @@ printf 'def main():\n    return "Hello, world!"' > testDir/hello.py
 zip -jr demo-deploy-pkg.zip testDir/
 pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \')
 
-log "Updating function " $fn
-fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main"
+
+log "Updating function " $fn5
+fission fn create --name $fn5 --pkg $pkgName --entrypoint "hello.main"
+
+log "Creating route"
+fission route create --function $fn5 --url /$fn5 --method GET
 
 log "Waiting for router to update cache"
 sleep 3
 
-checkFunctionResponse $fn 'Hello, world!'
+checkFunctionResponse $fn5 'Hello, world!'
+
+# 6) Deployment archive from a HTTP location
+# TBD
 
 # crappy cleanup, improve this later
 kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger

test/tests/test_spec_create/test_spec.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-#test:disabled
-
-set -euo pipefail
-
-fn=spec-$(date +%N)
-env=python-$fn
-
-# init
-fission spec init
-
-# verify init
-[ -d specs ]
-[ -f specs/README ]
-[ -f specs/fission-deployment-config.yaml ]
-
-# TODO replace with `fission env create --spec`
-cat > specs/env.yaml <<EOF
-apiVersion: fission.io/v1
-kind: Environment
-metadata:
-  name: $env
-  namespace: default
-spec:
-  version: 1
-  runtime:
-    image: fission/python-env:0.4.0
-EOF
-
-# create env
-fission spec apply
-trap "fission env delete --name $env" EXIT
-
-# verify env created
-fission env list | grep python
-
-# generate function spec
-fission fn create --spec --name $fn --env $env --code $(dirname $0)/hello.py
-
-# verify that function spec exists and has ArchiveUploadSpec, Package and Function
-[ -f specs/function-$fn.yaml ]
-grep ArchiveUploadSpec specs/*.yaml
-grep Package specs/*.yaml
-grep Function specs/*.yaml
-
-# create function
-fission spec apply
-trap "fission fn delete --name $fn" EXIT
-
-# verify function
-fission fn list | grep $fn
-
-sleep 3
-
-fission fn test --name $fn | grep -i hello
-
-

test/tests/test_specs/test_spec.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -euo pipefail
+
+fn=spec-$(date +%N)
+env=python-$fn
+
+# init
+fission spec init
+
+# verify init
+[ -d specs ]
+[ -f specs/README ]
+[ -f specs/fission-deployment-config.yaml ]
+
+fission env create --spec --name $env --image fission/python-env
+
+log "create env spec"
+fission spec apply
+trap "fission spec destroy" EXIT
+
+log "verify env created"
+fission env list | grep python
+
+log "generate function spec"
+fission fn create --spec --name $fn --env $env --code hello.py
+
+
+[ -f specs/function-$fn.yaml ]
+grep ArchiveUploadSpec specs/*.yaml
+grep Package specs/*.yaml
+grep Function specs/*.yaml
+
+log "Apply specs"
+fission spec apply
+trap "fission fn delete --name $fn" EXIT
+
+log "verify function exists"
+fission fn list | grep $fn
+
+sleep 3
+
+log "Test the function"
+fission fn test --name $fn | grep -i hello

test/tests/test_specs/test_spec_multifile.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ROOT=$(dirname $0)/../../..
+
+fn=spec-$(date +%N)
+env=python-$fn
+
+pushd $ROOT/examples/python
+
+fission spec init
+
+log "Creating environment spec"
+fission env create --spec --name $env --image fission/python-env --builder fission/python-builder
+fission env list | grep python
+
+log "Creating function spec"
+fission fn create --spec --name $fn --env $env --deploy "multifile/*" --entrypoint main.main
+
+log "Applying specs"
+fission spec apply
+
+log "Checking function's existance"
+fission fn list | grep $fn
+
+log "Testing function"
+fission fn test --name $fn | grep -i hello
+
+log "Destroying spec objects"
+fission spec destroy
+popd