diff --git a/common.go b/common.go index f3f16375..df591146 100644 --- a/common.go +++ b/common.go @@ -149,12 +149,11 @@ func MakeArchive(targetName string, globs ...string) (string, error) { for _, glob := range globs { f, err := filepath.Glob(glob) if err != nil { - log.Info(fmt.Sprintf("Invalid glob %v: %v", glob, err)) + log.Warn(fmt.Sprintf("Invalid glob %v: %v", glob, err)) return "", err } files = append(files, f...) } - // zip up the file list err := archiver.Zip.Make(targetName, files) if err != nil { diff --git a/fission/function.go b/fission/function.go index bb7df223..890b202e 100644 --- a/fission/function.go +++ b/fission/function.go @@ -252,10 +252,15 @@ func fnCreate(c *cli.Context) error { } } - srcArchiveName := c.String("src") - deployArchiveName := c.String("code") - if len(deployArchiveName) == 0 { - deployArchiveName = c.String("deploy") + srcArchiveName := c.StringSlice("src") + var deployArchiveName []string + codeFlag := false + code := c.String("code") + if len(code) == 0 { + deployArchiveName = c.StringSlice("deploy") + } else { + deployArchiveName = append(deployArchiveName, c.String("code")) + codeFlag = true } // fatal when both src & deploy archive are empty if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 { @@ -265,7 +270,7 @@ func fnCreate(c *cli.Context) error { buildcmd := c.String("buildcmd") // create new package in the same namespace as the function. - pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, specFile) + pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, specFile, codeFlag) fmt.Printf("package '%v' created\n", pkgMetadata.Name) } @@ -466,11 +471,17 @@ func fnUpdate(c *cli.Context) error { envNamespace = "" } - deployArchiveName := c.String("code") - if len(deployArchiveName) == 0 { - deployArchiveName = c.String("deploy") + var deployArchiveName []string + codeFlag := false + code := c.String("code") + if len(code) == 0 { + deployArchiveName = c.StringSlice("deploy") + } else { + deployArchiveName = append(deployArchiveName, c.String("code")) + codeFlag = true } - srcArchiveName := c.String("src") + + srcArchiveName := c.StringSlice("src") pkgName := c.String("pkg") entrypoint := c.String("entrypoint") buildcmd := c.String("buildcmd") @@ -563,7 +574,7 @@ func fnUpdate(c *cli.Context) error { log.Fatal("Package is used by multiple functions, use --force to force update") } - pkgMetadata, err = updatePackage(client, pkg, pkg.Spec.Environment.Name, pkg.Spec.Environment.Namespace, srcArchiveName, deployArchiveName, buildcmd, false) + pkgMetadata, err = updatePackage(client, pkg, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, false, codeFlag) util.CheckErr(err, fmt.Sprintf("update package '%v'", pkgName)) fmt.Printf("package '%v' updated\n", pkgMetadata.GetName()) diff --git a/fission/main.go b/fission/main.go index 31bc5072..83bd0620 100644 --- a/fission/main.go +++ b/fission/main.go @@ -90,8 +90,8 @@ func newCliApp() *cli.App { fnNameFlag := cli.StringFlag{Name: "name", Usage: "function name"} fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"} fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"} - fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"} - fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"} + fnDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"} + fnSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"} fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored), should be in the same namespace as the function"} fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"} fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"} @@ -234,8 +234,8 @@ func newCliApp() *cli.App { pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"} pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"} pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"} - pkgSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"} - pkgDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"} + pkgSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"} + pkgDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"} pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"} pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"} pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "orphan packages that are not referenced by any function"} diff --git a/fission/package.go b/fission/package.go index 64f2a006..39e5e25e 100644 --- a/fission/package.go +++ b/fission/package.go @@ -30,10 +30,12 @@ import ( "path/filepath" "strings" "text/tabwriter" + "time" "github.com/dchest/uniuri" "github.com/fission/fission/fission/util" storageSvcClient "github.com/fission/fission/storagesvc/client" + "github.com/mholt/archiver" "github.com/satori/go.uuid" "github.com/urfave/cli" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -82,15 +84,15 @@ func pkgCreate(c *cli.Context) error { log.Fatal("Need --env argument.") } envNamespace := c.String("envNamespace") - srcArchiveName := c.String("src") - deployArchiveName := c.String("deploy") + srcArchive := c.StringSlice("src") + deployArchive := c.StringSlice("deploy") buildcmd := c.String("buildcmd") - if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 { + if len(srcArchive) == 0 && len(deployArchive) == 0 { log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") } - meta := createPackage(client, pkgNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, "") + meta := createPackage(client, pkgNamespace, envName, envNamespace, srcArchive, deployArchive, buildcmd, "", false) fmt.Printf("Package '%v' created\n", meta.GetName()) return nil @@ -108,15 +110,15 @@ func pkgUpdate(c *cli.Context) error { force := c.Bool("f") envName := c.String("env") envNamespace := c.String("envNamespace") - srcArchiveName := c.String("src") - deployArchiveName := c.String("deploy") + srcArchive := c.StringSlice("src") + deployArchive := c.StringSlice("deploy") buildcmd := c.String("buildcmd") - if len(srcArchiveName) > 0 && len(deployArchiveName) > 0 { + if len(srcArchive) > 0 && len(deployArchive) > 0 { log.Fatal("Need either of --src or --deploy and not both arguments.") } - if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 && + if len(srcArchive) == 0 && len(deployArchive) == 0 && len(envName) == 0 && len(buildcmd) == 0 { log.Fatal("Need --env or --src or --deploy or --buildcmd argument.") } @@ -146,7 +148,7 @@ func pkgUpdate(c *cli.Context) error { } newPkgMeta, err := updatePackage(client, pkg, - envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd, false) + envName, envNamespace, srcArchive, deployArchive, buildcmd, false, false) if err != nil { util.CheckErr(err, "update package") } @@ -163,8 +165,8 @@ func pkgUpdate(c *cli.Context) error { return nil } -func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespace, - srcArchiveName, deployArchiveName, buildcmd string, forceRebuild bool) (*metav1.ObjectMeta, error) { +func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespace string, + srcArchive []string, deployArchive []string, buildcmd string, forceRebuild bool, codeFlag bool) (*metav1.ObjectMeta, error) { var srcArchiveMetadata, deployArchiveMetadata *fission.Archive needToBuild := false @@ -184,13 +186,20 @@ func updatePackage(client *client.Client, pkg *crd.Package, envName, envNamespac needToBuild = true } - if len(srcArchiveName) > 0 { + if len(srcArchive) > 0 { + srcArchiveName := archiveParser(srcArchive, envName) srcArchiveMetadata = createArchive(client, srcArchiveName, "") pkg.Spec.Source = *srcArchiveMetadata needToBuild = true } - if len(deployArchiveName) > 0 { + if len(deployArchive) > 0 { + var deployArchiveName string + if codeFlag { + deployArchiveName = deployArchive[0] + } else { + deployArchiveName = archiveParser(deployArchive, envName) + } deployArchiveMetadata = createArchive(client, deployArchiveName, "") pkg.Spec.Deployment = *deployArchiveMetadata // Users may update the env, envNS and deploy archive at the same time, @@ -439,7 +448,7 @@ func pkgRebuild(c *cli.Context) error { pkg.Metadata.Name, fission.BuildStatusFailed)) } - _, err = updatePackage(client, pkg, "", "", "", "", "", true) + _, err = updatePackage(client, pkg, "", "", nil, nil, "", true, false) util.CheckErr(err, "update package") fmt.Printf("Retrying build for pkg %v. Use \"fission pkg info --name %v\" to view status.\n", pkg.Metadata.Name, pkg.Metadata.Name) @@ -530,7 +539,7 @@ func createArchive(client *client.Client, fileName string, specFile string) *fis return &archive } -func createPackage(client *client.Client, pkgNamespace, envName, envNamespace, srcArchiveName, deployArchiveName, buildcmd string, specFile string) *metav1.ObjectMeta { +func createPackage(client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchive []string, deployArchive []string, buildcmd string, specFile string, codeFlag bool) *metav1.ObjectMeta { pkgSpec := fission.PackageSpec{ Environment: fission.EnvironmentReference{ Namespace: envNamespace, @@ -540,14 +549,21 @@ func createPackage(client *client.Client, pkgNamespace, envName, envNamespace, s var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded var pkgName string - if len(deployArchiveName) > 0 { + if len(deployArchive) > 0 { if len(specFile) > 0 { // we should do this in all cases, i think pkgStatus = fission.BuildStatusNone } + var deployArchiveName string + if codeFlag { + deployArchiveName = deployArchive[0] + } else { + deployArchiveName = archiveParser(deployArchive, envName) + } pkgSpec.Deployment = *createArchive(client, deployArchiveName, specFile) pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveName), uniuri.NewLen(4))) } - if len(srcArchiveName) > 0 { + if len(srcArchive) > 0 { + srcArchiveName := archiveParser(srcArchive, envName) pkgSpec.Source = *createArchive(client, srcArchiveName, specFile) // set pending status to package pkgStatus = fission.BuildStatusPending @@ -652,3 +668,23 @@ func downloadURL(fileUrl string) (io.ReadCloser, error) { } return resp.Body, nil } + +func archiveParser(archiveInput []string, envName string) string { + var archiveName = "" + + if (len(archiveInput) == 1 && archiver.Zip.Match(archiveInput[0])) || + (len(archiveInput) == 1 && (strings.HasPrefix(archiveInput[0], "http://") || strings.HasPrefix(archiveInput[0], "https://"))) { + return archiveInput[0] + } + + tmpDir, err := fission.GetTempDir() + if err != nil { + util.CheckErr(err, "create archive file") + } + archiveName, err = fission.MakeArchive(filepath.Join(tmpDir, fmt.Sprintf("%v-%v", envName, time.Now().Unix())), archiveInput...) + if err != nil { + util.CheckErr(err, "create archive file") + } + + return archiveName +} diff --git a/fission/spec.go b/fission/spec.go index 82c3e34e..259e5fd0 100644 --- a/fission/spec.go +++ b/fission/spec.go @@ -969,6 +969,9 @@ func localArchiveFromSpec(specDir string, aus *ArchiveUploadSpec) (*fission.Arch // XXX if there are lots of globs it's probably more efficient // to do a filepath.Walk and call path.Match on each path... files := make([]string, 0) + if len(aus.IncludeGlobs) == 1 && archiver.Zip.Match(aus.IncludeGlobs[0]) { + files = append(files, aus.IncludeGlobs[0]) + } for _, relativeGlob := range aus.IncludeGlobs { absGlob := rootDir + "/" + relativeGlob f, err := filepath.Glob(absGlob) diff --git a/test/tests/test_package_command.sh b/test/tests/test_package_command.sh index 419093d7..a5eeda20 100755 --- a/test/tests/test_package_command.sh +++ b/test/tests/test_package_command.sh @@ -2,16 +2,26 @@ set -euo pipefail -# Use package command to create two packages one with source -# archive and the other with deploy archive. Also, create a -# function to test the packages created by package command are +# Use package command to create packages of type: +# 1) Multiple source files from a directory +# 2) Source archive file +# TBD 3) Source file from a HTTP location +# 4) Deployment files from a directory +# 5) Deployment archive +# TBD 6) Deployment archive from a HTTP location +# TBD 7) Multiple files from multiple directories +# Then create a function to test the packages created by package command are # able to work. ROOT=$(dirname $0)/../.. PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python-env:test PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test -fn=python-srcbuild-$(date +%s) +fn1=python-srcbuild1-$(date +%s) +fn2=python-srcbuild2-$(date +%s) + +fn4=python-deploy4-$(date +%s) +fn5=python-deploy5-$(date +%s) waitBuild() { log "Waiting for builder manager to finish the build" @@ -55,7 +65,10 @@ export -f waitEnvBuilder cleanup() { log "Cleaning up..." fission env delete --name python || true - fission fn delete --name $fn || true + fission fn delete --name $fn1 || true + fission fn delete --name $fn2 || true + fission fn delete --name $fn4 || true + fission fn delete --name $fn5 || true } if [ -z "${TEST_NOCLEANUP:-}" ]; then @@ -71,24 +84,62 @@ log "Creating python env" fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE timeout 180s bash -c "waitEnvBuilder python" - -log "Creating pacakage with source archive" -zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/ -pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \') - +# 1) Multiple source files (multiple inputs, Using * expression, from a directory) +# Currently only * expression implemented as a test +pushd $ROOT/examples/python/ +pkg1=$(fission package create --src "sourcepkg/*" --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \') +popd # wait for build to finish at most 60s -timeout 60s bash -c "waitBuild $pkgName" - -log "Creating function " $fn -fission fn create --name $fn --pkg $pkgName --entrypoint "user.main" +timeout 60s bash -c "waitBuild $pkg1" +log "Creating function " $fn1 +fission fn create --name $fn1 --pkg $pkg1 --entrypoint "user.main" log "Creating route" -fission route create --function $fn --url /$fn --method GET +fission route create --function $fn1 --url /$fn1 --method GET log "Waiting for router to catch up" sleep 3 -checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}' +checkFunctionResponse $fn1 'a: 1 b: {c: 3, d: 4}' + +# 2) Source archive file +log "Creating pacakage with source archive" +zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/ +pkg2=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \') + +# wait for build to finish at most 60s +timeout 60s bash -c "waitBuild $pkg2" + +log "Creating function " $fn2 +fission fn create --name $fn2 --pkg $pkg2 --entrypoint "user.main" + +log "Creating route" +fission route create --function $fn2 --url /$fn2 --method GET + +log "Waiting for router to catch up" +sleep 3 + +checkFunctionResponse $fn2 'a: 1 b: {c: 3, d: 4}' + +# 3) Source file from a HTTP location +# TBD + +# 4) Deployment files from a directory +pushd $ROOT/examples/python/ +pkg4=$(fission package create --deploy "multifile/*" --env python| cut -f2 -d' '| tr -d \') +popd +log "Creating function " $fn4 +fission fn create --name $fn4 --pkg $pkg4 --entrypoint "main.main" + +log "Creating route" +fission route create --function $fn4 --url /$fn4 --method GET + +log "Waiting for router to catch up" +sleep 3 + +checkFunctionResponse $fn4 'Hello, world!' + +# 5) Deployment archive log "Creating package with deploy archive" mkdir testDir @@ -97,13 +148,20 @@ printf 'def main():\n return "Hello, world!"' > testDir/hello.py zip -jr demo-deploy-pkg.zip testDir/ pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \') -log "Updating function " $fn -fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main" + +log "Updating function " $fn5 +fission fn create --name $fn5 --pkg $pkgName --entrypoint "hello.main" + +log "Creating route" +fission route create --function $fn5 --url /$fn5 --method GET log "Waiting for router to update cache" sleep 3 -checkFunctionResponse $fn 'Hello, world!' +checkFunctionResponse $fn5 'Hello, world!' + +# 6) Deployment archive from a HTTP location +# TBD # crappy cleanup, improve this later kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger diff --git a/test/tests/test_spec_create/test_spec.sh b/test/tests/test_spec_create/test_spec.sh deleted file mode 100755 index f45830db..00000000 --- a/test/tests/test_spec_create/test_spec.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -#test:disabled - -set -euo pipefail - -fn=spec-$(date +%N) -env=python-$fn - -# init -fission spec init - -# verify init -[ -d specs ] -[ -f specs/README ] -[ -f specs/fission-deployment-config.yaml ] - -# TODO replace with `fission env create --spec` -cat > specs/env.yaml <