Allow using URL as archive source when creating functions (#1360)

In the case of large files, it takes a long time for the user to download
the source from the URL and upload it to StorgeSvc through CLI.

This PR allows a user to use URL as the function source when creating a function
and provides a new flag "--keeparchiveurl" to let the user to decided
whether the CLI should download the file first or store the file URL in the
archive directly. If "--keeparchiveurl" is true, then no checksum will be
generated, it's the user's responsibility to ensure the file won't be changed.

pkg/fetcher/fetcher.go

@@ -287,7 +287,10 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, pkg *fv1.Package, req types.F
 				return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus))
 			}
 			archive = &pkg.Spec.Deployment
+		} else {
+			return http.StatusBadRequest, fmt.Errorf("unkonwn fetch type: %v", req.FetchType)
 		}
+
 		// get package data as literal or by url
 		if len(archive.Literal) > 0 {
 			// write pkg.Literal into tmpPath
@@ -306,18 +309,20 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, pkg *fv1.Package, req types.F
 				return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url)
 			}
 
-			checksum, err := utils.FileChecksum(tmpPath)
-			if err != nil {
-				e := "failed to get checksum"
-				fetcher.logger.Error(e, zap.Error(err))
-				return http.StatusBadRequest, errors.Wrap(err, e)
-			}
-
-			err = verifyChecksum(checksum, &archive.Checksum)
-			if err != nil {
-				e := "failed to verify checksum"
-				fetcher.logger.Error(e, zap.Error(err))
-				return http.StatusBadRequest, errors.Wrap(err, e)
+			// check file integrity only if checksum is not empty.
+			if len(archive.Checksum.Sum) > 0 {
+				checksum, err := utils.GetFileChecksum(tmpPath)
+				if err != nil {
+					e := "failed to get checksum"
+					fetcher.logger.Error(e, zap.Error(err))
+					return http.StatusBadRequest, errors.Wrap(err, e)
+				}
+				err = verifyChecksum(checksum, &archive.Checksum)
+				if err != nil {
+					e := "failed to verify checksum"
+					fetcher.logger.Error(e, zap.Error(err))
+					return http.StatusBadRequest, errors.Wrap(err, e)
+				}
 			}
 		}
 	}
@@ -512,7 +517,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	sum, err := utils.FileChecksum(dstFilepath)
+	sum, err := utils.GetFileChecksum(dstFilepath)
 	if err != nil {
 		e := "error calculating checksum of zip file"
 		fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))

pkg/fission-cli/cli.go

@@ -104,9 +104,10 @@ func NewCliApp() *cli.App {
 	// functions
 	fnNameFlag := cli.StringFlag{Name: "name", Usage: "function name"}
 	fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"}
-	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
+	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for single file source code"}
 	fnDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
 	fnSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src, source", Usage: "local path or URL for source archive"}
+	fnKeepURLFlag := cli.BoolFlag{Name: "keeparchiveurl, keepurl", Usage: "Keep the providing URL in archive instead of downloading file from it. (If set, no checksum will be generated for file integrity check. You must ensure the file won't be changed.)"}
 	fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored), should be in the same namespace as the function"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
@@ -124,14 +125,13 @@ func NewCliApp() *cli.App {
 	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
 	fnExecutorTypeFlag := cli.StringFlag{Name: "executortype", Value: types.ExecutorTypePoolmgr, Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy' defaults to 'poolmgr'"}
 	fnExecutionTimeoutFlag := cli.IntFlag{Name: "fntimeout, ft", Value: 60, Usage: "Time duration to wait for the response while executing the function. If the flag is not provided, by default it will wait of 60s for the response."}
-
 	fnTimeoutFlag := cli.DurationFlag{Name: "timeout, t", Value: 30 * time.Second, Usage: "The length of time to wait for the response. If set to zero or negative number, no timeout is set."}
 
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, specSaveFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, specializationTimeoutFlag, fnExecutionTimeoutFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, specSaveFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnKeepURLFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag, specializationTimeoutFlag, fnExecutionTimeoutFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnGetMeta},
-		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, pkgNamespaceFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, specializationTimeoutFlag, fnExecutionTimeoutFlag}, Action: fnUpdate},
+		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnKeepURLFlag, fnEntryPointFlag, fnPkgNameFlag, pkgNamespaceFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, specializationTimeoutFlag, fnExecutionTimeoutFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnDelete},
 		// TODO : for fnList, i feel like it's nice to allow --fns all, to list functions across all namespaces for cluster admins, although, this is against ns isolation.
 		// so, in the future, if we end up using kubeconfig in fission cli and enforcing rolebindings to be created for users by admins etc, we can add this option at the time.
@@ -259,13 +259,14 @@ func NewCliApp() *cli.App {
 	pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"}
 	pkgSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
 	pkgDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
+	pkgKeepURLFlag := cli.BoolFlag{Name: "keeparchiveurl, keepurl", Usage: "Keep the providing URL in archive instead of downloading file from it. (If set, no checksum will be generated for file integrity check. You must ensure the file won't be changed.)"}
 	pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"}
 	pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"}
 	pkgStatusFlag := cli.StringFlag{Name: "status", Usage: `Filter packages by status`}
 	pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "Orphan packages that are not referenced by any function"}
 	pkgSubCommands := []cli.Command{
-		{Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: urfavecli.Wrapper(_package.Create)},
-		{Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: urfavecli.Wrapper(_package.Update)},
+		{Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgKeepURLFlag, pkgBuildCmdFlag}, Action: urfavecli.Wrapper(_package.Create)},
+		{Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgKeepURLFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: urfavecli.Wrapper(_package.Update)},
 		{Name: "rebuild", Usage: "Rebuild a failed package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: urfavecli.Wrapper(_package.Rebuild)},
 		{Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: urfavecli.Wrapper(_package.GetSrc)},
 		{Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: urfavecli.Wrapper(_package.GetDeploy)},

pkg/fission-cli/cmd/package/create.go

@@ -67,17 +67,21 @@ func (opts *CreateSubCommand) complete(flags cli.Input) error {
 	srcArchiveFiles := flags.StringSlice("src")
 	deployArchiveFiles := flags.StringSlice("deploy")
 	buildcmd := flags.String("buildcmd")
+	keepURL := flags.Bool("keepurl")
 
 	if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 {
 		log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
 	}
 
-	_, err := CreatePackage(flags, opts.client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false)
+	_, err := CreatePackage(flags, opts.client, pkgNamespace, envName, envNamespace,
+		srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false, keepURL)
 
 	return err
 }
 
-func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) (*metav1.ObjectMeta, error) {
+func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string, envName string, envNamespace string,
+	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool, keepURL bool) (*metav1.ObjectMeta, error) {
+
 	pkgSpec := fv1.PackageSpec{
 		Environment: fv1.EnvironmentReference{
 			Namespace: envNamespace,
@@ -91,7 +95,7 @@ func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string,
 		if len(specFile) > 0 { // we should do this in all cases, i think
 			pkgStatus = fv1.BuildStatusNone
 		}
-		deployment, err := CreateArchive(client, deployArchiveFiles, noZip, specDir, specFile)
+		deployment, err := CreateArchive(client, deployArchiveFiles, noZip, keepURL, specDir, specFile)
 		if err != nil {
 			return nil, err
 		}
@@ -99,7 +103,7 @@ func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string,
 		pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveFiles[0]), uniuri.NewLen(4)))
 	}
 	if len(srcArchiveFiles) > 0 {
-		source, err := CreateArchive(client, srcArchiveFiles, false, specDir, specFile)
+		source, err := CreateArchive(client, srcArchiveFiles, false, keepURL, specDir, specFile)
 		if err != nil {
 			return nil, err
 		}

pkg/fission-cli/cmd/package/package.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/dchest/uniuri"
 	"github.com/hashicorp/go-multierror"
@@ -41,15 +40,22 @@ import (
 // create an archive upload spec in the specs directory; otherwise
 // upload the archive using client.  noZip avoids zipping the
 // includeFiles, but is ignored if there's more than one includeFile.
-func CreateArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) (*fv1.Archive, error) {
+func CreateArchive(client *client.Client, includeFiles []string, noZip bool, keepURL bool, specDir string, specFile string) (*fv1.Archive, error) {
 
 	errs := &multierror.Error{}
+	fileURL := ""
 
 	// check files existence
 	for _, path := range includeFiles {
 		// ignore http files
-		if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-			continue
+		if utils.IsURL(path) {
+			if len(includeFiles) > 1 {
+				// It's intentional to disallow the user to provide file
+				// and URL at the same time even the keepurl is false.
+				return nil, errors.New("unable to create an archive that contains both file and URL")
+			}
+			fileURL = path
+			break
 		}
 
 		// Get files from inputs as number of files decide next steps
@@ -68,46 +74,67 @@ func CreateArchive(client *client.Client, includeFiles []string, noZip bool, spe
 	}
 
 	if len(specFile) > 0 {
-		// create an ArchiveUploadSpec and reference it from the archive
-		aus := &spectypes.ArchiveUploadSpec{
-			Name:         archiveName("", includeFiles),
-			IncludeGlobs: includeFiles,
-		}
+		var archive fv1.Archive
 
-		// check if this AUS exists in the specs; if so, don't create a new one
-		fr, err := spec.ReadSpecs(specDir)
-		util.CheckErr(err, "read specs")
-		if m := fr.SpecExists(aus, false, true); m != nil {
-			fmt.Printf("Re-using previously created archive %v\n", m.Name)
-			aus.Name = m.Name
+		if len(fileURL) > 0 {
+			archive = fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fileURL,
+			}
 		} else {
-			// save the uploadspec
-			err := spec.SpecSave(*aus, specFile)
-			util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile))
+			// create an ArchiveUploadSpec and reference it from the archive
+			aus := &spectypes.ArchiveUploadSpec{
+				Name:         archiveName("", includeFiles),
+				IncludeGlobs: includeFiles,
+			}
+
+			// check if this AUS exists in the specs; if so, don't create a new one
+			fr, err := spec.ReadSpecs(specDir)
+			util.CheckErr(err, "read specs")
+			if m := fr.SpecExists(aus, false, true); m != nil {
+				fmt.Printf("Re-using previously created archive %v\n", m.Name)
+				aus.Name = m.Name
+			} else {
+				// save the uploadspec
+				err := spec.SpecSave(*aus, specFile)
+				util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile))
+			}
+
+			// create the archive object
+			archive = fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name),
+			}
 		}
 
-		// create the archive object
-		ar := &fv1.Archive{
-			Type: fv1.ArchiveTypeUrl,
-			URL:  fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name),
-		}
-		return ar, nil
+		return &archive, nil
 	}
 
-	archivePath := makeArchiveFileIfNeeded("", includeFiles, noZip)
+	if len(fileURL) > 0 {
+		if keepURL {
+			return &fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fileURL,
+			}, nil
+		}
+		// download the file before we archive it
+		dst := pkgutil.DownloadToTempFile(fileURL)
+		includeFiles = []string{dst}
+	}
 
+	archivePath := makeArchiveFile("", includeFiles, noZip)
 	ctx := context.Background()
-	return pkgutil.UploadArchive(ctx, client, archivePath)
+	return pkgutil.UploadArchiveFile(ctx, client, archivePath)
 }
 
-// Create an archive from the given list of input files, unless that
-// list has only one item and that item is either a zip file or a URL.
+// makeArchiveFile creates a zip file from the given list of input files,
+// unless that list has only one item and that item is a zip file.
 //
 // If the inputs have only one file and noZip is true, the file is
 // returned as-is with no zipping.  (This is used for compatibility
 // with v1 envs.)  noZip is IGNORED if there is more than one input
 // file.
-func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZip bool) string {
+func makeArchiveFile(archiveNameHint string, archiveInput []string, noZip bool) string {
 
 	// Unique name for the archive
 	archiveName := archiveName(archiveNameHint, archiveInput)
@@ -118,7 +145,7 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		util.CheckErr(err, "finding all globs")
 	}
 
-	// We have one file; if it's a zip file or a URL, no need to archive it
+	// We have one file; if it's a zip file, no need to archive it
 	if len(files) == 1 {
 		// make sure it exists
 		if _, err := os.Stat(files[0]); err != nil {
@@ -129,11 +156,6 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		if archiver.Zip.Match(files[0]) || noZip {
 			return files[0]
 		}
-
-		// if it's an HTTP URL, just use the URL.
-		if strings.HasPrefix(files[0], "http://") || strings.HasPrefix(files[0], "https://") {
-			return files[0]
-		}
 	}
 
 	// For anything else, create a new archive
@@ -142,7 +164,7 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		util.CheckErr(err, "create temporary archive directory")
 	}
 
-	archivePath, err := utils.MakeArchive(filepath.Join(tmpDir, archiveName), archiveInput...)
+	archivePath, err := utils.MakeZipArchive(filepath.Join(tmpDir, archiveName), archiveInput...)
 	if err != nil {
 		util.CheckErr(err, "create archive file")
 	}

pkg/fission-cli/cmd/package/update.go

@@ -39,6 +39,7 @@ type UpdateSubCommand struct {
 	srcArchiveFiles    []string
 	deployArchiveFiles []string
 	buildcmd           string
+	keepURL            bool
 }
 
 func Update(flags cli.Input) error {
@@ -68,6 +69,7 @@ func (opts *UpdateSubCommand) complete(flags cli.Input) error {
 	opts.srcArchiveFiles = flags.StringSlice("src")
 	opts.deployArchiveFiles = flags.StringSlice("deploy")
 	opts.buildcmd = flags.String("buildcmd")
+	opts.keepURL = flags.Bool("keepurl")
 
 	if len(opts.srcArchiveFiles) > 0 && len(opts.deployArchiveFiles) > 0 {
 		return errors.New("Need either of --src or --deploy and not both arguments.")
@@ -111,7 +113,7 @@ func (opts *UpdateSubCommand) run(flags cli.Input) error {
 
 	newPkgMeta, err := UpdatePackage(opts.client, pkg,
 		opts.envName, opts.envNamespace, opts.srcArchiveFiles,
-		opts.deployArchiveFiles, opts.buildcmd, false, false)
+		opts.deployArchiveFiles, opts.buildcmd, false, false, opts.keepURL)
 	if err != nil {
 		return errors.Wrap(err, "update package")
 	}
@@ -130,7 +132,7 @@ func (opts *UpdateSubCommand) run(flags cli.Input) error {
 }
 
 func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string,
-	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) {
+	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool, keepURL bool) (*metav1.ObjectMeta, error) {
 
 	needToBuild := false
 
@@ -150,20 +152,20 @@ func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespac
 	}
 
 	if len(srcArchiveFiles) > 0 {
-		srcArchiveMetadata, err := CreateArchive(client, srcArchiveFiles, false, "", "")
+		srcArchive, err := CreateArchive(client, srcArchiveFiles, false, keepURL, "", "")
 		if err != nil {
 			return nil, err
 		}
-		pkg.Spec.Source = *srcArchiveMetadata
+		pkg.Spec.Source = *srcArchive
 		needToBuild = true
 	}
 
 	if len(deployArchiveFiles) > 0 {
-		deployArchiveMetadata, err := CreateArchive(client, deployArchiveFiles, noZip, "", "")
+		deployArchive, err := CreateArchive(client, deployArchiveFiles, noZip, keepURL, "", "")
 		if err != nil {
 			return nil, err
 		}
-		pkg.Spec.Deployment = *deployArchiveMetadata
+		pkg.Spec.Deployment = *deployArchive
 		// Users may update the env, envNS and deploy archive at the same time,
 		// but without the source archive. In this case, we should set needToBuild to false
 		needToBuild = false

pkg/fission-cli/cmd/package/util/util.go

@@ -36,14 +36,9 @@ import (
 	"github.com/fission/fission/pkg/utils"
 )
 
-func UploadArchive(ctx context.Context, client *client.Client, fileName string) (*fv1.Archive, error) {
+func UploadArchiveFile(ctx context.Context, client *client.Client, fileName string) (*fv1.Archive, error) {
 	var archive fv1.Archive
 
-	// If filename is a URL, download it first
-	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
-		fileName = DownloadToTempFile(fileName)
-	}
-
 	size, err := utils.FileSize(fileName)
 	if err != nil {
 		return nil, err
@@ -72,11 +67,12 @@ func UploadArchive(ctx context.Context, client *client.Client, fileName string)
 		archive.Type = fv1.ArchiveTypeUrl
 		archive.URL = archiveURL
 
-		csum, err := utils.FileChecksum(fileName)
+		csum, err := utils.GetFileChecksum(fileName)
 		util.CheckErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
 
 		archive.Checksum = *csum
 	}
+
 	return &archive, nil
 }
 
@@ -101,8 +97,6 @@ func DownloadToTempFile(fileUrl string) string {
 
 	tmpFilename := uuid.NewV4().String()
 	destination := filepath.Join(tmpDir, tmpFilename)
-	err = os.Mkdir(tmpDir, 0744)
-	util.CheckErr(err, "create temp directory")
 
 	err = WriteArchiveToFile(destination, reader)
 	util.CheckErr(err, "write archive to file")
@@ -127,8 +121,9 @@ func WriteArchiveToFile(fileName string, reader io.Reader) error {
 	if err != nil {
 		return err
 	}
+	tmpFileName := uuid.NewV4().String()
 
-	path := filepath.Join(tmpDir, fileName+".tmp")
+	path := filepath.Join(tmpDir, tmpFileName+".tmp")
 	w, err := os.Create(path)
 	if err != nil {
 		return err

pkg/fission-cli/cmd/spec/apply.go

@@ -281,7 +281,7 @@ func applyArchives(fclient *client.Client, specDir string, fr *FissionResources)
 			fmt.Printf("uploading archive %v\n", name)
 			// ar.URL is actually a local filename at this stage
 			ctx := context.Background()
-			uploadedAr, err := pkgutil.UploadArchive(ctx, fclient, ar.URL)
+			uploadedAr, err := pkgutil.UploadArchiveFile(ctx, fclient, ar.URL)
 			if err != nil {
 				return err
 			}
@@ -461,7 +461,7 @@ func localArchiveFromSpec(specDir string, aus *spectypes.ArchiveUploadSpec) (*fv
 		}, nil
 	} else {
 		// checksum
-		csum, err := utils.FileChecksum(archiveFileName)
+		csum, err := utils.GetFileChecksum(archiveFileName)
 		if err != nil {
 			return nil, fmt.Errorf("failed to calculate archive checksum for %v (%v): %v", aus.Name, archiveFileName, err)
 		}

pkg/fission-cli/cmd/support/dump.go

@@ -146,7 +146,7 @@ func (opts *DumpSubCommand) do(flags cli.Input) error {
 	if !nozip {
 		defer os.RemoveAll(tempDir)
 		path := filepath.Join(outputDir, fmt.Sprintf("%v.zip", dumpName))
-		_, err := utils.MakeArchive(path, tempDir)
+		_, err := utils.MakeZipArchive(path, tempDir)
 		if err != nil {
 			fmt.Printf("Error creating archive for dump files: %v", err)
 			return err

pkg/fission-cli/function.go

@@ -298,9 +298,11 @@ func fnCreate(c *cli.Context) error {
 		}
 
 		buildcmd := c.String("buildcmd")
+		keepURL := c.Bool("keepurl")
 
 		// create new package in the same namespace as the function.
-		pkgMetadata, err = _package.CreatePackage(c, client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip)
+		pkgMetadata, err = _package.CreatePackage(c, client, fnNamespace, envName, envNamespace,
+			srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip, keepURL)
 		util.CheckErr(err, "create package")
 	}
 
@@ -637,7 +639,9 @@ func fnUpdate(c *cli.Context) error {
 			log.Fatal("Package is used by multiple functions, use --force to force update")
 		}
 
-		pkgMetadata, err = _package.UpdatePackage(client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag)
+		keepURL := c.Bool("keepurl")
+
+		pkgMetadata, err = _package.UpdatePackage(client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag, keepURL)
 		util.CheckErr(err, fmt.Sprintf("update package '%v'", pkgName))
 
 		fmt.Printf("package '%v' updated\n", pkgMetadata.GetName())

pkg/utils/utils.go

@@ -19,12 +19,14 @@ package utils
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/mholt/archiver"
 	uuid "github.com/satori/go.uuid"
@@ -110,7 +112,7 @@ func FindAllGlobs(inputList []string) ([]string, error) {
 	return files, nil
 }
 
-func MakeArchive(targetName string, globs ...string) (string, error) {
+func MakeZipArchive(targetName string, globs ...string) (string, error) {
 	files, err := FindAllGlobs(globs)
 	if err != nil {
 		return "", err
@@ -160,21 +162,39 @@ func FileSize(filePath string) (int64, error) {
 	return info.Size(), err
 }
 
-func FileChecksum(fileName string) (*fv1.Checksum, error) {
+func GetFileChecksum(fileName string) (*fv1.Checksum, error) {
 	f, err := os.Open(fileName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open file %v: %v", fileName, err)
 	}
 	defer f.Close()
 
-	h := sha256.New()
-	_, err = io.Copy(h, f)
+	sum, err := GetChecksum(f)
 	if err != nil {
 		return nil, fmt.Errorf("failed to calculate checksum for %v", fileName)
 	}
 
+	return sum, nil
+}
+
+func GetChecksum(src io.Reader) (*fv1.Checksum, error) {
+	if src == nil {
+		return nil, errors.New("cannot read from nil reader")
+	}
+
+	h := sha256.New()
+
+	_, err := io.Copy(h, src)
+	if err != nil {
+		return nil, err
+	}
+
 	return &fv1.Checksum{
 		Type: fv1.ChecksumTypeSHA256,
 		Sum:  hex.EncodeToString(h.Sum(nil)),
 	}, nil
 }
+
+func IsURL(str string) bool {
+	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://")
+}

pkg/utils/utils_test.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func TestIsURL(t *testing.T) {
+	tests := []struct {
+		name string
+		url  string
+		want bool
+	}{
+		{"http", "http://example.com", true},
+		{"https", "https://example.com", true},
+		{"file", "file://example.com", false},
+		{"filename", "foobar.zip", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsURL(tt.url); got != tt.want {
+				t.Errorf("IsURL() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetChecksum(t *testing.T) {
+	tests := []struct {
+		name    string
+		src     io.Reader
+		want    *fv1.Checksum
+		wantErr bool
+	}{
+		{
+			name: "string case",
+			src:  bytes.NewReader([]byte("foobar hello world")),
+			want: &fv1.Checksum{
+				Type: "sha256",
+				Sum:  "99936be1902361c29745aef68bd818f5f08246fc695e2d6e4cc474daf79fed32",
+			},
+			wantErr: false,
+		},
+		{
+			name:    "empty reader",
+			src:     nil,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetChecksum(tt.src)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetChecksum() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetChecksum() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

test/init_tools.sh

@@ -54,6 +54,14 @@ if [ $(uname -s) == 'Darwin' ]; then
         exit 1
     fi
 
+    if command -v gsha256sum >/dev/null; then
+        sha256sum() { gsha256sum "$@"; }
+        export -f sha256sum
+    else
+        echo '"gsha256sum" command not found. Try "brew install coreutils".'
+        exit 1
+    fi
+
     find_executable() {
         path=$1; shift
         find $path -perm +111 -type f "$@"

test/tests/test_create_fn_with_url.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+ROOT=$(dirname $0)/../..
+
+env=nodejs-$TEST_ID
+fn=nodejs-hello-$TEST_ID
+code_url=https://raw.githubusercontent.com/fission/fission/master/examples/nodejs/hello.js
+base64val=$(wget -O- ${code_url} | base64)
+
+cleanup() {
+    log "Cleaning up..."
+    clean_resource_by_id $TEST_ID
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+# Create a hello world function in nodejs, test it with an http trigger
+log "Creating nodejs env"
+fission env create --name $env --image $NODE_RUNTIME_IMAGE
+
+log "Creating function with file"
+fission fn create --name $fn --env $env --code ${code_url}
+
+pkg=$(kubectl -n default get functions ${fn} -o yaml|grep hello-js|awk '{print $2}')
+literal=$(kubectl -n default get packages ${pkg} -o yaml|grep "literal:"|awk '{print $2}')
+
+if [ ${literal} != ${base64val}  ]; then
+    log "have different literal value: ${literal} vs. ${base64val}"
+    exit 1
+fi
+
+log "Creating route"
+fission route create --function $fn --url /$fn --method GET
+
+log "Waiting for router to catch up"
+sleep 3
+
+log "Doing an HTTP GET on the function's route"
+response=$(curl --retry 5 http://$FISSION_ROUTER/$fn)
+
+log "Checking for valid response"
+echo $response | grep -i hello
+
+log "Update function with file URL"
+fission fn update --name $fn --env $env --code ${code_url} --keepurl
+
+pkg=$(kubectl -n default get functions ${fn} -o yaml|grep hello-js|awk '{print $2}')
+url=$(kubectl -n default get packages ${pkg} -o yaml|grep "://"|awk '{print $2}')
+
+if [ ${url} != ${code_url} ]; then
+    log "have different code url: ${url} vs. ${code_url}"
+    exit 1
+fi
+
+log "Doing an HTTP GET on the function's route"
+response=$(curl --retry 5 http://$FISSION_ROUTER/$fn)
+
+log "Checking for valid response"
+echo $response | grep -i hello
+
+log "All done."