Allow using URL as archive source when creating functions (#1360)

In the case of large files, it takes a long time for the user to download
the source from the URL and upload it to StorgeSvc through CLI.

This PR allows a user to use URL as the function source when creating a function
and provides a new flag "--keeparchiveurl" to let the user to decided
whether the CLI should download the file first or store the file URL in the
archive directly. If "--keeparchiveurl" is true, then no checksum will be
generated, it's the user's responsibility to ensure the file won't be changed.

pkg/fission-cli/cmd/package/create.go

@@ -67,17 +67,21 @@ func (opts *CreateSubCommand) complete(flags cli.Input) error {
 	srcArchiveFiles := flags.StringSlice("src")
 	deployArchiveFiles := flags.StringSlice("deploy")
 	buildcmd := flags.String("buildcmd")
+	keepURL := flags.Bool("keepurl")
 
 	if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 {
 		log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
 	}
 
-	_, err := CreatePackage(flags, opts.client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false)
+	_, err := CreatePackage(flags, opts.client, pkgNamespace, envName, envNamespace,
+		srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false, keepURL)
 
 	return err
 }
 
-func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) (*metav1.ObjectMeta, error) {
+func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string, envName string, envNamespace string,
+	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool, keepURL bool) (*metav1.ObjectMeta, error) {
+
 	pkgSpec := fv1.PackageSpec{
 		Environment: fv1.EnvironmentReference{
 			Namespace: envNamespace,
@@ -91,7 +95,7 @@ func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string,
 		if len(specFile) > 0 { // we should do this in all cases, i think
 			pkgStatus = fv1.BuildStatusNone
 		}
-		deployment, err := CreateArchive(client, deployArchiveFiles, noZip, specDir, specFile)
+		deployment, err := CreateArchive(client, deployArchiveFiles, noZip, keepURL, specDir, specFile)
 		if err != nil {
 			return nil, err
 		}
@@ -99,7 +103,7 @@ func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string,
 		pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveFiles[0]), uniuri.NewLen(4)))
 	}
 	if len(srcArchiveFiles) > 0 {
-		source, err := CreateArchive(client, srcArchiveFiles, false, specDir, specFile)
+		source, err := CreateArchive(client, srcArchiveFiles, false, keepURL, specDir, specFile)
 		if err != nil {
 			return nil, err
 		}

pkg/fission-cli/cmd/package/package.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/dchest/uniuri"
 	"github.com/hashicorp/go-multierror"
@@ -41,15 +40,22 @@ import (
 // create an archive upload spec in the specs directory; otherwise
 // upload the archive using client.  noZip avoids zipping the
 // includeFiles, but is ignored if there's more than one includeFile.
-func CreateArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) (*fv1.Archive, error) {
+func CreateArchive(client *client.Client, includeFiles []string, noZip bool, keepURL bool, specDir string, specFile string) (*fv1.Archive, error) {
 
 	errs := &multierror.Error{}
+	fileURL := ""
 
 	// check files existence
 	for _, path := range includeFiles {
 		// ignore http files
-		if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
-			continue
+		if utils.IsURL(path) {
+			if len(includeFiles) > 1 {
+				// It's intentional to disallow the user to provide file
+				// and URL at the same time even the keepurl is false.
+				return nil, errors.New("unable to create an archive that contains both file and URL")
+			}
+			fileURL = path
+			break
 		}
 
 		// Get files from inputs as number of files decide next steps
@@ -68,46 +74,67 @@ func CreateArchive(client *client.Client, includeFiles []string, noZip bool, spe
 	}
 
 	if len(specFile) > 0 {
-		// create an ArchiveUploadSpec and reference it from the archive
-		aus := &spectypes.ArchiveUploadSpec{
-			Name:         archiveName("", includeFiles),
-			IncludeGlobs: includeFiles,
-		}
+		var archive fv1.Archive
 
-		// check if this AUS exists in the specs; if so, don't create a new one
-		fr, err := spec.ReadSpecs(specDir)
-		util.CheckErr(err, "read specs")
-		if m := fr.SpecExists(aus, false, true); m != nil {
-			fmt.Printf("Re-using previously created archive %v\n", m.Name)
-			aus.Name = m.Name
+		if len(fileURL) > 0 {
+			archive = fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fileURL,
+			}
 		} else {
-			// save the uploadspec
-			err := spec.SpecSave(*aus, specFile)
-			util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile))
+			// create an ArchiveUploadSpec and reference it from the archive
+			aus := &spectypes.ArchiveUploadSpec{
+				Name:         archiveName("", includeFiles),
+				IncludeGlobs: includeFiles,
+			}
+
+			// check if this AUS exists in the specs; if so, don't create a new one
+			fr, err := spec.ReadSpecs(specDir)
+			util.CheckErr(err, "read specs")
+			if m := fr.SpecExists(aus, false, true); m != nil {
+				fmt.Printf("Re-using previously created archive %v\n", m.Name)
+				aus.Name = m.Name
+			} else {
+				// save the uploadspec
+				err := spec.SpecSave(*aus, specFile)
+				util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile))
+			}
+
+			// create the archive object
+			archive = fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name),
+			}
 		}
 
-		// create the archive object
-		ar := &fv1.Archive{
-			Type: fv1.ArchiveTypeUrl,
-			URL:  fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name),
-		}
-		return ar, nil
+		return &archive, nil
 	}
 
-	archivePath := makeArchiveFileIfNeeded("", includeFiles, noZip)
+	if len(fileURL) > 0 {
+		if keepURL {
+			return &fv1.Archive{
+				Type: fv1.ArchiveTypeUrl,
+				URL:  fileURL,
+			}, nil
+		}
+		// download the file before we archive it
+		dst := pkgutil.DownloadToTempFile(fileURL)
+		includeFiles = []string{dst}
+	}
 
+	archivePath := makeArchiveFile("", includeFiles, noZip)
 	ctx := context.Background()
-	return pkgutil.UploadArchive(ctx, client, archivePath)
+	return pkgutil.UploadArchiveFile(ctx, client, archivePath)
 }
 
-// Create an archive from the given list of input files, unless that
-// list has only one item and that item is either a zip file or a URL.
+// makeArchiveFile creates a zip file from the given list of input files,
+// unless that list has only one item and that item is a zip file.
 //
 // If the inputs have only one file and noZip is true, the file is
 // returned as-is with no zipping.  (This is used for compatibility
 // with v1 envs.)  noZip is IGNORED if there is more than one input
 // file.
-func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZip bool) string {
+func makeArchiveFile(archiveNameHint string, archiveInput []string, noZip bool) string {
 
 	// Unique name for the archive
 	archiveName := archiveName(archiveNameHint, archiveInput)
@@ -118,7 +145,7 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		util.CheckErr(err, "finding all globs")
 	}
 
-	// We have one file; if it's a zip file or a URL, no need to archive it
+	// We have one file; if it's a zip file, no need to archive it
 	if len(files) == 1 {
 		// make sure it exists
 		if _, err := os.Stat(files[0]); err != nil {
@@ -129,11 +156,6 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		if archiver.Zip.Match(files[0]) || noZip {
 			return files[0]
 		}
-
-		// if it's an HTTP URL, just use the URL.
-		if strings.HasPrefix(files[0], "http://") || strings.HasPrefix(files[0], "https://") {
-			return files[0]
-		}
 	}
 
 	// For anything else, create a new archive
@@ -142,7 +164,7 @@ func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZi
 		util.CheckErr(err, "create temporary archive directory")
 	}
 
-	archivePath, err := utils.MakeArchive(filepath.Join(tmpDir, archiveName), archiveInput...)
+	archivePath, err := utils.MakeZipArchive(filepath.Join(tmpDir, archiveName), archiveInput...)
 	if err != nil {
 		util.CheckErr(err, "create archive file")
 	}

pkg/fission-cli/cmd/package/update.go

@@ -39,6 +39,7 @@ type UpdateSubCommand struct {
 	srcArchiveFiles    []string
 	deployArchiveFiles []string
 	buildcmd           string
+	keepURL            bool
 }
 
 func Update(flags cli.Input) error {
@@ -68,6 +69,7 @@ func (opts *UpdateSubCommand) complete(flags cli.Input) error {
 	opts.srcArchiveFiles = flags.StringSlice("src")
 	opts.deployArchiveFiles = flags.StringSlice("deploy")
 	opts.buildcmd = flags.String("buildcmd")
+	opts.keepURL = flags.Bool("keepurl")
 
 	if len(opts.srcArchiveFiles) > 0 && len(opts.deployArchiveFiles) > 0 {
 		return errors.New("Need either of --src or --deploy and not both arguments.")
@@ -111,7 +113,7 @@ func (opts *UpdateSubCommand) run(flags cli.Input) error {
 
 	newPkgMeta, err := UpdatePackage(opts.client, pkg,
 		opts.envName, opts.envNamespace, opts.srcArchiveFiles,
-		opts.deployArchiveFiles, opts.buildcmd, false, false)
+		opts.deployArchiveFiles, opts.buildcmd, false, false, opts.keepURL)
 	if err != nil {
 		return errors.Wrap(err, "update package")
 	}
@@ -130,7 +132,7 @@ func (opts *UpdateSubCommand) run(flags cli.Input) error {
 }
 
 func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string,
-	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) {
+	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool, keepURL bool) (*metav1.ObjectMeta, error) {
 
 	needToBuild := false
 
@@ -150,20 +152,20 @@ func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespac
 	}
 
 	if len(srcArchiveFiles) > 0 {
-		srcArchiveMetadata, err := CreateArchive(client, srcArchiveFiles, false, "", "")
+		srcArchive, err := CreateArchive(client, srcArchiveFiles, false, keepURL, "", "")
 		if err != nil {
 			return nil, err
 		}
-		pkg.Spec.Source = *srcArchiveMetadata
+		pkg.Spec.Source = *srcArchive
 		needToBuild = true
 	}
 
 	if len(deployArchiveFiles) > 0 {
-		deployArchiveMetadata, err := CreateArchive(client, deployArchiveFiles, noZip, "", "")
+		deployArchive, err := CreateArchive(client, deployArchiveFiles, noZip, keepURL, "", "")
 		if err != nil {
 			return nil, err
 		}
-		pkg.Spec.Deployment = *deployArchiveMetadata
+		pkg.Spec.Deployment = *deployArchive
 		// Users may update the env, envNS and deploy archive at the same time,
 		// but without the source archive. In this case, we should set needToBuild to false
 		needToBuild = false

pkg/fission-cli/cmd/package/util/util.go

@@ -36,14 +36,9 @@ import (
 	"github.com/fission/fission/pkg/utils"
 )
 
-func UploadArchive(ctx context.Context, client *client.Client, fileName string) (*fv1.Archive, error) {
+func UploadArchiveFile(ctx context.Context, client *client.Client, fileName string) (*fv1.Archive, error) {
 	var archive fv1.Archive
 
-	// If filename is a URL, download it first
-	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
-		fileName = DownloadToTempFile(fileName)
-	}
-
 	size, err := utils.FileSize(fileName)
 	if err != nil {
 		return nil, err
@@ -72,11 +67,12 @@ func UploadArchive(ctx context.Context, client *client.Client, fileName string)
 		archive.Type = fv1.ArchiveTypeUrl
 		archive.URL = archiveURL
 
-		csum, err := utils.FileChecksum(fileName)
+		csum, err := utils.GetFileChecksum(fileName)
 		util.CheckErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
 
 		archive.Checksum = *csum
 	}
+
 	return &archive, nil
 }
 
@@ -101,8 +97,6 @@ func DownloadToTempFile(fileUrl string) string {
 
 	tmpFilename := uuid.NewV4().String()
 	destination := filepath.Join(tmpDir, tmpFilename)
-	err = os.Mkdir(tmpDir, 0744)
-	util.CheckErr(err, "create temp directory")
 
 	err = WriteArchiveToFile(destination, reader)
 	util.CheckErr(err, "write archive to file")
@@ -127,8 +121,9 @@ func WriteArchiveToFile(fileName string, reader io.Reader) error {
 	if err != nil {
 		return err
 	}
+	tmpFileName := uuid.NewV4().String()
 
-	path := filepath.Join(tmpDir, fileName+".tmp")
+	path := filepath.Join(tmpDir, tmpFileName+".tmp")
 	w, err := os.Create(path)
 	if err != nil {
 		return err

pkg/fission-cli/cmd/spec/apply.go

@@ -281,7 +281,7 @@ func applyArchives(fclient *client.Client, specDir string, fr *FissionResources)
 			fmt.Printf("uploading archive %v\n", name)
 			// ar.URL is actually a local filename at this stage
 			ctx := context.Background()
-			uploadedAr, err := pkgutil.UploadArchive(ctx, fclient, ar.URL)
+			uploadedAr, err := pkgutil.UploadArchiveFile(ctx, fclient, ar.URL)
 			if err != nil {
 				return err
 			}
@@ -461,7 +461,7 @@ func localArchiveFromSpec(specDir string, aus *spectypes.ArchiveUploadSpec) (*fv
 		}, nil
 	} else {
 		// checksum
-		csum, err := utils.FileChecksum(archiveFileName)
+		csum, err := utils.GetFileChecksum(archiveFileName)
 		if err != nil {
 			return nil, fmt.Errorf("failed to calculate archive checksum for %v (%v): %v", aus.Name, archiveFileName, err)
 		}

pkg/fission-cli/cmd/support/dump.go

@@ -146,7 +146,7 @@ func (opts *DumpSubCommand) do(flags cli.Input) error {
 	if !nozip {
 		defer os.RemoveAll(tempDir)
 		path := filepath.Join(outputDir, fmt.Sprintf("%v.zip", dumpName))
-		_, err := utils.MakeArchive(path, tempDir)
+		_, err := utils.MakeZipArchive(path, tempDir)
 		if err != nil {
 			fmt.Printf("Error creating archive for dump files: %v", err)
 			return err