Allow using URL as archive source when creating functions (#1360)

In the case of large files, it takes a long time for the user to download
the source from the URL and upload it to StorgeSvc through CLI.

This PR allows a user to use URL as the function source when creating a function
and provides a new flag "--keeparchiveurl" to let the user to decided
whether the CLI should download the file first or store the file URL in the
archive directly. If "--keeparchiveurl" is true, then no checksum will be
generated, it's the user's responsibility to ensure the file won't be changed.

pkg/utils/utils.go

@@ -19,12 +19,14 @@ package utils
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/mholt/archiver"
 	uuid "github.com/satori/go.uuid"
@@ -110,7 +112,7 @@ func FindAllGlobs(inputList []string) ([]string, error) {
 	return files, nil
 }
 
-func MakeArchive(targetName string, globs ...string) (string, error) {
+func MakeZipArchive(targetName string, globs ...string) (string, error) {
 	files, err := FindAllGlobs(globs)
 	if err != nil {
 		return "", err
@@ -160,21 +162,39 @@ func FileSize(filePath string) (int64, error) {
 	return info.Size(), err
 }
 
-func FileChecksum(fileName string) (*fv1.Checksum, error) {
+func GetFileChecksum(fileName string) (*fv1.Checksum, error) {
 	f, err := os.Open(fileName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open file %v: %v", fileName, err)
 	}
 	defer f.Close()
 
-	h := sha256.New()
-	_, err = io.Copy(h, f)
+	sum, err := GetChecksum(f)
 	if err != nil {
 		return nil, fmt.Errorf("failed to calculate checksum for %v", fileName)
 	}
 
+	return sum, nil
+}
+
+func GetChecksum(src io.Reader) (*fv1.Checksum, error) {
+	if src == nil {
+		return nil, errors.New("cannot read from nil reader")
+	}
+
+	h := sha256.New()
+
+	_, err := io.Copy(h, src)
+	if err != nil {
+		return nil, err
+	}
+
 	return &fv1.Checksum{
 		Type: fv1.ChecksumTypeSHA256,
 		Sum:  hex.EncodeToString(h.Sum(nil)),
 	}, nil
 }
+
+func IsURL(str string) bool {
+	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://")
+}

pkg/utils/utils_test.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func TestIsURL(t *testing.T) {
+	tests := []struct {
+		name string
+		url  string
+		want bool
+	}{
+		{"http", "http://example.com", true},
+		{"https", "https://example.com", true},
+		{"file", "file://example.com", false},
+		{"filename", "foobar.zip", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsURL(tt.url); got != tt.want {
+				t.Errorf("IsURL() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetChecksum(t *testing.T) {
+	tests := []struct {
+		name    string
+		src     io.Reader
+		want    *fv1.Checksum
+		wantErr bool
+	}{
+		{
+			name: "string case",
+			src:  bytes.NewReader([]byte("foobar hello world")),
+			want: &fv1.Checksum{
+				Type: "sha256",
+				Sum:  "99936be1902361c29745aef68bd818f5f08246fc695e2d6e4cc474daf79fed32",
+			},
+			wantErr: false,
+		},
+		{
+			name:    "empty reader",
+			src:     nil,
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetChecksum(tt.src)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetChecksum() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetChecksum() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}