Add package command (#385)

Add package command support: This PR adds package command to CLI. package provides some useful subcommands to use such as packages CRUD and display package detail information. Also it is able to reuse existing package at function creation.

* Check function existence before creating package

* Check package existence

* Add support for downloading archive from given url

The functionality was supported before e238776bf7. Add this functionality back for more flexible usage.

* Add output flag to save archive content in specific file

* Add logs when package create/update

* Fix fnCreate requires environment argument when a pkg is specified

* Fix fnUpdate failed to update function pkg info when a package is specified

* Add package command test

* Fix wrong python test image in test script

* Remove package description fields

* Fix package command not update package but creating a new one

* Set package as pending status only when there is no deploy archive

* Rename function from fetchArchiveFromArbitraryURL to downloadToTempFile

* Use io.Copy to prevent loading all body into memory

* Revise some messages

* Allow user to update package build command

* Allow user to update package content when using function update

* Retrieve pkgName from function packageref if it’s not specified

* Fix function failed to update due to resource conflict

* Fix test case failure due to single quote

* kick ci

* Fix failed to update function packageRef

* kick ci

* kick ci

environments/fetcher/fetcher.go

@@ -78,12 +78,15 @@ func downloadUrl(url string, localPath string) error {
 	}
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
+	w, err := os.Create(localPath)
 	if err != nil {
 		return err
 	}
-
-	err = ioutil.WriteFile(localPath, body, 0600)
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(localPath, 0600)
 	if err != nil {
 		return err
 	}

fission/common.go

@@ -17,13 +17,24 @@ limitations under the License.
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 
+	uuid "github.com/satori/go.uuid"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
 	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+	storageSvcClient "github.com/fission/fission/storagesvc/client"
 )
 
 func fatal(msg string) {
@@ -86,3 +97,169 @@ func httpRequest(method, url, body string, headers []string) *http.Response {
 
 	return resp
 }
+
+func fileSize(filePath string) int64 {
+	info, err := os.Stat(filePath)
+	checkErr(err, fmt.Sprintf("stat %v", filePath))
+	return info.Size()
+}
+
+// upload a file and return a fission.Archive
+func createArchive(client *client.Client, fileName string) *fission.Archive {
+	var archive fission.Archive
+
+	// fetch archive from arbitrary url if fileName is a url
+	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
+		fileName = downloadToTempFile(fileName)
+	}
+
+	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
+		contents := getContents(fileName)
+		archive.Type = fission.ArchiveTypeLiteral
+		archive.Literal = contents
+	} else {
+		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
+		ssClient := storageSvcClient.MakeClient(u)
+
+		// TODO add a progress bar
+		id, err := ssClient.Upload(fileName, nil)
+		checkErr(err, fmt.Sprintf("upload file %v", fileName))
+
+		archiveUrl := ssClient.GetUrl(id)
+
+		archive.Type = fission.ArchiveTypeUrl
+		archive.URL = archiveUrl
+
+		f, err := os.Open(fileName)
+		if err != nil {
+			checkErr(err, fmt.Sprintf("find file %v", fileName))
+		}
+		defer f.Close()
+
+		h := sha256.New()
+		if _, err := io.Copy(h, f); err != nil {
+			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
+		}
+
+		archive.Checksum = fission.Checksum{
+			Type: fission.ChecksumTypeSHA256,
+			Sum:  hex.EncodeToString(h.Sum(nil)),
+		}
+	}
+	return &archive
+}
+
+func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+	pkgSpec := fission.PackageSpec{
+		Environment: fission.EnvironmentReference{
+			Namespace: metav1.NamespaceDefault,
+			Name:      envName,
+		},
+	}
+	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
+
+	if len(deployArchiveName) > 0 {
+		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
+	}
+	if len(srcArchiveName) > 0 {
+		pkgSpec.Source = *createArchive(client, srcArchiveName)
+		// set pending status to package
+		pkgStatus = fission.BuildStatusPending
+	}
+
+	if len(buildcmd) > 0 {
+		pkgSpec.BuildCommand = buildcmd
+	}
+
+	pkgName := strings.ToLower(uuid.NewV4().String())
+	pkg := &crd.Package{
+		Metadata: metav1.ObjectMeta{
+			Name:      pkgName,
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: pkgSpec,
+		Status: fission.PackageStatus{
+			BuildStatus: pkgStatus,
+		},
+	}
+	pkgMetadata, err := client.PackageCreate(pkg)
+	checkErr(err, "create package")
+	return pkgMetadata
+}
+
+func getContents(filePath string) []byte {
+	var code []byte
+	var err error
+
+	code, err = ioutil.ReadFile(filePath)
+	checkErr(err, fmt.Sprintf("read %v", filePath))
+	return code
+}
+
+func getTempDir() (string, error) {
+	tmpDir := uuid.NewV4().String()
+	tmpPath := filepath.Join(os.TempDir(), tmpDir)
+	err := os.Mkdir(tmpPath, 0744)
+	return tmpPath, err
+}
+
+func writeArchiveToFile(fileName string, reader io.Reader) error {
+	tmpDir, err := getTempDir()
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(tmpDir, fileName+".tmp")
+	w, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, reader)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(path, 0644)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(path, fileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// downloadToTempFile fetches archive file from arbitrary url
+// and write it to temp file for further usage
+func downloadToTempFile(fileUrl string) string {
+	reader, err := downloadURL(fileUrl)
+	defer reader.Close()
+	checkErr(err, fmt.Sprintf("download from url: %v", fileUrl))
+
+	tmpDir, err := getTempDir()
+	checkErr(err, "create temp directory")
+
+	tmpFilename := uuid.NewV4().String()
+	destination := filepath.Join(tmpDir, tmpFilename)
+	err = os.Mkdir(tmpDir, 0744)
+	checkErr(err, "create temp directory")
+
+	err = writeArchiveToFile(destination, reader)
+	checkErr(err, "write archive to file")
+
+	return destination
+}
+
+// downloadURL downloads file from given url
+func downloadURL(fileUrl string) (io.ReadCloser, error) {
+	resp, err := http.Get(fileUrl)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode)
+	}
+	return resp.Body, nil
+}

fission/function.go

@@ -18,139 +18,24 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
 	"text/tabwriter"
 	"time"
 
-	"github.com/dchest/uniuri"
 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
-	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/crd"
 	"github.com/fission/fission/fission/logdb"
-	storageSvcClient "github.com/fission/fission/storagesvc/client"
 )
 
-func fileSize(filePath string) int64 {
-	info, err := os.Stat(filePath)
-	checkErr(err, fmt.Sprintf("stat %v", filePath))
-	return info.Size()
-}
-
-// upload a file and return a fission.Archive
-func createArchive(client *client.Client, fileName string) *fission.Archive {
-	var archive fission.Archive
-	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
-		contents := getContents(fileName)
-		archive.Type = fission.ArchiveTypeLiteral
-		archive.Literal = contents
-	} else {
-		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
-		ssClient := storageSvcClient.MakeClient(u)
-
-		// TODO add a progress bar
-		id, err := ssClient.Upload(fileName, nil)
-		checkErr(err, fmt.Sprintf("upload file %v", fileName))
-
-		archiveUrl := ssClient.GetUrl(id)
-
-		archive.Type = fission.ArchiveTypeUrl
-		archive.URL = archiveUrl
-
-		f, err := os.Open(fileName)
-		if err != nil {
-			checkErr(err, fmt.Sprintf("find file %v", fileName))
-		}
-		defer f.Close()
-
-		h := sha256.New()
-		if _, err := io.Copy(h, f); err != nil {
-			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
-		}
-
-		archive.Checksum = fission.Checksum{
-			Type: fission.ChecksumTypeSHA256,
-			Sum:  hex.EncodeToString(h.Sum(nil)),
-		}
-	}
-	return &archive
-}
-
-func createPackage(client *client.Client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
-	pkgSpec := fission.PackageSpec{
-		Environment: fission.EnvironmentReference{
-			Namespace: metav1.NamespaceDefault,
-			Name:      envName,
-		},
-	}
-
-	var pkgStatus fission.BuildStatus
-
-	if len(deployArchiveName) > 0 {
-		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
-	}
-	if len(srcArchiveName) > 0 {
-		pkgSpec.Source = *createArchive(client, srcArchiveName)
-	}
-
-	// start a build only when a package has no deploy archive
-	if len(srcArchiveName) > 0 && len(deployArchiveName) == 0 {
-		pkgStatus = fission.BuildStatusPending
-	} else {
-		pkgStatus = fission.BuildStatusNone
-	}
-
-	if len(buildcmd) > 0 {
-		pkgSpec.BuildCommand = buildcmd
-	}
-
-	fnList, err := json.Marshal([]string{fnName})
-	checkErr(err, "encode json")
-
-	annotation := map[string]string{
-		"createdForFunction": fnName,
-		"usedByFunctions":    string(fnList),
-	}
-
-	pkgName := strings.ToLower(fmt.Sprintf("%v-%v", fnName, uniuri.NewLen(6)))
-	pkg := &crd.Package{
-		Metadata: metav1.ObjectMeta{
-			Name:        pkgName,
-			Namespace:   metav1.NamespaceDefault,
-			Annotations: annotation,
-		},
-		Spec: pkgSpec,
-		Status: fission.PackageStatus{
-			BuildStatus: pkgStatus,
-		},
-	}
-	pkgMetadata, err := client.PackageCreate(pkg)
-	checkErr(err, "create package")
-	return pkgMetadata
-}
-
-func getContents(filePath string) []byte {
-	var code []byte
-	var err error
-
-	code, err = ioutil.ReadFile(filePath)
-	checkErr(err, fmt.Sprintf("read %v", filePath))
-	return code
-}
-
 func printPodLogs(c *cli.Context) error {
 	fnName := c.String("name")
 	if len(fnName) == 0 {
@@ -191,25 +76,51 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --name argument.")
 	}
 
-	envName := c.String("env")
-	if len(envName) == 0 {
-		fatal("Need --env argument.")
+	fnList, err := client.FunctionList()
+	checkErr(err, "get function list")
+	// check function existence before creating package
+	for _, fn := range fnList {
+		if fn.Metadata.Name == fnName {
+			fatal("A function with the same name already exists.")
+		}
 	}
-
-	srcArchiveName := c.String("src")
-	deployArchiveName := c.String("code")
-	if len(deployArchiveName) == 0 {
-		deployArchiveName = c.String("deploy")
-	}
-
-	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
-		fatal("Need --code or --deploy to specify deployment archive, or use --src to specify source archive.")
-	}
-
 	entrypoint := c.String("entrypoint")
-	buildcmd := c.String("buildcmd")
+	pkgName := c.String("pkg")
 
-	pkgMetadata := createPackage(client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd)
+	var pkgMetadata *metav1.ObjectMeta
+	var envName string
+
+	if len(pkgName) > 0 {
+		// use existing package
+		pkg, err := client.PackageGet(&metav1.ObjectMeta{
+			Namespace: metav1.NamespaceDefault,
+			Name:      pkgName,
+		})
+		checkErr(err, fmt.Sprintf("read package '%v'", pkgName))
+		pkgMetadata = &pkg.Metadata
+		envName = pkg.Spec.Environment.Name
+	} else {
+		// need to specify environment for creating new package
+		envName = c.String("env")
+		if len(envName) == 0 {
+			fatal("Need --env argument.")
+		}
+
+		srcArchiveName := c.String("src")
+		deployArchiveName := c.String("code")
+		if len(deployArchiveName) == 0 {
+			deployArchiveName = c.String("deploy")
+		}
+		// fatal when both src & deploy archive are empty
+		if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+			fatal("Need --deploy or --src argument.")
+		}
+
+		buildcmd := c.String("buildcmd")
+
+		// create new package
+		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	}
 
 	function := &crd.Function{
 		Metadata: metav1.ObjectMeta{
@@ -232,7 +143,7 @@ func fnCreate(c *cli.Context) error {
 		},
 	}
 
-	_, err := client.FunctionCreate(function)
+	_, err = client.FunctionCreate(function)
 	checkErr(err, "create function")
 
 	fmt.Printf("function '%v' created\n", fnName)
@@ -319,15 +230,19 @@ func fnGetMeta(c *cli.Context) error {
 func fnUpdate(c *cli.Context) error {
 	client := getClient(c.GlobalString("server"))
 
+	if len(c.String("package")) > 0 {
+		fatal("--package is deprecated, please use --deploy instead.")
+	}
+
+	if len(c.String("srcpkg")) > 0 {
+		fatal("--srcpkg is deprecated, please use --src instead.")
+	}
+
 	fnName := c.String("name")
 	if len(fnName) == 0 {
 		fatal("Need name of function, use --name")
 	}
 
-	if len(c.String("package")) > 0 {
-		fatal("--package is deprecated, please use --deploy instead.")
-	}
-
 	function, err := client.FunctionGet(&metav1.ObjectMeta{
 		Name:      fnName,
 		Namespace: metav1.NamespaceDefault,
@@ -340,43 +255,65 @@ func fnUpdate(c *cli.Context) error {
 		deployArchiveName = c.String("deploy")
 	}
 	srcArchiveName := c.String("src")
+	pkgName := c.String("pkg")
+	entrypoint := c.String("entrypoint")
+	buildcmd := c.String("buildcmd")
+	force := c.Bool("force")
 
-	if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 {
-		fatal("Need --env or --code or --package or --deploy argument.")
+	if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 && len(pkgName) == 0 &&
+		len(entrypoint) == 0 && len(buildcmd) == 0 {
+		fatal("Need --env or --deploy or --src or --pkg or --entrypoint or --buildcmd argument.")
 	}
 
 	if len(envName) > 0 {
 		function.Spec.Environment.Name = envName
 	}
 
-	entrypoint := c.String("entrypoint")
 	if len(entrypoint) > 0 {
 		function.Spec.Package.FunctionName = entrypoint
 	}
 
-	pkg, err := client.PackageGet(&metav1.ObjectMeta{
-		Name:      function.Spec.Package.PackageRef.Name,
-		Namespace: function.Spec.Package.PackageRef.Namespace,
-	})
-	checkErr(err, fmt.Sprintf("read package '%v'", function.Spec.Package.PackageRef.Name))
-
-	buildcmd := c.String("buildcmd")
-	if len(buildcmd) == 0 {
-		// use previous build command if not specified.
-		buildcmd = pkg.Spec.BuildCommand
+	if len(pkgName) == 0 {
+		pkgName = function.Spec.Package.PackageRef.Name
 	}
 
-	if len(deployArchiveName) > 0 || len(srcArchiveName) > 0 {
-		// create a new package for function
-		pkgMetadata := createPackage(client, function.Metadata.Name,
-			function.Spec.Environment.Name, srcArchiveName, deployArchiveName, buildcmd)
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	checkErr(err, fmt.Sprintf("read package '%v'", pkgName))
 
-		// update function spec with resource version
-		function.Spec.Package.PackageRef = fission.PackageRef{
-			Namespace:       pkgMetadata.Namespace,
-			Name:            pkgMetadata.Name,
-			ResourceVersion: pkgMetadata.ResourceVersion,
+	pkgMetadata := &pkg.Metadata
+
+	if len(deployArchiveName) != 0 || len(srcArchiveName) != 0 || len(buildcmd) != 0 || len(envName) != 0 {
+		fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name)
+		checkErr(err, "get function list")
+
+		if !force && len(fnList) > 1 {
+			fatal("Package is used by multiple functions, use --force to force update")
 		}
+
+		pkgMetadata = updatePackage(client, pkg, envName, srcArchiveName, deployArchiveName, buildcmd)
+		checkErr(err, fmt.Sprintf("update package '%v'", pkgName))
+
+		fmt.Printf("package '%v' updated\n", pkgMetadata.GetName())
+
+		// update resource version of package reference of functions that shared the same package
+		for _, fn := range fnList {
+			// ignore the update for current function here, it will be updated later.
+			if fn.Metadata.Name != fnName {
+				fn.Spec.Package.PackageRef.ResourceVersion = pkgMetadata.ResourceVersion
+				_, err := client.FunctionUpdate(&fn)
+				checkErr(err, "update function")
+			}
+		}
+	}
+
+	// update function spec with new package metadata
+	function.Spec.Package.PackageRef = fission.PackageRef{
+		Namespace:       pkgMetadata.Namespace,
+		Name:            pkgMetadata.Name,
+		ResourceVersion: pkgMetadata.ResourceVersion,
 	}
 
 	_, err = client.FunctionUpdate(function)

fission/main.go

@@ -43,6 +43,7 @@ func main() {
 	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "(Deprecated) local path or URL for binary package"}
 	fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
 	fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"}
+	fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored)"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
 	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
@@ -52,12 +53,13 @@ func main() {
 	fnEntryPointFlag := cli.StringFlag{Name: "entrypoint", Usage: "entry point for environment v2 to load with"}
 	fnBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "build command for builder to run with"}
 	fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"}
+	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
 
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
-		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag}, Action: fnUpdate},
+		{Name: "update", Usage: "Update function", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete},
 		{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
 		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag, fnLogCountFlag}, Action: fnLogs},
@@ -131,6 +133,24 @@ func main() {
 		{Name: "list", Usage: "List all watches", Flags: []cli.Flag{}, Action: wList},
 	}
 
+	// packages
+	pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"}
+	pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"}
+	pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"}
+	pkgSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
+	pkgDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
+	pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"}
+	pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"}
+	pkgSubCommands := []cli.Command{
+		{Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: pkgCreate},
+		{Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: pkgUpdate},
+		{Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgSourceGet},
+		{Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgDeployGet},
+		{Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag}, Action: pkgInfo},
+		{Name: "list", Usage: "List all packages", Flags: []cli.Flag{}, Action: pkgList},
+		{Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete},
+	}
+
 	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
 	upgradeSubCommands := []cli.Command{
 		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
@@ -151,6 +171,7 @@ func main() {
 		{Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands},
 		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
 		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
+		{Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands},
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands},
 	}

fission/package.go

@@ -0,0 +1,325 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+)
+
+func getFunctionsByPackage(client *client.Client, pkgName string) ([]crd.Function, error) {
+	fnList, err := client.FunctionList()
+	if err != nil {
+		return nil, err
+	}
+	fns := []crd.Function{}
+	for _, fn := range fnList {
+		if fn.Spec.Package.PackageRef.Name == pkgName {
+			fns = append(fns, fn)
+		}
+	}
+	return fns, nil
+}
+
+// downloadStoragesvcURL downloads and return archive content with given storage service url
+func downloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser {
+	u, err := url.ParseRequestURI(fileUrl)
+	if err != nil {
+		return nil
+	}
+	// replace in-cluster storage service host with controller server url
+	fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" + u.RequestURI()
+	reader, err := downloadURL(fileDownloadUrl)
+	checkErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl))
+	return reader
+}
+
+func pkgCreate(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	envName := c.String("env")
+	if len(envName) == 0 {
+		fatal("Need --env argument.")
+	}
+
+	srcArchiveName := c.String("src")
+	deployArchiveName := c.String("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+		fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
+	}
+
+	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	fmt.Printf("Package '%v' created\n", meta.GetName())
+
+	return nil
+}
+
+func pkgUpdate(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need --name argument.")
+	}
+
+	force := c.Bool("f")
+	envName := c.String("env")
+	srcArchiveName := c.String("src")
+	deployArchiveName := c.String("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 &&
+		len(envName) == 0 && len(buildcmd) == 0 {
+		fatal("Need --env or --src or --deploy or --buildcmd argument.")
+	}
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	checkErr(err, "get package")
+
+	fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name)
+	checkErr(err, "get function list")
+
+	if !force && len(fnList) > 1 {
+		fatal("Package is used by multiple functions, use --force to force update")
+	}
+
+	newPkgMeta := updatePackage(client, pkg,
+		envName, srcArchiveName, deployArchiveName, buildcmd)
+
+	// update resource version of package reference of functions that shared the same package
+	for _, fn := range fnList {
+		fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion
+		_, err := client.FunctionUpdate(&fn)
+		checkErr(err, "update function")
+	}
+
+	fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName())
+
+	return nil
+}
+
+func updatePackage(client *client.Client, pkg *crd.Package, envName,
+	srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+
+	var srcArchiveMetadata, deployArchiveMetadata *fission.Archive
+	needToBuild := false
+
+	if len(envName) > 0 {
+		pkg.Spec.Environment.Name = envName
+		needToBuild = true
+	}
+
+	if len(buildcmd) > 0 {
+		pkg.Spec.BuildCommand = buildcmd
+		needToBuild = true
+	}
+
+	if len(srcArchiveName) > 0 {
+		srcArchiveMetadata = createArchive(client, srcArchiveName)
+		pkg.Spec.Source = *srcArchiveMetadata
+		needToBuild = true
+	}
+
+	if len(deployArchiveName) > 0 {
+		deployArchiveMetadata = createArchive(client, deployArchiveName)
+		pkg.Spec.Deployment = *deployArchiveMetadata
+	}
+
+	// Set package as pending status only when there is no
+	// deploy archive.
+	if needToBuild && len(pkg.Spec.Deployment.Type) == 0 {
+		// change into pending state to trigger package build
+		pkg.Status = fission.PackageStatus{
+			BuildStatus: fission.BuildStatusPending,
+		}
+	}
+
+	newPkgMeta, err := client.PackageUpdate(pkg)
+	checkErr(err, "update package")
+
+	return newPkgMeta
+}
+
+func pkgSourceGet(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Source.Type == fission.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Source.Literal)
+	} else if pkg.Spec.Source.Type == fission.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Source.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgDeployGet(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Deployment.Type == fission.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Deployment.Literal)
+	} else if pkg.Spec.Deployment.Type == fission.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Deployment.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgInfo(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus)
+	fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog)
+	w.Flush()
+
+	return nil
+}
+
+func pkgList(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgList, err := client.PackageList()
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV")
+	for _, pkg := range pkgList {
+		fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name,
+			pkg.Status.BuildStatus, pkg.Spec.Environment.Name)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func pkgDelete(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fmt.Println("Need --name argument.")
+		return nil
+	}
+
+	force := c.Bool("f")
+
+	_, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	checkErr(err, "find package")
+
+	fnList, err := getFunctionsByPackage(client, pkgName)
+
+	if !force && len(fnList) > 0 {
+		fatal("Package is used by at least one function, use -f to force delete")
+	}
+
+	err = client.PackageDelete(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Package '%v' deleted\n", pkgName)
+
+	return nil
+}

test/test_utils.sh

@@ -237,10 +237,13 @@ dump_fission_crd() {
 }
 
 dump_fission_crds() {
-    dump_fission_crd function.fission.io    
-    dump_fission_crd package.fission.io    
-    dump_fission_crd httptrigger.fission.io    
-    dump_fission_crd environment.fission.io    
+    dump_fission_crd environments.fission.io
+    dump_fission_crd functions.fission.io
+    dump_fission_crd httptriggers.fission.io
+    dump_fission_crd kuberneteswatchtriggers.fission.io
+    dump_fission_crd messagequeuetriggers.fission.io
+    dump_fission_crd packages.fission.io
+    dump_fission_crd timetriggers.fission.io
 }
 
 dump_env_pods() {

test/tests/test_package_command.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Use package command to create two packages one with source 
+# archive and the other with deploy archive. Also, create a 
+# function to test the packages created by package command are 
+# able to work.
+
+ROOT=$(dirname $0)/../..
+PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python3-env:test
+PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python3-env-builder:test
+
+fn=python-srcbuild-$(date +%s)
+
+waitBuild() {
+    echo "Waiting for builder manager to finish the build"
+    
+    while true; do
+      kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
+      if [[ $? -eq 0 ]]; then
+          break
+      fi
+    done
+}
+export -f waitBuild
+
+checkFunctionResponse() {
+    echo "Doing an HTTP GET on the function's route"
+    response=$(curl http://$FISSION_ROUTER/$1)
+
+    echo "Checking for valid response"
+    echo $response
+    echo $response | grep -i "$2"
+}
+
+echo "Pre-test cleanup"
+fission env delete --name python || true
+
+echo "Creating python env"
+fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
+trap "fission env delete --name python" EXIT
+
+echo "Waiting for env builder to catch up"
+sleep 30
+
+echo "Creating pacakage with source archive"
+zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
+pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+
+# wait for build to finish at most 60s
+timeout 60s bash -c "waitBuild $pkgName"
+
+echo "Creating function " $fn
+fission fn create --name $fn --pkg $pkgName --entrypoint "user.main"
+trap "fission fn delete --name $fn" EXIT
+
+echo "Creating route"
+fission route create --function $fn --url /$fn --method GET
+
+echo "Waiting for router to catch up"
+sleep 3
+  
+checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}'
+
+echo "Creating package with deploy archive"
+mkdir testDir
+touch testDir/__init__.py
+printf 'def main():\n    return "Hello, world!"' > testDir/hello.py
+zip -jr demo-deploy-pkg.zip testDir/
+pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \')
+
+echo "Updating function " $fn
+fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main"
+trap "fission fn delete --name $fn" EXIT
+
+echo "Waiting for router to update cache"
+sleep 3
+
+checkFunctionResponse $fn 'Hello, world!'
+
+# crappy cleanup, improve this later
+kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
+
+echo "All done."