diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 80fecd01..6671b892 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -78,12 +78,15 @@ func downloadUrl(url string, localPath string) error { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + w, err := os.Create(localPath) if err != nil { return err } - - err = ioutil.WriteFile(localPath, body, 0600) + _, err = io.Copy(w, resp.Body) + if err != nil { + return err + } + err = os.Chmod(localPath, 0600) if err != nil { return err } diff --git a/fission/common.go b/fission/common.go index ab705777..b67ae12b 100644 --- a/fission/common.go +++ b/fission/common.go @@ -17,13 +17,24 @@ limitations under the License. package main import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" + "io/ioutil" "net/http" "os" + "path/filepath" "strings" + uuid "github.com/satori/go.uuid" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission" "github.com/fission/fission/controller/client" + "github.com/fission/fission/crd" + storageSvcClient "github.com/fission/fission/storagesvc/client" ) func fatal(msg string) { @@ -86,3 +97,169 @@ func httpRequest(method, url, body string, headers []string) *http.Response { return resp } + +func fileSize(filePath string) int64 { + info, err := os.Stat(filePath) + checkErr(err, fmt.Sprintf("stat %v", filePath)) + return info.Size() +} + +// upload a file and return a fission.Archive +func createArchive(client *client.Client, fileName string) *fission.Archive { + var archive fission.Archive + + // fetch archive from arbitrary url if fileName is a url + if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") { + fileName = downloadToTempFile(fileName) + } + + if fileSize(fileName) < fission.ArchiveLiteralSizeLimit { + contents := getContents(fileName) + archive.Type = fission.ArchiveTypeLiteral + archive.Literal = contents + } else { + u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" + ssClient := storageSvcClient.MakeClient(u) + + // TODO add a progress bar + id, err := ssClient.Upload(fileName, nil) + checkErr(err, fmt.Sprintf("upload file %v", fileName)) + + archiveUrl := ssClient.GetUrl(id) + + archive.Type = fission.ArchiveTypeUrl + archive.URL = archiveUrl + + f, err := os.Open(fileName) + if err != nil { + checkErr(err, fmt.Sprintf("find file %v", fileName)) + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) + } + + archive.Checksum = fission.Checksum{ + Type: fission.ChecksumTypeSHA256, + Sum: hex.EncodeToString(h.Sum(nil)), + } + } + return &archive +} + +func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta { + pkgSpec := fission.PackageSpec{ + Environment: fission.EnvironmentReference{ + Namespace: metav1.NamespaceDefault, + Name: envName, + }, + } + var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded + + if len(deployArchiveName) > 0 { + pkgSpec.Deployment = *createArchive(client, deployArchiveName) + } + if len(srcArchiveName) > 0 { + pkgSpec.Source = *createArchive(client, srcArchiveName) + // set pending status to package + pkgStatus = fission.BuildStatusPending + } + + if len(buildcmd) > 0 { + pkgSpec.BuildCommand = buildcmd + } + + pkgName := strings.ToLower(uuid.NewV4().String()) + pkg := &crd.Package{ + Metadata: metav1.ObjectMeta{ + Name: pkgName, + Namespace: metav1.NamespaceDefault, + }, + Spec: pkgSpec, + Status: fission.PackageStatus{ + BuildStatus: pkgStatus, + }, + } + pkgMetadata, err := client.PackageCreate(pkg) + checkErr(err, "create package") + return pkgMetadata +} + +func getContents(filePath string) []byte { + var code []byte + var err error + + code, err = ioutil.ReadFile(filePath) + checkErr(err, fmt.Sprintf("read %v", filePath)) + return code +} + +func getTempDir() (string, error) { + tmpDir := uuid.NewV4().String() + tmpPath := filepath.Join(os.TempDir(), tmpDir) + err := os.Mkdir(tmpPath, 0744) + return tmpPath, err +} + +func writeArchiveToFile(fileName string, reader io.Reader) error { + tmpDir, err := getTempDir() + if err != nil { + return err + } + + path := filepath.Join(tmpDir, fileName+".tmp") + w, err := os.Create(path) + if err != nil { + return err + } + _, err = io.Copy(w, reader) + if err != nil { + return err + } + err = os.Chmod(path, 0644) + if err != nil { + return err + } + + err = os.Rename(path, fileName) + if err != nil { + return err + } + + return nil +} + +// downloadToTempFile fetches archive file from arbitrary url +// and write it to temp file for further usage +func downloadToTempFile(fileUrl string) string { + reader, err := downloadURL(fileUrl) + defer reader.Close() + checkErr(err, fmt.Sprintf("download from url: %v", fileUrl)) + + tmpDir, err := getTempDir() + checkErr(err, "create temp directory") + + tmpFilename := uuid.NewV4().String() + destination := filepath.Join(tmpDir, tmpFilename) + err = os.Mkdir(tmpDir, 0744) + checkErr(err, "create temp directory") + + err = writeArchiveToFile(destination, reader) + checkErr(err, "write archive to file") + + return destination +} + +// downloadURL downloads file from given url +func downloadURL(fileUrl string) (io.ReadCloser, error) { + resp, err := http.Get(fileUrl) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) + } + return resp.Body, nil +} diff --git a/fission/function.go b/fission/function.go index 25f602fd..bfbf72e5 100644 --- a/fission/function.go +++ b/fission/function.go @@ -18,139 +18,24 @@ package main import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net/http" "net/url" "os" - "strings" "text/tabwriter" "time" - "github.com/dchest/uniuri" "github.com/satori/go.uuid" "github.com/urfave/cli" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" - "github.com/fission/fission/controller/client" "github.com/fission/fission/crd" "github.com/fission/fission/fission/logdb" - storageSvcClient "github.com/fission/fission/storagesvc/client" ) -func fileSize(filePath string) int64 { - info, err := os.Stat(filePath) - checkErr(err, fmt.Sprintf("stat %v", filePath)) - return info.Size() -} - -// upload a file and return a fission.Archive -func createArchive(client *client.Client, fileName string) *fission.Archive { - var archive fission.Archive - if fileSize(fileName) < fission.ArchiveLiteralSizeLimit { - contents := getContents(fileName) - archive.Type = fission.ArchiveTypeLiteral - archive.Literal = contents - } else { - u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" - ssClient := storageSvcClient.MakeClient(u) - - // TODO add a progress bar - id, err := ssClient.Upload(fileName, nil) - checkErr(err, fmt.Sprintf("upload file %v", fileName)) - - archiveUrl := ssClient.GetUrl(id) - - archive.Type = fission.ArchiveTypeUrl - archive.URL = archiveUrl - - f, err := os.Open(fileName) - if err != nil { - checkErr(err, fmt.Sprintf("find file %v", fileName)) - } - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) - } - - archive.Checksum = fission.Checksum{ - Type: fission.ChecksumTypeSHA256, - Sum: hex.EncodeToString(h.Sum(nil)), - } - } - return &archive -} - -func createPackage(client *client.Client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta { - pkgSpec := fission.PackageSpec{ - Environment: fission.EnvironmentReference{ - Namespace: metav1.NamespaceDefault, - Name: envName, - }, - } - - var pkgStatus fission.BuildStatus - - if len(deployArchiveName) > 0 { - pkgSpec.Deployment = *createArchive(client, deployArchiveName) - } - if len(srcArchiveName) > 0 { - pkgSpec.Source = *createArchive(client, srcArchiveName) - } - - // start a build only when a package has no deploy archive - if len(srcArchiveName) > 0 && len(deployArchiveName) == 0 { - pkgStatus = fission.BuildStatusPending - } else { - pkgStatus = fission.BuildStatusNone - } - - if len(buildcmd) > 0 { - pkgSpec.BuildCommand = buildcmd - } - - fnList, err := json.Marshal([]string{fnName}) - checkErr(err, "encode json") - - annotation := map[string]string{ - "createdForFunction": fnName, - "usedByFunctions": string(fnList), - } - - pkgName := strings.ToLower(fmt.Sprintf("%v-%v", fnName, uniuri.NewLen(6))) - pkg := &crd.Package{ - Metadata: metav1.ObjectMeta{ - Name: pkgName, - Namespace: metav1.NamespaceDefault, - Annotations: annotation, - }, - Spec: pkgSpec, - Status: fission.PackageStatus{ - BuildStatus: pkgStatus, - }, - } - pkgMetadata, err := client.PackageCreate(pkg) - checkErr(err, "create package") - return pkgMetadata -} - -func getContents(filePath string) []byte { - var code []byte - var err error - - code, err = ioutil.ReadFile(filePath) - checkErr(err, fmt.Sprintf("read %v", filePath)) - return code -} - func printPodLogs(c *cli.Context) error { fnName := c.String("name") if len(fnName) == 0 { @@ -191,25 +76,51 @@ func fnCreate(c *cli.Context) error { fatal("Need --name argument.") } - envName := c.String("env") - if len(envName) == 0 { - fatal("Need --env argument.") + fnList, err := client.FunctionList() + checkErr(err, "get function list") + // check function existence before creating package + for _, fn := range fnList { + if fn.Metadata.Name == fnName { + fatal("A function with the same name already exists.") + } } - - srcArchiveName := c.String("src") - deployArchiveName := c.String("code") - if len(deployArchiveName) == 0 { - deployArchiveName = c.String("deploy") - } - - if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 { - fatal("Need --code or --deploy to specify deployment archive, or use --src to specify source archive.") - } - entrypoint := c.String("entrypoint") - buildcmd := c.String("buildcmd") + pkgName := c.String("pkg") - pkgMetadata := createPackage(client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd) + var pkgMetadata *metav1.ObjectMeta + var envName string + + if len(pkgName) > 0 { + // use existing package + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + checkErr(err, fmt.Sprintf("read package '%v'", pkgName)) + pkgMetadata = &pkg.Metadata + envName = pkg.Spec.Environment.Name + } else { + // need to specify environment for creating new package + envName = c.String("env") + if len(envName) == 0 { + fatal("Need --env argument.") + } + + srcArchiveName := c.String("src") + deployArchiveName := c.String("code") + if len(deployArchiveName) == 0 { + deployArchiveName = c.String("deploy") + } + // fatal when both src & deploy archive are empty + if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 { + fatal("Need --deploy or --src argument.") + } + + buildcmd := c.String("buildcmd") + + // create new package + pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd) + } function := &crd.Function{ Metadata: metav1.ObjectMeta{ @@ -232,7 +143,7 @@ func fnCreate(c *cli.Context) error { }, } - _, err := client.FunctionCreate(function) + _, err = client.FunctionCreate(function) checkErr(err, "create function") fmt.Printf("function '%v' created\n", fnName) @@ -319,15 +230,19 @@ func fnGetMeta(c *cli.Context) error { func fnUpdate(c *cli.Context) error { client := getClient(c.GlobalString("server")) + if len(c.String("package")) > 0 { + fatal("--package is deprecated, please use --deploy instead.") + } + + if len(c.String("srcpkg")) > 0 { + fatal("--srcpkg is deprecated, please use --src instead.") + } + fnName := c.String("name") if len(fnName) == 0 { fatal("Need name of function, use --name") } - if len(c.String("package")) > 0 { - fatal("--package is deprecated, please use --deploy instead.") - } - function, err := client.FunctionGet(&metav1.ObjectMeta{ Name: fnName, Namespace: metav1.NamespaceDefault, @@ -340,43 +255,65 @@ func fnUpdate(c *cli.Context) error { deployArchiveName = c.String("deploy") } srcArchiveName := c.String("src") + pkgName := c.String("pkg") + entrypoint := c.String("entrypoint") + buildcmd := c.String("buildcmd") + force := c.Bool("force") - if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 { - fatal("Need --env or --code or --package or --deploy argument.") + if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 && len(pkgName) == 0 && + len(entrypoint) == 0 && len(buildcmd) == 0 { + fatal("Need --env or --deploy or --src or --pkg or --entrypoint or --buildcmd argument.") } if len(envName) > 0 { function.Spec.Environment.Name = envName } - entrypoint := c.String("entrypoint") if len(entrypoint) > 0 { function.Spec.Package.FunctionName = entrypoint } - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Name: function.Spec.Package.PackageRef.Name, - Namespace: function.Spec.Package.PackageRef.Namespace, - }) - checkErr(err, fmt.Sprintf("read package '%v'", function.Spec.Package.PackageRef.Name)) - - buildcmd := c.String("buildcmd") - if len(buildcmd) == 0 { - // use previous build command if not specified. - buildcmd = pkg.Spec.BuildCommand + if len(pkgName) == 0 { + pkgName = function.Spec.Package.PackageRef.Name } - if len(deployArchiveName) > 0 || len(srcArchiveName) > 0 { - // create a new package for function - pkgMetadata := createPackage(client, function.Metadata.Name, - function.Spec.Environment.Name, srcArchiveName, deployArchiveName, buildcmd) + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + checkErr(err, fmt.Sprintf("read package '%v'", pkgName)) - // update function spec with resource version - function.Spec.Package.PackageRef = fission.PackageRef{ - Namespace: pkgMetadata.Namespace, - Name: pkgMetadata.Name, - ResourceVersion: pkgMetadata.ResourceVersion, + pkgMetadata := &pkg.Metadata + + if len(deployArchiveName) != 0 || len(srcArchiveName) != 0 || len(buildcmd) != 0 || len(envName) != 0 { + fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name) + checkErr(err, "get function list") + + if !force && len(fnList) > 1 { + fatal("Package is used by multiple functions, use --force to force update") } + + pkgMetadata = updatePackage(client, pkg, envName, srcArchiveName, deployArchiveName, buildcmd) + checkErr(err, fmt.Sprintf("update package '%v'", pkgName)) + + fmt.Printf("package '%v' updated\n", pkgMetadata.GetName()) + + // update resource version of package reference of functions that shared the same package + for _, fn := range fnList { + // ignore the update for current function here, it will be updated later. + if fn.Metadata.Name != fnName { + fn.Spec.Package.PackageRef.ResourceVersion = pkgMetadata.ResourceVersion + _, err := client.FunctionUpdate(&fn) + checkErr(err, "update function") + } + } + } + + // update function spec with new package metadata + function.Spec.Package.PackageRef = fission.PackageRef{ + Namespace: pkgMetadata.Namespace, + Name: pkgMetadata.Name, + ResourceVersion: pkgMetadata.ResourceVersion, } _, err = client.FunctionUpdate(function) diff --git a/fission/main.go b/fission/main.go index ad4a7778..8983ad4e 100644 --- a/fission/main.go +++ b/fission/main.go @@ -43,6 +43,7 @@ func main() { fnPackageFlag := cli.StringFlag{Name: "package", Usage: "(Deprecated) local path or URL for binary package"} fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"} fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"} + fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored)"} fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"} fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"} fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"} @@ -52,12 +53,13 @@ func main() { fnEntryPointFlag := cli.StringFlag{Name: "entrypoint", Usage: "entry point for environment v2 to load with"} fnBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "build command for builder to run with"} fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"} + fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"} fnSubcommands := []cli.Command{ - {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, htUrlFlag, htMethodFlag}, Action: fnCreate}, + {Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag}, Action: fnCreate}, {Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet}, {Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta}, - {Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag}, Action: fnUpdate}, + {Name: "update", Usage: "Update function", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag}, Action: fnUpdate}, {Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete}, {Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList}, {Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag, fnLogCountFlag}, Action: fnLogs}, @@ -131,6 +133,24 @@ func main() { {Name: "list", Usage: "List all watches", Flags: []cli.Flag{}, Action: wList}, } + // packages + pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"} + pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"} + pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"} + pkgSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"} + pkgDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"} + pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"} + pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"} + pkgSubCommands := []cli.Command{ + {Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: pkgCreate}, + {Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: pkgUpdate}, + {Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgSourceGet}, + {Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgDeployGet}, + {Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag}, Action: pkgInfo}, + {Name: "list", Usage: "List all packages", Flags: []cli.Flag{}, Action: pkgList}, + {Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete}, + } + upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"} upgradeSubCommands := []cli.Command{ {Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState}, @@ -151,6 +171,7 @@ func main() { {Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands}, {Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands}, {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, + {Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands}, {Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands}, {Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands}, } diff --git a/fission/package.go b/fission/package.go new file mode 100644 index 00000000..cd11fe00 --- /dev/null +++ b/fission/package.go @@ -0,0 +1,325 @@ +/* +Copyright 2017 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os" + "strings" + "text/tabwriter" + + "github.com/urfave/cli" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission" + "github.com/fission/fission/controller/client" + "github.com/fission/fission/crd" +) + +func getFunctionsByPackage(client *client.Client, pkgName string) ([]crd.Function, error) { + fnList, err := client.FunctionList() + if err != nil { + return nil, err + } + fns := []crd.Function{} + for _, fn := range fnList { + if fn.Spec.Package.PackageRef.Name == pkgName { + fns = append(fns, fn) + } + } + return fns, nil +} + +// downloadStoragesvcURL downloads and return archive content with given storage service url +func downloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser { + u, err := url.ParseRequestURI(fileUrl) + if err != nil { + return nil + } + // replace in-cluster storage service host with controller server url + fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" + u.RequestURI() + reader, err := downloadURL(fileDownloadUrl) + checkErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl)) + return reader +} + +func pkgCreate(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + envName := c.String("env") + if len(envName) == 0 { + fatal("Need --env argument.") + } + + srcArchiveName := c.String("src") + deployArchiveName := c.String("deploy") + buildcmd := c.String("buildcmd") + + if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 { + fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") + } + + meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd) + fmt.Printf("Package '%v' created\n", meta.GetName()) + + return nil +} + +func pkgUpdate(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgName := c.String("name") + if len(pkgName) == 0 { + fatal("Need --name argument.") + } + + force := c.Bool("f") + envName := c.String("env") + srcArchiveName := c.String("src") + deployArchiveName := c.String("deploy") + buildcmd := c.String("buildcmd") + + if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 && + len(envName) == 0 && len(buildcmd) == 0 { + fatal("Need --env or --src or --deploy or --buildcmd argument.") + } + + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + checkErr(err, "get package") + + fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name) + checkErr(err, "get function list") + + if !force && len(fnList) > 1 { + fatal("Package is used by multiple functions, use --force to force update") + } + + newPkgMeta := updatePackage(client, pkg, + envName, srcArchiveName, deployArchiveName, buildcmd) + + // update resource version of package reference of functions that shared the same package + for _, fn := range fnList { + fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion + _, err := client.FunctionUpdate(&fn) + checkErr(err, "update function") + } + + fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName()) + + return nil +} + +func updatePackage(client *client.Client, pkg *crd.Package, envName, + srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta { + + var srcArchiveMetadata, deployArchiveMetadata *fission.Archive + needToBuild := false + + if len(envName) > 0 { + pkg.Spec.Environment.Name = envName + needToBuild = true + } + + if len(buildcmd) > 0 { + pkg.Spec.BuildCommand = buildcmd + needToBuild = true + } + + if len(srcArchiveName) > 0 { + srcArchiveMetadata = createArchive(client, srcArchiveName) + pkg.Spec.Source = *srcArchiveMetadata + needToBuild = true + } + + if len(deployArchiveName) > 0 { + deployArchiveMetadata = createArchive(client, deployArchiveName) + pkg.Spec.Deployment = *deployArchiveMetadata + } + + // Set package as pending status only when there is no + // deploy archive. + if needToBuild && len(pkg.Spec.Deployment.Type) == 0 { + // change into pending state to trigger package build + pkg.Status = fission.PackageStatus{ + BuildStatus: fission.BuildStatusPending, + } + } + + newPkgMeta, err := client.PackageUpdate(pkg) + checkErr(err, "update package") + + return newPkgMeta +} + +func pkgSourceGet(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgName := c.String("name") + if len(pkgName) == 0 { + fatal("Need name of package, use --name") + } + + output := c.String("output") + + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + if err != nil { + return err + } + + var reader io.Reader + + if pkg.Spec.Source.Type == fission.ArchiveTypeLiteral { + reader = bytes.NewReader(pkg.Spec.Source.Literal) + } else if pkg.Spec.Source.Type == fission.ArchiveTypeUrl { + readCloser := downloadStoragesvcURL(client, pkg.Spec.Source.URL) + defer readCloser.Close() + reader = readCloser + } + + if len(output) > 0 { + return writeArchiveToFile(output, reader) + } else { + _, err := io.Copy(os.Stdout, reader) + return err + } +} + +func pkgDeployGet(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgName := c.String("name") + if len(pkgName) == 0 { + fatal("Need name of package, use --name") + } + + output := c.String("output") + + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + if err != nil { + return err + } + + var reader io.Reader + + if pkg.Spec.Deployment.Type == fission.ArchiveTypeLiteral { + reader = bytes.NewReader(pkg.Spec.Deployment.Literal) + } else if pkg.Spec.Deployment.Type == fission.ArchiveTypeUrl { + readCloser := downloadStoragesvcURL(client, pkg.Spec.Deployment.URL) + defer readCloser.Close() + reader = readCloser + } + + if len(output) > 0 { + return writeArchiveToFile(output, reader) + } else { + _, err := io.Copy(os.Stdout, reader) + return err + } +} + +func pkgInfo(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgName := c.String("name") + if len(pkgName) == 0 { + fatal("Need name of package, use --name") + } + + pkg, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name) + fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name) + fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus) + fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog) + w.Flush() + + return nil +} + +func pkgList(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgList, err := client.PackageList() + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV") + for _, pkg := range pkgList { + fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, + pkg.Status.BuildStatus, pkg.Spec.Environment.Name) + } + w.Flush() + + return nil +} + +func pkgDelete(c *cli.Context) error { + client := getClient(c.GlobalString("server")) + + pkgName := c.String("name") + if len(pkgName) == 0 { + fmt.Println("Need --name argument.") + return nil + } + + force := c.Bool("f") + + _, err := client.PackageGet(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + checkErr(err, "find package") + + fnList, err := getFunctionsByPackage(client, pkgName) + + if !force && len(fnList) > 0 { + fatal("Package is used by at least one function, use -f to force delete") + } + + err = client.PackageDelete(&metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: pkgName, + }) + if err != nil { + return err + } + + fmt.Printf("Package '%v' deleted\n", pkgName) + + return nil +} diff --git a/test/test_utils.sh b/test/test_utils.sh index ec0a86c2..d2925aee 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -237,10 +237,13 @@ dump_fission_crd() { } dump_fission_crds() { - dump_fission_crd function.fission.io - dump_fission_crd package.fission.io - dump_fission_crd httptrigger.fission.io - dump_fission_crd environment.fission.io + dump_fission_crd environments.fission.io + dump_fission_crd functions.fission.io + dump_fission_crd httptriggers.fission.io + dump_fission_crd kuberneteswatchtriggers.fission.io + dump_fission_crd messagequeuetriggers.fission.io + dump_fission_crd packages.fission.io + dump_fission_crd timetriggers.fission.io } dump_env_pods() { diff --git a/test/tests/test_package_command.sh b/test/tests/test_package_command.sh new file mode 100755 index 00000000..52152b1c --- /dev/null +++ b/test/tests/test_package_command.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +set -euo pipefail + +# Use package command to create two packages one with source +# archive and the other with deploy archive. Also, create a +# function to test the packages created by package command are +# able to work. + +ROOT=$(dirname $0)/../.. +PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python3-env:test +PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python3-env-builder:test + +fn=python-srcbuild-$(date +%s) + +waitBuild() { + echo "Waiting for builder manager to finish the build" + + while true; do + kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded + if [[ $? -eq 0 ]]; then + break + fi + done +} +export -f waitBuild + +checkFunctionResponse() { + echo "Doing an HTTP GET on the function's route" + response=$(curl http://$FISSION_ROUTER/$1) + + echo "Checking for valid response" + echo $response + echo $response | grep -i "$2" +} + +echo "Pre-test cleanup" +fission env delete --name python || true + +echo "Creating python env" +fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE +trap "fission env delete --name python" EXIT + +echo "Waiting for env builder to catch up" +sleep 30 + +echo "Creating pacakage with source archive" +zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/ +pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \') + +# wait for build to finish at most 60s +timeout 60s bash -c "waitBuild $pkgName" + +echo "Creating function " $fn +fission fn create --name $fn --pkg $pkgName --entrypoint "user.main" +trap "fission fn delete --name $fn" EXIT + +echo "Creating route" +fission route create --function $fn --url /$fn --method GET + +echo "Waiting for router to catch up" +sleep 3 + +checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}' + +echo "Creating package with deploy archive" +mkdir testDir +touch testDir/__init__.py +printf 'def main():\n return "Hello, world!"' > testDir/hello.py +zip -jr demo-deploy-pkg.zip testDir/ +pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \') + +echo "Updating function " $fn +fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main" +trap "fission fn delete --name $fn" EXIT + +echo "Waiting for router to update cache" +sleep 3 + +checkFunctionResponse $fn 'Hello, world!' + +# crappy cleanup, improve this later +kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger + +echo "All done."