diff --git a/pkg/fetcher/fetcher.go b/pkg/fetcher/fetcher.go index 6a4d0096..7f567ca8 100644 --- a/pkg/fetcher/fetcher.go +++ b/pkg/fetcher/fetcher.go @@ -19,8 +19,6 @@ package fetcher import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "io" @@ -47,6 +45,7 @@ import ( "github.com/fission/fission/pkg/info" storageSvcClient "github.com/fission/fission/pkg/storagesvc/client" "github.com/fission/fission/pkg/types" + "github.com/fission/fission/pkg/utils" ) type ( @@ -129,27 +128,6 @@ func downloadUrl(ctx context.Context, httpClient *http.Client, url string, local return nil } -func getChecksum(path string) (*fv1.Checksum, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - hasher := sha256.New() - _, err = io.Copy(hasher, f) - if err != nil { - return nil, err - } - - c := hex.EncodeToString(hasher.Sum(nil)) - - return &fv1.Checksum{ - Type: fv1.ChecksumTypeSHA256, - Sum: c, - }, nil -} - func verifyChecksum(fileChecksum, checksum *fv1.Checksum) error { if checksum.Type != fv1.ChecksumTypeSHA256 { return ferror.MakeError(ferror.ErrorInvalidArgument, "Unsupported checksum type") @@ -328,7 +306,7 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, pkg *fv1.Package, req types.F return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url) } - checksum, err := getChecksum(tmpPath) + checksum, err := utils.FileChecksum(tmpPath) if err != nil { e := "failed to get checksum" fetcher.logger.Error(e, zap.Error(err)) @@ -534,7 +512,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { return } - sum, err := getChecksum(dstFilepath) + sum, err := utils.FileChecksum(dstFilepath) if err != nil { e := "error calculating checksum of zip file" fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath)) diff --git a/pkg/fission-cli/cmd/environment/create.go b/pkg/fission-cli/cmd/environment/create.go index 7310b580..734dcd4e 100644 --- a/pkg/fission-cli/cmd/environment/create.go +++ b/pkg/fission-cli/cmd/environment/create.go @@ -52,6 +52,7 @@ func (opts *CreateSubCommand) do(flags cli.Input) error { return opts.run(flags) } +// complete creates a environment objects and populates it with default value and CLI inputs. func (opts *CreateSubCommand) complete(flags cli.Input) error { env, err := createEnvironmentFromCmd(flags) if err != nil { @@ -61,6 +62,8 @@ func (opts *CreateSubCommand) complete(flags cli.Input) error { return nil } +// run write the resource to a spec file or create a fission CRD with remote fission server. +// It also prints warning/error if necessary. func (opts *CreateSubCommand) run(flags cli.Input) error { m, err := cmd.GetMetadata(flags) if err != nil { diff --git a/pkg/fission-cli/cmd/package/create.go b/pkg/fission-cli/cmd/package/create.go new file mode 100644 index 00000000..d3918388 --- /dev/null +++ b/pkg/fission-cli/cmd/package/create.go @@ -0,0 +1,148 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + "path" + "strings" + "time" + + "github.com/dchest/uniuri" + uuid "github.com/satori/go.uuid" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" + "github.com/fission/fission/pkg/fission-cli/log" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type CreateSubCommand struct { + client *client.Client +} + +func Create(flags cli.Input) error { + opts := CreateSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *CreateSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + //return opts.run(flags) + return nil +} + +// complete creates a environment objects and populates it with default value and CLI inputs. +func (opts *CreateSubCommand) complete(flags cli.Input) error { + pkgNamespace := flags.String("pkgNamespace") + envName := flags.String("env") + if len(envName) == 0 { + log.Fatal("Need --env argument.") + } + envNamespace := flags.String("envNamespace") + srcArchiveFiles := flags.StringSlice("src") + deployArchiveFiles := flags.StringSlice("deploy") + buildcmd := flags.String("buildcmd") + + if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 { + log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") + } + + _, err := CreatePackage(flags, opts.client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false) + + return err +} + +func CreatePackage(flags cli.Input, client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) (*metav1.ObjectMeta, error) { + pkgSpec := fv1.PackageSpec{ + Environment: fv1.EnvironmentReference{ + Namespace: envNamespace, + Name: envName, + }, + } + var pkgStatus fv1.BuildStatus = fv1.BuildStatusSucceeded + + var pkgName string + if len(deployArchiveFiles) > 0 { + if len(specFile) > 0 { // we should do this in all cases, i think + pkgStatus = fv1.BuildStatusNone + } + deployment, err := CreateArchive(client, deployArchiveFiles, noZip, specDir, specFile) + if err != nil { + return nil, err + } + pkgSpec.Deployment = *deployment + pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveFiles[0]), uniuri.NewLen(4))) + } + if len(srcArchiveFiles) > 0 { + source, err := CreateArchive(client, srcArchiveFiles, false, specDir, specFile) + if err != nil { + return nil, err + } + pkgSpec.Source = *source + pkgStatus = fv1.BuildStatusPending // set package build status to pending + pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveFiles[0]), uniuri.NewLen(4))) + } + + if len(buildcmd) > 0 { + pkgSpec.BuildCommand = buildcmd + } + + if len(pkgName) == 0 { + pkgName = strings.ToLower(uuid.NewV4().String()) + } + pkg := &fv1.Package{ + Metadata: metav1.ObjectMeta{ + Name: pkgName, + Namespace: pkgNamespace, + }, + Spec: pkgSpec, + Status: fv1.PackageStatus{ + BuildStatus: pkgStatus, + LastUpdateTimestamp: time.Now().UTC(), + }, + } + + if len(specFile) > 0 { + // if a package sith the same spec exists, don't create a new spec file + fr, err := spec.ReadSpecs(cmdutils.GetSpecDir(flags)) + util.CheckErr(err, "read specs") + if m := fr.SpecExists(pkg, false, true); m != nil { + fmt.Printf("Re-using previously created package %v\n", m.Name) + return m, nil + } + + err = spec.SpecSave(*pkg, specFile) + util.CheckErr(err, "save package spec") + return &pkg.Metadata, nil + } else { + pkgMetadata, err := client.PackageCreate(pkg) + util.CheckErr(err, "create package") + fmt.Printf("Package '%v' created\n", pkgMetadata.GetName()) + return pkgMetadata, nil + } +} diff --git a/pkg/fission-cli/cmd/package/delete.go b/pkg/fission-cli/cmd/package/delete.go new file mode 100644 index 00000000..4ac3e01f --- /dev/null +++ b/pkg/fission-cli/cmd/package/delete.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" +) + +type DeleteSubCommand struct { + client *client.Client + name string + namespace string + deleteOrphans bool + force bool +} + +func Delete(flags cli.Input) error { + opts := DeleteSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *DeleteSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *DeleteSubCommand) complete(flags cli.Input) error { + opts.name = flags.String("name") + opts.namespace = flags.String("pkgNamespace") + opts.deleteOrphans = flags.Bool("orphan") + opts.force = flags.Bool("f") + + if len(opts.name) == 0 && !opts.deleteOrphans { + return errors.New("need --name argument or --orphan flag") + } + if len(opts.name) != 0 && opts.deleteOrphans { + return errors.New("need either --name argument or --orphan flag") + } + + return nil +} + +func (opts *DeleteSubCommand) run(flags cli.Input) error { + if len(opts.name) != 0 { + _, err := opts.client.PackageGet(&metav1.ObjectMeta{ + Namespace: opts.namespace, + Name: opts.name, + }) + if err != nil { + return errors.Wrap(err, "find package") + } + + fnList, err := GetFunctionsByPackage(opts.client, opts.name, opts.namespace) + if err != nil { + return err + } + + if !opts.force && len(fnList) > 0 { + return errors.New("Package is used by at least one function, use -f to force delete") + } + err = deletePackage(opts.client, opts.name, opts.namespace) + if err != nil { + return err + } + fmt.Printf("Package '%v' deleted\n", opts.name) + } else { + err := deleteOrphanPkgs(opts.client, opts.namespace) + if err != nil { + return errors.Wrap(err, "deleting orphan packages") + } + fmt.Println("Orphan packages deleted") + } + + return nil +} + +func deleteOrphanPkgs(client *client.Client, pkgNamespace string) error { + pkgList, err := client.PackageList(pkgNamespace) + if err != nil { + return err + } + + // range through all packages and find out the ones not referenced by any function + for _, pkg := range pkgList { + fnList, err := GetFunctionsByPackage(client, pkg.Metadata.Name, pkgNamespace) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name)) + } + if len(fnList) == 0 { + err = deletePackage(client, pkg.Metadata.Name, pkgNamespace) + if err != nil { + return err + } + } + } + return nil +} + +func deletePackage(client *client.Client, pkgName string, pkgNamespace string) error { + return client.PackageDelete(&metav1.ObjectMeta{ + Namespace: pkgNamespace, + Name: pkgName, + }) +} diff --git a/pkg/fission-cli/cmd/package/get.go b/pkg/fission-cli/cmd/package/get.go new file mode 100644 index 00000000..9e0864bf --- /dev/null +++ b/pkg/fission-cli/cmd/package/get.go @@ -0,0 +1,110 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "bytes" + "errors" + "io" + "os" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + pkgutil "github.com/fission/fission/pkg/fission-cli/cmd/package/util" +) + +const ( + deployArchive = iota + sourceArchive +) + +type GetSubCommand struct { + client *client.Client + name string + namespace string + output string + archiveType int +} + +func GetSrc(flags cli.Input) error { + opts := GetSubCommand{ + client: cmdutils.GetServer(flags), + archiveType: sourceArchive, + } + return opts.do(flags) +} + +func GetDeploy(flags cli.Input) error { + opts := GetSubCommand{ + client: cmdutils.GetServer(flags), + archiveType: deployArchive, + } + return opts.do(flags) +} + +func (opts *GetSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *GetSubCommand) complete(flags cli.Input) error { + opts.name = flags.String("name") + if len(opts.name) == 0 { + return errors.New("need name of package, use --name") + } + opts.namespace = flags.String("pkgNamespace") + opts.output = flags.String("output") + return nil +} + +func (opts *GetSubCommand) run(flags cli.Input) error { + pkg, err := opts.client.PackageGet(&metav1.ObjectMeta{ + Namespace: opts.namespace, + Name: opts.name, + }) + if err != nil { + return err + } + + var reader io.Reader + archive := pkg.Spec.Source + if opts.archiveType == deployArchive { + archive = pkg.Spec.Deployment + } + + if pkg.Spec.Deployment.Type == fv1.ArchiveTypeLiteral { + reader = bytes.NewReader(archive.Literal) + } else if pkg.Spec.Deployment.Type == fv1.ArchiveTypeUrl { + readCloser := pkgutil.DownloadStoragesvcURL(opts.client, archive.URL) + defer readCloser.Close() + reader = readCloser + } + + if len(opts.output) > 0 { + return pkgutil.WriteArchiveToFile(opts.output, reader) + } else { + _, err := io.Copy(os.Stdout, reader) + return err + } +} diff --git a/pkg/fission-cli/cmd/package/info.go b/pkg/fission-cli/cmd/package/info.go new file mode 100644 index 00000000..fe0c0359 --- /dev/null +++ b/pkg/fission-cli/cmd/package/info.go @@ -0,0 +1,80 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + "os" + "text/tabwriter" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/log" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type InfoSubCommand struct { + client *client.Client + name string + namespace string +} + +func Info(flags cli.Input) error { + opts := InfoSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *InfoSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *InfoSubCommand) complete(flags cli.Input) error { + opts.name = flags.String("name") + if len(opts.name) == 0 { + log.Fatal("Need name of package, use --name") + } + opts.namespace = flags.String("pkgNamespace") + return nil +} + +func (opts *InfoSubCommand) run(flags cli.Input) error { + pkg, err := opts.client.PackageGet(&metav1.ObjectMeta{ + Namespace: opts.namespace, + Name: opts.name, + }) + if err != nil { + util.CheckErr(err, fmt.Sprintf("find package %s", opts.name)) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name) + fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name) + fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus) + fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog) + w.Flush() + + return nil +} diff --git a/pkg/fission-cli/cmd/package/list.go b/pkg/fission-cli/cmd/package/list.go new file mode 100644 index 00000000..0fd90167 --- /dev/null +++ b/pkg/fission-cli/cmd/package/list.go @@ -0,0 +1,98 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + "os" + "sort" + "text/tabwriter" + + "github.com/pkg/errors" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" +) + +type ListSubCommand struct { + client *client.Client + listOrphans bool + status string + pkgNamespace string +} + +func List(flags cli.Input) error { + opts := ListSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *ListSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *ListSubCommand) complete(flags cli.Input) error { + // option for the user to list all orphan packages (not referenced by any function) + opts.listOrphans = flags.Bool("orphan") + opts.status = flags.String("status") + opts.pkgNamespace = flags.String("pkgNamespace") + return nil +} + +func (opts *ListSubCommand) run(flags cli.Input) error { + pkgList, err := opts.client.PackageList(opts.pkgNamespace) + if err != nil { + return err + } + + // sort the package list by lastUpdatedTimestamp + sort.Slice(pkgList, func(i, j int) bool { + return pkgList[i].Status.LastUpdateTimestamp.After(pkgList[j].Status.LastUpdateTimestamp) + }) + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV", "LASTUPDATEDAT") + + for _, pkg := range pkgList { + show := true + if opts.listOrphans { + fnList, err := GetFunctionsByPackage(opts.client, pkg.Metadata.Name, pkg.Metadata.Namespace) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name)) + } + if len(fnList) > 0 { + show = false + } + } + if len(opts.status) > 0 && opts.status != string(pkg.Status.BuildStatus) { + show = false + } + if show { + fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name) + } + } + + w.Flush() + + return nil +} diff --git a/pkg/fission-cli/cmd/package/package.go b/pkg/fission-cli/cmd/package/package.go new file mode 100644 index 00000000..c6207fb2 --- /dev/null +++ b/pkg/fission-cli/cmd/package/package.go @@ -0,0 +1,176 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/dchest/uniuri" + "github.com/hashicorp/go-multierror" + "github.com/mholt/archiver" + "github.com/pkg/errors" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + pkgutil "github.com/fission/fission/pkg/fission-cli/cmd/package/util" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" + spectypes "github.com/fission/fission/pkg/fission-cli/cmd/spec/types" + "github.com/fission/fission/pkg/fission-cli/util" + "github.com/fission/fission/pkg/utils" +) + +// CreateArchive returns a fv1.Archive made from an archive . If specFile, then +// create an archive upload spec in the specs directory; otherwise +// upload the archive using client. noZip avoids zipping the +// includeFiles, but is ignored if there's more than one includeFile. +func CreateArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) (*fv1.Archive, error) { + + errs := &multierror.Error{} + + // check files existence + for _, path := range includeFiles { + // ignore http files + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + continue + } + + // Get files from inputs as number of files decide next steps + files, err := utils.FindAllGlobs([]string{path}) + if err != nil { + util.CheckErr(err, "finding all globs") + } + + if len(files) == 0 { + errs = multierror.Append(errs, errors.New(fmt.Sprintf("Error finding any files with path \"%v\"", path))) + } + } + + if errs.ErrorOrNil() != nil { + return nil, errs.ErrorOrNil() + } + + if len(specFile) > 0 { + // create an ArchiveUploadSpec and reference it from the archive + aus := &spectypes.ArchiveUploadSpec{ + Name: archiveName("", includeFiles), + IncludeGlobs: includeFiles, + } + + // check if this AUS exists in the specs; if so, don't create a new one + fr, err := spec.ReadSpecs(specDir) + util.CheckErr(err, "read specs") + if m := fr.SpecExists(aus, false, true); m != nil { + fmt.Printf("Re-using previously created archive %v\n", m.Name) + aus.Name = m.Name + } else { + // save the uploadspec + err := spec.SpecSave(*aus, specFile) + util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile)) + } + + // create the archive object + ar := &fv1.Archive{ + Type: fv1.ArchiveTypeUrl, + URL: fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name), + } + return ar, nil + } + + archivePath := makeArchiveFileIfNeeded("", includeFiles, noZip) + + ctx := context.Background() + return pkgutil.UploadArchive(ctx, client, archivePath) +} + +// Create an archive from the given list of input files, unless that +// list has only one item and that item is either a zip file or a URL. +// +// If the inputs have only one file and noZip is true, the file is +// returned as-is with no zipping. (This is used for compatibility +// with v1 envs.) noZip is IGNORED if there is more than one input +// file. +func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZip bool) string { + + // Unique name for the archive + archiveName := archiveName(archiveNameHint, archiveInput) + + // Get files from inputs as number of files decide next steps + files, err := utils.FindAllGlobs(archiveInput) + if err != nil { + util.CheckErr(err, "finding all globs") + } + + // We have one file; if it's a zip file or a URL, no need to archive it + if len(files) == 1 { + // make sure it exists + if _, err := os.Stat(files[0]); err != nil { + util.CheckErr(err, fmt.Sprintf("open input file %v", files[0])) + } + + // if it's an existing zip file OR we're not supposed to zip it, don't do anything + if archiver.Zip.Match(files[0]) || noZip { + return files[0] + } + + // if it's an HTTP URL, just use the URL. + if strings.HasPrefix(files[0], "http://") || strings.HasPrefix(files[0], "https://") { + return files[0] + } + } + + // For anything else, create a new archive + tmpDir, err := utils.GetTempDir() + if err != nil { + util.CheckErr(err, "create temporary archive directory") + } + + archivePath, err := utils.MakeArchive(filepath.Join(tmpDir, archiveName), archiveInput...) + if err != nil { + util.CheckErr(err, "create archive file") + } + + return archivePath +} + +// Name an archive +func archiveName(givenNameHint string, includedFiles []string) string { + if len(givenNameHint) > 0 { + return fmt.Sprintf("%v-%v", givenNameHint, uniuri.NewLen(4)) + } + if len(includedFiles) == 0 { + return uniuri.NewLen(8) + } + return fmt.Sprintf("%v-%v", util.KubifyName(includedFiles[0]), uniuri.NewLen(4)) +} + +func GetFunctionsByPackage(client *client.Client, pkgName, pkgNamespace string) ([]fv1.Function, error) { + fnList, err := client.FunctionList(pkgNamespace) + if err != nil { + return nil, err + } + fns := []fv1.Function{} + for _, fn := range fnList { + if fn.Spec.Package.PackageRef.Name == pkgName { + fns = append(fns, fn) + } + } + return fns, nil +} diff --git a/pkg/fission-cli/cmd/package/rebuild.go b/pkg/fission-cli/cmd/package/rebuild.go new file mode 100644 index 00000000..62cc9579 --- /dev/null +++ b/pkg/fission-cli/cmd/package/rebuild.go @@ -0,0 +1,83 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" +) + +type RebuildSubCommand struct { + client *client.Client + name string + namespace string +} + +func Rebuild(flags cli.Input) error { + opts := RebuildSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *RebuildSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *RebuildSubCommand) complete(flags cli.Input) error { + opts.name = flags.String("name") + if len(opts.name) == 0 { + return errors.New("Need name of package, use --name") + } + opts.namespace = flags.String("pkgNamespace") + return nil +} + +func (opts *RebuildSubCommand) run(flags cli.Input) error { + pkg, err := opts.client.PackageGet(&metav1.ObjectMeta{ + Name: opts.name, + Namespace: opts.namespace, + }) + if err != nil { + return errors.Wrap(err, "find package") + } + + if pkg.Status.BuildStatus != fv1.BuildStatusFailed { + return errors.New(fmt.Sprintf("Package %v is not in %v state.", + pkg.Metadata.Name, fv1.BuildStatusFailed)) + } + + _, err = updatePackageStatus(opts.client, pkg, fv1.BuildStatusPending) + if err != nil { + return errors.Wrap(err, "update package status") + } + + fmt.Printf("Retrying build for pkg %v. Use \"fission pkg info --name %v\" to view status.\n", pkg.Metadata.Name, pkg.Metadata.Name) + + return nil +} diff --git a/pkg/fission-cli/cmd/package/update.go b/pkg/fission-cli/cmd/package/update.go new file mode 100644 index 00000000..f34b4623 --- /dev/null +++ b/pkg/fission-cli/cmd/package/update.go @@ -0,0 +1,200 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package _package + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" +) + +type UpdateSubCommand struct { + client *client.Client + pkgName string + pkgNamespace string + force bool + envName string + envNamespace string + srcArchiveFiles []string + deployArchiveFiles []string + buildcmd string +} + +func Update(flags cli.Input) error { + opts := UpdateSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *UpdateSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *UpdateSubCommand) complete(flags cli.Input) error { + opts.pkgName = flags.String("name") + if len(opts.pkgName) == 0 { + return errors.New("Need --name argument.") + } + opts.pkgNamespace = flags.String("pkgNamespace") + opts.force = flags.Bool("f") + opts.envName = flags.String("env") + opts.envNamespace = flags.String("envNamespace") + opts.srcArchiveFiles = flags.StringSlice("src") + opts.deployArchiveFiles = flags.StringSlice("deploy") + opts.buildcmd = flags.String("buildcmd") + + if len(opts.srcArchiveFiles) > 0 && len(opts.deployArchiveFiles) > 0 { + return errors.New("Need either of --src or --deploy and not both arguments.") + } + + if len(opts.srcArchiveFiles) == 0 && len(opts.deployArchiveFiles) == 0 && + len(opts.envName) == 0 && len(opts.buildcmd) == 0 { + return errors.New("Need --env or --src or --deploy or --buildcmd argument.") + } + return nil +} + +func (opts *UpdateSubCommand) run(flags cli.Input) error { + pkg, err := opts.client.PackageGet(&metav1.ObjectMeta{ + Namespace: opts.pkgNamespace, + Name: opts.pkgName, + }) + if err != nil { + return errors.Wrap(err, "get package") + } + + // if the new env specified is the same as the old one, no need to update package + // same is true for all update parameters, but, for now, we dont check all of them - because, its ok to + // re-write the object with same old values, we just end up getting a new resource version for the object. + if len(opts.envName) > 0 && opts.envName == pkg.Spec.Environment.Name { + opts.envName = "" + } + + if opts.envNamespace == pkg.Spec.Environment.Namespace { + opts.envNamespace = "" + } + + fnList, err := GetFunctionsByPackage(opts.client, pkg.Metadata.Name, pkg.Metadata.Namespace) + if err != nil { + return errors.Wrap(err, "get function list") + } + + if !opts.force && len(fnList) > 1 { + return errors.New("Package is used by multiple functions, use --force to force update") + } + + newPkgMeta, err := UpdatePackage(opts.client, pkg, + opts.envName, opts.envNamespace, opts.srcArchiveFiles, + opts.deployArchiveFiles, opts.buildcmd, false, false) + if err != nil { + return errors.Wrap(err, "update package") + } + + // update resource version of package reference of functions that shared the same package + for _, fn := range fnList { + fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion + _, err := opts.client.FunctionUpdate(&fn) + if err != nil { + return errors.Wrap(err, "update function") + } + } + + fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName()) + return nil +} + +func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string, + srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) { + + needToBuild := false + + if len(envName) > 0 { + pkg.Spec.Environment.Name = envName + needToBuild = true + } + + if len(envNamespace) > 0 { + pkg.Spec.Environment.Namespace = envNamespace + needToBuild = true + } + + if len(buildcmd) > 0 { + pkg.Spec.BuildCommand = buildcmd + needToBuild = true + } + + if len(srcArchiveFiles) > 0 { + srcArchiveMetadata, err := CreateArchive(client, srcArchiveFiles, false, "", "") + if err != nil { + return nil, err + } + pkg.Spec.Source = *srcArchiveMetadata + needToBuild = true + } + + if len(deployArchiveFiles) > 0 { + deployArchiveMetadata, err := CreateArchive(client, deployArchiveFiles, noZip, "", "") + if err != nil { + return nil, err + } + pkg.Spec.Deployment = *deployArchiveMetadata + // Users may update the env, envNS and deploy archive at the same time, + // but without the source archive. In this case, we should set needToBuild to false + needToBuild = false + } + + // Set package as pending status when needToBuild is true + if needToBuild || forceRebuild { + // change into pending state to trigger package build + pkg.Status = fv1.PackageStatus{ + BuildStatus: fv1.BuildStatusPending, + LastUpdateTimestamp: time.Now().UTC(), + } + } + + newPkgMeta, err := client.PackageUpdate(pkg) + if err != nil { + return nil, errors.Wrap(err, "update package") + } + + return newPkgMeta, err +} + +func updatePackageStatus(client *client.Client, pkg *fv1.Package, status fv1.BuildStatus) (*metav1.ObjectMeta, error) { + switch status { + case fv1.BuildStatusNone, fv1.BuildStatusPending, fv1.BuildStatusRunning, fv1.BuildStatusSucceeded, fv1.CanaryConfigStatusAborted: + pkg.Status = fv1.PackageStatus{ + BuildStatus: status, + LastUpdateTimestamp: time.Now().UTC(), + } + pkg, err := client.PackageUpdate(pkg) + return pkg, err + } + return nil, errors.New("unknown package status") +} diff --git a/pkg/fission-cli/cmd/package/util/util.go b/pkg/fission-cli/cmd/package/util/util.go new file mode 100644 index 00000000..adeef2e9 --- /dev/null +++ b/pkg/fission-cli/cmd/package/util/util.go @@ -0,0 +1,166 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package util + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + uuid "github.com/satori/go.uuid" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/util" + storageSvcClient "github.com/fission/fission/pkg/storagesvc/client" + "github.com/fission/fission/pkg/types" + "github.com/fission/fission/pkg/utils" +) + +func UploadArchive(ctx context.Context, client *client.Client, fileName string) (*fv1.Archive, error) { + var archive fv1.Archive + + // If filename is a URL, download it first + if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") { + fileName = DownloadToTempFile(fileName) + } + + size, err := utils.FileSize(fileName) + if err != nil { + return nil, err + } + + if size < types.ArchiveLiteralSizeLimit { + archive.Type = fv1.ArchiveTypeLiteral + archive.Literal = GetContents(fileName) + } else { + u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" + ssClient := storageSvcClient.MakeClient(u) + + // TODO add a progress bar + id, err := ssClient.Upload(ctx, fileName, nil) + util.CheckErr(err, fmt.Sprintf("upload file %v", fileName)) + + storageSvc, err := client.GetSvcURL("application=fission-storage") + storageSvcURL := "http://" + storageSvc + util.CheckErr(err, "get fission storage service name") + + // We make a new client with actual URL of Storage service so that the URL is not + // pointing to 127.0.0.1 i.e. proxy. DON'T reuse previous ssClient + pkgClient := storageSvcClient.MakeClient(storageSvcURL) + archiveURL := pkgClient.GetUrl(id) + + archive.Type = fv1.ArchiveTypeUrl + archive.URL = archiveURL + + csum, err := utils.FileChecksum(fileName) + util.CheckErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) + + archive.Checksum = *csum + } + return &archive, nil +} + +func GetContents(filePath string) []byte { + var code []byte + var err error + + code, err = ioutil.ReadFile(filePath) + util.CheckErr(err, fmt.Sprintf("read %v", filePath)) + return code +} + +// DownloadToTempFile fetches archive file from arbitrary url +// and write it to temp file for further usage +func DownloadToTempFile(fileUrl string) string { + reader, err := DownloadURL(fileUrl) + util.CheckErr(err, fmt.Sprintf("download from url: %v", fileUrl)) + defer reader.Close() + + tmpDir, err := utils.GetTempDir() + util.CheckErr(err, "create temp directory") + + tmpFilename := uuid.NewV4().String() + destination := filepath.Join(tmpDir, tmpFilename) + err = os.Mkdir(tmpDir, 0744) + util.CheckErr(err, "create temp directory") + + err = WriteArchiveToFile(destination, reader) + util.CheckErr(err, "write archive to file") + + return destination +} + +// DownloadURL downloads file from given url +func DownloadURL(fileUrl string) (io.ReadCloser, error) { + resp, err := http.Get(fileUrl) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) + } + return resp.Body, nil +} + +func WriteArchiveToFile(fileName string, reader io.Reader) error { + tmpDir, err := utils.GetTempDir() + if err != nil { + return err + } + + path := filepath.Join(tmpDir, fileName+".tmp") + w, err := os.Create(path) + if err != nil { + return err + } + _, err = io.Copy(w, reader) + if err != nil { + return err + } + err = os.Chmod(path, 0644) + if err != nil { + return err + } + + err = os.Rename(path, fileName) + if err != nil { + return err + } + + return nil +} + +// DownloadStoragesvcURL downloads and return archive content with given storage service url +func DownloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser { + u, err := url.ParseRequestURI(fileUrl) + if err != nil { + return nil + } + + // replace in-cluster storage service host with controller server url + fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage/" + u.RequestURI() + reader, err := DownloadURL(fileDownloadUrl) + + util.CheckErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl)) + return reader +} diff --git a/pkg/fission-cli/spec.go b/pkg/fission-cli/cmd/spec/apply.go similarity index 75% rename from pkg/fission-cli/spec.go rename to pkg/fission-cli/cmd/spec/apply.go index e25f5d4a..156e8cbc 100644 --- a/pkg/fission-cli/spec.go +++ b/pkg/fission-cli/cmd/spec/apply.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Fission Authors. +Copyright 2019 The Fission Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fission_cli +package spec import ( - "bytes" "context" "fmt" "io/ioutil" @@ -28,236 +27,59 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/ghodss/yaml" - multierror "github.com/hashicorp/go-multierror" "github.com/mholt/archiver" "github.com/pkg/errors" - uuid "github.com/satori/go.uuid" - "github.com/urfave/cli" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" "github.com/fission/fission/pkg/controller/client" - "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" "github.com/fission/fission/pkg/fission-cli/cmd" - "github.com/fission/fission/pkg/fission-cli/cmd/spec" + pkgutil "github.com/fission/fission/pkg/fission-cli/cmd/package/util" + spectypes "github.com/fission/fission/pkg/fission-cli/cmd/spec/types" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" "github.com/fission/fission/pkg/types" + "github.com/fission/fission/pkg/utils" ) -// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new -// fission-config.yaml in specDir. -func writeDeploymentConfig(specDir string, dc *spec.DeploymentConfig) error { - y, err := yaml.Marshal(dc) - if err != nil { - return err - } - - msg := []byte("# This file is generated by the 'fission spec init' command.\n" + - "# See the README in this directory for background and usage information.\n" + - "# Do not edit the UID below: that will break 'fission spec apply'\n") - - err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644) - if err != nil { - return err - } - return nil +type ApplySubCommand struct { + client *client.Client } -// specInit just initializes an empty spec directory and adds some -// sample YAMLs in there that might be useful. -func specInit(c *cli.Context) error { - // Figure out spec directory - specDir := cmd.GetSpecDir(urfavecli.Parse(c)) - - name := c.String("name") - if len(name) == 0 { - // come up with a name using the current dir - dir, err := filepath.Abs(".") - util.CheckErr(err, "get current working directory") - basename := filepath.Base(dir) - name = util.KubifyName(basename) - } - - deployID := c.String("deployid") - if len(deployID) == 0 { - deployID = uuid.NewV4().String() - } - - // Create spec dir - fmt.Printf("Creating fission spec directory '%v'\n", specDir) - err := os.MkdirAll(specDir, 0755) - util.CheckErr(err, fmt.Sprintf("create spec directory '%v'", specDir)) - - // Add a bit of documentation to the spec dir here - err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(spec.SPEC_README), 0644) - if err != nil { - return err - } - - // Write the deployment config - dc := spec.DeploymentConfig{ - TypeMeta: spec.TypeMeta{ - APIVersion: spec.SPEC_API_VERSION, - Kind: "DeploymentConfig", - }, - Name: name, - - // All resources will be annotated with the UID when they're created. This allows - // us to be idempotent, as well as to delete resources when their specs are - // removed. - UID: deployID, - } - err = writeDeploymentConfig(specDir, &dc) - util.CheckErr(err, "write deployment config") - - // Other possible things to do here: - // - add example specs to the dir to make it easy to manually - // add new ones - - return nil -} - -// specValidate parses a set of specs and checks for references to -// resources that don't exist. -func specValidate(c *cli.Context) error { - // this will error on parse errors and on duplicates - specDir := cmd.GetSpecDir(urfavecli.Parse(c)) - fr, err := readSpecs(specDir) - util.CheckErr(err, "read specs") - - // this does the rest of the checks, like dangling refs - err = fr.Validate(c) - if err != nil { - fmt.Printf("Error validating specs: %v", err) - } - - return nil -} - -// readSpecs reads all specs in the specified directory and returns a parsed set of -// fission resources. -func readSpecs(specDir string) (*spec.FissionResources, error) { - - // make sure spec directory exists before continue - if _, err := os.Stat(specDir); os.IsNotExist(err) { - log.Fatal(fmt.Sprintf("Spec directory %v doesn't exist. "+ - "Please check directory path or run \"fission spec init\" to create it.", specDir)) - } - - fr := spec.FissionResources{ - Packages: make([]fv1.Package, 0), - Functions: make([]fv1.Function, 0), - Environments: make([]fv1.Environment, 0), - HttpTriggers: make([]fv1.HTTPTrigger, 0), - KubernetesWatchTriggers: make([]fv1.KubernetesWatchTrigger, 0), - TimeTriggers: make([]fv1.TimeTrigger, 0), - MessageQueueTriggers: make([]fv1.MessageQueueTrigger, 0), - - SourceMap: spec.SourceMap{ - Locations: make(map[string](map[string](map[string]spec.Location))), - }, - } - - result := &multierror.Error{} - - // Users can organize the specdir into subdirs if they want to. - err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // For now just read YAML files. We'll add jsonnet at some point. Skip - // unsupported files. - if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { - return nil - } - // read - b, err := ioutil.ReadFile(path) - if err != nil { - result = multierror.Append(result, err) - return nil - } - // handle the case where there are multiple YAML docs per file. go-yaml - // doesn't support this directly, yet. - docs := bytes.Split(b, []byte("\n---")) - lines := 1 - for _, doc := range docs { - d := []byte(strings.TrimSpace(string(doc))) - if len(d) != 0 { - // parse this document and add whatever is in it to fr - err = fr.ParseYaml(d, &spec.Location{ - Path: path, - Line: lines, - }) - if err != nil { - // collect all errors so user can fix them all - result = multierror.Append(result, err) - } - } - // the separator occupies one line, hence the +1 - lines += strings.Count(string(doc), "\n") + 1 - } - return nil - }) - - if err != nil { - return nil, err - } - if err = result.ErrorOrNil(); err != nil { - return nil, err - } - - return &fr, nil -} - -func ignoreFile(path string) bool { - return (strings.Contains(path, "/.#") || // editor autosave files - strings.HasSuffix(path, "~")) // editor backups, usually -} - -func waitForFileWatcherToSettleDown(watcher *fsnotify.Watcher) error { - // Wait a bit for things to settle down in case a bunch of - // files changed; also drain all events that queue up during - // the wait interval. - time.Sleep(500 * time.Millisecond) - for { - select { - case <-watcher.Events: - time.Sleep(200 * time.Millisecond) - continue - case err := <-watcher.Errors: - return err - default: - return nil - } - } -} - -// specApply compares the specs in the spec/config/ directory to the +// Apply compares the specs in the spec/config/ directory to the // deployed resources on the cluster, and reconciles the differences // by creating, updating or deleting resources on the cluster. // -// specApply is idempotent. +// Apply is idempotent. // -// specApply is *not* transactional -- if the user hits Ctrl-C, or their laptop dies +// Apply is *not* transactional -- if the user hits Ctrl-C, or their laptop dies // etc, while doing an apply, they will get a partially applied deployment. However, // they can retry their apply command once they're back online. -func specApply(c *cli.Context) error { - fclient := util.GetApiClient(c.GlobalString("server")) - specDir := cmd.GetSpecDir(urfavecli.Parse(c)) +func Apply(flags cli.Input) error { + opts := ApplySubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} - deleteResources := c.Bool("delete") - watchResources := c.Bool("watch") - waitForBuild := c.Bool("wait") +func (opts *ApplySubCommand) do(flags cli.Input) error { + return opts.run(flags) +} + +func (opts *ApplySubCommand) run(flags cli.Input) error { + specDir := cmd.GetSpecDir(flags) + + deleteResources := flags.Bool("delete") + watchResources := flags.Bool("watch") + waitForBuild := flags.Bool("wait") var watcher *fsnotify.Watcher var pbw *packageBuildWatcher if watchResources || waitForBuild { // init package build watcher - pbw = makePackageBuildWatcher(fclient) + pbw = makePackageBuildWatcher(opts.client) } if watchResources { @@ -282,15 +104,15 @@ func specApply(c *cli.Context) error { for { // read all specs - fr, err := readSpecs(specDir) + fr, err := ReadSpecs(specDir) util.CheckErr(err, "read specs") // validate - err = fr.Validate(c) + err = fr.Validate(flags) util.CheckErr(err, "validate specs") // make changes to the cluster based on the specs - pkgMetas, as, err := applyResources(fclient, specDir, fr, deleteResources) + pkgMetas, as, err := applyResources(opts.client, specDir, fr, deleteResources) util.CheckErr(err, "apply specs") printApplyStatus(as) @@ -340,12 +162,36 @@ func specApply(c *cli.Context) error { } } } + return nil } -// printApplyStatus prints a summary of what changed on the cluster as the result of a spec apply -// operation. -func printApplyStatus(applyStatus map[string]spec.ResourceApplyStatus) { +func ignoreFile(path string) bool { + return (strings.Contains(path, "/.#") || // editor autosave files + strings.HasSuffix(path, "~")) // editor backups, usually +} + +func waitForFileWatcherToSettleDown(watcher *fsnotify.Watcher) error { + // Wait a bit for things to settle down in case a bunch of + // files changed; also drain all events that queue up during + // the wait interval. + time.Sleep(500 * time.Millisecond) + for { + select { + case <-watcher.Events: + time.Sleep(200 * time.Millisecond) + continue + case err := <-watcher.Errors: + return err + default: + return nil + } + } +} + +// printApplyStatus prints a summary of what changed on the +// cluster as the result of a spec apply operation. +func printApplyStatus(applyStatus map[string]ResourceApplyStatus) { changed := false for typ, ras := range applyStatus { n := len(ras.Created) @@ -387,30 +233,8 @@ func pluralize(num int, word string) string { return word + "s" } -// specDestroy destroys everything in the spec. -func specDestroy(c *cli.Context) error { - fclient := util.GetApiClient(c.GlobalString("server")) - - // get specdir - specDir := cmd.GetSpecDir(urfavecli.Parse(c)) - - // read everything - fr, err := readSpecs(specDir) - util.CheckErr(err, "read specs") - - // set desired state to nothing, but keep the UID so "apply" can find it - emptyFr := spec.FissionResources{} - emptyFr.DeploymentConfig = fr.DeploymentConfig - - // "apply" the empty state - _, _, err = applyResources(fclient, specDir, &emptyFr, true) - util.CheckErr(err, "delete resources") - - return nil -} - // applyArchives figures out the set of archives that need to be uploaded, and uploads them. -func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResources) error { +func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) error { // archive:// URL -> archive map. archiveFiles := make(map[string]fv1.Archive) @@ -424,7 +248,7 @@ func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResou if err != nil { return err } - archiveUrl := fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name) + archiveUrl := fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name) archiveFiles[archiveUrl] = *ar } @@ -457,7 +281,10 @@ func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResou fmt.Printf("uploading archive %v\n", name) // ar.URL is actually a local filename at this stage ctx := context.Background() - uploadedAr := uploadArchive(ctx, fclient, ar.URL) + uploadedAr, err := pkgutil.UploadArchive(ctx, fclient, ar.URL) + if err != nil { + return err + } archiveFiles[name] = *uploadedAr } } @@ -465,10 +292,10 @@ func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResou // resolve references to urls in packages to be applied for i := range fr.Packages { for _, ar := range []*fv1.Archive{&fr.Packages[i].Spec.Source, &fr.Packages[i].Spec.Deployment} { - if strings.HasPrefix(ar.URL, spec.ARCHIVE_URL_PREFIX) { + if strings.HasPrefix(ar.URL, ARCHIVE_URL_PREFIX) { availableAr, ok := archiveFiles[ar.URL] if !ok { - return fmt.Errorf("unknown archive name %v", strings.TrimPrefix(ar.URL, spec.ARCHIVE_URL_PREFIX)) + return fmt.Errorf("unknown archive name %v", strings.TrimPrefix(ar.URL, ARCHIVE_URL_PREFIX)) } ar.Type = availableAr.Type ar.Literal = availableAr.Literal @@ -481,9 +308,9 @@ func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResou } // applyResources applies the given set of fission resources. -func applyResources(fclient *client.Client, specDir string, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, map[string]spec.ResourceApplyStatus, error) { +func applyResources(fclient *client.Client, specDir string, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, map[string]ResourceApplyStatus, error) { - applyStatus := make(map[string]spec.ResourceApplyStatus) + applyStatus := make(map[string]ResourceApplyStatus) // upload archives that need to be uploaded. Changes archive references in fr.Packages. err := applyArchives(fclient, specDir, fr) @@ -558,7 +385,7 @@ func applyResources(fclient *client.Client, specDir string, fr *spec.FissionReso // localArchiveFromSpec creates an archive on the local filesystem from the given spec, // and returns its path and checksum. -func localArchiveFromSpec(specDir string, aus *spec.ArchiveUploadSpec) (*fv1.Archive, error) { +func localArchiveFromSpec(specDir string, aus *spectypes.ArchiveUploadSpec) (*fv1.Archive, error) { // get root dir var rootDir string if len(aus.RootDir) == 0 { @@ -620,16 +447,21 @@ func localArchiveFromSpec(specDir string, aus *spec.ArchiveUploadSpec) (*fv1.Arc } } + size, err := utils.FileSize(archiveFileName) + if err != nil { + return nil, err + } + // figure out if we're making a literal or a URL-based archive - if fileSize(archiveFileName) < types.ArchiveLiteralSizeLimit { - contents := getContents(archiveFileName) + if size < types.ArchiveLiteralSizeLimit { + contents := pkgutil.GetContents(archiveFileName) return &fv1.Archive{ Type: fv1.ArchiveTypeLiteral, Literal: contents, }, nil } else { // checksum - csum, err := fileChecksum(archiveFileName) + csum, err := utils.FileChecksum(archiveFileName) if err != nil { return nil, fmt.Errorf("failed to calculate archive checksum for %v (%v): %v", aus.Name, archiveFileName, err) } @@ -646,29 +478,23 @@ func localArchiveFromSpec(specDir string, aus *spec.ArchiveUploadSpec) (*fv1.Arc } } -// specHelm creates a helm chart from a spec directory and a -// deployment config. -func specHelm(c *cli.Context) error { - return nil -} - func mapKey(m *metav1.ObjectMeta) string { return fmt.Sprintf("%v:%v", m.Namespace, m.Name) } -func applyDeploymentConfig(m *metav1.ObjectMeta, fr *spec.FissionResources) { +func applyDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) { if m.Annotations == nil { m.Annotations = make(map[string]string) } - m.Annotations[spec.FISSION_DEPLOYMENT_NAME_KEY] = fr.DeploymentConfig.Name - m.Annotations[spec.FISSION_DEPLOYMENT_UID_KEY] = fr.DeploymentConfig.UID + m.Annotations[FISSION_DEPLOYMENT_NAME_KEY] = fr.DeploymentConfig.Name + m.Annotations[FISSION_DEPLOYMENT_UID_KEY] = fr.DeploymentConfig.UID } -func hasDeploymentConfig(m *metav1.ObjectMeta, fr *spec.FissionResources) bool { +func hasDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) bool { if m.Annotations == nil { return false } - uid, ok := m.Annotations[spec.FISSION_DEPLOYMENT_UID_KEY] + uid, ok := m.Annotations[FISSION_DEPLOYMENT_UID_KEY] if ok && uid == fr.DeploymentConfig.UID { return true } @@ -696,7 +522,7 @@ func waitForPackageBuild(fclient *client.Client, pkg *fv1.Package) (*fv1.Package } } -func applyPackages(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.PackageList(metav1.NamespaceAll) if err != nil { @@ -721,7 +547,7 @@ func applyPackages(fclient *client.Client, fr *spec.FissionResources, delete boo // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.Packages { @@ -807,7 +633,7 @@ func applyPackages(fclient *client.Client, fr *spec.FissionResources, delete boo return metadataMap, &ras, nil } -func applyFunctions(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.FunctionList(metav1.NamespaceAll) if err != nil { @@ -832,7 +658,7 @@ func applyFunctions(fclient *client.Client, fr *spec.FissionResources, delete bo // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.Functions { @@ -890,7 +716,7 @@ func applyFunctions(fclient *client.Client, fr *spec.FissionResources, delete bo return metadataMap, &ras, nil } -func applyEnvironments(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.EnvironmentList(metav1.NamespaceAll) if err != nil { @@ -915,7 +741,7 @@ func applyEnvironments(fclient *client.Client, fr *spec.FissionResources, delete // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.Environments { @@ -973,7 +799,7 @@ func applyEnvironments(fclient *client.Client, fr *spec.FissionResources, delete return metadataMap, &ras, nil } -func applyHTTPTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.HTTPTriggerList(metav1.NamespaceAll) if err != nil { @@ -998,7 +824,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *spec.FissionResources, delete // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.HttpTriggers { @@ -1056,7 +882,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *spec.FissionResources, delete return metadataMap, &ras, nil } -func applyKubernetesWatchTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.WatchList(metav1.NamespaceAll) if err != nil { @@ -1081,7 +907,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *spec.FissionResour // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.KubernetesWatchTriggers { @@ -1139,7 +965,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *spec.FissionResour return metadataMap, &ras, nil } -func applyTimeTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.TimeTriggerList(metav1.NamespaceAll) if err != nil { @@ -1164,7 +990,7 @@ func applyTimeTriggers(fclient *client.Client, fr *spec.FissionResources, delete // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.TimeTriggers { @@ -1222,7 +1048,7 @@ func applyTimeTriggers(fclient *client.Client, fr *spec.FissionResources, delete return metadataMap, &ras, nil } -func applyMessageQueueTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { +func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *ResourceApplyStatus, error) { // get list allObjs, err := fclient.MessageQueueTriggerList("", metav1.NamespaceAll) if err != nil { @@ -1247,7 +1073,7 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *spec.FissionResources // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras spec.ResourceApplyStatus + var ras ResourceApplyStatus // create or update desired state for _, o := range fr.MessageQueueTriggers { diff --git a/pkg/fission-cli/buildwatch.go b/pkg/fission-cli/cmd/spec/buildwatch.go similarity index 96% rename from pkg/fission-cli/buildwatch.go rename to pkg/fission-cli/cmd/spec/buildwatch.go index 51651071..58137659 100644 --- a/pkg/fission-cli/buildwatch.go +++ b/pkg/fission-cli/cmd/spec/buildwatch.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Fission Authors. +Copyright 2019 The Fission Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fission_cli +package spec import ( "context" @@ -66,7 +66,7 @@ func (w *packageBuildWatcher) watch(ctx context.Context) { default: } - // poll list of packages (TODO: convert to watch) + // pull list of packages (TODO: convert to watch) pkgs, err := w.fclient.PackageList(metav1.NamespaceAll) util.CheckErr(err, "Getting list of packages") diff --git a/pkg/fission-cli/cmd/spec/destroy.go b/pkg/fission-cli/cmd/spec/destroy.go new file mode 100644 index 00000000..0a8e1ff5 --- /dev/null +++ b/pkg/fission-cli/cmd/spec/destroy.go @@ -0,0 +1,59 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type DestroySubCommand struct { + client *client.Client +} + +// Destroy destroys everything in the spec. +func Destroy(flags cli.Input) error { + opts := &DestroySubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *DestroySubCommand) do(flags cli.Input) error { + return opts.run(flags) +} + +func (opts *DestroySubCommand) run(flags cli.Input) error { + // get specdir + specDir := cmd.GetSpecDir(flags) + + // read everything + fr, err := ReadSpecs(specDir) + util.CheckErr(err, "read specs") + + // set desired state to nothing, but keep the UID so "apply" can find it + emptyFr := FissionResources{} + emptyFr.DeploymentConfig = fr.DeploymentConfig + + // "apply" the empty state + _, _, err = applyResources(opts.client, specDir, &emptyFr, true) + util.CheckErr(err, "delete resources") + + return nil +} diff --git a/pkg/fission-cli/cmd/spec/init.go b/pkg/fission-cli/cmd/spec/init.go new file mode 100644 index 00000000..74749b88 --- /dev/null +++ b/pkg/fission-cli/cmd/spec/init.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/ghodss/yaml" + uuid "github.com/satori/go.uuid" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + spectypes "github.com/fission/fission/pkg/fission-cli/cmd/spec/types" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type InitSubCommand struct { + client *client.Client + deployConfig *spectypes.DeploymentConfig +} + +func Init(flags cli.Input) error { + opts := InitSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *InitSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *InitSubCommand) complete(flags cli.Input) error { + // Figure out spec directory + specDir := cmd.GetSpecDir(flags) + + name := flags.String("name") + if len(name) == 0 { + // come up with a name using the current dir + dir, err := filepath.Abs(".") + util.CheckErr(err, "get current working directory") + basename := filepath.Base(dir) + name = util.KubifyName(basename) + } + + deployID := flags.String("deployid") + if len(deployID) == 0 { + deployID = uuid.NewV4().String() + } + + // Create spec dir + fmt.Printf("Creating fission spec directory '%v'\n", specDir) + err := os.MkdirAll(specDir, 0755) + util.CheckErr(err, fmt.Sprintf("create spec directory '%v'", specDir)) + + // Write the deployment config + opts.deployConfig = &spectypes.DeploymentConfig{ + TypeMeta: spectypes.TypeMeta{ + APIVersion: SPEC_API_VERSION, + Kind: "DeploymentConfig", + }, + Name: name, + + // All resources will be annotated with the UID when they're created. This allows + // us to be idempotent, as well as to delete resources when their specs are + // removed. + UID: deployID, + } + return nil +} + +// run just initializes an empty spec directory and adds some +// sample YAMLs in there that might be useful. +func (opts *InitSubCommand) run(flags cli.Input) error { + specDir := cmd.GetSpecDir(flags) + + // Add a bit of documentation to the spec dir here + err := ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644) + if err != nil { + return err + } + + err = writeDeploymentConfig(specDir, opts.deployConfig) + util.CheckErr(err, "write deployment config") + + // Other possible things to do here: + // - add example specs to the dir to make it easy to manually + // add new ones + return nil +} + +// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new +// fission-config.yaml in specDir. +func writeDeploymentConfig(specDir string, dc *spectypes.DeploymentConfig) error { + y, err := yaml.Marshal(dc) + if err != nil { + return err + } + + msg := []byte("# This file is generated by the 'fission spec init' command.\n" + + "# See the README in this directory for background and usage information.\n" + + "# Do not edit the UID below: that will break 'fission spec apply'\n") + + err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644) + if err != nil { + return err + } + return nil +} diff --git a/pkg/fission-cli/cmd/spec/spec.go b/pkg/fission-cli/cmd/spec/spec.go index 9b6e4ac6..1c54bf03 100644 --- a/pkg/fission-cli/cmd/spec/spec.go +++ b/pkg/fission-cli/cmd/spec/spec.go @@ -26,13 +26,14 @@ import ( "github.com/ghodss/yaml" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" - "github.com/urfave/cli" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec/types" "github.com/fission/fission/pkg/fission-cli/log" - "github.com/fission/fission/pkg/fission-cli/util" "github.com/fission/fission/pkg/generator/encoder" v1generator "github.com/fission/fission/pkg/generator/v1" ) @@ -92,60 +93,8 @@ fission. // CLI spec types type ( - // DeploymentConfig is the global configuration for a set of Fission specs. - DeploymentConfig struct { - // TypeMeta describes the type of this object. It is inlined. The Kind - // field should always be "DeploymentConfig". - TypeMeta `json:",inline"` - - // Name is a user-friendly name for the deployment. It is also stored in - // all uploaded resources as an annotation. - Name string `json:"name"` - - // UID uniquely identifies the deployment. It is stored as a label and - // used to find resources to clean up when local specs are changed. - UID string `json:"uid"` - } - - // ArchiveUploadSpec specifies a set of files to be archived and uploaded. - // - // The resulting archive can be referenced as archive:// in PackageSpecs, - // using the name specified in the archive. The fission spec applier will - // replace the archive:// URL with a real HTTP URL after uploading the file. - ArchiveUploadSpec struct { - // TypeMeta describes the type of this object. It is inlined. The Kind - // field should always be "ArchiveUploadSpec". - TypeMeta `json:",inline"` - - // Name is a local name that can be used to reference this archive. It - // must be unique; duplicate names will cause an error while handling - // specs. - Name string `json:"name"` - - // RootDir specifies the root that the globs below are relative to. It - // is optional and defaults to the parent directory of the spec - // directory: for example, if the deployment config is at - // /path/to/project/specs/config.yaml, the RootDir is /path/to/project. - RootDir string `json:"rootdir,omitempty"` - - // IncludeGlobs is a list of Unix shell globs to include - IncludeGlobs []string `json:"include,omitempty"` - - // ExcludeGlobs is a list of globs to exclude from the set specified by - // IncludeGlobs. - ExcludeGlobs []string `json:"exclude,omitempty"` - } - - // TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and - // unmarshal local-only objects (like ArchiveUploadSpec) the same way that - // Kubernetes does. - TypeMeta struct { - Kind string `json:"kind,omitempty"` - APIVersion string `json:"apiVersion,omitempty"` - } - FissionResources struct { - DeploymentConfig DeploymentConfig + DeploymentConfig types.DeploymentConfig Packages []fv1.Package Functions []fv1.Function Environments []fv1.Environment @@ -153,7 +102,7 @@ type ( KubernetesWatchTriggers []fv1.KubernetesWatchTrigger TimeTriggers []fv1.TimeTrigger MessageQueueTriggers []fv1.MessageQueueTrigger - ArchiveUploadSpecs []ArchiveUploadSpec + ArchiveUploadSpecs []types.ArchiveUploadSpec SourceMap SourceMap } @@ -224,7 +173,7 @@ func SpecSave(resource interface{}, specFile string) error { var data []byte var err error switch typedres := resource.(type) { - case ArchiveUploadSpec: + case types.ArchiveUploadSpec: typedres.Kind = "ArchiveUploadSpec" data, err = yaml.Marshal(typedres) case fv1.Package: @@ -296,7 +245,7 @@ func (fr *FissionResources) validateFunctionReference(functions map[string]bool, return nil } -func (fr *FissionResources) Validate(c *cli.Context) error { +func (fr *FissionResources) Validate(flags cli.Input) error { result := &multierror.Error{} // check references: both dangling refs + garbage @@ -398,7 +347,7 @@ func (fr *FissionResources) Validate(c *cli.Context) error { packages[MapKey(pkgMeta)] = true } - client := util.GetApiClient(c.GlobalString("server")) + client := cmd.GetServer(flags) for _, cm := range f.Spec.ConfigMaps { _, err := client.ConfigMapGet(&metav1.ObjectMeta{ Name: cm.Name, @@ -532,7 +481,7 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error { // Figure out the object type by unmarshaling into the TypeMeta struct; then // unmarshal again into the "real" struct once we know the type. - var tm TypeMeta + var tm types.TypeMeta err := yaml.Unmarshal(b, &tm) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b))) @@ -605,14 +554,14 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error { // The following are not CRDs case "DeploymentConfig": - var v DeploymentConfig + var v types.DeploymentConfig err = yaml.Unmarshal(b, &v) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) } fr.DeploymentConfig = v case "ArchiveUploadSpec": - var v ArchiveUploadSpec + var v types.ArchiveUploadSpec err = yaml.Unmarshal(b, &v) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) @@ -644,7 +593,7 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error { // equality check is performed. func (fr *FissionResources) SpecExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta { switch typedres := resource.(type) { - case *ArchiveUploadSpec: + case *types.ArchiveUploadSpec: for _, aus := range fr.ArchiveUploadSpecs { if compareMetadata && aus.Name != typedres.Name { continue diff --git a/pkg/fission-cli/cmd/spec/types/types.go b/pkg/fission-cli/cmd/spec/types/types.go new file mode 100644 index 00000000..20b28fe9 --- /dev/null +++ b/pkg/fission-cli/cmd/spec/types/types.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// CLI spec types +type ( + // DeploymentConfig is the global configuration for a set of Fission specs. + DeploymentConfig struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "DeploymentConfig". + TypeMeta `json:",inline"` + + // Name is a user-friendly name for the deployment. It is also stored in + // all uploaded resources as an annotation. + Name string `json:"name"` + + // UID uniquely identifies the deployment. It is stored as a label and + // used to find resources to clean up when local specs are changed. + UID string `json:"uid"` + } + + // ArchiveUploadSpec specifies a set of files to be archived and uploaded. + // + // The resulting archive can be referenced as archive:// in PackageSpecs, + // using the name specified in the archive. The fission spec applier will + // replace the archive:// URL with a real HTTP URL after uploading the file. + ArchiveUploadSpec struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "ArchiveUploadSpec". + TypeMeta `json:",inline"` + + // Name is a local name that can be used to reference this archive. It + // must be unique; duplicate names will cause an error while handling + // specs. + Name string `json:"name"` + + // RootDir specifies the root that the globs below are relative to. It + // is optional and defaults to the parent directory of the spec + // directory: for example, if the deployment config is at + // /path/to/project/specs/config.yaml, the RootDir is /path/to/project. + RootDir string `json:"rootdir,omitempty"` + + // IncludeGlobs is a list of Unix shell globs to include + IncludeGlobs []string `json:"include,omitempty"` + + // ExcludeGlobs is a list of globs to exclude from the set specified by + // IncludeGlobs. + ExcludeGlobs []string `json:"exclude,omitempty"` + } + + // TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and + // unmarshal local-only objects (like ArchiveUploadSpec) the same way that + // Kubernetes does. + TypeMeta struct { + Kind string `json:"kind,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + } +) diff --git a/pkg/fission-cli/cmd/spec/validate.go b/pkg/fission-cli/cmd/spec/validate.go new file mode 100644 index 00000000..1536a4e7 --- /dev/null +++ b/pkg/fission-cli/cmd/spec/validate.go @@ -0,0 +1,146 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/go-multierror" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/log" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type ValidateSubCommand struct { + client *client.Client +} + +// Validate parses a set of specs and checks for references to +// resources that don't exist. +func Validate(flags cli.Input) error { + opts := &ValidateSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *ValidateSubCommand) do(flags cli.Input) error { + return opts.run(flags) +} + +func (opts *ValidateSubCommand) run(flags cli.Input) error { + + // this will error on parse errors and on duplicates + specDir := cmd.GetSpecDir(flags) + fr, err := ReadSpecs(specDir) + util.CheckErr(err, "read specs") + + // this does the rest of the checks, like dangling refs + err = fr.Validate(flags) + if err != nil { + fmt.Printf("Error validating specs: %v", err) + + return nil + } + + return nil +} + +// ReadSpecs reads all specs in the specified directory and returns a parsed set of +// fission resources. +func ReadSpecs(specDir string) (*FissionResources, error) { + + // make sure spec directory exists before continue + if _, err := os.Stat(specDir); os.IsNotExist(err) { + log.Fatal(fmt.Sprintf("Spec directory %v doesn't exist. "+ + "Please check directory path or run \"fission spec init\" to create it.", specDir)) + } + + fr := FissionResources{ + Packages: make([]fv1.Package, 0), + Functions: make([]fv1.Function, 0), + Environments: make([]fv1.Environment, 0), + HttpTriggers: make([]fv1.HTTPTrigger, 0), + KubernetesWatchTriggers: make([]fv1.KubernetesWatchTrigger, 0), + TimeTriggers: make([]fv1.TimeTrigger, 0), + MessageQueueTriggers: make([]fv1.MessageQueueTrigger, 0), + + SourceMap: SourceMap{ + Locations: make(map[string](map[string](map[string]Location))), + }, + } + + var result *multierror.Error + + // Users can organize the specdir into subdirs if they want to. + err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // For now just read YAML files. We'll add jsonnet at some point. Skip + // unsupported files. + if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + return nil + } + // read + b, err := ioutil.ReadFile(path) + if err != nil { + result = multierror.Append(result, err) + return nil + } + // handle the case where there are multiple YAML docs per file. go-yaml + // doesn't support this directly, yet. + docs := bytes.Split(b, []byte("\n---")) + lines := 1 + for _, doc := range docs { + d := []byte(strings.TrimSpace(string(doc))) + if len(d) != 0 { + // parse this document and add whatever is in it to fr + err = fr.ParseYaml(d, &Location{ + Path: path, + Line: lines, + }) + if err != nil { + // collect all errors so user can fix them all + result = multierror.Append(result, err) + } + } + // the separator occupies one line, hence the +1 + lines += strings.Count(string(doc), "\n") + 1 + } + return nil + }) + + if err != nil { + return nil, err + } + if err = result.ErrorOrNil(); err != nil { + return nil, err + } + + return &fr, nil +} diff --git a/pkg/fission-cli/function.go b/pkg/fission-cli/function.go index d70b2f14..71c1e52c 100644 --- a/pkg/fission-cli/function.go +++ b/pkg/fission-cli/function.go @@ -39,6 +39,7 @@ import ( "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" "github.com/fission/fission/pkg/fission-cli/cmd" cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + _package "github.com/fission/fission/pkg/fission-cli/cmd/package" "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/logdb" @@ -299,7 +300,8 @@ func fnCreate(c *cli.Context) error { buildcmd := c.String("buildcmd") // create new package in the same namespace as the function. - pkgMetadata = createPackage(c, client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip) + pkgMetadata, err = _package.CreatePackage(c, client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip) + util.CheckErr(err, "create package") } var secrets []fv1.SecretReference @@ -629,14 +631,14 @@ func fnUpdate(c *cli.Context) error { pkgMetadata := &pkg.Metadata if len(deployArchiveFiles) != 0 || len(srcArchiveFiles) != 0 || len(buildcmd) != 0 || len(envName) != 0 || len(envNamespace) != 0 { - fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace) + fnList, err := _package.GetFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace) util.CheckErr(err, "get function list") if !force && len(fnList) > 1 { log.Fatal("Package is used by multiple functions, use --force to force update") } - pkgMetadata, err = updatePackage(client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag) + pkgMetadata, err = _package.UpdatePackage(client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag) util.CheckErr(err, fmt.Sprintf("update package '%v'", pkgName)) fmt.Printf("package '%v' updated\n", pkgMetadata.GetName()) diff --git a/pkg/fission-cli/main.go b/pkg/fission-cli/main.go index a774b2a9..0474f8dd 100644 --- a/pkg/fission-cli/main.go +++ b/pkg/fission-cli/main.go @@ -31,6 +31,8 @@ import ( "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" "github.com/fission/fission/pkg/fission-cli/cmd" "github.com/fission/fission/pkg/fission-cli/cmd/environment" + _package "github.com/fission/fission/pkg/fission-cli/cmd/package" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/cmd/support" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/plugin" @@ -261,16 +263,16 @@ func NewCliApp() *cli.App { pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"} pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"} pkgStatusFlag := cli.StringFlag{Name: "status", Usage: `Filter packages by status`} - pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "orphan packages that are not referenced by any function"} + pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "Orphan packages that are not referenced by any function"} pkgSubCommands := []cli.Command{ - {Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: pkgCreate}, - {Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: pkgUpdate}, - {Name: "rebuild", Usage: "Rebuild a failed package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: pkgRebuild}, - {Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: pkgSourceGet}, - {Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: pkgDeployGet}, - {Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: pkgInfo}, - {Name: "list", Usage: "List all packages", Flags: []cli.Flag{pkgOrphanFlag, pkgStatusFlag, pkgNamespaceFlag}, Action: pkgList}, - {Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgForceFlag, pkgOrphanFlag}, Action: pkgDelete}, + {Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: urfavecli.Wrapper(_package.Create)}, + {Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: urfavecli.Wrapper(_package.Update)}, + {Name: "rebuild", Usage: "Rebuild a failed package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: urfavecli.Wrapper(_package.Rebuild)}, + {Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: urfavecli.Wrapper(_package.GetSrc)}, + {Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: urfavecli.Wrapper(_package.GetDeploy)}, + {Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: urfavecli.Wrapper(_package.Info)}, + {Name: "list", Usage: "List all packages", Flags: []cli.Flag{pkgOrphanFlag, pkgStatusFlag, pkgNamespaceFlag}, Action: urfavecli.Wrapper(_package.List)}, + {Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgForceFlag, pkgOrphanFlag}, Action: urfavecli.Wrapper(_package.Delete)}, } // specs @@ -281,11 +283,10 @@ func NewCliApp() *cli.App { specWatchFlag := cli.BoolFlag{Name: "watch", Usage: "Watch local files for change, and re-apply specs as necessary"} specDeleteFlag := cli.BoolFlag{Name: "delete", Usage: "Allow apply to delete resources that no longer exist in the specification"} specSubCommands := []cli.Command{ - {Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag, specDeployIDFlag}, Action: specInit}, - {Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: specValidate}, - {Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: specApply}, - {Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: specDestroy}, - {Name: "helm", Usage: "Create a helm chart from the app specification", Flags: []cli.Flag{specDirFlag}, Action: specHelm, Hidden: true}, + {Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag, specDeployIDFlag}, Action: urfavecli.Wrapper(spec.Init)}, + {Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: urfavecli.Wrapper(spec.Validate)}, + {Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: urfavecli.Wrapper(spec.Apply)}, + {Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: urfavecli.Wrapper(spec.Destroy)}, } // support diff --git a/pkg/fission-cli/package.go b/pkg/fission-cli/package.go deleted file mode 100644 index 3d2d56d5..00000000 --- a/pkg/fission-cli/package.go +++ /dev/null @@ -1,794 +0,0 @@ -/* -Copyright 2017 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fission_cli - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "sort" - "strings" - "text/tabwriter" - "time" - - "github.com/dchest/uniuri" - "github.com/fission/fission/pkg/utils" - "github.com/hashicorp/go-multierror" - "github.com/mholt/archiver" - "github.com/pkg/errors" - "github.com/satori/go.uuid" - "github.com/urfave/cli" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" - "github.com/fission/fission/pkg/controller/client" - "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" - cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" - "github.com/fission/fission/pkg/fission-cli/cmd/spec" - "github.com/fission/fission/pkg/fission-cli/log" - "github.com/fission/fission/pkg/fission-cli/util" - storageSvcClient "github.com/fission/fission/pkg/storagesvc/client" - "github.com/fission/fission/pkg/types" -) - -func getFunctionsByPackage(client *client.Client, pkgName, pkgNamespace string) ([]fv1.Function, error) { - fnList, err := client.FunctionList(pkgNamespace) - if err != nil { - return nil, err - } - fns := []fv1.Function{} - for _, fn := range fnList { - if fn.Spec.Package.PackageRef.Name == pkgName { - fns = append(fns, fn) - } - } - return fns, nil -} - -// downloadStoragesvcURL downloads and return archive content with given storage service url -func downloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser { - u, err := url.ParseRequestURI(fileUrl) - if err != nil { - return nil - } - - // replace in-cluster storage service host with controller server url - fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage/" + u.RequestURI() - reader, err := downloadURL(fileDownloadUrl) - - util.CheckErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl)) - return reader -} - -func pkgCreate(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgNamespace := c.String("pkgNamespace") - envName := c.String("env") - if len(envName) == 0 { - log.Fatal("Need --env argument.") - } - envNamespace := c.String("envNamespace") - srcArchiveFiles := c.StringSlice("src") - deployArchiveFiles := c.StringSlice("deploy") - buildcmd := c.String("buildcmd") - - if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 { - log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") - } - - createPackage(c, client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false) - - return nil -} - -func pkgUpdate(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - if len(pkgName) == 0 { - log.Fatal("Need --name argument.") - } - pkgNamespace := c.String("pkgNamespace") - - force := c.Bool("f") - envName := c.String("env") - envNamespace := c.String("envNamespace") - srcArchiveFiles := c.StringSlice("src") - deployArchiveFiles := c.StringSlice("deploy") - buildcmd := c.String("buildcmd") - - if len(srcArchiveFiles) > 0 && len(deployArchiveFiles) > 0 { - log.Fatal("Need either of --src or --deploy and not both arguments.") - } - - if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 && - len(envName) == 0 && len(buildcmd) == 0 { - log.Fatal("Need --env or --src or --deploy or --buildcmd argument.") - } - - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) - util.CheckErr(err, "get package") - - // if the new env specified is the same as the old one, no need to update package - // same is true for all update parameters, but, for now, we dont check all of them - because, its ok to - // re-write the object with same old values, we just end up getting a new resource version for the object. - if len(envName) > 0 && envName == pkg.Spec.Environment.Name { - envName = "" - } - - if envNamespace == pkg.Spec.Environment.Namespace { - envNamespace = "" - } - - fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace) - util.CheckErr(err, "get function list") - - if !force && len(fnList) > 1 { - log.Fatal("Package is used by multiple functions, use --force to force update") - } - - newPkgMeta, err := updatePackage(client, pkg, - envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, false) - if err != nil { - util.CheckErr(err, "update package") - } - - // update resource version of package reference of functions that shared the same package - for _, fn := range fnList { - fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion - _, err := client.FunctionUpdate(&fn) - util.CheckErr(err, "update function") - } - - fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName()) - - return nil -} - -func updatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string, - srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) { - - var srcArchiveMetadata, deployArchiveMetadata *fv1.Archive - needToBuild := false - - if len(envName) > 0 { - pkg.Spec.Environment.Name = envName - needToBuild = true - } - - if len(envNamespace) > 0 { - pkg.Spec.Environment.Namespace = envNamespace - needToBuild = true - } - - if len(buildcmd) > 0 { - pkg.Spec.BuildCommand = buildcmd - needToBuild = true - } - - if len(srcArchiveFiles) > 0 { - srcArchiveMetadata = createArchive(client, srcArchiveFiles, false, "", "") - pkg.Spec.Source = *srcArchiveMetadata - needToBuild = true - } - - if len(deployArchiveFiles) > 0 { - deployArchiveMetadata = createArchive(client, deployArchiveFiles, noZip, "", "") - pkg.Spec.Deployment = *deployArchiveMetadata - // Users may update the env, envNS and deploy archive at the same time, - // but without the source archive. In this case, we should set needToBuild to false - needToBuild = false - } - - // Set package as pending status when needToBuild is true - if needToBuild || forceRebuild { - // change into pending state to trigger package build - pkg.Status = fv1.PackageStatus{ - BuildStatus: fv1.BuildStatusPending, - LastUpdateTimestamp: time.Now().UTC(), - } - } - - newPkgMeta, err := client.PackageUpdate(pkg) - util.CheckErr(err, "update package") - - return newPkgMeta, err -} - -func pkgSourceGet(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - if len(pkgName) == 0 { - log.Fatal("Need name of package, use --name") - } - pkgNamespace := c.String("pkgNamespace") - - output := c.String("output") - - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) - if err != nil { - return err - } - - var reader io.Reader - - if pkg.Spec.Source.Type == fv1.ArchiveTypeLiteral { - reader = bytes.NewReader(pkg.Spec.Source.Literal) - } else if pkg.Spec.Source.Type == fv1.ArchiveTypeUrl { - readCloser := downloadStoragesvcURL(client, pkg.Spec.Source.URL) - defer readCloser.Close() - reader = readCloser - } - - if len(output) > 0 { - return writeArchiveToFile(output, reader) - } else { - _, err := io.Copy(os.Stdout, reader) - return err - } -} - -func pkgDeployGet(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - if len(pkgName) == 0 { - log.Fatal("Need name of package, use --name") - } - pkgNamespace := c.String("pkgNamespace") - - output := c.String("output") - - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) - if err != nil { - return err - } - - var reader io.Reader - - if pkg.Spec.Deployment.Type == fv1.ArchiveTypeLiteral { - reader = bytes.NewReader(pkg.Spec.Deployment.Literal) - } else if pkg.Spec.Deployment.Type == fv1.ArchiveTypeUrl { - readCloser := downloadStoragesvcURL(client, pkg.Spec.Deployment.URL) - defer readCloser.Close() - reader = readCloser - } - - if len(output) > 0 { - return writeArchiveToFile(output, reader) - } else { - _, err := io.Copy(os.Stdout, reader) - return err - } -} - -func pkgInfo(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - if len(pkgName) == 0 { - log.Fatal("Need name of package, use --name") - } - pkgNamespace := c.String("pkgNamespace") - - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) - if err != nil { - util.CheckErr(err, fmt.Sprintf("find package %s", pkgName)) - } - - w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name) - fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name) - fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus) - fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog) - w.Flush() - - return nil -} - -func pkgList(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - // option for the user to list all orphan packages (not referenced by any function) - listOrphans := c.Bool("orphan") - status := c.String("status") - pkgNamespace := c.String("pkgNamespace") - - pkgList, err := client.PackageList(pkgNamespace) - if err != nil { - return err - } - - // sort the package list by lastUpdatedTimestamp - sort.Slice(pkgList, func(i, j int) bool { - return pkgList[i].Status.LastUpdateTimestamp.After(pkgList[j].Status.LastUpdateTimestamp) - }) - - w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV", "LASTUPDATEDAT") - if listOrphans { - for _, pkg := range pkgList { - fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace) - util.CheckErr(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name)) - if len(fnList) == 0 { - if status == "" { - fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name) - } else if status == string(pkg.Status.BuildStatus) { - fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name) - } - } - } - } else { - for _, pkg := range pkgList { - if status == "" { - fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name) - } else if status == string(pkg.Status.BuildStatus) { - fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name) - } - } - } - - w.Flush() - - return nil -} - -func deleteOrphanPkgs(client *client.Client, pkgNamespace string) error { - pkgList, err := client.PackageList(pkgNamespace) - if err != nil { - return err - } - - // range through all packages and find out the ones not referenced by any function - for _, pkg := range pkgList { - fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkgNamespace) - util.CheckErr(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name)) - if len(fnList) == 0 { - err = deletePackage(client, pkg.Metadata.Name, pkgNamespace) - if err != nil { - return err - } - } - } - return nil -} - -func deletePackage(client *client.Client, pkgName string, pkgNamespace string) error { - return client.PackageDelete(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) -} - -func pkgDelete(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - pkgNamespace := c.String("pkgNamespace") - deleteOrphans := c.Bool("orphan") - - if len(pkgName) == 0 && !deleteOrphans { - fmt.Println("Need --name argument or --orphan flag.") - return nil - } - if len(pkgName) != 0 && deleteOrphans { - fmt.Println("Need either --name argument or --orphan flag") - return nil - } - - if len(pkgName) != 0 { - force := c.Bool("f") - - _, err := client.PackageGet(&metav1.ObjectMeta{ - Namespace: pkgNamespace, - Name: pkgName, - }) - util.CheckErr(err, "find package") - - fnList, err := getFunctionsByPackage(client, pkgName, pkgNamespace) - if err != nil { - return err - } - - if !force && len(fnList) > 0 { - log.Fatal("Package is used by at least one function, use -f to force delete") - } - - err = deletePackage(client, pkgName, pkgNamespace) - if err != nil { - return err - } - - fmt.Printf("Package '%v' deleted\n", pkgName) - } else { - err := deleteOrphanPkgs(client, pkgNamespace) - util.CheckErr(err, "error deleting orphan packages") - fmt.Println("Orphan packages deleted") - } - - return nil -} - -func pkgRebuild(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - pkgName := c.String("name") - if len(pkgName) == 0 { - log.Fatal("Need name of package, use --name") - } - pkgNamespace := c.String("pkgNamespace") - - pkg, err := client.PackageGet(&metav1.ObjectMeta{ - Name: pkgName, - Namespace: pkgNamespace, - }) - util.CheckErr(err, "find package") - - if pkg.Status.BuildStatus != fv1.BuildStatusFailed { - log.Fatal(fmt.Sprintf("Package %v is not in %v state.", - pkg.Metadata.Name, fv1.BuildStatusFailed)) - } - - _, err = updatePackage(client, pkg, "", "", nil, nil, "", true, false) - util.CheckErr(err, "update package") - - fmt.Printf("Retrying build for pkg %v. Use \"fission pkg info --name %v\" to view status.\n", pkg.Metadata.Name, pkg.Metadata.Name) - - return nil -} - -func fileSize(filePath string) int64 { - info, err := os.Stat(filePath) - util.CheckErr(err, fmt.Sprintf("stat %v", filePath)) - return info.Size() -} - -func fileChecksum(fileName string) (*fv1.Checksum, error) { - f, err := os.Open(fileName) - if err != nil { - return nil, fmt.Errorf("failed to open file %v: %v", fileName, err) - } - defer f.Close() - - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return nil, fmt.Errorf("failed to calculate checksum for %v", fileName) - } - - return &fv1.Checksum{ - Type: fv1.ChecksumTypeSHA256, - Sum: hex.EncodeToString(h.Sum(nil)), - }, nil -} - -// Return a fv1.Archive made from an archive . If specFile, then -// create an archive upload spec in the specs directory; otherwise -// upload the archive using client. noZip avoids zipping the -// includeFiles, but is ignored if there's more than one includeFile. -func createArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) *fv1.Archive { - - errs := &multierror.Error{} - - // check files existence - for _, path := range includeFiles { - // ignore http files - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - continue - } - - // Get files from inputs as number of files decide next steps - files, err := utils.FindAllGlobs([]string{path}) - if err != nil { - util.CheckErr(err, "finding all globs") - } - - if len(files) == 0 { - errs = multierror.Append(errs, errors.New(fmt.Sprintf("Error finding any files with path \"%v\"", path))) - } - } - - if errs.ErrorOrNil() != nil { - log.Fatal(errs.Error()) - } - - if len(specFile) > 0 { - // create an ArchiveUploadSpec and reference it from the archive - aus := &spec.ArchiveUploadSpec{ - Name: archiveName("", includeFiles), - IncludeGlobs: includeFiles, - } - - // check if this AUS exists in the specs; if so, don't create a new one - fr, err := readSpecs(specDir) - util.CheckErr(err, "read specs") - if m := fr.SpecExists(aus, false, true); m != nil { - fmt.Printf("Re-using previously created archive %v\n", m.Name) - aus.Name = m.Name - } else { - // save the uploadspec - err := spec.SpecSave(*aus, specFile) - util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile)) - } - - // create the archive object - ar := &fv1.Archive{ - Type: fv1.ArchiveTypeUrl, - URL: fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name), - } - return ar - } - - archivePath := makeArchiveFileIfNeeded("", includeFiles, noZip) - - ctx := context.Background() - return uploadArchive(ctx, client, archivePath) -} - -func uploadArchive(ctx context.Context, client *client.Client, fileName string) *fv1.Archive { - var archive fv1.Archive - - // If filename is a URL, download it first - if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") { - fileName = downloadToTempFile(fileName) - } - - if fileSize(fileName) < types.ArchiveLiteralSizeLimit { - archive.Type = fv1.ArchiveTypeLiteral - archive.Literal = getContents(fileName) - } else { - u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" - ssClient := storageSvcClient.MakeClient(u) - - // TODO add a progress bar - id, err := ssClient.Upload(ctx, fileName, nil) - util.CheckErr(err, fmt.Sprintf("upload file %v", fileName)) - - storageSvc, err := client.GetSvcURL("application=fission-storage") - storageSvcURL := "http://" + storageSvc - util.CheckErr(err, "get fission storage service name") - - // We make a new client with actual URL of Storage service so that the URL is not - // pointing to 127.0.0.1 i.e. proxy. DON'T reuse previous ssClient - pkgClient := storageSvcClient.MakeClient(storageSvcURL) - archiveURL := pkgClient.GetUrl(id) - - archive.Type = fv1.ArchiveTypeUrl - archive.URL = archiveURL - - csum, err := fileChecksum(fileName) - util.CheckErr(err, fmt.Sprintf("calculate checksum for file %v", fileName)) - - archive.Checksum = *csum - } - return &archive -} - -func createPackage(c *cli.Context, client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) *metav1.ObjectMeta { - pkgSpec := fv1.PackageSpec{ - Environment: fv1.EnvironmentReference{ - Namespace: envNamespace, - Name: envName, - }, - } - var pkgStatus fv1.BuildStatus = fv1.BuildStatusSucceeded - - var pkgName string - if len(deployArchiveFiles) > 0 { - if len(specFile) > 0 { // we should do this in all cases, i think - pkgStatus = fv1.BuildStatusNone - } - pkgSpec.Deployment = *createArchive(client, deployArchiveFiles, noZip, specDir, specFile) - pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveFiles[0]), uniuri.NewLen(4))) - } - if len(srcArchiveFiles) > 0 { - pkgSpec.Source = *createArchive(client, srcArchiveFiles, false, specDir, specFile) - pkgStatus = fv1.BuildStatusPending // set package build status to pending - pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveFiles[0]), uniuri.NewLen(4))) - } - - if len(buildcmd) > 0 { - pkgSpec.BuildCommand = buildcmd - } - - if len(pkgName) == 0 { - pkgName = strings.ToLower(uuid.NewV4().String()) - } - pkg := &fv1.Package{ - Metadata: metav1.ObjectMeta{ - Name: pkgName, - Namespace: pkgNamespace, - }, - Spec: pkgSpec, - Status: fv1.PackageStatus{ - BuildStatus: pkgStatus, - LastUpdateTimestamp: time.Now().UTC(), - }, - } - - if len(specFile) > 0 { - // if a package with the same spec exists, don't create a new spec file - fr, err := readSpecs(cmdutils.GetSpecDir(urfavecli.Parse(c))) - util.CheckErr(err, "read specs") - if m := fr.SpecExists(pkg, false, true); m != nil { - fmt.Printf("Re-using previously created package %v\n", m.Name) - return m - } - - err = spec.SpecSave(*pkg, specFile) - util.CheckErr(err, "save package spec") - return &pkg.Metadata - } else { - pkgMetadata, err := client.PackageCreate(pkg) - util.CheckErr(err, "create package") - fmt.Printf("Package '%v' created\n", pkgMetadata.GetName()) - return pkgMetadata - } -} - -func getContents(filePath string) []byte { - var code []byte - var err error - - code, err = ioutil.ReadFile(filePath) - util.CheckErr(err, fmt.Sprintf("read %v", filePath)) - return code -} - -func writeArchiveToFile(fileName string, reader io.Reader) error { - tmpDir, err := utils.GetTempDir() - if err != nil { - return err - } - - path := filepath.Join(tmpDir, fileName+".tmp") - w, err := os.Create(path) - if err != nil { - return err - } - _, err = io.Copy(w, reader) - if err != nil { - return err - } - err = os.Chmod(path, 0644) - if err != nil { - return err - } - - err = os.Rename(path, fileName) - if err != nil { - return err - } - - return nil -} - -// downloadToTempFile fetches archive file from arbitrary url -// and write it to temp file for further usage -func downloadToTempFile(fileUrl string) string { - reader, err := downloadURL(fileUrl) - util.CheckErr(err, fmt.Sprintf("download from url: %v", fileUrl)) - defer reader.Close() - - tmpDir, err := utils.GetTempDir() - util.CheckErr(err, "create temp directory") - - tmpFilename := uuid.NewV4().String() - destination := filepath.Join(tmpDir, tmpFilename) - err = os.Mkdir(tmpDir, 0744) - util.CheckErr(err, "create temp directory") - - err = writeArchiveToFile(destination, reader) - util.CheckErr(err, "write archive to file") - - return destination -} - -// downloadURL downloads file from given url -func downloadURL(fileUrl string) (io.ReadCloser, error) { - resp, err := http.Get(fileUrl) - if err != nil { - return nil, err - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode) - } - return resp.Body, nil -} - -// Create an archive from the given list of input files, unless that -// list has only one item and that item is either a zip file or a URL. -// -// If the inputs have only one file and noZip is true, the file is -// returned as-is with no zipping. (This is used for compatibility -// with v1 envs.) noZip is IGNORED if there is more than one input -// file. -func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZip bool) string { - - // Unique name for the archive - archiveName := archiveName(archiveNameHint, archiveInput) - - // Get files from inputs as number of files decide next steps - files, err := utils.FindAllGlobs(archiveInput) - if err != nil { - util.CheckErr(err, "finding all globs") - } - - // We have one file; if it's a zip file or a URL, no need to archive it - if len(files) == 1 { - // make sure it exists - if _, err := os.Stat(files[0]); err != nil { - util.CheckErr(err, fmt.Sprintf("open input file %v", files[0])) - } - - // if it's an existing zip file OR we're not supposed to zip it, don't do anything - if archiver.Zip.Match(files[0]) || noZip { - return files[0] - } - - // if it's an HTTP URL, just use the URL. - if strings.HasPrefix(files[0], "http://") || strings.HasPrefix(files[0], "https://") { - return files[0] - } - } - - // For anything else, create a new archive - tmpDir, err := utils.GetTempDir() - if err != nil { - util.CheckErr(err, "create temporary archive directory") - } - - archivePath, err := utils.MakeArchive(filepath.Join(tmpDir, archiveName), archiveInput...) - if err != nil { - util.CheckErr(err, "create archive file") - } - - return archivePath -} - -// Name an archive -func archiveName(givenNameHint string, includedFiles []string) string { - if len(givenNameHint) > 0 { - return fmt.Sprintf("%v-%v", givenNameHint, uniuri.NewLen(4)) - } - if len(includedFiles) == 0 { - return uniuri.NewLen(8) - } - return fmt.Sprintf("%v-%v", util.KubifyName(includedFiles[0]), uniuri.NewLen(4)) -} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index b0283db2..8350b7c8 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -17,15 +17,21 @@ limitations under the License. package utils import ( + "crypto/sha256" + "encoding/hex" "fmt" + "io" "io/ioutil" "net" + "os" "path/filepath" "github.com/mholt/archiver" uuid "github.com/satori/go.uuid" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" ) func UrlForFunction(name, namespace string) string { @@ -145,3 +151,30 @@ func GetImagePullPolicy(policy string) apiv1.PullPolicy { return apiv1.PullIfNotPresent } } + +func FileSize(filePath string) (int64, error) { + info, err := os.Stat(filePath) + if err != nil { + return 0, err + } + return info.Size(), err +} + +func FileChecksum(fileName string) (*fv1.Checksum, error) { + f, err := os.Open(fileName) + if err != nil { + return nil, fmt.Errorf("failed to open file %v: %v", fileName, err) + } + defer f.Close() + + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return nil, fmt.Errorf("failed to calculate checksum for %v", fileName) + } + + return &fv1.Checksum{ + Type: fv1.ChecksumTypeSHA256, + Sum: hex.EncodeToString(h.Sum(nil)), + }, nil +}