From a5fb5b99016d73b7bbf7d57c6ff9b55c60118a28 Mon Sep 17 00:00:00 2001 From: Pradeep Lakshmi Narasimha Date: Fri, 29 Oct 2021 13:26:42 +0530 Subject: [PATCH] feature: Added '--specignore' option to pass file containing specs to be ignored for fission spec commands (#2239) - New flag `--specignore` has been added for command fission spec [list | validate | apply | destroy] - This flag is optional. - The default spec ignore file name is `.specignore`, if the flag is not used - The spec ignore file existence will be checked in the `--specdir` path - If spec ignore file is passed to the flag, with a name other than .specignore and if it does not exist in the `--specdir` path then an error will be returned. - `--specdir` will be the root path against which the patterns will be evaluated - The behaviour of the spec ignore file will be similar to .gitignore (only for yml and yaml files). For reference: http://git-scm.com/docs/gitignore --- go.mod | 1 + go.sum | 2 ++ pkg/fission-cli/cmd/function/create.go | 8 +++--- pkg/fission-cli/cmd/httptrigger/create.go | 3 ++- pkg/fission-cli/cmd/kubewatch/create.go | 3 ++- pkg/fission-cli/cmd/mqtrigger/create.go | 3 ++- pkg/fission-cli/cmd/package/create.go | 5 ++-- pkg/fission-cli/cmd/package/package.go | 3 ++- pkg/fission-cli/cmd/spec/apply.go | 3 ++- pkg/fission-cli/cmd/spec/command.go | 8 +++--- pkg/fission-cli/cmd/spec/destroy.go | 5 ++-- pkg/fission-cli/cmd/spec/list.go | 5 ++-- pkg/fission-cli/cmd/spec/spec.go | 2 +- pkg/fission-cli/cmd/spec/validate.go | 17 ++++++++++--- pkg/fission-cli/cmd/timetrigger/create.go | 3 ++- pkg/fission-cli/flag/flag.go | 3 +++ pkg/fission-cli/flag/key/key.go | 1 + pkg/fission-cli/util/constants.go | 22 +++++++++++++++++ pkg/fission-cli/util/util.go | 30 ++++++++++++++++++++++- 19 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 pkg/fission-cli/util/constants.go diff --git a/go.mod b/go.mod index cb2fc51d..e61fbc19 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.26.0 github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 3ab385db..ebe22088 100644 --- a/go.sum +++ b/go.sum @@ -720,6 +720,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= +github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102 h1:WAQaHPfnpevd8SKXCcy5nk3JzEv2h5Q0kSwvoMqXiZs= github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= diff --git a/pkg/fission-cli/cmd/function/create.go b/pkg/fission-cli/cmd/function/create.go index 0e67ff5d..d3c5ccdd 100644 --- a/pkg/fission-cli/cmd/function/create.go +++ b/pkg/fission-cli/cmd/function/create.go @@ -73,6 +73,7 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { opts.specFile = fmt.Sprintf("function-%v.yaml", fnName) } specDir := util.GetSpecDir(input) + specIgnore := util.GetSpecIgnore(input) if !toSpec { // check for unique function names within a namespace @@ -126,7 +127,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { var pkg *fv1.Package if toSpec { - fr, err := spec.ReadSpecs(specDir) + + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } @@ -166,8 +168,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { } if toSpec { - specDir := util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/cmd/httptrigger/create.go b/pkg/fission-cli/cmd/httptrigger/create.go index d0a7f373..c466b5f4 100644 --- a/pkg/fission-cli/cmd/httptrigger/create.go +++ b/pkg/fission-cli/cmd/httptrigger/create.go @@ -134,7 +134,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { // For Specs, the spec validate checks for function reference if input.Bool(flagkey.SpecSave) { specDir := util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/cmd/kubewatch/create.go b/pkg/fission-cli/cmd/kubewatch/create.go index e4792df2..3b802e2e 100644 --- a/pkg/fission-cli/cmd/kubewatch/create.go +++ b/pkg/fission-cli/cmd/kubewatch/create.go @@ -66,7 +66,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { if input.Bool(flagkey.SpecSave) { specDir := util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/cmd/mqtrigger/create.go b/pkg/fission-cli/cmd/mqtrigger/create.go index 5c14e557..1f688ce7 100644 --- a/pkg/fission-cli/cmd/mqtrigger/create.go +++ b/pkg/fission-cli/cmd/mqtrigger/create.go @@ -127,7 +127,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { if input.Bool(flagkey.SpecSave) { specDir := util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/cmd/package/create.go b/pkg/fission-cli/cmd/package/create.go index b15975ac..a0c47dbf 100644 --- a/pkg/fission-cli/cmd/package/create.go +++ b/pkg/fission-cli/cmd/package/create.go @@ -86,7 +86,8 @@ func (opts *CreateSubCommand) run(input cli.Input) error { if input.Bool(flagkey.SpecSave) { specDir = util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } @@ -185,7 +186,7 @@ func CreatePackage(input cli.Input, client client.Interface, pkgName string, pkg if input.Bool(flagkey.SpecSave) { // if a package with the same spec exists, don't create a new spec file - fr, err := spec.ReadSpecs(util.GetSpecDir(input)) + fr, err := spec.ReadSpecs(util.GetSpecDir(input), util.GetSpecIgnore(input)) if err != nil { return nil, errors.Wrap(err, "error reading specs") } diff --git a/pkg/fission-cli/cmd/package/package.go b/pkg/fission-cli/cmd/package/package.go index 15ca96de..91aaa0fd 100644 --- a/pkg/fission-cli/cmd/package/package.go +++ b/pkg/fission-cli/cmd/package/package.go @@ -161,7 +161,8 @@ func CreateArchive(client client.Interface, input cli.Input, includeFiles []stri } } else if input.Bool(flagkey.SpecSave) { // check if this AUS exists in the specs; if so, don't create a new one - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return nil, errors.Wrap(err, "error reading specs") } diff --git a/pkg/fission-cli/cmd/spec/apply.go b/pkg/fission-cli/cmd/spec/apply.go index 2f161867..dcc83557 100644 --- a/pkg/fission-cli/cmd/spec/apply.go +++ b/pkg/fission-cli/cmd/spec/apply.go @@ -66,6 +66,7 @@ func (opts *ApplySubCommand) do(input cli.Input) error { func (opts *ApplySubCommand) run(input cli.Input) error { specDir := util.GetSpecDir(input) + specIgnore := util.GetSpecIgnore(input) deleteResources := input.Bool(flagkey.SpecDelete) watchResources := input.Bool(flagkey.SpecWatch) @@ -111,7 +112,7 @@ func (opts *ApplySubCommand) run(input cli.Input) error { for { // read all specs - fr, err := ReadSpecs(specDir) + fr, err := ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, "error reading specs") } diff --git a/pkg/fission-cli/cmd/spec/command.go b/pkg/fission-cli/cmd/spec/command.go index b45a1889..c1fafcd5 100644 --- a/pkg/fission-cli/cmd/spec/command.go +++ b/pkg/fission-cli/cmd/spec/command.go @@ -39,7 +39,7 @@ func Commands() *cobra.Command { RunE: wrapper.Wrapper(Validate), } wrapper.SetFlags(validateCmd, flag.FlagSet{ - Optional: []flag.Flag{flag.SpecDir}, + Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore}, }) applyCmd := &cobra.Command{ @@ -48,7 +48,7 @@ func Commands() *cobra.Command { RunE: wrapper.Wrapper(Apply), } wrapper.SetFlags(applyCmd, flag.FlagSet{ - Optional: []flag.Flag{flag.SpecDir, flag.SpecDelete, flag.SpecWait, flag.SpecWatch, flag.SpecValidation}, + Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore, flag.SpecDelete, flag.SpecWait, flag.SpecWatch, flag.SpecValidation}, }) destroyCmd := &cobra.Command{ @@ -57,7 +57,7 @@ func Commands() *cobra.Command { RunE: wrapper.Wrapper(Destroy), } wrapper.SetFlags(destroyCmd, flag.FlagSet{ - Optional: []flag.Flag{flag.SpecDir}, + Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore}, }) listCmd := &cobra.Command{ @@ -66,7 +66,7 @@ func Commands() *cobra.Command { RunE: wrapper.Wrapper(List), } wrapper.SetFlags(listCmd, flag.FlagSet{ - Optional: []flag.Flag{flag.SpecDeployID, flag.SpecDir}, + Optional: []flag.Flag{flag.SpecDeployID, flag.SpecDir, flag.SpecIgnore}, }) command := &cobra.Command{ diff --git a/pkg/fission-cli/cmd/spec/destroy.go b/pkg/fission-cli/cmd/spec/destroy.go index fd64fa2f..7c5d955f 100644 --- a/pkg/fission-cli/cmd/spec/destroy.go +++ b/pkg/fission-cli/cmd/spec/destroy.go @@ -38,11 +38,12 @@ func (opts *DestroySubCommand) do(input cli.Input) error { } func (opts *DestroySubCommand) run(input cli.Input) error { - // get specdir + // get specdir and specignore specDir := util.GetSpecDir(input) + specIgnore := util.GetSpecIgnore(input) // read everything - fr, err := ReadSpecs(specDir) + fr, err := ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, "error reading specs") } diff --git a/pkg/fission-cli/cmd/spec/list.go b/pkg/fission-cli/cmd/spec/list.go index d5c44caf..f752a983 100644 --- a/pkg/fission-cli/cmd/spec/list.go +++ b/pkg/fission-cli/cmd/spec/list.go @@ -50,9 +50,10 @@ func (opts *ListSubCommand) do(input cli.Input) error { func (opts *ListSubCommand) run(input cli.Input) error { deployID := input.String(flagkey.SpecDeployID) if len(deployID) == 0 { - // get specdir and read the deployID + // get specdir, specignore and read the deployID specDir := util.GetSpecDir(input) - fr, err := ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, "error reading specs") } diff --git a/pkg/fission-cli/cmd/spec/spec.go b/pkg/fission-cli/cmd/spec/spec.go index 7579db3b..48229b7c 100644 --- a/pkg/fission-cli/cmd/spec/spec.go +++ b/pkg/fission-cli/cmd/spec/spec.go @@ -171,7 +171,7 @@ func SpecSave(resource interface{}, specFile string) error { return err } - fr, err := ReadSpecs(specDir) + fr, err := ReadSpecs(specDir, util.SPEC_IGNORE_FILE) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/cmd/spec/validate.go b/pkg/fission-cli/cmd/spec/validate.go index cdda38e3..eb87e551 100644 --- a/pkg/fission-cli/cmd/spec/validate.go +++ b/pkg/fission-cli/cmd/spec/validate.go @@ -53,7 +53,8 @@ func (opts *ValidateSubCommand) run(input cli.Input) error { // this will error on parse errors and on duplicates specDir := util.GetSpecDir(input) - fr, err := ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, "error reading specs") } @@ -197,7 +198,7 @@ func isResourceConflicts(deployUID string, specObj fv1.MetadataAccessor, cluster // ReadSpecs reads all specs in the specified directory and returns a parsed set of // fission resources. -func ReadSpecs(specDir string) (*FissionResources, error) { +func ReadSpecs(specDir, specIgnore string) (*FissionResources, error) { // make sure spec directory exists before continue if _, err := os.Stat(specDir); os.IsNotExist(err) { @@ -205,6 +206,11 @@ func ReadSpecs(specDir string) (*FissionResources, error) { "Please check directory path or run \"fission spec init\" to create it.", specDir) } + ignoreParser, err := util.GetSpecIgnoreParser(specDir, specIgnore) + if err != nil { + return nil, err + } + fr := FissionResources{ Packages: make([]fv1.Package, 0), Functions: make([]fv1.Function, 0), @@ -222,7 +228,7 @@ func ReadSpecs(specDir string) (*FissionResources, error) { var result *multierror.Error // Users can organize the specdir into subdirs if they want to. - err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -232,6 +238,11 @@ func ReadSpecs(specDir string) (*FissionResources, error) { if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { return nil } + + if ignoreParser.MatchesPath(path) { + return nil + } + // read b, err := os.ReadFile(path) if err != nil { diff --git a/pkg/fission-cli/cmd/timetrigger/create.go b/pkg/fission-cli/cmd/timetrigger/create.go index 903da849..07b4f69d 100644 --- a/pkg/fission-cli/cmd/timetrigger/create.go +++ b/pkg/fission-cli/cmd/timetrigger/create.go @@ -77,7 +77,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error { if input.Bool(flagkey.SpecSave) { specDir := util.GetSpecDir(input) - fr, err := spec.ReadSpecs(specDir) + specIgnore := util.GetSpecIgnore(input) + fr, err := spec.ReadSpecs(specDir, specIgnore) if err != nil { return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir)) } diff --git a/pkg/fission-cli/flag/flag.go b/pkg/fission-cli/flag/flag.go index 48f6048b..abecf02c 100644 --- a/pkg/fission-cli/flag/flag.go +++ b/pkg/fission-cli/flag/flag.go @@ -17,6 +17,7 @@ limitations under the License. package flag import ( + "fmt" "net/http" "time" @@ -24,6 +25,7 @@ import ( fv1 "github.com/fission/fission/pkg/apis/core/v1" flagkey "github.com/fission/fission/pkg/fission-cli/flag/key" + "github.com/fission/fission/pkg/fission-cli/util" ) type ( @@ -198,6 +200,7 @@ var ( SpecDelete = Flag{Type: Bool, Name: flagkey.SpecDelete, Usage: "Allow apply to delete resources that no longer exist in the specification"} SpecDry = Flag{Type: Bool, Name: flagkey.SpecDry, Usage: "View the generated specs"} SpecValidation = Flag{Type: String, Name: flagkey.SpecValidate, Usage: "Turns server side validations of Fission objects on/off"} + SpecIgnore = Flag{Type: String, Name: flagkey.SpecIgnore, Usage: fmt.Sprintf("File containing specs to be ingored inside --specdir, defaults to %v", util.SPEC_IGNORE_FILE)} SupportOutput = Flag{Type: String, Name: flagkey.SupportOutput, Short: "o", Usage: "Output directory to save dump archive/files", DefaultValue: flagkey.DefaultSpecOutputDir} SupportNoZip = Flag{Type: Bool, Name: flagkey.SupportNoZip, Usage: "Save dump information into multiple files instead of single zip file"} diff --git a/pkg/fission-cli/flag/key/key.go b/pkg/fission-cli/flag/key/key.go index a45b71ad..34f80650 100644 --- a/pkg/fission-cli/flag/key/key.go +++ b/pkg/fission-cli/flag/key/key.go @@ -151,6 +151,7 @@ const ( SpecDelete = "delete" SpecDry = "dry" SpecValidate = "validation" + SpecIgnore = "specignore" SupportOutput = Output SupportNoZip = "nozip" diff --git a/pkg/fission-cli/util/constants.go b/pkg/fission-cli/util/constants.go new file mode 100644 index 00000000..fb070c44 --- /dev/null +++ b/pkg/fission-cli/util/constants.go @@ -0,0 +1,22 @@ +/* +Copyright 2021 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +// fission-cli options +const ( + SPEC_IGNORE_FILE = ".specignore" +) diff --git a/pkg/fission-cli/util/util.go b/pkg/fission-cli/util/util.go index 37ec661c..b9877bbd 100644 --- a/pkg/fission-cli/util/util.go +++ b/pkg/fission-cli/util/util.go @@ -25,7 +25,7 @@ import ( "strconv" "strings" - "github.com/fission/fission/pkg/controller/client/rest" + ignore "github.com/sabhiram/go-gitignore" "github.com/hashicorp/go-multierror" "github.com/pkg/errors" @@ -38,6 +38,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/controller/client/rest" "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" "github.com/fission/fission/pkg/fission-cli/console" flagkey "github.com/fission/fission/pkg/fission-cli/flag/key" @@ -317,6 +318,33 @@ func GetSpecDir(input cli.Input) string { return specDir } +func GetSpecIgnore(input cli.Input) string { + specIgnoreFile := input.String(flagkey.SpecIgnore) + if len(specIgnoreFile) == 0 { + specIgnoreFile = SPEC_IGNORE_FILE + } + return specIgnoreFile +} + +// GetSpecIgnoreParser reads the specignore file and returns the ignore.IgnoreParser +// if the specignore file does not exist it returns empty ignore.IgnoreParser +func GetSpecIgnoreParser(specDir, specIgnore string) (ignore.IgnoreParser, error) { + + specIgnorePath := filepath.Join(specDir, specIgnore) + + // check for existence of spec ignore file + if _, err := os.Stat(specIgnorePath); errors.Is(err, os.ErrNotExist) { + // return error if it's custom spec ignore file + if specIgnore != SPEC_IGNORE_FILE { + return nil, errors.Errorf("Spec ignore file '%s' doesn't exist. "+ + "Please check the file path: '%s'", specIgnore, specIgnorePath) + } + return ignore.CompileIgnoreLines(), nil + } + + return ignore.CompileIgnoreFile(specIgnorePath) +} + func GetValidationFlag(input cli.Input) bool { validationFlag := input.String(flagkey.SpecValidate) // if flag has not been set, we return true to turn on validation by default