feature: Added '--specignore' option to pass file containing specs to be ignored for fission spec commands (#2239)

- New flag `--specignore` has been added for command fission spec [list | validate | apply | destroy]
- This flag is optional.
- The default spec ignore file name is `.specignore`, if the flag is not used
- The spec ignore file existence will be checked in the `--specdir` path
- If spec ignore file is passed to the flag, with a name other than .specignore and if it does not exist in the `--specdir` path then an error will be returned.
- `--specdir` will be the root path against which the patterns will be evaluated
- The behaviour of the spec ignore file will be similar to .gitignore (only for yml and yaml files). For reference: http://git-scm.com/docs/gitignore

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.26.0
 	github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -720,6 +720,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102 h1:WAQaHPfnpevd8SKXCcy5nk3JzEv2h5Q0kSwvoMqXiZs=
 github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=

pkg/fission-cli/cmd/function/create.go

@@ -73,6 +73,7 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 		opts.specFile = fmt.Sprintf("function-%v.yaml", fnName)
 	}
 	specDir := util.GetSpecDir(input)
+	specIgnore := util.GetSpecIgnore(input)
 
 	if !toSpec {
 		// check for unique function names within a namespace
@@ -126,7 +127,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 		var pkg *fv1.Package
 
 		if toSpec {
-			fr, err := spec.ReadSpecs(specDir)
+
+			fr, err := spec.ReadSpecs(specDir, specIgnore)
 			if err != nil {
 				return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 			}
@@ -166,8 +168,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 		}
 
 		if toSpec {
-			specDir := util.GetSpecDir(input)
+
-			fr, err := spec.ReadSpecs(specDir)
+			fr, err := spec.ReadSpecs(specDir, specIgnore)
 			if err != nil {
 				return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 			}

pkg/fission-cli/cmd/httptrigger/create.go

@@ -134,7 +134,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 	// For Specs, the spec validate checks for function reference
 	if input.Bool(flagkey.SpecSave) {
 		specDir := util.GetSpecDir(input)
-		fr, err := spec.ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := spec.ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 		}

pkg/fission-cli/cmd/kubewatch/create.go

@@ -66,7 +66,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 
 	if input.Bool(flagkey.SpecSave) {
 		specDir := util.GetSpecDir(input)
-		fr, err := spec.ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := spec.ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 		}

pkg/fission-cli/cmd/mqtrigger/create.go

@@ -127,7 +127,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 
 	if input.Bool(flagkey.SpecSave) {
 		specDir := util.GetSpecDir(input)
-		fr, err := spec.ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := spec.ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 		}

pkg/fission-cli/cmd/package/create.go

@@ -86,7 +86,8 @@ func (opts *CreateSubCommand) run(input cli.Input) error {
 
 	if input.Bool(flagkey.SpecSave) {
 		specDir = util.GetSpecDir(input)
-		fr, err := spec.ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := spec.ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 		}
@@ -185,7 +186,7 @@ func CreatePackage(input cli.Input, client client.Interface, pkgName string, pkg
 
 	if input.Bool(flagkey.SpecSave) {
 		// if a package with the same spec exists, don't create a new spec file
-		fr, err := spec.ReadSpecs(util.GetSpecDir(input))
+		fr, err := spec.ReadSpecs(util.GetSpecDir(input), util.GetSpecIgnore(input))
 		if err != nil {
 			return nil, errors.Wrap(err, "error reading specs")
 		}

pkg/fission-cli/cmd/package/package.go

@@ -161,7 +161,8 @@ func CreateArchive(client client.Interface, input cli.Input, includeFiles []stri
 			}
 		} else if input.Bool(flagkey.SpecSave) {
 			// check if this AUS exists in the specs; if so, don't create a new one
-			fr, err := spec.ReadSpecs(specDir)
+			specIgnore := util.GetSpecIgnore(input)
+			fr, err := spec.ReadSpecs(specDir, specIgnore)
 			if err != nil {
 				return nil, errors.Wrap(err, "error reading specs")
 			}

pkg/fission-cli/cmd/spec/apply.go

@@ -66,6 +66,7 @@ func (opts *ApplySubCommand) do(input cli.Input) error {
 
 func (opts *ApplySubCommand) run(input cli.Input) error {
 	specDir := util.GetSpecDir(input)
+	specIgnore := util.GetSpecIgnore(input)
 
 	deleteResources := input.Bool(flagkey.SpecDelete)
 	watchResources := input.Bool(flagkey.SpecWatch)
@@ -111,7 +112,7 @@ func (opts *ApplySubCommand) run(input cli.Input) error {
 
 	for {
 		// read all specs
-		fr, err := ReadSpecs(specDir)
+		fr, err := ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, "error reading specs")
 		}

pkg/fission-cli/cmd/spec/command.go

@@ -39,7 +39,7 @@ func Commands() *cobra.Command {
 		RunE:  wrapper.Wrapper(Validate),
 	}
 	wrapper.SetFlags(validateCmd, flag.FlagSet{
-		Optional: []flag.Flag{flag.SpecDir},
+		Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore},
 	})
 
 	applyCmd := &cobra.Command{
@@ -48,7 +48,7 @@ func Commands() *cobra.Command {
 		RunE:  wrapper.Wrapper(Apply),
 	}
 	wrapper.SetFlags(applyCmd, flag.FlagSet{
-		Optional: []flag.Flag{flag.SpecDir, flag.SpecDelete, flag.SpecWait, flag.SpecWatch, flag.SpecValidation},
+		Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore, flag.SpecDelete, flag.SpecWait, flag.SpecWatch, flag.SpecValidation},
 	})
 
 	destroyCmd := &cobra.Command{
@@ -57,7 +57,7 @@ func Commands() *cobra.Command {
 		RunE:  wrapper.Wrapper(Destroy),
 	}
 	wrapper.SetFlags(destroyCmd, flag.FlagSet{
-		Optional: []flag.Flag{flag.SpecDir},
+		Optional: []flag.Flag{flag.SpecDir, flag.SpecIgnore},
 	})
 
 	listCmd := &cobra.Command{
@@ -66,7 +66,7 @@ func Commands() *cobra.Command {
 		RunE:  wrapper.Wrapper(List),
 	}
 	wrapper.SetFlags(listCmd, flag.FlagSet{
-		Optional: []flag.Flag{flag.SpecDeployID, flag.SpecDir},
+		Optional: []flag.Flag{flag.SpecDeployID, flag.SpecDir, flag.SpecIgnore},
 	})
 
 	command := &cobra.Command{

pkg/fission-cli/cmd/spec/destroy.go

@@ -38,11 +38,12 @@ func (opts *DestroySubCommand) do(input cli.Input) error {
 }
 
 func (opts *DestroySubCommand) run(input cli.Input) error {
-	// get specdir
+	// get specdir and specignore
 	specDir := util.GetSpecDir(input)
+	specIgnore := util.GetSpecIgnore(input)
 
 	// read everything
-	fr, err := ReadSpecs(specDir)
+	fr, err := ReadSpecs(specDir, specIgnore)
 	if err != nil {
 		return errors.Wrap(err, "error reading specs")
 	}

pkg/fission-cli/cmd/spec/list.go

@@ -50,9 +50,10 @@ func (opts *ListSubCommand) do(input cli.Input) error {
 func (opts *ListSubCommand) run(input cli.Input) error {
 	deployID := input.String(flagkey.SpecDeployID)
 	if len(deployID) == 0 {
-		// get specdir and read the deployID
+		// get specdir, specignore and read the deployID
 		specDir := util.GetSpecDir(input)
-		fr, err := ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, "error reading specs")
 		}

pkg/fission-cli/cmd/spec/spec.go

@@ -171,7 +171,7 @@ func SpecSave(resource interface{}, specFile string) error {
 		return err
 	}
 
-	fr, err := ReadSpecs(specDir)
+	fr, err := ReadSpecs(specDir, util.SPEC_IGNORE_FILE)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 	}

pkg/fission-cli/cmd/spec/validate.go

@@ -53,7 +53,8 @@ func (opts *ValidateSubCommand) run(input cli.Input) error {
 
 	// this will error on parse errors and on duplicates
 	specDir := util.GetSpecDir(input)
-	fr, err := ReadSpecs(specDir)
+	specIgnore := util.GetSpecIgnore(input)
+	fr, err := ReadSpecs(specDir, specIgnore)
 	if err != nil {
 		return errors.Wrap(err, "error reading specs")
 	}
@@ -197,7 +198,7 @@ func isResourceConflicts(deployUID string, specObj fv1.MetadataAccessor, cluster
 
 // ReadSpecs reads all specs in the specified directory and returns a parsed set of
 // fission resources.
-func ReadSpecs(specDir string) (*FissionResources, error) {
+func ReadSpecs(specDir, specIgnore string) (*FissionResources, error) {
 
 	// make sure spec directory exists before continue
 	if _, err := os.Stat(specDir); os.IsNotExist(err) {
@@ -205,6 +206,11 @@ func ReadSpecs(specDir string) (*FissionResources, error) {
 			"Please check directory path or run \"fission spec init\" to create it.", specDir)
 	}
 
+	ignoreParser, err := util.GetSpecIgnoreParser(specDir, specIgnore)
+	if err != nil {
+		return nil, err
+	}
+
 	fr := FissionResources{
 		Packages:                make([]fv1.Package, 0),
 		Functions:               make([]fv1.Function, 0),
@@ -222,7 +228,7 @@ func ReadSpecs(specDir string) (*FissionResources, error) {
 	var result *multierror.Error
 
 	// Users can organize the specdir into subdirs if they want to.
-	err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error {
+	err = filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -232,6 +238,11 @@ func ReadSpecs(specDir string) (*FissionResources, error) {
 		if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) {
 			return nil
 		}
+
+		if ignoreParser.MatchesPath(path) {
+			return nil
+		}
+
 		// read
 		b, err := os.ReadFile(path)
 		if err != nil {

pkg/fission-cli/cmd/timetrigger/create.go

@@ -77,7 +77,8 @@ func (opts *CreateSubCommand) complete(input cli.Input) error {
 
 	if input.Bool(flagkey.SpecSave) {
 		specDir := util.GetSpecDir(input)
-		fr, err := spec.ReadSpecs(specDir)
+		specIgnore := util.GetSpecIgnore(input)
+		fr, err := spec.ReadSpecs(specDir, specIgnore)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("error reading spec in '%v'", specDir))
 		}

pkg/fission-cli/flag/flag.go

@@ -17,6 +17,7 @@ limitations under the License.
 package flag
 
 import (
+	"fmt"
 	"net/http"
 	"time"
 
@@ -24,6 +25,7 @@ import (
 
 	fv1 "github.com/fission/fission/pkg/apis/core/v1"
 	flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
+	"github.com/fission/fission/pkg/fission-cli/util"
 )
 
 type (
@@ -198,6 +200,7 @@ var (
 	SpecDelete     = Flag{Type: Bool, Name: flagkey.SpecDelete, Usage: "Allow apply to delete resources that no longer exist in the specification"}
 	SpecDry        = Flag{Type: Bool, Name: flagkey.SpecDry, Usage: "View the generated specs"}
 	SpecValidation = Flag{Type: String, Name: flagkey.SpecValidate, Usage: "Turns server side validations of Fission objects on/off"}
+	SpecIgnore     = Flag{Type: String, Name: flagkey.SpecIgnore, Usage: fmt.Sprintf("File containing specs to be ingored inside --specdir, defaults to %v", util.SPEC_IGNORE_FILE)}
 
 	SupportOutput = Flag{Type: String, Name: flagkey.SupportOutput, Short: "o", Usage: "Output directory to save dump archive/files", DefaultValue: flagkey.DefaultSpecOutputDir}
 	SupportNoZip  = Flag{Type: Bool, Name: flagkey.SupportNoZip, Usage: "Save dump information into multiple files instead of single zip file"}

pkg/fission-cli/flag/key/key.go

@@ -151,6 +151,7 @@ const (
 	SpecDelete   = "delete"
 	SpecDry      = "dry"
 	SpecValidate = "validation"
+	SpecIgnore   = "specignore"
 
 	SupportOutput = Output
 	SupportNoZip  = "nozip"

pkg/fission-cli/util/constants.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2021 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+// fission-cli options
+const (
+	SPEC_IGNORE_FILE = ".specignore"
+)

pkg/fission-cli/util/util.go

@@ -25,7 +25,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/fission/fission/pkg/controller/client/rest"
+	ignore "github.com/sabhiram/go-gitignore"
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
@@ -38,6 +38,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/controller/client/rest"
 	"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
 	"github.com/fission/fission/pkg/fission-cli/console"
 	flagkey "github.com/fission/fission/pkg/fission-cli/flag/key"
@@ -317,6 +318,33 @@ func GetSpecDir(input cli.Input) string {
 	return specDir
 }
 
+func GetSpecIgnore(input cli.Input) string {
+	specIgnoreFile := input.String(flagkey.SpecIgnore)
+	if len(specIgnoreFile) == 0 {
+		specIgnoreFile = SPEC_IGNORE_FILE
+	}
+	return specIgnoreFile
+}
+
+// GetSpecIgnoreParser reads the specignore file and returns the ignore.IgnoreParser
+// if the specignore file does not exist it returns empty ignore.IgnoreParser
+func GetSpecIgnoreParser(specDir, specIgnore string) (ignore.IgnoreParser, error) {
+
+	specIgnorePath := filepath.Join(specDir, specIgnore)
+
+	// check for existence of spec ignore file
+	if _, err := os.Stat(specIgnorePath); errors.Is(err, os.ErrNotExist) {
+		// return error if it's custom spec ignore file
+		if specIgnore != SPEC_IGNORE_FILE {
+			return nil, errors.Errorf("Spec ignore file '%s' doesn't exist. "+
+				"Please check the file path: '%s'", specIgnore, specIgnorePath)
+		}
+		return ignore.CompileIgnoreLines(), nil
+	}
+
+	return ignore.CompileIgnoreFile(specIgnorePath)
+}
+
 func GetValidationFlag(input cli.Input) bool {
 	validationFlag := input.String(flagkey.SpecValidate)
 	// if flag has not been set, we return true to turn on validation by default