Refactor environment CLI command (#1265)

2019-08-27 15:43:59 +08:00
parent 560aa34ce2
commit c49377569e
37 changed files with 1976 additions and 1599 deletions
@@ -0,0 +1,673 @@
+/*
+Copyright 2019 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package spec
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/generator/encoder"
+	v1generator "github.com/fission/fission/pkg/generator/v1"
+)
+
+var specDefaultEncoder = encoder.DefaultYAMLEncoder()
+
+const (
+	FISSION_DEPLOYMENT_NAME_KEY = "fission-name"
+	FISSION_DEPLOYMENT_UID_KEY  = "fission-uid"
+
+	SPEC_API_VERSION          = "fission.io/v1"
+	ARCHIVE_URL_PREFIX string = "archive://"
+	SPEC_README               = `
+Fission Specs
+=============
+
+This is a set of specifications for a Fission app.  This includes functions,
+environments, and triggers; we collectively call these things "resources".
+
+How to use these specs
+----------------------
+
+These specs are handled with the 'fission spec' command.  See 'fission spec --help'.
+
+'fission spec apply' will "apply" all resources specified in this directory to your
+cluster.  That means it checks what resources exist on your cluster, what resources are
+specified in the specs directory, and reconciles the difference by creating, updating or
+deleting resources on the cluster.
+
+'fission spec apply' will also package up your source code (or compiled binaries) and
+upload the archives to the cluster if needed.  It uses 'ArchiveUploadSpec' resources in
+this directory to figure out which files to archive.
+
+You can use 'fission spec apply --watch' to watch for file changes and continuously keep
+the cluster updated.
+
+You can add YAMLs to this directory by writing them manually, but it's easier to generate
+them.  Use 'fission function create --spec' to generate a function spec,
+'fission environment create --spec' to generate an environment spec, and so on.
+
+You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
+which contains a UID that you should never change.  To apply your changes simply use
+'fission spec apply'.
+
+fission-deployment-config.yaml
+------------------------------
+
+fission-deployment-config.yaml contains a UID.  This UID is what fission uses to correlate
+resources on the cluster to resources in this directory.
+
+All resources created by 'fission spec apply' are annotated with this UID.  Resources on
+the cluster that are _not_ annotated with this UID are never modified or deleted by
+fission.
+
+`
+)
+
+// CLI spec types
+type (
+	// DeploymentConfig is the global configuration for a set of Fission specs.
+	DeploymentConfig struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "DeploymentConfig".
+		TypeMeta `json:",inline"`
+
+		// Name is a user-friendly name for the deployment. It is also stored in
+		// all uploaded resources as an annotation.
+		Name string `json:"name"`
+
+		// UID uniquely identifies the deployment. It is stored as a label and
+		// used to find resources to clean up when local specs are changed.
+		UID string `json:"uid"`
+	}
+
+	// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
+	//
+	// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
+	// using the name specified in the archive.  The fission spec applier will
+	// replace the archive:// URL with a real HTTP URL after uploading the file.
+	ArchiveUploadSpec struct {
+		// TypeMeta describes the type of this object. It is inlined. The Kind
+		// field should always be "ArchiveUploadSpec".
+		TypeMeta `json:",inline"`
+
+		// Name is a local name that can be used to reference this archive. It
+		// must be unique; duplicate names will cause an error while handling
+		// specs.
+		Name string `json:"name"`
+
+		// RootDir specifies the root that the globs below are relative to. It
+		// is optional and defaults to the parent directory of the spec
+		// directory: for example, if the deployment config is at
+		// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
+		RootDir string `json:"rootdir,omitempty"`
+
+		// IncludeGlobs is a list of Unix shell globs to include
+		IncludeGlobs []string `json:"include,omitempty"`
+
+		// ExcludeGlobs is a list of globs to exclude from the set specified by
+		// IncludeGlobs.
+		ExcludeGlobs []string `json:"exclude,omitempty"`
+	}
+
+	// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
+	// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
+	// Kubernetes does.
+	TypeMeta struct {
+		Kind       string `json:"kind,omitempty"`
+		APIVersion string `json:"apiVersion,omitempty"`
+	}
+
+	FissionResources struct {
+		DeploymentConfig        DeploymentConfig
+		Packages                []fv1.Package
+		Functions               []fv1.Function
+		Environments            []fv1.Environment
+		HttpTriggers            []fv1.HTTPTrigger
+		KubernetesWatchTriggers []fv1.KubernetesWatchTrigger
+		TimeTriggers            []fv1.TimeTrigger
+		MessageQueueTriggers    []fv1.MessageQueueTrigger
+		ArchiveUploadSpecs      []ArchiveUploadSpec
+
+		SourceMap SourceMap
+	}
+
+	ResourceApplyStatus struct {
+		Created []*metav1.ObjectMeta
+		Updated []*metav1.ObjectMeta
+		Deleted []*metav1.ObjectMeta
+	}
+
+	Location struct {
+		Path string
+		Line int
+	}
+
+	SourceMap struct {
+		// kind -> namespace -> name -> location
+		Locations map[string](map[string](map[string]Location))
+	}
+)
+
+func MapKey(m *metav1.ObjectMeta) string {
+	return fmt.Sprintf("%v:%v", m.Namespace, m.Name)
+}
+
+// Save saves object encoded value to spec file under given spec directory
+func Save(data []byte, specDir string, specFile string) error {
+	// verify
+	if _, err := os.Stat(filepath.Join(specDir, "fission-deployment-config.yaml")); os.IsNotExist(err) {
+		return errors.Wrap(err, "Couldn't find specs, run `fission spec init` first")
+	}
+
+	filename := filepath.Join(specDir, specFile)
+	// check if the file is new
+	newFile := false
+	if _, err := os.Stat(filename); os.IsNotExist(err) {
+		newFile = true
+	}
+
+	// open spec file to append or write
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return errors.Wrap(err, "couldn't create spec file")
+	}
+	defer f.Close()
+
+	// if we're appending, add a yaml document separator
+	if !newFile {
+		_, err = f.Write([]byte("\n---\n"))
+		if err != nil {
+			return errors.Wrap(err, "couldn't write to spec file")
+		}
+	}
+
+	// write our resource
+	_, err = f.Write(data)
+	if err != nil {
+		return errors.Wrap(err, "couldn't write to spec file")
+	}
+	return nil
+}
+
+// called from `fission * create --spec`
+func SpecSave(resource interface{}, specFile string) error {
+	specDir := "specs"
+
+	// make sure we're writing a known type
+	var data []byte
+	var err error
+	switch typedres := resource.(type) {
+	case ArchiveUploadSpec:
+		typedres.Kind = "ArchiveUploadSpec"
+		data, err = yaml.Marshal(typedres)
+	case fv1.Package:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "Package"
+		data, err = yaml.Marshal(typedres)
+	case fv1.Function:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "Function"
+		data, err = yaml.Marshal(typedres)
+	case fv1.Environment:
+		env := resource.(fv1.Environment)
+		var generator *v1generator.EnvironmentGenerator
+		generator, err = v1generator.CreateEnvironmentGeneratorFromObj(&env)
+		if err != nil {
+			return err
+		}
+		data, err = generator.StructuredGenerate(specDefaultEncoder)
+	case fv1.HTTPTrigger:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "HTTPTrigger"
+		data, err = yaml.Marshal(typedres)
+	case fv1.KubernetesWatchTrigger:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "KubernetesWatchTrigger"
+		data, err = yaml.Marshal(typedres)
+	case fv1.MessageQueueTrigger:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "MessageQueueTrigger"
+		data, err = yaml.Marshal(typedres)
+	case fv1.TimeTrigger:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "TimeTrigger"
+		data, err = yaml.Marshal(typedres)
+	case fv1.Recorder:
+		typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
+		typedres.TypeMeta.Kind = "Recorder"
+		data, err = yaml.Marshal(typedres)
+	default:
+		return fmt.Errorf("can't save resource %#v", resource)
+	}
+	if err != nil {
+		return errors.Wrap(err, "Couldn't marshal YAML")
+	}
+
+	return Save(data, specDir, specFile)
+}
+
+// validateFunctionReference checks a function reference
+func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fv1.FunctionReference) error {
+	if funcRef.Type == fv1.FunctionReferenceTypeFunctionName {
+		// triggers only reference functions in their own namespace
+		namespace := meta.Namespace
+		name := funcRef.Name
+		m := &metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		}
+		if _, ok := functions[MapKey(m)]; !ok {
+			return fmt.Errorf("%v: %v '%v' references unknown function '%v'",
+				fr.SourceMap.Locations[kind][meta.Namespace][meta.Name],
+				kind,
+				meta.Name,
+				name)
+		} else {
+			functions[MapKey(m)] = true
+		}
+	}
+	return nil
+}
+
+func (fr *FissionResources) Validate(c *cli.Context) error {
+	var result *multierror.Error
+
+	// check references: both dangling refs + garbage
+	//   packages -> archives
+	//   functions -> packages
+	//   functions -> environments + shared environments between functions [TODO]
+	//   functions -> secrets + configmaps (same ns) [TODO]
+	//   triggers -> functions
+
+	// index archives
+	archives := make(map[string]bool)
+	for _, a := range fr.ArchiveUploadSpecs {
+		archives[a.Name] = false
+	}
+
+	// index packages, check outgoing refs, mark archives that are referenced
+	packages := make(map[string]bool)
+	for _, p := range fr.Packages {
+		packages[MapKey(&p.Metadata)] = false
+
+		// check archive refs from package
+		aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX)
+		if len(aname) > 0 {
+			if _, ok := archives[aname]; !ok {
+				result = multierror.Append(result, fmt.Errorf(
+					"%v: package '%v' references unknown source archive %v%v",
+					fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
+					p.Metadata.Name,
+					ARCHIVE_URL_PREFIX,
+					aname))
+			} else {
+				archives[aname] = true
+			}
+		}
+
+		aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX)
+		if len(aname) > 0 {
+			if _, ok := archives[aname]; !ok {
+				result = multierror.Append(result, fmt.Errorf(
+					"%v: package '%v' references unknown deployment archive %v%v",
+					fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
+					p.Metadata.Name,
+					ARCHIVE_URL_PREFIX,
+					aname))
+			} else {
+				archives[aname] = true
+			}
+		}
+
+		result = multierror.Append(result, p.Validate())
+	}
+
+	// error on unreferenced archives
+	for name, referenced := range archives {
+		if !referenced {
+			result = multierror.Append(result, fmt.Errorf(
+				"%v: archive '%v' is not used in any package",
+				fr.SourceMap.Locations["ArchiveUploadSpec"][""][name],
+				name))
+		}
+	}
+
+	// index functions, check function package refs, mark referenced packages
+	functions := make(map[string]bool)
+	for _, f := range fr.Functions {
+		functions[MapKey(&f.Metadata)] = false
+
+		pkgMeta := &metav1.ObjectMeta{
+			Name:      f.Spec.Package.PackageRef.Name,
+			Namespace: f.Spec.Package.PackageRef.Namespace,
+		}
+
+		// check package ref from function
+		packageRefExists := func() bool {
+			_, ok := packages[MapKey(pkgMeta)]
+			return ok
+		}
+
+		// check that the package referenced by each function is in the same ns as the function
+		packageRefInFuncNs := func(f *fv1.Function) bool {
+			return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace
+		}
+
+		if !packageRefInFuncNs(&f) {
+			result = multierror.Append(result, fmt.Errorf(
+				"%v: function '%v' references a package outside of its namespace %v/%v",
+				fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
+				f.Metadata.Name,
+				f.Spec.Package.PackageRef.Namespace,
+				f.Spec.Package.PackageRef.Name))
+		} else if !packageRefExists() {
+			result = multierror.Append(result, fmt.Errorf(
+				"%v: function '%v' references unknown package %v/%v",
+				fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
+				f.Metadata.Name,
+				pkgMeta.Namespace,
+				pkgMeta.Name))
+		} else {
+			packages[MapKey(pkgMeta)] = true
+		}
+
+		client := util.GetApiClient(c.GlobalString("server"))
+		for _, cm := range f.Spec.ConfigMaps {
+			_, err := client.ConfigMapGet(&metav1.ObjectMeta{
+				Name:      cm.Name,
+				Namespace: cm.Namespace,
+			})
+			if k8serrors.IsNotFound(err) {
+				log.Warn(fmt.Sprintf("Configmap %s is referred in the spec but not present in the cluster", cm.Name))
+			}
+		}
+
+		for _, s := range f.Spec.Secrets {
+			_, err := client.SecretGet(&metav1.ObjectMeta{
+				Name:      s.Name,
+				Namespace: s.Namespace,
+			})
+			if k8serrors.IsNotFound(err) {
+				log.Warn(fmt.Sprintf("Secret %s is referred in the spec but not present in the cluster", s.Name))
+			}
+
+		}
+
+		result = multierror.Append(result, f.Validate())
+	}
+
+	// error on unreferenced packages
+	for key, referenced := range packages {
+		ks := strings.Split(key, ":")
+		namespace, name := ks[0], ks[1]
+		if !referenced {
+			result = multierror.Append(result, fmt.Errorf(
+				"%v: package '%v' is not used in any function",
+				fr.SourceMap.Locations["Package"][namespace][name],
+				name))
+		}
+	}
+
+	// check function refs from triggers
+	for _, t := range fr.HttpTriggers {
+		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
+		if err != nil {
+			result = multierror.Append(result, err)
+		}
+		result = multierror.Append(result, t.Validate())
+	}
+	for _, t := range fr.KubernetesWatchTriggers {
+		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
+		if err != nil {
+			result = multierror.Append(result, err)
+		}
+		result = multierror.Append(result, t.Validate())
+	}
+	for _, t := range fr.TimeTriggers {
+		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
+		if err != nil {
+			result = multierror.Append(result, err)
+		}
+		result = multierror.Append(result, t.Validate())
+	}
+	for _, t := range fr.MessageQueueTriggers {
+		err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
+		if err != nil {
+			result = multierror.Append(result, err)
+		}
+		result = multierror.Append(result, t.Validate())
+	}
+
+	// we do not error on unreferenced functions (you can call a function through workflows,
+	// `fission function test`, etc.)
+
+	// Index envs, warn on functions referencing an environment for which spes does not exist
+	environments := make(map[string]struct{})
+	for _, e := range fr.Environments {
+		environments[fmt.Sprintf("%s:%s", e.Metadata.Name, e.Metadata.Namespace)] = struct{}{}
+		if (e.Spec.Runtime.Container != nil) && (e.Spec.Runtime.PodSpec != nil) {
+			log.Warn("You have provided both - container spec and pod spec and while merging the pod spec will take precedence.")
+		}
+		// Unlike CLI can change the environment version silently,
+		// we have to warn the user to modify spec file when this takes place.
+		if e.Spec.Version < 3 && e.Spec.Poolsize != 0 {
+			log.Warn("Poolsize can only be configured when environment version equals to 3, default poolsize 3 will be used for creating environment pool.")
+		}
+	}
+
+	for _, f := range fr.Functions {
+		if _, ok := environments[fmt.Sprintf("%s:%s", f.Spec.Environment.Name, f.Spec.Environment.Namespace)]; !ok {
+			log.Warn(fmt.Sprintf("Environment %s is referenced in function %s but not declared in specs", f.Spec.Environment.Name, f.Metadata.Name))
+		}
+		strategy := f.Spec.InvokeStrategy.ExecutionStrategy
+		if strategy.ExecutorType == fv1.ExecutorTypeNewdeploy && strategy.SpecializationTimeout < fv1.DefaultSpecializationTimeOut {
+			log.Warn(fmt.Sprintf("SpecializationTimeout in function spec.InvokeStrategy.ExecutionStrategy should be a value equal to or greater than %v", fv1.DefaultSpecializationTimeOut))
+		}
+	}
+
+	// (ErrorOrNil returns nil if there were no errors appended.)
+	return result.ErrorOrNil()
+}
+
+// Keep track of source location of resources, and track duplicates
+func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *Location) error {
+	if _, exists := fr.SourceMap.Locations[kind]; !exists {
+		fr.SourceMap.Locations[kind] = make(map[string](map[string]Location))
+	}
+	if _, exists := fr.SourceMap.Locations[kind][newobj.Namespace]; !exists {
+		fr.SourceMap.Locations[kind][newobj.Namespace] = make(map[string]Location)
+	}
+
+	// check for duplicate resources
+	oldloc, exists := fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name]
+	if exists {
+		return fmt.Errorf("%v: Duplicate %v '%v', first defined in %v", loc, kind, newobj.Name, oldloc)
+	}
+
+	// track new resource
+	fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name] = *loc
+
+	return nil
+}
+
+// ParseYaml takes one yaml document, figures out its type, parses it, and puts it in
+// the right list in the given fission resources set.
+func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error {
+	var m *metav1.ObjectMeta
+
+	// Figure out the object type by unmarshaling into the TypeMeta struct; then
+	// unmarshal again into the "real" struct once we know the type.
+	var tm TypeMeta
+	err := yaml.Unmarshal(b, &tm)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b)))
+	}
+
+	switch tm.Kind {
+	case "Package":
+		var v fv1.Package
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.Packages = append(fr.Packages, v)
+	case "Function":
+		var v fv1.Function
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.Functions = append(fr.Functions, v)
+	case "Environment":
+		var v fv1.Environment
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.Environments = append(fr.Environments, v)
+	case "HTTPTrigger":
+		var v fv1.HTTPTrigger
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+
+		// TODO move to validator
+		if !strings.HasPrefix(v.Spec.RelativeURL, "/") {
+			v.Spec.RelativeURL = fmt.Sprintf("/%s", v.Spec.RelativeURL)
+		}
+
+		m = &v.Metadata
+		fr.HttpTriggers = append(fr.HttpTriggers, v)
+	case "KubernetesWatchTrigger":
+		var v fv1.KubernetesWatchTrigger
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.KubernetesWatchTriggers = append(fr.KubernetesWatchTriggers, v)
+	case "TimeTrigger":
+		var v fv1.TimeTrigger
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.TimeTriggers = append(fr.TimeTriggers, v)
+	case "MessageQueueTrigger":
+		var v fv1.MessageQueueTrigger
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &v.Metadata
+		fr.MessageQueueTriggers = append(fr.MessageQueueTriggers, v)
+
+	// The following are not CRDs
+
+	case "DeploymentConfig":
+		var v DeploymentConfig
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		fr.DeploymentConfig = v
+	case "ArchiveUploadSpec":
+		var v ArchiveUploadSpec
+		err = yaml.Unmarshal(b, &v)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
+		}
+		m = &metav1.ObjectMeta{
+			Name:      v.Name,
+			Namespace: "",
+		}
+		fr.ArchiveUploadSpecs = append(fr.ArchiveUploadSpecs, v)
+	default:
+		// no need to error out just because there's some extra files around;
+		// also good for compatibility.
+		log.Warn(fmt.Sprintf("Ignoring unknown type %v in %v", tm.Kind, loc))
+	}
+
+	// add to source map, check for duplicates
+	if m != nil {
+		err = fr.trackSourceMap(tm.Kind, m, loc)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Returns metadata if the given resource exists in the specs, nil
+// otherwise.  compareMetadata and compareSpec control how the
+// equality check is performed.
+func (fr *FissionResources) SpecExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta {
+	switch typedres := resource.(type) {
+	case *ArchiveUploadSpec:
+		for _, aus := range fr.ArchiveUploadSpecs {
+			if compareMetadata && aus.Name != typedres.Name {
+				continue
+			}
+			if compareSpec &&
+				!(reflect.DeepEqual(aus.RootDir, typedres.RootDir) &&
+					reflect.DeepEqual(aus.IncludeGlobs, typedres.IncludeGlobs) &&
+					reflect.DeepEqual(aus.ExcludeGlobs, typedres.ExcludeGlobs)) {
+				continue
+			}
+			return &metav1.ObjectMeta{Name: aus.Name}
+		}
+		return nil
+	case *fv1.Package:
+		for _, p := range fr.Packages {
+			if compareMetadata && !reflect.DeepEqual(p.Metadata, typedres.Metadata) {
+				continue
+			}
+			if compareSpec && !reflect.DeepEqual(p.Spec, typedres.Spec) {
+				continue
+			}
+			return &p.Metadata
+		}
+		return nil
+
+	default:
+		// XXX not implemented
+		return nil
+	}
+}
+
+func (loc Location) String() string {
+	return fmt.Sprintf("%v:%v", loc.Path, loc.Line)
+}