Refactor environment CLI command (#1265)

This commit is contained in:
Ta-Ching Chen
2019-08-27 15:43:59 +08:00
committed by GitHub
parent 560aa34ce2
commit c49377569e
37 changed files with 1976 additions and 1599 deletions
+25
View File
@@ -0,0 +1,25 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
)
type (
CommandAction func(flags cli.Input) error
)
+56
View File
@@ -0,0 +1,56 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import "strings"
const (
GLOBAL_VERBOSITY = "verbosity"
GLOBAL_PLUGIN = "plugin"
FISSION_SERVER = "server"
RESOURCE_NAME = "name"
ENVIRONMENT_NAMESPACE = "envNamespace"
ENVIRONMENT_NAMESPACE_ALIAS = "envns"
ENVIRONMENT_POOLSIZE = "poolsize"
ENVIRONMENT_IMAGE = "image"
ENVIRONMENT_BUILDER = "builder"
ENVIRONMENT_BUILDCOMMAND = "buildcmd"
ENVIRONMENT_KEEPARCHIVE = "keeparchive"
ENVIRONMENT_EXTERNAL_NETWORK = "externalnetwork"
ENVIRONMENT_GRACE_PERIOD = "graceperiod"
ENVIRONMENT_GRACE_PERIOD_ALIAS = "period"
ENVIRONMENT_VERSION = "version"
SPEC_SPEC = "spec"
SPEC_SPECDIR = "specdir"
RUNTIME_MINCPU = "mincpu"
RUNTIME_MAXCPU = "maxcpu"
RUNTIME_MINMEMORY = "minmemory"
RUNTIME_MAXMEMORY = "maxmemory"
RUNTIME_MINSCALE = "minscale"
RUNTIME_MAXSCALE = "maxscale"
RUNTIME_TARGETCPU = "targetcpu"
)
// GetCliFlagName concatenates flag and its alias into a command flag name.
func GetCliFlagName(flags ...string) string {
return strings.Join(flags, ", ")
}
+184
View File
@@ -0,0 +1,184 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package environment
import (
"fmt"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/cmd/spec"
"github.com/fission/fission/pkg/fission-cli/log"
"github.com/fission/fission/pkg/fission-cli/util"
)
type CreateSubCommand struct {
client *client.Client
env *fv1.Environment
}
func Create(flags cli.Input) error {
opts := CreateSubCommand{
client: cmd.GetServer(flags),
}
return opts.do(flags)
}
func (opts *CreateSubCommand) do(flags cli.Input) error {
err := opts.complete(flags)
if err != nil {
return err
}
return opts.run(flags)
}
func (opts *CreateSubCommand) complete(flags cli.Input) error {
env, err := createEnvironmentFromCmd(flags)
if err != nil {
return err
}
opts.env = env
return nil
}
func (opts *CreateSubCommand) run(flags cli.Input) error {
m, err := cmd.GetMetadata(flags)
if err != nil {
return err
}
envList, err := opts.client.EnvironmentList(m.Namespace)
if err != nil {
return err
} else if len(envList) > 0 {
log.Verbose(2, "%d environment(s) are present in the %s namespace. "+
"These environments are not isolated from each other; use separate namespaces if you need isolation.",
len(envList), m.Namespace)
}
// if we're writing a spec, don't call the API
// save to spec file
if flags.Bool(cmd.SPEC_SPEC) {
specFile := fmt.Sprintf("env-%v.yaml", m.Name)
err = spec.SpecSave(*opts.env, specFile)
util.CheckErr(err, "create environment spec")
return nil
}
_, err = opts.client.EnvironmentCreate(opts.env)
util.CheckErr(err, "create environment")
fmt.Printf("environment '%v' created\n", m.Name)
return nil
}
// createEnvironmentFromCmd creates environment initialized with CLI input.
func createEnvironmentFromCmd(flags cli.Input) (*fv1.Environment, error) {
e := &multierror.Error{}
envNamespace := flags.String(cmd.ENVIRONMENT_NAMESPACE)
envBuildCmd := flags.String(cmd.ENVIRONMENT_BUILDCOMMAND)
envExternalNetwork := flags.Bool(cmd.ENVIRONMENT_EXTERNAL_NETWORK)
keepArchive := flags.Bool(cmd.ENVIRONMENT_KEEPARCHIVE)
envName := flags.String(cmd.RESOURCE_NAME)
if len(envName) == 0 {
e = multierror.Append(e, errors.New("Need a name, use --name."))
}
envImg := flags.String(cmd.ENVIRONMENT_IMAGE)
if len(envImg) == 0 {
e = multierror.Append(e, errors.New("Need an image, use --image."))
}
envGracePeriod := flags.Int64(cmd.ENVIRONMENT_GRACE_PERIOD)
if envGracePeriod <= 0 {
envGracePeriod = 360
}
envVersion := flags.Int(cmd.ENVIRONMENT_VERSION)
// Environment API interface version is not specified and
// builder image is empty, set default interface version
if envVersion == 0 {
envVersion = 1
}
envBuilderImg := flags.String(cmd.ENVIRONMENT_BUILDER)
if len(envBuilderImg) > 0 {
if !flags.IsSet(cmd.ENVIRONMENT_VERSION) {
// TODO: remove set env version to 2 silently, we need to warn user to set it explicitly.
envVersion = 2
}
if len(envBuildCmd) == 0 {
envBuildCmd = "build"
}
}
poolsize := 3
if flags.IsSet(cmd.ENVIRONMENT_POOLSIZE) {
poolsize = flags.Int(cmd.ENVIRONMENT_POOLSIZE)
// TODO: remove silently version 3 assignment, we need to warn user to set it explicitly.
envVersion = 3
}
resourceReq, err := cmd.GetResourceReqs(flags, nil)
if err != nil {
e = multierror.Append(e, err)
}
if e.ErrorOrNil() != nil {
return nil, e.ErrorOrNil()
}
env := &fv1.Environment{
TypeMeta: metav1.TypeMeta{
Kind: fv1.CRD_NAME_ENVIRONMENT,
APIVersion: fv1.CRD_VERSION,
},
Metadata: metav1.ObjectMeta{
Name: envName,
Namespace: envNamespace,
},
Spec: fv1.EnvironmentSpec{
Version: envVersion,
Runtime: fv1.Runtime{
Image: envImg,
},
Builder: fv1.Builder{
Image: envBuilderImg,
Command: envBuildCmd,
},
Poolsize: poolsize,
Resources: *resourceReq,
AllowAccessToExternalNetwork: envExternalNetwork,
TerminationGracePeriod: envGracePeriod,
KeepArchive: keepArchive,
},
}
err = env.Validate()
if err != nil {
return nil, fv1.AggregateValidationErrors("Environment", err)
}
return env, nil
}
+50
View File
@@ -0,0 +1,50 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package environment
import (
"fmt"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
cmdutils "github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/util"
)
type DeleteSubCommand struct {
client *client.Client
}
func Delete(flags cli.Input) error {
opts := DeleteSubCommand{
client: cmdutils.GetServer(flags),
}
return opts.do(flags)
}
func (opts *DeleteSubCommand) do(flags cli.Input) error {
m, err := cmdutils.GetMetadata(flags)
if err != nil {
return err
}
err = opts.client.EnvironmentDelete(m)
util.CheckErr(err, "delete environment")
fmt.Printf("environment '%v' deleted\n", m.Name)
return nil
}
+58
View File
@@ -0,0 +1,58 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package environment
import (
"fmt"
"os"
"text/tabwriter"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
cmdutils "github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/util"
)
type GetSubCommand struct {
client *client.Client
}
func Get(flags cli.Input) error {
opts := GetSubCommand{
client: cmdutils.GetServer(flags),
}
return opts.do(flags)
}
func (opts *GetSubCommand) do(flags cli.Input) error {
m, err := cmdutils.GetMetadata(flags)
if err != nil {
return err
}
env, err := opts.client.EnvironmentGet(m)
util.CheckErr(err, "get environment")
w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE")
fmt.Fprintf(w, "%v\t%v\t%v\n",
env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image)
w.Flush()
return nil
}
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package environment
import (
"fmt"
"os"
"text/tabwriter"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
cmdutils "github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/util"
)
type ListSubCommand struct {
client *client.Client
}
func List(flags cli.Input) error {
opts := ListSubCommand{
client: cmdutils.GetServer(flags),
}
return opts.do(flags)
}
func (opts *ListSubCommand) do(flags cli.Input) error {
envNamespace := flags.String(cmdutils.ENVIRONMENT_NAMESPACE)
envs, err := opts.client.EnvironmentList(envNamespace)
util.CheckErr(err, "list environments")
w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME")
for _, env := range envs {
fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize,
env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(),
env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(),
env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod)
}
w.Flush()
return nil
}
+131
View File
@@ -0,0 +1,131 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package environment
import (
"errors"
"fmt"
"github.com/hashicorp/go-multierror"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/util"
)
type UpdateSubCommand struct {
client *client.Client
env *fv1.Environment
}
func Update(flags cli.Input) error {
opts := UpdateSubCommand{
client: cmd.GetServer(flags),
}
return opts.do(flags)
}
func (opts *UpdateSubCommand) do(flags cli.Input) error {
err := opts.complete(flags)
if err != nil {
return err
}
return opts.run(flags)
}
func (opts *UpdateSubCommand) complete(flags cli.Input) error {
m, err := cmd.GetMetadata(flags)
if err != nil {
return err
}
env, err := opts.client.EnvironmentGet(m)
util.CheckErr(err, "find environment")
env, err = updateExistingEnvironmentWithCmd(env, flags)
if err != nil {
return err
}
opts.env = env
return nil
}
func (opts *UpdateSubCommand) run(flags cli.Input) error {
_, err := opts.client.EnvironmentUpdate(opts.env)
util.CheckErr(err, "update environment")
fmt.Printf("environment '%v' updated\n", opts.env.Metadata.Name)
return nil
}
// updateExistingEnvironmentWithCmd updates a existing environment's value based on CLI input.
func updateExistingEnvironmentWithCmd(env *fv1.Environment, flags cli.Input) (*fv1.Environment, error) {
e := &multierror.Error{}
envImg := flags.String(cmd.ENVIRONMENT_IMAGE)
envBuilderImg := flags.String(cmd.ENVIRONMENT_BUILDER)
envBuildCmd := flags.String(cmd.ENVIRONMENT_BUILDCOMMAND)
envExternalNetwork := flags.Bool(cmd.ENVIRONMENT_EXTERNAL_NETWORK)
if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 {
e = multierror.Append(e, errors.New("need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command"))
}
if len(envImg) > 0 {
env.Spec.Runtime.Image = envImg
}
if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) {
e = multierror.Append(e, errors.New("version 1 Environments do not support builders. Must specify --version=2"))
}
if len(envBuilderImg) > 0 {
env.Spec.Builder.Image = envBuilderImg
}
if len(envBuildCmd) > 0 {
env.Spec.Builder.Command = envBuildCmd
}
if flags.IsSet(cmd.ENVIRONMENT_POOLSIZE) {
env.Spec.Poolsize = flags.Int(cmd.ENVIRONMENT_POOLSIZE)
}
if flags.IsSet(cmd.ENVIRONMENT_GRACE_PERIOD) {
env.Spec.TerminationGracePeriod = flags.Int64(cmd.ENVIRONMENT_GRACE_PERIOD)
}
if flags.IsSet(cmd.ENVIRONMENT_KEEPARCHIVE) {
env.Spec.KeepArchive = flags.Bool(cmd.ENVIRONMENT_KEEPARCHIVE)
}
env.Spec.AllowAccessToExternalNetwork = envExternalNetwork
if flags.IsSet(cmd.RUNTIME_MINCPU) || flags.IsSet(cmd.RUNTIME_MAXCPU) ||
flags.IsSet(cmd.RUNTIME_MINMEMORY) || flags.IsSet(cmd.RUNTIME_MAXMEMORY) ||
flags.IsSet(cmd.RUNTIME_MINSCALE) || flags.IsSet(cmd.RUNTIME_MAXSCALE) {
e = multierror.Append(e, errors.New("updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead"))
}
if e.ErrorOrNil() != nil {
return nil, e.ErrorOrNil()
}
return env, nil
}
+673
View File
@@ -0,0 +1,673 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"github.com/ghodss/yaml"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/urfave/cli"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/fission-cli/log"
"github.com/fission/fission/pkg/fission-cli/util"
"github.com/fission/fission/pkg/generator/encoder"
v1generator "github.com/fission/fission/pkg/generator/v1"
)
var specDefaultEncoder = encoder.DefaultYAMLEncoder()
const (
FISSION_DEPLOYMENT_NAME_KEY = "fission-name"
FISSION_DEPLOYMENT_UID_KEY = "fission-uid"
SPEC_API_VERSION = "fission.io/v1"
ARCHIVE_URL_PREFIX string = "archive://"
SPEC_README = `
Fission Specs
=============
This is a set of specifications for a Fission app. This includes functions,
environments, and triggers; we collectively call these things "resources".
How to use these specs
----------------------
These specs are handled with the 'fission spec' command. See 'fission spec --help'.
'fission spec apply' will "apply" all resources specified in this directory to your
cluster. That means it checks what resources exist on your cluster, what resources are
specified in the specs directory, and reconciles the difference by creating, updating or
deleting resources on the cluster.
'fission spec apply' will also package up your source code (or compiled binaries) and
upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in
this directory to figure out which files to archive.
You can use 'fission spec apply --watch' to watch for file changes and continuously keep
the cluster updated.
You can add YAMLs to this directory by writing them manually, but it's easier to generate
them. Use 'fission function create --spec' to generate a function spec,
'fission environment create --spec' to generate an environment spec, and so on.
You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
which contains a UID that you should never change. To apply your changes simply use
'fission spec apply'.
fission-deployment-config.yaml
------------------------------
fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate
resources on the cluster to resources in this directory.
All resources created by 'fission spec apply' are annotated with this UID. Resources on
the cluster that are _not_ annotated with this UID are never modified or deleted by
fission.
`
)
// CLI spec types
type (
// DeploymentConfig is the global configuration for a set of Fission specs.
DeploymentConfig struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "DeploymentConfig".
TypeMeta `json:",inline"`
// Name is a user-friendly name for the deployment. It is also stored in
// all uploaded resources as an annotation.
Name string `json:"name"`
// UID uniquely identifies the deployment. It is stored as a label and
// used to find resources to clean up when local specs are changed.
UID string `json:"uid"`
}
// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
//
// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
// using the name specified in the archive. The fission spec applier will
// replace the archive:// URL with a real HTTP URL after uploading the file.
ArchiveUploadSpec struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "ArchiveUploadSpec".
TypeMeta `json:",inline"`
// Name is a local name that can be used to reference this archive. It
// must be unique; duplicate names will cause an error while handling
// specs.
Name string `json:"name"`
// RootDir specifies the root that the globs below are relative to. It
// is optional and defaults to the parent directory of the spec
// directory: for example, if the deployment config is at
// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
RootDir string `json:"rootdir,omitempty"`
// IncludeGlobs is a list of Unix shell globs to include
IncludeGlobs []string `json:"include,omitempty"`
// ExcludeGlobs is a list of globs to exclude from the set specified by
// IncludeGlobs.
ExcludeGlobs []string `json:"exclude,omitempty"`
}
// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
// Kubernetes does.
TypeMeta struct {
Kind string `json:"kind,omitempty"`
APIVersion string `json:"apiVersion,omitempty"`
}
FissionResources struct {
DeploymentConfig DeploymentConfig
Packages []fv1.Package
Functions []fv1.Function
Environments []fv1.Environment
HttpTriggers []fv1.HTTPTrigger
KubernetesWatchTriggers []fv1.KubernetesWatchTrigger
TimeTriggers []fv1.TimeTrigger
MessageQueueTriggers []fv1.MessageQueueTrigger
ArchiveUploadSpecs []ArchiveUploadSpec
SourceMap SourceMap
}
ResourceApplyStatus struct {
Created []*metav1.ObjectMeta
Updated []*metav1.ObjectMeta
Deleted []*metav1.ObjectMeta
}
Location struct {
Path string
Line int
}
SourceMap struct {
// kind -> namespace -> name -> location
Locations map[string](map[string](map[string]Location))
}
)
func MapKey(m *metav1.ObjectMeta) string {
return fmt.Sprintf("%v:%v", m.Namespace, m.Name)
}
// Save saves object encoded value to spec file under given spec directory
func Save(data []byte, specDir string, specFile string) error {
// verify
if _, err := os.Stat(filepath.Join(specDir, "fission-deployment-config.yaml")); os.IsNotExist(err) {
return errors.Wrap(err, "Couldn't find specs, run `fission spec init` first")
}
filename := filepath.Join(specDir, specFile)
// check if the file is new
newFile := false
if _, err := os.Stat(filename); os.IsNotExist(err) {
newFile = true
}
// open spec file to append or write
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return errors.Wrap(err, "couldn't create spec file")
}
defer f.Close()
// if we're appending, add a yaml document separator
if !newFile {
_, err = f.Write([]byte("\n---\n"))
if err != nil {
return errors.Wrap(err, "couldn't write to spec file")
}
}
// write our resource
_, err = f.Write(data)
if err != nil {
return errors.Wrap(err, "couldn't write to spec file")
}
return nil
}
// called from `fission * create --spec`
func SpecSave(resource interface{}, specFile string) error {
specDir := "specs"
// make sure we're writing a known type
var data []byte
var err error
switch typedres := resource.(type) {
case ArchiveUploadSpec:
typedres.Kind = "ArchiveUploadSpec"
data, err = yaml.Marshal(typedres)
case fv1.Package:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "Package"
data, err = yaml.Marshal(typedres)
case fv1.Function:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "Function"
data, err = yaml.Marshal(typedres)
case fv1.Environment:
env := resource.(fv1.Environment)
var generator *v1generator.EnvironmentGenerator
generator, err = v1generator.CreateEnvironmentGeneratorFromObj(&env)
if err != nil {
return err
}
data, err = generator.StructuredGenerate(specDefaultEncoder)
case fv1.HTTPTrigger:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "HTTPTrigger"
data, err = yaml.Marshal(typedres)
case fv1.KubernetesWatchTrigger:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "KubernetesWatchTrigger"
data, err = yaml.Marshal(typedres)
case fv1.MessageQueueTrigger:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "MessageQueueTrigger"
data, err = yaml.Marshal(typedres)
case fv1.TimeTrigger:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "TimeTrigger"
data, err = yaml.Marshal(typedres)
case fv1.Recorder:
typedres.TypeMeta.APIVersion = fv1.CRD_VERSION
typedres.TypeMeta.Kind = "Recorder"
data, err = yaml.Marshal(typedres)
default:
return fmt.Errorf("can't save resource %#v", resource)
}
if err != nil {
return errors.Wrap(err, "Couldn't marshal YAML")
}
return Save(data, specDir, specFile)
}
// validateFunctionReference checks a function reference
func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fv1.FunctionReference) error {
if funcRef.Type == fv1.FunctionReferenceTypeFunctionName {
// triggers only reference functions in their own namespace
namespace := meta.Namespace
name := funcRef.Name
m := &metav1.ObjectMeta{
Namespace: namespace,
Name: name,
}
if _, ok := functions[MapKey(m)]; !ok {
return fmt.Errorf("%v: %v '%v' references unknown function '%v'",
fr.SourceMap.Locations[kind][meta.Namespace][meta.Name],
kind,
meta.Name,
name)
} else {
functions[MapKey(m)] = true
}
}
return nil
}
func (fr *FissionResources) Validate(c *cli.Context) error {
var result *multierror.Error
// check references: both dangling refs + garbage
// packages -> archives
// functions -> packages
// functions -> environments + shared environments between functions [TODO]
// functions -> secrets + configmaps (same ns) [TODO]
// triggers -> functions
// index archives
archives := make(map[string]bool)
for _, a := range fr.ArchiveUploadSpecs {
archives[a.Name] = false
}
// index packages, check outgoing refs, mark archives that are referenced
packages := make(map[string]bool)
for _, p := range fr.Packages {
packages[MapKey(&p.Metadata)] = false
// check archive refs from package
aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX)
if len(aname) > 0 {
if _, ok := archives[aname]; !ok {
result = multierror.Append(result, fmt.Errorf(
"%v: package '%v' references unknown source archive %v%v",
fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
p.Metadata.Name,
ARCHIVE_URL_PREFIX,
aname))
} else {
archives[aname] = true
}
}
aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX)
if len(aname) > 0 {
if _, ok := archives[aname]; !ok {
result = multierror.Append(result, fmt.Errorf(
"%v: package '%v' references unknown deployment archive %v%v",
fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
p.Metadata.Name,
ARCHIVE_URL_PREFIX,
aname))
} else {
archives[aname] = true
}
}
result = multierror.Append(result, p.Validate())
}
// error on unreferenced archives
for name, referenced := range archives {
if !referenced {
result = multierror.Append(result, fmt.Errorf(
"%v: archive '%v' is not used in any package",
fr.SourceMap.Locations["ArchiveUploadSpec"][""][name],
name))
}
}
// index functions, check function package refs, mark referenced packages
functions := make(map[string]bool)
for _, f := range fr.Functions {
functions[MapKey(&f.Metadata)] = false
pkgMeta := &metav1.ObjectMeta{
Name: f.Spec.Package.PackageRef.Name,
Namespace: f.Spec.Package.PackageRef.Namespace,
}
// check package ref from function
packageRefExists := func() bool {
_, ok := packages[MapKey(pkgMeta)]
return ok
}
// check that the package referenced by each function is in the same ns as the function
packageRefInFuncNs := func(f *fv1.Function) bool {
return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace
}
if !packageRefInFuncNs(&f) {
result = multierror.Append(result, fmt.Errorf(
"%v: function '%v' references a package outside of its namespace %v/%v",
fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
f.Metadata.Name,
f.Spec.Package.PackageRef.Namespace,
f.Spec.Package.PackageRef.Name))
} else if !packageRefExists() {
result = multierror.Append(result, fmt.Errorf(
"%v: function '%v' references unknown package %v/%v",
fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name],
f.Metadata.Name,
pkgMeta.Namespace,
pkgMeta.Name))
} else {
packages[MapKey(pkgMeta)] = true
}
client := util.GetApiClient(c.GlobalString("server"))
for _, cm := range f.Spec.ConfigMaps {
_, err := client.ConfigMapGet(&metav1.ObjectMeta{
Name: cm.Name,
Namespace: cm.Namespace,
})
if k8serrors.IsNotFound(err) {
log.Warn(fmt.Sprintf("Configmap %s is referred in the spec but not present in the cluster", cm.Name))
}
}
for _, s := range f.Spec.Secrets {
_, err := client.SecretGet(&metav1.ObjectMeta{
Name: s.Name,
Namespace: s.Namespace,
})
if k8serrors.IsNotFound(err) {
log.Warn(fmt.Sprintf("Secret %s is referred in the spec but not present in the cluster", s.Name))
}
}
result = multierror.Append(result, f.Validate())
}
// error on unreferenced packages
for key, referenced := range packages {
ks := strings.Split(key, ":")
namespace, name := ks[0], ks[1]
if !referenced {
result = multierror.Append(result, fmt.Errorf(
"%v: package '%v' is not used in any function",
fr.SourceMap.Locations["Package"][namespace][name],
name))
}
}
// check function refs from triggers
for _, t := range fr.HttpTriggers {
err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
if err != nil {
result = multierror.Append(result, err)
}
result = multierror.Append(result, t.Validate())
}
for _, t := range fr.KubernetesWatchTriggers {
err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
if err != nil {
result = multierror.Append(result, err)
}
result = multierror.Append(result, t.Validate())
}
for _, t := range fr.TimeTriggers {
err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
if err != nil {
result = multierror.Append(result, err)
}
result = multierror.Append(result, t.Validate())
}
for _, t := range fr.MessageQueueTriggers {
err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference)
if err != nil {
result = multierror.Append(result, err)
}
result = multierror.Append(result, t.Validate())
}
// we do not error on unreferenced functions (you can call a function through workflows,
// `fission function test`, etc.)
// Index envs, warn on functions referencing an environment for which spes does not exist
environments := make(map[string]struct{})
for _, e := range fr.Environments {
environments[fmt.Sprintf("%s:%s", e.Metadata.Name, e.Metadata.Namespace)] = struct{}{}
if (e.Spec.Runtime.Container != nil) && (e.Spec.Runtime.PodSpec != nil) {
log.Warn("You have provided both - container spec and pod spec and while merging the pod spec will take precedence.")
}
// Unlike CLI can change the environment version silently,
// we have to warn the user to modify spec file when this takes place.
if e.Spec.Version < 3 && e.Spec.Poolsize != 0 {
log.Warn("Poolsize can only be configured when environment version equals to 3, default poolsize 3 will be used for creating environment pool.")
}
}
for _, f := range fr.Functions {
if _, ok := environments[fmt.Sprintf("%s:%s", f.Spec.Environment.Name, f.Spec.Environment.Namespace)]; !ok {
log.Warn(fmt.Sprintf("Environment %s is referenced in function %s but not declared in specs", f.Spec.Environment.Name, f.Metadata.Name))
}
strategy := f.Spec.InvokeStrategy.ExecutionStrategy
if strategy.ExecutorType == fv1.ExecutorTypeNewdeploy && strategy.SpecializationTimeout < fv1.DefaultSpecializationTimeOut {
log.Warn(fmt.Sprintf("SpecializationTimeout in function spec.InvokeStrategy.ExecutionStrategy should be a value equal to or greater than %v", fv1.DefaultSpecializationTimeOut))
}
}
// (ErrorOrNil returns nil if there were no errors appended.)
return result.ErrorOrNil()
}
// Keep track of source location of resources, and track duplicates
func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *Location) error {
if _, exists := fr.SourceMap.Locations[kind]; !exists {
fr.SourceMap.Locations[kind] = make(map[string](map[string]Location))
}
if _, exists := fr.SourceMap.Locations[kind][newobj.Namespace]; !exists {
fr.SourceMap.Locations[kind][newobj.Namespace] = make(map[string]Location)
}
// check for duplicate resources
oldloc, exists := fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name]
if exists {
return fmt.Errorf("%v: Duplicate %v '%v', first defined in %v", loc, kind, newobj.Name, oldloc)
}
// track new resource
fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name] = *loc
return nil
}
// ParseYaml takes one yaml document, figures out its type, parses it, and puts it in
// the right list in the given fission resources set.
func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error {
var m *metav1.ObjectMeta
// Figure out the object type by unmarshaling into the TypeMeta struct; then
// unmarshal again into the "real" struct once we know the type.
var tm TypeMeta
err := yaml.Unmarshal(b, &tm)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b)))
}
switch tm.Kind {
case "Package":
var v fv1.Package
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.Packages = append(fr.Packages, v)
case "Function":
var v fv1.Function
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.Functions = append(fr.Functions, v)
case "Environment":
var v fv1.Environment
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.Environments = append(fr.Environments, v)
case "HTTPTrigger":
var v fv1.HTTPTrigger
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
// TODO move to validator
if !strings.HasPrefix(v.Spec.RelativeURL, "/") {
v.Spec.RelativeURL = fmt.Sprintf("/%s", v.Spec.RelativeURL)
}
m = &v.Metadata
fr.HttpTriggers = append(fr.HttpTriggers, v)
case "KubernetesWatchTrigger":
var v fv1.KubernetesWatchTrigger
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.KubernetesWatchTriggers = append(fr.KubernetesWatchTriggers, v)
case "TimeTrigger":
var v fv1.TimeTrigger
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.TimeTriggers = append(fr.TimeTriggers, v)
case "MessageQueueTrigger":
var v fv1.MessageQueueTrigger
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &v.Metadata
fr.MessageQueueTriggers = append(fr.MessageQueueTriggers, v)
// The following are not CRDs
case "DeploymentConfig":
var v DeploymentConfig
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
fr.DeploymentConfig = v
case "ArchiveUploadSpec":
var v ArchiveUploadSpec
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
m = &metav1.ObjectMeta{
Name: v.Name,
Namespace: "",
}
fr.ArchiveUploadSpecs = append(fr.ArchiveUploadSpecs, v)
default:
// no need to error out just because there's some extra files around;
// also good for compatibility.
log.Warn(fmt.Sprintf("Ignoring unknown type %v in %v", tm.Kind, loc))
}
// add to source map, check for duplicates
if m != nil {
err = fr.trackSourceMap(tm.Kind, m, loc)
if err != nil {
return err
}
}
return nil
}
// Returns metadata if the given resource exists in the specs, nil
// otherwise. compareMetadata and compareSpec control how the
// equality check is performed.
func (fr *FissionResources) SpecExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta {
switch typedres := resource.(type) {
case *ArchiveUploadSpec:
for _, aus := range fr.ArchiveUploadSpecs {
if compareMetadata && aus.Name != typedres.Name {
continue
}
if compareSpec &&
!(reflect.DeepEqual(aus.RootDir, typedres.RootDir) &&
reflect.DeepEqual(aus.IncludeGlobs, typedres.IncludeGlobs) &&
reflect.DeepEqual(aus.ExcludeGlobs, typedres.ExcludeGlobs)) {
continue
}
return &metav1.ObjectMeta{Name: aus.Name}
}
return nil
case *fv1.Package:
for _, p := range fr.Packages {
if compareMetadata && !reflect.DeepEqual(p.Metadata, typedres.Metadata) {
continue
}
if compareSpec && !reflect.DeepEqual(p.Spec, typedres.Spec) {
continue
}
return &p.Metadata
}
return nil
default:
// XXX not implemented
return nil
}
}
func (loc Location) String() string {
return fmt.Sprintf("%v:%v", loc.Path, loc.Line)
}
+143
View File
@@ -0,0 +1,143 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"strconv"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/util"
)
func GetServer(flags cli.Input) *client.Client {
return util.GetApiClient(flags.GlobalString(FISSION_SERVER))
}
func GetResourceReqs(flags cli.Input, resReqs *v1.ResourceRequirements) (*v1.ResourceRequirements, error) {
r := &v1.ResourceRequirements{}
if resReqs != nil {
r.Requests = resReqs.Requests
r.Limits = resReqs.Limits
}
if len(r.Requests) == 0 {
r.Requests = make(map[v1.ResourceName]resource.Quantity)
}
if len(r.Limits) == 0 {
r.Limits = make(map[v1.ResourceName]resource.Quantity)
}
e := &multierror.Error{}
if flags.IsSet(RUNTIME_MINCPU) {
mincpu := flags.Int(RUNTIME_MINCPU)
cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m")
if err != nil {
e = multierror.Append(e, errors.Wrap(err, "Failed to parse mincpu"))
}
r.Requests[v1.ResourceCPU] = cpuRequest
}
if flags.IsSet(RUNTIME_MINMEMORY) {
minmem := flags.Int(RUNTIME_MINMEMORY)
memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi")
if err != nil {
e = multierror.Append(e, errors.Wrap(err, "Failed to parse minmemory"))
}
r.Requests[v1.ResourceMemory] = memRequest
}
if flags.IsSet(RUNTIME_MAXCPU) {
maxcpu := flags.Int(RUNTIME_MAXCPU)
cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m")
if err != nil {
e = multierror.Append(e, errors.Wrap(err, "Failed to parse maxcpu"))
}
r.Limits[v1.ResourceCPU] = cpuLimit
}
if flags.IsSet(RUNTIME_MAXMEMORY) {
maxmem := flags.Int(RUNTIME_MAXMEMORY)
memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi")
if err != nil {
e = multierror.Append(e, errors.Wrap(err, "Failed to parse maxmemory"))
}
r.Limits[v1.ResourceMemory] = memLimit
}
limitCPU := r.Limits[v1.ResourceCPU]
requestCPU := r.Requests[v1.ResourceCPU]
if limitCPU.IsZero() && !requestCPU.IsZero() {
r.Limits[v1.ResourceCPU] = requestCPU
} else if limitCPU.Cmp(requestCPU) < 0 {
e = multierror.Append(e, fmt.Errorf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String()))
}
limitMem := r.Limits[v1.ResourceMemory]
requestMem := r.Requests[v1.ResourceMemory]
if limitMem.IsZero() && !requestMem.IsZero() {
r.Limits[v1.ResourceMemory] = requestMem
} else if limitMem.Cmp(requestMem) < 0 {
e = multierror.Append(e, fmt.Errorf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String()))
}
if e.ErrorOrNil() != nil {
return nil, e
}
return &v1.ResourceRequirements{
Requests: r.Requests,
Limits: r.Limits,
}, nil
}
func GetSpecDir(flags cli.Input) string {
specDir := flags.String(SPEC_SPECDIR)
if len(specDir) == 0 {
specDir = "specs"
}
return specDir
}
// GetMetadata returns a pointer to ObjectMeta which initialized with command line input.
func GetMetadata(flags cli.Input) (*metav1.ObjectMeta, error) {
name := flags.String(RESOURCE_NAME)
if len(name) == 0 {
return nil, errors.New("Need a resource name, use --name.")
}
ns := flags.String(ENVIRONMENT_NAMESPACE)
m := &metav1.ObjectMeta{
Name: name,
Namespace: ns,
}
return m, nil
}