From c49377569e392a1d9905ef65a7d8c273987cd184 Mon Sep 17 00:00:00 2001 From: Ta-Ching Chen Date: Tue, 27 Aug 2019 15:43:59 +0800 Subject: [PATCH] Refactor environment CLI command (#1265) --- go.mod | 2 +- go.sum | 5 +- pkg/apis/fission.io/v1/const.go | 4 + pkg/apis/fission.io/v1/doc.go | 6 + pkg/controller/api_test.go | 1 + pkg/controller/client/environment.go | 25 +- pkg/executor/newdeploy/newdeploy.go | 9 +- pkg/fission-cli/cliwrapper/cli/cli.go | 70 ++ pkg/fission-cli/cliwrapper/doc.go | 23 + pkg/fission-cli/cliwrapper/driver/doc.go | 1 + .../cliwrapper/driver/urfavecli/urfavecli.go | 109 +++ pkg/fission-cli/cmd/cmd.go | 25 + pkg/fission-cli/cmd/doc.go | 56 ++ pkg/fission-cli/cmd/environment/create.go | 184 +++++ pkg/fission-cli/cmd/environment/delete.go | 50 ++ pkg/fission-cli/cmd/environment/get.go | 58 ++ pkg/fission-cli/cmd/environment/list.go | 59 ++ pkg/fission-cli/cmd/environment/update.go | 131 +++ pkg/fission-cli/cmd/spec/spec.go | 673 +++++++++++++++ pkg/fission-cli/cmd/util.go | 143 ++++ pkg/fission-cli/environment.go | 338 -------- pkg/fission-cli/function.go | 45 +- pkg/fission-cli/function_test.go | 24 +- pkg/fission-cli/httptrigger.go | 9 +- pkg/fission-cli/main.go | 61 +- pkg/fission-cli/mqtrigger.go | 3 +- pkg/fission-cli/package.go | 21 +- pkg/fission-cli/recorder.go | 3 +- pkg/fission-cli/spec.go | 768 +++--------------- pkg/fission-cli/timetrigger.go | 3 +- pkg/fission-cli/types.go | 77 -- pkg/fission-cli/upgrade.go | 416 ---------- pkg/fission-cli/watch.go | 3 +- pkg/generator/doc.go | 19 + pkg/generator/encoder/encoder.go | 62 ++ pkg/generator/generate.go | 27 + pkg/generator/v1/environment.go | 62 ++ 37 files changed, 1976 insertions(+), 1599 deletions(-) create mode 100644 pkg/fission-cli/cliwrapper/cli/cli.go create mode 100644 pkg/fission-cli/cliwrapper/doc.go create mode 100644 pkg/fission-cli/cliwrapper/driver/doc.go create mode 100644 pkg/fission-cli/cliwrapper/driver/urfavecli/urfavecli.go create mode 100644 pkg/fission-cli/cmd/cmd.go create mode 100644 pkg/fission-cli/cmd/doc.go create mode 100644 pkg/fission-cli/cmd/environment/create.go create mode 100644 pkg/fission-cli/cmd/environment/delete.go create mode 100644 pkg/fission-cli/cmd/environment/get.go create mode 100644 pkg/fission-cli/cmd/environment/list.go create mode 100644 pkg/fission-cli/cmd/environment/update.go create mode 100644 pkg/fission-cli/cmd/spec/spec.go create mode 100644 pkg/fission-cli/cmd/util.go delete mode 100644 pkg/fission-cli/environment.go delete mode 100644 pkg/fission-cli/types.go delete mode 100644 pkg/fission-cli/upgrade.go create mode 100644 pkg/generator/doc.go create mode 100644 pkg/generator/encoder/encoder.go create mode 100644 pkg/generator/generate.go create mode 100644 pkg/generator/v1/environment.go diff --git a/go.mod b/go.mod index 1a4ca01c..0e45360a 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/go-openapi/spec v0.17.2 github.com/go-sql-driver/mysql v1.4.1 // indirect - github.com/gogo/protobuf v1.2.0 // indirect + github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/protobuf v1.3.1 diff --git a/go.sum b/go.sum index 6ad973a4..eaf18160 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,9 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -202,6 +203,7 @@ github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwK github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -395,6 +397,7 @@ golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/apis/fission.io/v1/const.go b/pkg/apis/fission.io/v1/const.go index 18569609..b68142d2 100644 --- a/pkg/apis/fission.io/v1/const.go +++ b/pkg/apis/fission.io/v1/const.go @@ -101,3 +101,7 @@ const ( // set a max number for iterations to prevent infinite processing of canary config MaxIterationsForCanaryConfig = 10 ) + +const ( + DefaultSpecializationTimeOut = 120 +) diff --git a/pkg/apis/fission.io/v1/doc.go b/pkg/apis/fission.io/v1/doc.go index 9cfe080f..05eef000 100644 --- a/pkg/apis/fission.io/v1/doc.go +++ b/pkg/apis/fission.io/v1/doc.go @@ -21,3 +21,9 @@ limitations under the License. // +k8s:defaulter-gen=TypeMeta // +groupName=fission.io package v1 + +const ( + CRD_VERSION = "fission.io/v1" + + CRD_NAME_ENVIRONMENT = "Environment" +) diff --git a/pkg/controller/api_test.go b/pkg/controller/api_test.go index 13f35e91..5eccca20 100644 --- a/pkg/controller/api_test.go +++ b/pkg/controller/api_test.go @@ -235,6 +235,7 @@ func TestEnvironmentApi(t *testing.T) { testEnv.Metadata.ResourceVersion = "" testEnv.Metadata.Name = "bar" + m2, err := g.client.EnvironmentCreate(testEnv) panicIf(err) defer g.client.EnvironmentDelete(m2) diff --git a/pkg/controller/client/environment.go b/pkg/controller/client/environment.go index 83856205..1c67d4d0 100644 --- a/pkg/controller/client/environment.go +++ b/pkg/controller/client/environment.go @@ -25,20 +25,25 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/generator/encoder" + v1generator "github.com/fission/fission/pkg/generator/v1" ) -func (c *Client) EnvironmentCreate(env *fv1.Environment) (*metav1.ObjectMeta, error) { - err := env.Validate() +func getEnvEncodingPayload(env *fv1.Environment) ([]byte, error) { + generator, err := v1generator.CreateEnvironmentGeneratorFromObj(env) if err != nil { - return nil, fv1.AggregateValidationErrors("Environment", err) + return nil, err } + return generator.StructuredGenerate(encoder.DefaultJSONEncoder()) +} - reqbody, err := json.Marshal(env) +func (c *Client) EnvironmentCreate(env *fv1.Environment) (*metav1.ObjectMeta, error) { + data, err := getEnvEncodingPayload(env) if err != nil { return nil, err } - resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody)) + resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(data)) if err != nil { return nil, err } @@ -83,18 +88,14 @@ func (c *Client) EnvironmentGet(m *metav1.ObjectMeta) (*fv1.Environment, error) } func (c *Client) EnvironmentUpdate(env *fv1.Environment) (*metav1.ObjectMeta, error) { - err := env.Validate() - if err != nil { - return nil, fv1.AggregateValidationErrors("Environment", err) - } - - reqbody, err := json.Marshal(env) + data, err := getEnvEncodingPayload(env) if err != nil { return nil, err } + relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name) - resp, err := c.put(relativeUrl, "application/json", reqbody) + resp, err := c.put(relativeUrl, "application/json", data) if err != nil { return nil, err } diff --git a/pkg/executor/newdeploy/newdeploy.go b/pkg/executor/newdeploy/newdeploy.go index f2e6ba2b..25adffe7 100644 --- a/pkg/executor/newdeploy/newdeploy.go +++ b/pkg/executor/newdeploy/newdeploy.go @@ -38,9 +38,8 @@ import ( ) const ( - DeploymentKind = "Deployment" - DeploymentVersion = "extensions/v1beta1" - DEFAULT_SPECIALIZATION_TIMEOUT = 120 + DeploymentKind = "Deployment" + DeploymentVersion = "extensions/v1beta1" ) func (deploy *NewDeploy) createOrGetDeployment(fn *fv1.Function, env *fv1.Environment, @@ -418,8 +417,8 @@ func (deploy *NewDeploy) deleteSvc(ns string, name string) error { func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32, specializationTimeout int) (*v1beta1.Deployment, error) { // if no specializationTimeout is set, use default value - if specializationTimeout < DEFAULT_SPECIALIZATION_TIMEOUT { - specializationTimeout = DEFAULT_SPECIALIZATION_TIMEOUT + if specializationTimeout < fv1.DefaultSpecializationTimeOut { + specializationTimeout = fv1.DefaultSpecializationTimeOut } for i := 0; i < specializationTimeout; i++ { diff --git a/pkg/fission-cli/cliwrapper/cli/cli.go b/pkg/fission-cli/cliwrapper/cli/cli.go new file mode 100644 index 00000000..8bfd96ca --- /dev/null +++ b/pkg/fission-cli/cliwrapper/cli/cli.go @@ -0,0 +1,70 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +type ( + Input interface { + //Parse(input interface{}) error + + // IsSet checks whether a flag has been set by the user + IsSet(key string) bool + + // Bool returns true if given flag has been set; + // otherwise, return false. + Bool(key string) bool + + // String returns string value of given flag. + String(key string) string + + // StringSlice returns string slice of given flag.. + StringSlice(key string) []string + + // Int returns int value of given flag.nd false. + Int(key string) int + + // IntSlice returns int slice of given flag.lse. + IntSlice(key string) []int + + // Int64 returns int64 value of given flag. false. + Int64(key string) int64 + + // Int64Slice returns int64 slice of given flag.e. + Int64Slice(key string) []int64 + + // GlobalBool returns true if given global flag has been set; + // otherwise, return false. + GlobalBool(key string) bool + + // GlobalString returns global string value of given flag. + GlobalString(key string) string + + // GlobalStringSlice returns global string slice of given flag. + GlobalStringSlice(key string) []string + + // GlobalInt returns global int value of given flag. + GlobalInt(key string) int + + // GlobalIntSlice returns global int slice of given flag. + GlobalIntSlice(key string) []int + + // GlobalInt64 returns global int64 value of given flag. + GlobalInt64(key string) int64 + + // GlobalInt64Slice returns global int64 slice of given flag. + GlobalInt64Slice(key string) []int64 + } +) diff --git a/pkg/fission-cli/cliwrapper/doc.go b/pkg/fission-cli/cliwrapper/doc.go new file mode 100644 index 00000000..e245b05a --- /dev/null +++ b/pkg/fission-cli/cliwrapper/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cliwrapper is a wrapper that allowing functions to access flag value in an +// identical way no matter what underlying CLI package used. It brings couple benefits +// for doing this: +// 1. Separate CLI function and CLI package. +// 2. Easier to write test no matter what CLI package actually used. +// 3. Migrate to new CLI package without changing the way for CLI function to access flag value. +package cliwrapper diff --git a/pkg/fission-cli/cliwrapper/driver/doc.go b/pkg/fission-cli/cliwrapper/driver/doc.go new file mode 100644 index 00000000..bce7c468 --- /dev/null +++ b/pkg/fission-cli/cliwrapper/driver/doc.go @@ -0,0 +1 @@ +package driver diff --git a/pkg/fission-cli/cliwrapper/driver/urfavecli/urfavecli.go b/pkg/fission-cli/cliwrapper/driver/urfavecli/urfavecli.go new file mode 100644 index 00000000..dd7a758e --- /dev/null +++ b/pkg/fission-cli/cliwrapper/driver/urfavecli/urfavecli.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package urfavecli + +import ( + "log" + + "github.com/urfave/cli" + + fCli "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" +) + +var _ fCli.Input = &Cli{} + +type Cli struct { + c *cli.Context +} + +// Parse is only for converting urfave *cli.Context to Input and will be removed in future. +func Parse(c *cli.Context) fCli.Input { + return Cli{c: c} +} + +func Wrapper(action cmd.CommandAction) func(*cli.Context) error { + return func(c *cli.Context) error { + e := action(Cli{c: c}) + // Urfave cli doesn't exit with error code even error is not nil. + // We have to check whether error is empty and print error log here. + if e != nil { + log.Fatalf("%v", e) + } + return e + } +} + +func (u Cli) IsSet(key string) bool { + return u.c.IsSet(key) +} + +func (u Cli) Bool(key string) bool { + return u.c.Bool(key) +} + +func (u Cli) String(key string) string { + return u.c.String(key) +} + +func (u Cli) StringSlice(key string) []string { + return u.c.StringSlice(key) +} + +func (u Cli) Int(key string) int { + return u.c.Int(key) +} + +func (u Cli) IntSlice(key string) []int { + return u.c.IntSlice(key) +} + +func (u Cli) Int64(key string) int64 { + return u.c.Int64(key) +} + +func (u Cli) Int64Slice(key string) []int64 { + return u.c.Int64Slice(key) +} + +func (u Cli) GlobalBool(key string) bool { + return u.c.GlobalBool(key) +} + +func (u Cli) GlobalString(key string) string { + return u.c.GlobalString(key) +} + +func (u Cli) GlobalStringSlice(key string) []string { + return u.c.GlobalStringSlice(key) +} + +func (u Cli) GlobalInt(key string) int { + return u.c.GlobalInt(key) +} + +func (u Cli) GlobalIntSlice(key string) []int { + return u.c.GlobalIntSlice(key) +} + +func (u Cli) GlobalInt64(key string) int64 { + return u.c.GlobalInt64(key) +} + +func (u Cli) GlobalInt64Slice(key string) []int64 { + return u.c.GlobalInt64Slice(key) +} diff --git a/pkg/fission-cli/cmd/cmd.go b/pkg/fission-cli/cmd/cmd.go new file mode 100644 index 00000000..067bc276 --- /dev/null +++ b/pkg/fission-cli/cmd/cmd.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" +) + +type ( + CommandAction func(flags cli.Input) error +) diff --git a/pkg/fission-cli/cmd/doc.go b/pkg/fission-cli/cmd/doc.go new file mode 100644 index 00000000..cfb5c623 --- /dev/null +++ b/pkg/fission-cli/cmd/doc.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import "strings" + +const ( + GLOBAL_VERBOSITY = "verbosity" + GLOBAL_PLUGIN = "plugin" + + FISSION_SERVER = "server" + + RESOURCE_NAME = "name" + + ENVIRONMENT_NAMESPACE = "envNamespace" + ENVIRONMENT_NAMESPACE_ALIAS = "envns" + ENVIRONMENT_POOLSIZE = "poolsize" + ENVIRONMENT_IMAGE = "image" + ENVIRONMENT_BUILDER = "builder" + ENVIRONMENT_BUILDCOMMAND = "buildcmd" + ENVIRONMENT_KEEPARCHIVE = "keeparchive" + ENVIRONMENT_EXTERNAL_NETWORK = "externalnetwork" + ENVIRONMENT_GRACE_PERIOD = "graceperiod" + ENVIRONMENT_GRACE_PERIOD_ALIAS = "period" + ENVIRONMENT_VERSION = "version" + + SPEC_SPEC = "spec" + SPEC_SPECDIR = "specdir" + + RUNTIME_MINCPU = "mincpu" + RUNTIME_MAXCPU = "maxcpu" + RUNTIME_MINMEMORY = "minmemory" + RUNTIME_MAXMEMORY = "maxmemory" + RUNTIME_MINSCALE = "minscale" + RUNTIME_MAXSCALE = "maxscale" + RUNTIME_TARGETCPU = "targetcpu" +) + +// GetCliFlagName concatenates flag and its alias into a command flag name. +func GetCliFlagName(flags ...string) string { + return strings.Join(flags, ", ") +} diff --git a/pkg/fission-cli/cmd/environment/create.go b/pkg/fission-cli/cmd/environment/create.go new file mode 100644 index 00000000..7310b580 --- /dev/null +++ b/pkg/fission-cli/cmd/environment/create.go @@ -0,0 +1,184 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package environment + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" + "github.com/fission/fission/pkg/fission-cli/log" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type CreateSubCommand struct { + client *client.Client + env *fv1.Environment +} + +func Create(flags cli.Input) error { + opts := CreateSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *CreateSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *CreateSubCommand) complete(flags cli.Input) error { + env, err := createEnvironmentFromCmd(flags) + if err != nil { + return err + } + opts.env = env + return nil +} + +func (opts *CreateSubCommand) run(flags cli.Input) error { + m, err := cmd.GetMetadata(flags) + if err != nil { + return err + } + envList, err := opts.client.EnvironmentList(m.Namespace) + if err != nil { + return err + } else if len(envList) > 0 { + log.Verbose(2, "%d environment(s) are present in the %s namespace. "+ + "These environments are not isolated from each other; use separate namespaces if you need isolation.", + len(envList), m.Namespace) + } + + // if we're writing a spec, don't call the API + // save to spec file + if flags.Bool(cmd.SPEC_SPEC) { + specFile := fmt.Sprintf("env-%v.yaml", m.Name) + err = spec.SpecSave(*opts.env, specFile) + util.CheckErr(err, "create environment spec") + return nil + } + + _, err = opts.client.EnvironmentCreate(opts.env) + util.CheckErr(err, "create environment") + + fmt.Printf("environment '%v' created\n", m.Name) + return nil +} + +// createEnvironmentFromCmd creates environment initialized with CLI input. +func createEnvironmentFromCmd(flags cli.Input) (*fv1.Environment, error) { + e := &multierror.Error{} + + envNamespace := flags.String(cmd.ENVIRONMENT_NAMESPACE) + envBuildCmd := flags.String(cmd.ENVIRONMENT_BUILDCOMMAND) + envExternalNetwork := flags.Bool(cmd.ENVIRONMENT_EXTERNAL_NETWORK) + keepArchive := flags.Bool(cmd.ENVIRONMENT_KEEPARCHIVE) + + envName := flags.String(cmd.RESOURCE_NAME) + if len(envName) == 0 { + e = multierror.Append(e, errors.New("Need a name, use --name.")) + } + + envImg := flags.String(cmd.ENVIRONMENT_IMAGE) + if len(envImg) == 0 { + e = multierror.Append(e, errors.New("Need an image, use --image.")) + } + + envGracePeriod := flags.Int64(cmd.ENVIRONMENT_GRACE_PERIOD) + if envGracePeriod <= 0 { + envGracePeriod = 360 + } + + envVersion := flags.Int(cmd.ENVIRONMENT_VERSION) + // Environment API interface version is not specified and + // builder image is empty, set default interface version + if envVersion == 0 { + envVersion = 1 + } + + envBuilderImg := flags.String(cmd.ENVIRONMENT_BUILDER) + if len(envBuilderImg) > 0 { + if !flags.IsSet(cmd.ENVIRONMENT_VERSION) { + // TODO: remove set env version to 2 silently, we need to warn user to set it explicitly. + envVersion = 2 + } + if len(envBuildCmd) == 0 { + envBuildCmd = "build" + } + } + + poolsize := 3 + if flags.IsSet(cmd.ENVIRONMENT_POOLSIZE) { + poolsize = flags.Int(cmd.ENVIRONMENT_POOLSIZE) + // TODO: remove silently version 3 assignment, we need to warn user to set it explicitly. + envVersion = 3 + } + + resourceReq, err := cmd.GetResourceReqs(flags, nil) + if err != nil { + e = multierror.Append(e, err) + } + + if e.ErrorOrNil() != nil { + return nil, e.ErrorOrNil() + } + + env := &fv1.Environment{ + TypeMeta: metav1.TypeMeta{ + Kind: fv1.CRD_NAME_ENVIRONMENT, + APIVersion: fv1.CRD_VERSION, + }, + Metadata: metav1.ObjectMeta{ + Name: envName, + Namespace: envNamespace, + }, + Spec: fv1.EnvironmentSpec{ + Version: envVersion, + Runtime: fv1.Runtime{ + Image: envImg, + }, + Builder: fv1.Builder{ + Image: envBuilderImg, + Command: envBuildCmd, + }, + Poolsize: poolsize, + Resources: *resourceReq, + AllowAccessToExternalNetwork: envExternalNetwork, + TerminationGracePeriod: envGracePeriod, + KeepArchive: keepArchive, + }, + } + + err = env.Validate() + if err != nil { + return nil, fv1.AggregateValidationErrors("Environment", err) + } + + return env, nil +} diff --git a/pkg/fission-cli/cmd/environment/delete.go b/pkg/fission-cli/cmd/environment/delete.go new file mode 100644 index 00000000..af4b240b --- /dev/null +++ b/pkg/fission-cli/cmd/environment/delete.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package environment + +import ( + "fmt" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type DeleteSubCommand struct { + client *client.Client +} + +func Delete(flags cli.Input) error { + opts := DeleteSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *DeleteSubCommand) do(flags cli.Input) error { + m, err := cmdutils.GetMetadata(flags) + if err != nil { + return err + } + + err = opts.client.EnvironmentDelete(m) + util.CheckErr(err, "delete environment") + + fmt.Printf("environment '%v' deleted\n", m.Name) + return nil +} diff --git a/pkg/fission-cli/cmd/environment/get.go b/pkg/fission-cli/cmd/environment/get.go new file mode 100644 index 00000000..f83c179c --- /dev/null +++ b/pkg/fission-cli/cmd/environment/get.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package environment + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type GetSubCommand struct { + client *client.Client +} + +func Get(flags cli.Input) error { + opts := GetSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *GetSubCommand) do(flags cli.Input) error { + m, err := cmdutils.GetMetadata(flags) + if err != nil { + return err + } + + env, err := opts.client.EnvironmentGet(m) + util.CheckErr(err, "get environment") + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + + fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") + fmt.Fprintf(w, "%v\t%v\t%v\n", + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) + + w.Flush() + return nil +} diff --git a/pkg/fission-cli/cmd/environment/list.go b/pkg/fission-cli/cmd/environment/list.go new file mode 100644 index 00000000..0dbe67a4 --- /dev/null +++ b/pkg/fission-cli/cmd/environment/list.go @@ -0,0 +1,59 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package environment + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type ListSubCommand struct { + client *client.Client +} + +func List(flags cli.Input) error { + opts := ListSubCommand{ + client: cmdutils.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *ListSubCommand) do(flags cli.Input) error { + envNamespace := flags.String(cmdutils.ENVIRONMENT_NAMESPACE) + + envs, err := opts.client.EnvironmentList(envNamespace) + util.CheckErr(err, "list environments") + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME") + for _, env := range envs { + fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", + env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize, + env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(), + env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(), + env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod) + } + w.Flush() + + return nil +} diff --git a/pkg/fission-cli/cmd/environment/update.go b/pkg/fission-cli/cmd/environment/update.go new file mode 100644 index 00000000..09dd068d --- /dev/null +++ b/pkg/fission-cli/cmd/environment/update.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package environment + +import ( + "errors" + "fmt" + + "github.com/hashicorp/go-multierror" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/util" +) + +type UpdateSubCommand struct { + client *client.Client + env *fv1.Environment +} + +func Update(flags cli.Input) error { + opts := UpdateSubCommand{ + client: cmd.GetServer(flags), + } + return opts.do(flags) +} + +func (opts *UpdateSubCommand) do(flags cli.Input) error { + err := opts.complete(flags) + if err != nil { + return err + } + return opts.run(flags) +} + +func (opts *UpdateSubCommand) complete(flags cli.Input) error { + m, err := cmd.GetMetadata(flags) + if err != nil { + return err + } + + env, err := opts.client.EnvironmentGet(m) + util.CheckErr(err, "find environment") + + env, err = updateExistingEnvironmentWithCmd(env, flags) + if err != nil { + return err + } + + opts.env = env + return nil +} + +func (opts *UpdateSubCommand) run(flags cli.Input) error { + _, err := opts.client.EnvironmentUpdate(opts.env) + util.CheckErr(err, "update environment") + + fmt.Printf("environment '%v' updated\n", opts.env.Metadata.Name) + return nil +} + +// updateExistingEnvironmentWithCmd updates a existing environment's value based on CLI input. +func updateExistingEnvironmentWithCmd(env *fv1.Environment, flags cli.Input) (*fv1.Environment, error) { + e := &multierror.Error{} + + envImg := flags.String(cmd.ENVIRONMENT_IMAGE) + envBuilderImg := flags.String(cmd.ENVIRONMENT_BUILDER) + envBuildCmd := flags.String(cmd.ENVIRONMENT_BUILDCOMMAND) + envExternalNetwork := flags.Bool(cmd.ENVIRONMENT_EXTERNAL_NETWORK) + + if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 { + e = multierror.Append(e, errors.New("need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command")) + } + + if len(envImg) > 0 { + env.Spec.Runtime.Image = envImg + } + + if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) { + e = multierror.Append(e, errors.New("version 1 Environments do not support builders. Must specify --version=2")) + } + + if len(envBuilderImg) > 0 { + env.Spec.Builder.Image = envBuilderImg + } + if len(envBuildCmd) > 0 { + env.Spec.Builder.Command = envBuildCmd + } + + if flags.IsSet(cmd.ENVIRONMENT_POOLSIZE) { + env.Spec.Poolsize = flags.Int(cmd.ENVIRONMENT_POOLSIZE) + } + + if flags.IsSet(cmd.ENVIRONMENT_GRACE_PERIOD) { + env.Spec.TerminationGracePeriod = flags.Int64(cmd.ENVIRONMENT_GRACE_PERIOD) + } + + if flags.IsSet(cmd.ENVIRONMENT_KEEPARCHIVE) { + env.Spec.KeepArchive = flags.Bool(cmd.ENVIRONMENT_KEEPARCHIVE) + } + + env.Spec.AllowAccessToExternalNetwork = envExternalNetwork + + if flags.IsSet(cmd.RUNTIME_MINCPU) || flags.IsSet(cmd.RUNTIME_MAXCPU) || + flags.IsSet(cmd.RUNTIME_MINMEMORY) || flags.IsSet(cmd.RUNTIME_MAXMEMORY) || + flags.IsSet(cmd.RUNTIME_MINSCALE) || flags.IsSet(cmd.RUNTIME_MAXSCALE) { + e = multierror.Append(e, errors.New("updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead")) + } + + if e.ErrorOrNil() != nil { + return nil, e.ErrorOrNil() + } + + return env, nil +} diff --git a/pkg/fission-cli/cmd/spec/spec.go b/pkg/fission-cli/cmd/spec/spec.go new file mode 100644 index 00000000..d22724f9 --- /dev/null +++ b/pkg/fission-cli/cmd/spec/spec.go @@ -0,0 +1,673 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package spec + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + + "github.com/ghodss/yaml" + multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/urfave/cli" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/fission-cli/log" + "github.com/fission/fission/pkg/fission-cli/util" + "github.com/fission/fission/pkg/generator/encoder" + v1generator "github.com/fission/fission/pkg/generator/v1" +) + +var specDefaultEncoder = encoder.DefaultYAMLEncoder() + +const ( + FISSION_DEPLOYMENT_NAME_KEY = "fission-name" + FISSION_DEPLOYMENT_UID_KEY = "fission-uid" + + SPEC_API_VERSION = "fission.io/v1" + ARCHIVE_URL_PREFIX string = "archive://" + SPEC_README = ` +Fission Specs +============= + +This is a set of specifications for a Fission app. This includes functions, +environments, and triggers; we collectively call these things "resources". + +How to use these specs +---------------------- + +These specs are handled with the 'fission spec' command. See 'fission spec --help'. + +'fission spec apply' will "apply" all resources specified in this directory to your +cluster. That means it checks what resources exist on your cluster, what resources are +specified in the specs directory, and reconciles the difference by creating, updating or +deleting resources on the cluster. + +'fission spec apply' will also package up your source code (or compiled binaries) and +upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in +this directory to figure out which files to archive. + +You can use 'fission spec apply --watch' to watch for file changes and continuously keep +the cluster updated. + +You can add YAMLs to this directory by writing them manually, but it's easier to generate +them. Use 'fission function create --spec' to generate a function spec, +'fission environment create --spec' to generate an environment spec, and so on. + +You can edit any of the files in this directory, except 'fission-deployment-config.yaml', +which contains a UID that you should never change. To apply your changes simply use +'fission spec apply'. + +fission-deployment-config.yaml +------------------------------ + +fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate +resources on the cluster to resources in this directory. + +All resources created by 'fission spec apply' are annotated with this UID. Resources on +the cluster that are _not_ annotated with this UID are never modified or deleted by +fission. + +` +) + +// CLI spec types +type ( + // DeploymentConfig is the global configuration for a set of Fission specs. + DeploymentConfig struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "DeploymentConfig". + TypeMeta `json:",inline"` + + // Name is a user-friendly name for the deployment. It is also stored in + // all uploaded resources as an annotation. + Name string `json:"name"` + + // UID uniquely identifies the deployment. It is stored as a label and + // used to find resources to clean up when local specs are changed. + UID string `json:"uid"` + } + + // ArchiveUploadSpec specifies a set of files to be archived and uploaded. + // + // The resulting archive can be referenced as archive:// in PackageSpecs, + // using the name specified in the archive. The fission spec applier will + // replace the archive:// URL with a real HTTP URL after uploading the file. + ArchiveUploadSpec struct { + // TypeMeta describes the type of this object. It is inlined. The Kind + // field should always be "ArchiveUploadSpec". + TypeMeta `json:",inline"` + + // Name is a local name that can be used to reference this archive. It + // must be unique; duplicate names will cause an error while handling + // specs. + Name string `json:"name"` + + // RootDir specifies the root that the globs below are relative to. It + // is optional and defaults to the parent directory of the spec + // directory: for example, if the deployment config is at + // /path/to/project/specs/config.yaml, the RootDir is /path/to/project. + RootDir string `json:"rootdir,omitempty"` + + // IncludeGlobs is a list of Unix shell globs to include + IncludeGlobs []string `json:"include,omitempty"` + + // ExcludeGlobs is a list of globs to exclude from the set specified by + // IncludeGlobs. + ExcludeGlobs []string `json:"exclude,omitempty"` + } + + // TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and + // unmarshal local-only objects (like ArchiveUploadSpec) the same way that + // Kubernetes does. + TypeMeta struct { + Kind string `json:"kind,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` + } + + FissionResources struct { + DeploymentConfig DeploymentConfig + Packages []fv1.Package + Functions []fv1.Function + Environments []fv1.Environment + HttpTriggers []fv1.HTTPTrigger + KubernetesWatchTriggers []fv1.KubernetesWatchTrigger + TimeTriggers []fv1.TimeTrigger + MessageQueueTriggers []fv1.MessageQueueTrigger + ArchiveUploadSpecs []ArchiveUploadSpec + + SourceMap SourceMap + } + + ResourceApplyStatus struct { + Created []*metav1.ObjectMeta + Updated []*metav1.ObjectMeta + Deleted []*metav1.ObjectMeta + } + + Location struct { + Path string + Line int + } + + SourceMap struct { + // kind -> namespace -> name -> location + Locations map[string](map[string](map[string]Location)) + } +) + +func MapKey(m *metav1.ObjectMeta) string { + return fmt.Sprintf("%v:%v", m.Namespace, m.Name) +} + +// Save saves object encoded value to spec file under given spec directory +func Save(data []byte, specDir string, specFile string) error { + // verify + if _, err := os.Stat(filepath.Join(specDir, "fission-deployment-config.yaml")); os.IsNotExist(err) { + return errors.Wrap(err, "Couldn't find specs, run `fission spec init` first") + } + + filename := filepath.Join(specDir, specFile) + // check if the file is new + newFile := false + if _, err := os.Stat(filename); os.IsNotExist(err) { + newFile = true + } + + // open spec file to append or write + f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return errors.Wrap(err, "couldn't create spec file") + } + defer f.Close() + + // if we're appending, add a yaml document separator + if !newFile { + _, err = f.Write([]byte("\n---\n")) + if err != nil { + return errors.Wrap(err, "couldn't write to spec file") + } + } + + // write our resource + _, err = f.Write(data) + if err != nil { + return errors.Wrap(err, "couldn't write to spec file") + } + return nil +} + +// called from `fission * create --spec` +func SpecSave(resource interface{}, specFile string) error { + specDir := "specs" + + // make sure we're writing a known type + var data []byte + var err error + switch typedres := resource.(type) { + case ArchiveUploadSpec: + typedres.Kind = "ArchiveUploadSpec" + data, err = yaml.Marshal(typedres) + case fv1.Package: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "Package" + data, err = yaml.Marshal(typedres) + case fv1.Function: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "Function" + data, err = yaml.Marshal(typedres) + case fv1.Environment: + env := resource.(fv1.Environment) + var generator *v1generator.EnvironmentGenerator + generator, err = v1generator.CreateEnvironmentGeneratorFromObj(&env) + if err != nil { + return err + } + data, err = generator.StructuredGenerate(specDefaultEncoder) + case fv1.HTTPTrigger: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "HTTPTrigger" + data, err = yaml.Marshal(typedres) + case fv1.KubernetesWatchTrigger: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "KubernetesWatchTrigger" + data, err = yaml.Marshal(typedres) + case fv1.MessageQueueTrigger: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "MessageQueueTrigger" + data, err = yaml.Marshal(typedres) + case fv1.TimeTrigger: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "TimeTrigger" + data, err = yaml.Marshal(typedres) + case fv1.Recorder: + typedres.TypeMeta.APIVersion = fv1.CRD_VERSION + typedres.TypeMeta.Kind = "Recorder" + data, err = yaml.Marshal(typedres) + default: + return fmt.Errorf("can't save resource %#v", resource) + } + if err != nil { + return errors.Wrap(err, "Couldn't marshal YAML") + } + + return Save(data, specDir, specFile) +} + +// validateFunctionReference checks a function reference +func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fv1.FunctionReference) error { + if funcRef.Type == fv1.FunctionReferenceTypeFunctionName { + // triggers only reference functions in their own namespace + namespace := meta.Namespace + name := funcRef.Name + m := &metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + } + if _, ok := functions[MapKey(m)]; !ok { + return fmt.Errorf("%v: %v '%v' references unknown function '%v'", + fr.SourceMap.Locations[kind][meta.Namespace][meta.Name], + kind, + meta.Name, + name) + } else { + functions[MapKey(m)] = true + } + } + return nil +} + +func (fr *FissionResources) Validate(c *cli.Context) error { + var result *multierror.Error + + // check references: both dangling refs + garbage + // packages -> archives + // functions -> packages + // functions -> environments + shared environments between functions [TODO] + // functions -> secrets + configmaps (same ns) [TODO] + // triggers -> functions + + // index archives + archives := make(map[string]bool) + for _, a := range fr.ArchiveUploadSpecs { + archives[a.Name] = false + } + + // index packages, check outgoing refs, mark archives that are referenced + packages := make(map[string]bool) + for _, p := range fr.Packages { + packages[MapKey(&p.Metadata)] = false + + // check archive refs from package + aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX) + if len(aname) > 0 { + if _, ok := archives[aname]; !ok { + result = multierror.Append(result, fmt.Errorf( + "%v: package '%v' references unknown source archive %v%v", + fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name], + p.Metadata.Name, + ARCHIVE_URL_PREFIX, + aname)) + } else { + archives[aname] = true + } + } + + aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX) + if len(aname) > 0 { + if _, ok := archives[aname]; !ok { + result = multierror.Append(result, fmt.Errorf( + "%v: package '%v' references unknown deployment archive %v%v", + fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name], + p.Metadata.Name, + ARCHIVE_URL_PREFIX, + aname)) + } else { + archives[aname] = true + } + } + + result = multierror.Append(result, p.Validate()) + } + + // error on unreferenced archives + for name, referenced := range archives { + if !referenced { + result = multierror.Append(result, fmt.Errorf( + "%v: archive '%v' is not used in any package", + fr.SourceMap.Locations["ArchiveUploadSpec"][""][name], + name)) + } + } + + // index functions, check function package refs, mark referenced packages + functions := make(map[string]bool) + for _, f := range fr.Functions { + functions[MapKey(&f.Metadata)] = false + + pkgMeta := &metav1.ObjectMeta{ + Name: f.Spec.Package.PackageRef.Name, + Namespace: f.Spec.Package.PackageRef.Namespace, + } + + // check package ref from function + packageRefExists := func() bool { + _, ok := packages[MapKey(pkgMeta)] + return ok + } + + // check that the package referenced by each function is in the same ns as the function + packageRefInFuncNs := func(f *fv1.Function) bool { + return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace + } + + if !packageRefInFuncNs(&f) { + result = multierror.Append(result, fmt.Errorf( + "%v: function '%v' references a package outside of its namespace %v/%v", + fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name], + f.Metadata.Name, + f.Spec.Package.PackageRef.Namespace, + f.Spec.Package.PackageRef.Name)) + } else if !packageRefExists() { + result = multierror.Append(result, fmt.Errorf( + "%v: function '%v' references unknown package %v/%v", + fr.SourceMap.Locations["Function"][f.Metadata.Namespace][f.Metadata.Name], + f.Metadata.Name, + pkgMeta.Namespace, + pkgMeta.Name)) + } else { + packages[MapKey(pkgMeta)] = true + } + + client := util.GetApiClient(c.GlobalString("server")) + for _, cm := range f.Spec.ConfigMaps { + _, err := client.ConfigMapGet(&metav1.ObjectMeta{ + Name: cm.Name, + Namespace: cm.Namespace, + }) + if k8serrors.IsNotFound(err) { + log.Warn(fmt.Sprintf("Configmap %s is referred in the spec but not present in the cluster", cm.Name)) + } + } + + for _, s := range f.Spec.Secrets { + _, err := client.SecretGet(&metav1.ObjectMeta{ + Name: s.Name, + Namespace: s.Namespace, + }) + if k8serrors.IsNotFound(err) { + log.Warn(fmt.Sprintf("Secret %s is referred in the spec but not present in the cluster", s.Name)) + } + + } + + result = multierror.Append(result, f.Validate()) + } + + // error on unreferenced packages + for key, referenced := range packages { + ks := strings.Split(key, ":") + namespace, name := ks[0], ks[1] + if !referenced { + result = multierror.Append(result, fmt.Errorf( + "%v: package '%v' is not used in any function", + fr.SourceMap.Locations["Package"][namespace][name], + name)) + } + } + + // check function refs from triggers + for _, t := range fr.HttpTriggers { + err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) + if err != nil { + result = multierror.Append(result, err) + } + result = multierror.Append(result, t.Validate()) + } + for _, t := range fr.KubernetesWatchTriggers { + err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) + if err != nil { + result = multierror.Append(result, err) + } + result = multierror.Append(result, t.Validate()) + } + for _, t := range fr.TimeTriggers { + err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) + if err != nil { + result = multierror.Append(result, err) + } + result = multierror.Append(result, t.Validate()) + } + for _, t := range fr.MessageQueueTriggers { + err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) + if err != nil { + result = multierror.Append(result, err) + } + result = multierror.Append(result, t.Validate()) + } + + // we do not error on unreferenced functions (you can call a function through workflows, + // `fission function test`, etc.) + + // Index envs, warn on functions referencing an environment for which spes does not exist + environments := make(map[string]struct{}) + for _, e := range fr.Environments { + environments[fmt.Sprintf("%s:%s", e.Metadata.Name, e.Metadata.Namespace)] = struct{}{} + if (e.Spec.Runtime.Container != nil) && (e.Spec.Runtime.PodSpec != nil) { + log.Warn("You have provided both - container spec and pod spec and while merging the pod spec will take precedence.") + } + // Unlike CLI can change the environment version silently, + // we have to warn the user to modify spec file when this takes place. + if e.Spec.Version < 3 && e.Spec.Poolsize != 0 { + log.Warn("Poolsize can only be configured when environment version equals to 3, default poolsize 3 will be used for creating environment pool.") + } + } + + for _, f := range fr.Functions { + if _, ok := environments[fmt.Sprintf("%s:%s", f.Spec.Environment.Name, f.Spec.Environment.Namespace)]; !ok { + log.Warn(fmt.Sprintf("Environment %s is referenced in function %s but not declared in specs", f.Spec.Environment.Name, f.Metadata.Name)) + } + strategy := f.Spec.InvokeStrategy.ExecutionStrategy + if strategy.ExecutorType == fv1.ExecutorTypeNewdeploy && strategy.SpecializationTimeout < fv1.DefaultSpecializationTimeOut { + log.Warn(fmt.Sprintf("SpecializationTimeout in function spec.InvokeStrategy.ExecutionStrategy should be a value equal to or greater than %v", fv1.DefaultSpecializationTimeOut)) + } + } + + // (ErrorOrNil returns nil if there were no errors appended.) + return result.ErrorOrNil() +} + +// Keep track of source location of resources, and track duplicates +func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *Location) error { + if _, exists := fr.SourceMap.Locations[kind]; !exists { + fr.SourceMap.Locations[kind] = make(map[string](map[string]Location)) + } + if _, exists := fr.SourceMap.Locations[kind][newobj.Namespace]; !exists { + fr.SourceMap.Locations[kind][newobj.Namespace] = make(map[string]Location) + } + + // check for duplicate resources + oldloc, exists := fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name] + if exists { + return fmt.Errorf("%v: Duplicate %v '%v', first defined in %v", loc, kind, newobj.Name, oldloc) + } + + // track new resource + fr.SourceMap.Locations[kind][newobj.Namespace][newobj.Name] = *loc + + return nil +} + +// ParseYaml takes one yaml document, figures out its type, parses it, and puts it in +// the right list in the given fission resources set. +func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error { + var m *metav1.ObjectMeta + + // Figure out the object type by unmarshaling into the TypeMeta struct; then + // unmarshal again into the "real" struct once we know the type. + var tm TypeMeta + err := yaml.Unmarshal(b, &tm) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b))) + } + + switch tm.Kind { + case "Package": + var v fv1.Package + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.Packages = append(fr.Packages, v) + case "Function": + var v fv1.Function + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.Functions = append(fr.Functions, v) + case "Environment": + var v fv1.Environment + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.Environments = append(fr.Environments, v) + case "HTTPTrigger": + var v fv1.HTTPTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + + // TODO move to validator + if !strings.HasPrefix(v.Spec.RelativeURL, "/") { + v.Spec.RelativeURL = fmt.Sprintf("/%s", v.Spec.RelativeURL) + } + + m = &v.Metadata + fr.HttpTriggers = append(fr.HttpTriggers, v) + case "KubernetesWatchTrigger": + var v fv1.KubernetesWatchTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.KubernetesWatchTriggers = append(fr.KubernetesWatchTriggers, v) + case "TimeTrigger": + var v fv1.TimeTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.TimeTriggers = append(fr.TimeTriggers, v) + case "MessageQueueTrigger": + var v fv1.MessageQueueTrigger + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &v.Metadata + fr.MessageQueueTriggers = append(fr.MessageQueueTriggers, v) + + // The following are not CRDs + + case "DeploymentConfig": + var v DeploymentConfig + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + fr.DeploymentConfig = v + case "ArchiveUploadSpec": + var v ArchiveUploadSpec + err = yaml.Unmarshal(b, &v) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) + } + m = &metav1.ObjectMeta{ + Name: v.Name, + Namespace: "", + } + fr.ArchiveUploadSpecs = append(fr.ArchiveUploadSpecs, v) + default: + // no need to error out just because there's some extra files around; + // also good for compatibility. + log.Warn(fmt.Sprintf("Ignoring unknown type %v in %v", tm.Kind, loc)) + } + + // add to source map, check for duplicates + if m != nil { + err = fr.trackSourceMap(tm.Kind, m, loc) + if err != nil { + return err + } + } + + return nil +} + +// Returns metadata if the given resource exists in the specs, nil +// otherwise. compareMetadata and compareSpec control how the +// equality check is performed. +func (fr *FissionResources) SpecExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta { + switch typedres := resource.(type) { + case *ArchiveUploadSpec: + for _, aus := range fr.ArchiveUploadSpecs { + if compareMetadata && aus.Name != typedres.Name { + continue + } + if compareSpec && + !(reflect.DeepEqual(aus.RootDir, typedres.RootDir) && + reflect.DeepEqual(aus.IncludeGlobs, typedres.IncludeGlobs) && + reflect.DeepEqual(aus.ExcludeGlobs, typedres.ExcludeGlobs)) { + continue + } + return &metav1.ObjectMeta{Name: aus.Name} + } + return nil + case *fv1.Package: + for _, p := range fr.Packages { + if compareMetadata && !reflect.DeepEqual(p.Metadata, typedres.Metadata) { + continue + } + if compareSpec && !reflect.DeepEqual(p.Spec, typedres.Spec) { + continue + } + return &p.Metadata + } + return nil + + default: + // XXX not implemented + return nil + } +} + +func (loc Location) String() string { + return fmt.Sprintf("%v:%v", loc.Path, loc.Line) +} diff --git a/pkg/fission-cli/cmd/util.go b/pkg/fission-cli/cmd/util.go new file mode 100644 index 00000000..211c51d4 --- /dev/null +++ b/pkg/fission-cli/cmd/util.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "strconv" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/cli" + "github.com/fission/fission/pkg/fission-cli/util" +) + +func GetServer(flags cli.Input) *client.Client { + return util.GetApiClient(flags.GlobalString(FISSION_SERVER)) +} + +func GetResourceReqs(flags cli.Input, resReqs *v1.ResourceRequirements) (*v1.ResourceRequirements, error) { + r := &v1.ResourceRequirements{} + + if resReqs != nil { + r.Requests = resReqs.Requests + r.Limits = resReqs.Limits + } + + if len(r.Requests) == 0 { + r.Requests = make(map[v1.ResourceName]resource.Quantity) + } + + if len(r.Limits) == 0 { + r.Limits = make(map[v1.ResourceName]resource.Quantity) + } + + e := &multierror.Error{} + + if flags.IsSet(RUNTIME_MINCPU) { + mincpu := flags.Int(RUNTIME_MINCPU) + cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m") + if err != nil { + e = multierror.Append(e, errors.Wrap(err, "Failed to parse mincpu")) + } + r.Requests[v1.ResourceCPU] = cpuRequest + } + + if flags.IsSet(RUNTIME_MINMEMORY) { + minmem := flags.Int(RUNTIME_MINMEMORY) + memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi") + if err != nil { + e = multierror.Append(e, errors.Wrap(err, "Failed to parse minmemory")) + } + r.Requests[v1.ResourceMemory] = memRequest + } + + if flags.IsSet(RUNTIME_MAXCPU) { + maxcpu := flags.Int(RUNTIME_MAXCPU) + cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m") + if err != nil { + e = multierror.Append(e, errors.Wrap(err, "Failed to parse maxcpu")) + } + r.Limits[v1.ResourceCPU] = cpuLimit + } + + if flags.IsSet(RUNTIME_MAXMEMORY) { + maxmem := flags.Int(RUNTIME_MAXMEMORY) + memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi") + if err != nil { + e = multierror.Append(e, errors.Wrap(err, "Failed to parse maxmemory")) + } + r.Limits[v1.ResourceMemory] = memLimit + } + + limitCPU := r.Limits[v1.ResourceCPU] + requestCPU := r.Requests[v1.ResourceCPU] + + if limitCPU.IsZero() && !requestCPU.IsZero() { + r.Limits[v1.ResourceCPU] = requestCPU + } else if limitCPU.Cmp(requestCPU) < 0 { + e = multierror.Append(e, fmt.Errorf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String())) + } + + limitMem := r.Limits[v1.ResourceMemory] + requestMem := r.Requests[v1.ResourceMemory] + + if limitMem.IsZero() && !requestMem.IsZero() { + r.Limits[v1.ResourceMemory] = requestMem + } else if limitMem.Cmp(requestMem) < 0 { + e = multierror.Append(e, fmt.Errorf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String())) + } + + if e.ErrorOrNil() != nil { + return nil, e + } + + return &v1.ResourceRequirements{ + Requests: r.Requests, + Limits: r.Limits, + }, nil +} + +func GetSpecDir(flags cli.Input) string { + specDir := flags.String(SPEC_SPECDIR) + if len(specDir) == 0 { + specDir = "specs" + } + return specDir +} + +// GetMetadata returns a pointer to ObjectMeta which initialized with command line input. +func GetMetadata(flags cli.Input) (*metav1.ObjectMeta, error) { + name := flags.String(RESOURCE_NAME) + if len(name) == 0 { + return nil, errors.New("Need a resource name, use --name.") + } + + ns := flags.String(ENVIRONMENT_NAMESPACE) + + m := &metav1.ObjectMeta{ + Name: name, + Namespace: ns, + } + + return m, nil +} diff --git a/pkg/fission-cli/environment.go b/pkg/fission-cli/environment.go deleted file mode 100644 index 1490e0eb..00000000 --- a/pkg/fission-cli/environment.go +++ /dev/null @@ -1,338 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fission_cli - -import ( - "fmt" - "os" - "strconv" - "text/tabwriter" - - "github.com/urfave/cli" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" - "github.com/fission/fission/pkg/fission-cli/log" - "github.com/fission/fission/pkg/fission-cli/util" -) - -func envCreate(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - envName := c.String("name") - if len(envName) == 0 { - log.Fatal("Need a name, use --name.") - } - envNamespace := c.String("envNamespace") - - envList, err := client.EnvironmentList(envNamespace) - if err == nil && len(envList) > 0 { - log.Verbose(2, "%d environment(s) are present in the %s namespace. "+ - "These environments are not isolated from each other; use separate namespaces if you need isolation.", - len(envList), envNamespace) - } - - var poolsize int - if c.IsSet("poolsize") { - poolsize = c.Int("poolsize") - } else { - poolsize = 3 - } - - envImg := c.String("image") - if len(envImg) == 0 { - log.Fatal("Need an image, use --image.") - } - - envVersion := c.Int("version") - envBuilderImg := c.String("builder") - envBuildCmd := c.String("buildcmd") - envExternalNetwork := c.Bool("externalnetwork") - envGracePeriod := c.Int64("period") - if envGracePeriod <= 0 { - envGracePeriod = 360 - } - - if len(envBuilderImg) > 0 { - if !c.IsSet("version") { - envVersion = 2 - } - if len(envBuildCmd) == 0 { - envBuildCmd = "build" - } - } - if c.IsSet("poolsize") { - envVersion = 3 - } - - keepArchive := c.Bool("keeparchive") - - // Environment API interface version is not specified and - // builder image is empty, set default interface version - if envVersion == 0 { - envVersion = 1 - } - - resourceReq := getResourceReq(c, v1.ResourceRequirements{}) - - env := &fv1.Environment{ - Metadata: metav1.ObjectMeta{ - Name: envName, - Namespace: envNamespace, - }, - Spec: fv1.EnvironmentSpec{ - Version: envVersion, - Runtime: fv1.Runtime{ - Image: envImg, - }, - Builder: fv1.Builder{ - Image: envBuilderImg, - Command: envBuildCmd, - }, - Poolsize: poolsize, - Resources: resourceReq, - AllowAccessToExternalNetwork: envExternalNetwork, - TerminationGracePeriod: envGracePeriod, - KeepArchive: keepArchive, - }, - } - - // if we're writing a spec, don't call the API - if c.Bool("spec") { - specFile := fmt.Sprintf("env-%v.yaml", envName) - err := specSave(*env, specFile) - util.CheckErr(err, "create environment spec") - return nil - } - - _, err = client.EnvironmentCreate(env) - util.CheckErr(err, "create environment") - - fmt.Printf("environment '%v' created\n", envName) - return err -} - -func envGet(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - envName := c.String("name") - if len(envName) == 0 { - log.Fatal("Need a name, use --name.") - } - envNamespace := c.String("envNamespace") - - m := &metav1.ObjectMeta{ - Name: envName, - Namespace: envNamespace, - } - env, err := client.EnvironmentGet(m) - util.CheckErr(err, "get environment") - - w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE") - fmt.Fprintf(w, "%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image) - w.Flush() - return nil -} - -func envUpdate(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - envName := c.String("name") - if len(envName) == 0 { - log.Fatal("Need a name, use --name.") - } - envNamespace := c.String("envNamespace") - - envImg := c.String("image") - envBuilderImg := c.String("builder") - envBuildCmd := c.String("buildcmd") - envExternalNetwork := c.Bool("externalnetwork") - - if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 { - log.Fatal("Need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command.") - } - - env, err := client.EnvironmentGet(&metav1.ObjectMeta{ - Name: envName, - Namespace: envNamespace, - }) - util.CheckErr(err, "find environment") - - if len(envImg) > 0 { - env.Spec.Runtime.Image = envImg - } - - if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) { - log.Fatal("Version 1 Environments do not support builders. Must specify --version=2.") - } - - if len(envBuilderImg) > 0 { - env.Spec.Builder.Image = envBuilderImg - } - if len(envBuildCmd) > 0 { - env.Spec.Builder.Command = envBuildCmd - } - - if c.IsSet("poolsize") { - env.Spec.Poolsize = c.Int("poolsize") - } - - if c.IsSet("period") { - env.Spec.TerminationGracePeriod = c.Int64("period") - } - - if c.IsSet("keeparchive") { - env.Spec.KeepArchive = c.Bool("keeparchive") - } - - env.Spec.AllowAccessToExternalNetwork = envExternalNetwork - - if c.IsSet("mincpu") || c.IsSet("maxcpu") || c.IsSet("minmemory") || c.IsSet("maxmemory") || c.IsSet("minscale") || c.IsSet("maxscale") { - log.Fatal("Updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead.") - } - - _, err = client.EnvironmentUpdate(env) - util.CheckErr(err, "update environment") - - fmt.Printf("environment '%v' updated\n", envName) - return nil -} - -func envDelete(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - - envName := c.String("name") - if len(envName) == 0 { - log.Fatal("Need a name , use --name.") - } - envNamespace := c.String("envNamespace") - - m := &metav1.ObjectMeta{ - Name: envName, - Namespace: envNamespace, - } - err := client.EnvironmentDelete(m) - util.CheckErr(err, "delete environment") - - fmt.Printf("environment '%v' deleted\n", envName) - return nil -} - -func envList(c *cli.Context) error { - client := util.GetApiClient(c.GlobalString("server")) - envNamespace := c.String("envNamespace") - - envs, err := client.EnvironmentList(envNamespace) - util.CheckErr(err, "list environments") - - w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME") - for _, env := range envs { - fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", - env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize, - env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(), - env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(), - env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod) - } - w.Flush() - - return nil -} - -func getResourceReq(c *cli.Context, resources v1.ResourceRequirements) v1.ResourceRequirements { - - var requestResources map[v1.ResourceName]resource.Quantity - - if len(resources.Requests) == 0 { - requestResources = make(map[v1.ResourceName]resource.Quantity) - } else { - requestResources = resources.Requests - } - - if c.IsSet("mincpu") { - mincpu := c.Int("mincpu") - cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m") - if err != nil { - log.Fatal("Failed to parse mincpu") - } - requestResources[v1.ResourceCPU] = cpuRequest - } - - if c.IsSet("minmemory") { - minmem := c.Int("minmemory") - memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi") - if err != nil { - log.Fatal("Failed to parse minmemory") - } - requestResources[v1.ResourceMemory] = memRequest - } - - var limitResources map[v1.ResourceName]resource.Quantity - - if len(resources.Limits) == 0 { - limitResources = make(map[v1.ResourceName]resource.Quantity) - } else { - limitResources = resources.Limits - } - - if c.IsSet("maxcpu") { - maxcpu := c.Int("maxcpu") - cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m") - if err != nil { - log.Fatal("Failed to parse maxcpu") - } - limitResources[v1.ResourceCPU] = cpuLimit - } - - if c.IsSet("maxmemory") { - maxmem := c.Int("maxmemory") - memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi") - if err != nil { - log.Fatal("Failed to parse maxmemory") - } - limitResources[v1.ResourceMemory] = memLimit - } - - limitCPU := limitResources[v1.ResourceCPU] - requestCPU := requestResources[v1.ResourceCPU] - - if limitCPU.IsZero() && !requestCPU.IsZero() { - limitResources[v1.ResourceCPU] = requestCPU - } else if limitCPU.Cmp(requestCPU) < 0 { - log.Fatal(fmt.Sprintf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String())) - } - - limitMem := limitResources[v1.ResourceMemory] - requestMem := requestResources[v1.ResourceMemory] - - if limitMem.IsZero() && !requestMem.IsZero() { - limitResources[v1.ResourceMemory] = requestMem - } else if limitMem.Cmp(requestMem) < 0 { - log.Fatal(fmt.Sprintf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String())) - } - - resources = v1.ResourceRequirements{ - Requests: requestResources, - Limits: limitResources, - } - - return resources -} diff --git a/pkg/fission-cli/function.go b/pkg/fission-cli/function.go index 71131de8..28d89a8b 100644 --- a/pkg/fission-cli/function.go +++ b/pkg/fission-cli/function.go @@ -28,7 +28,6 @@ import ( "text/tabwriter" "time" - "github.com/fission/fission/pkg/types" uuid "github.com/satori/go.uuid" "github.com/urfave/cli" apiv1 "k8s.io/api/core/v1" @@ -37,15 +36,19 @@ import ( fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" ferror "github.com/fission/fission/pkg/error" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" + "github.com/fission/fission/pkg/fission-cli/cmd" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/logdb" "github.com/fission/fission/pkg/fission-cli/util" + "github.com/fission/fission/pkg/types" ) const ( - DEFAULT_MIN_SCALE = 1 - DEFAULT_TARGET_CPU_PERCENTAGE = 80 - DEFAULT_SPECIALIZATION_TIMEOUT = 120 + DEFAULT_MIN_SCALE = 1 + DEFAULT_TARGET_CPU_PERCENTAGE = 80 ) func printPodLogs(c *cli.Context) error { @@ -125,7 +128,7 @@ func getInvokeStrategy(c *cli.Context, existingInvokeStrategy *fv1.InvokeStrateg targetCPU := DEFAULT_TARGET_CPU_PERCENTAGE minScale := DEFAULT_MIN_SCALE maxScale := minScale - specializationTimeout := DEFAULT_SPECIALIZATION_TIMEOUT + specializationTimeout := fv1.DefaultSpecializationTimeOut if existingInvokeStrategy != nil && existingInvokeStrategy.ExecutionStrategy.ExecutorType == types.ExecutorTypeNewdeploy { minScale = existingInvokeStrategy.ExecutionStrategy.MinScale @@ -151,7 +154,7 @@ func getInvokeStrategy(c *cli.Context, existingInvokeStrategy *fv1.InvokeStrateg if c.IsSet("specializationtimeout") { specializationTimeout = c.Int("specializationtimeout") - if specializationTimeout < DEFAULT_SPECIALIZATION_TIMEOUT { + if specializationTimeout < fv1.DefaultSpecializationTimeOut { return nil, errors.New("specializationtimeout must be greater than or equal to 120 seconds") } } @@ -203,13 +206,13 @@ func fnCreate(c *cli.Context) error { } // user wants a spec, create a yaml file with package and function - spec := false + toSpec := false specFile := "" if c.Bool("spec") { - spec = true + toSpec = true specFile = fmt.Sprintf("function-%v.yaml", fnName) } - specDir := getSpecDir(c) + specDir := cmdutils.GetSpecDir(urfavecli.Parse(c)) // check for unique function names within a namespace fnList, err := client.FunctionList(fnNamespace) @@ -230,7 +233,10 @@ func fnCreate(c *cli.Context) error { if err != nil { log.Fatal(err) } - resourceReq := getResourceReq(c, apiv1.ResourceRequirements{}) + resourceReq, err := cmd.GetResourceReqs(urfavecli.Parse(c), &apiv1.ResourceRequirements{}) + if err != nil { + log.Fatal(err) + } var pkgMetadata *metav1.ObjectMeta var envName string @@ -255,7 +261,7 @@ func fnCreate(c *cli.Context) error { } // examine existence of given environment. If specs - then spec validate will do it, don't check here. - if !spec { + if !toSpec { _, err := client.EnvironmentGet(&metav1.ObjectMeta{ Namespace: envNamespace, Name: envName, @@ -287,7 +293,7 @@ func fnCreate(c *cli.Context) error { buildcmd := c.String("buildcmd") // create new package in the same namespace as the function. - pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip) + pkgMetadata = createPackage(c, client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip) } var secrets []fv1.SecretReference @@ -347,14 +353,14 @@ func fnCreate(c *cli.Context) error { }, Secrets: secrets, ConfigMaps: cfgmaps, - Resources: resourceReq, + Resources: *resourceReq, InvokeStrategy: *invokeStrategy, }, } // if we're writing a spec, don't create the function - if spec { - err = specSave(*function, specFile) + if toSpec { + err = spec.SpecSave(*function, specFile) util.CheckErr(err, "create function spec") return nil @@ -578,14 +584,19 @@ func fnUpdate(c *cli.Context) error { log.Fatal("specializationtimeout flag is only applicable for newdeploy type of executor") } - if specializationTimeout < DEFAULT_SPECIALIZATION_TIMEOUT { + if specializationTimeout < fv1.DefaultSpecializationTimeOut { log.Fatal("specializationtimeout must be greater than or equal to 120 seconds") } else { function.Spec.InvokeStrategy.ExecutionStrategy.SpecializationTimeout = specializationTimeout } } - function.Spec.Resources = getResourceReq(c, function.Spec.Resources) + resReqs, err := cmd.GetResourceReqs(urfavecli.Parse(c), &function.Spec.Resources) + if err != nil { + log.Fatal(err) + } + + function.Spec.Resources = *resReqs pkg, err := client.PackageGet(&metav1.ObjectMeta{ Namespace: fnNamespace, diff --git a/pkg/fission-cli/function_test.go b/pkg/fission-cli/function_test.go index 24957f2a..df87a9ac 100644 --- a/pkg/fission-cli/function_test.go +++ b/pkg/fission-cli/function_test.go @@ -53,7 +53,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: DEFAULT_MIN_SCALE, MaxScale: DEFAULT_MIN_SCALE, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -74,7 +74,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: DEFAULT_MIN_SCALE, MaxScale: DEFAULT_MIN_SCALE, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -89,7 +89,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: DEFAULT_MIN_SCALE, MaxScale: DEFAULT_MIN_SCALE, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectedResult: &fv1.InvokeStrategy{ @@ -115,7 +115,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 3, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -155,7 +155,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: DEFAULT_MIN_SCALE, MaxScale: 3, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -183,7 +183,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 5, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectedResult: &fv1.InvokeStrategy{ @@ -193,7 +193,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 9, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -210,7 +210,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 5, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectedResult: &fv1.InvokeStrategy{ @@ -220,7 +220,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 5, TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -239,7 +239,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: DEFAULT_MIN_SCALE, MaxScale: DEFAULT_MIN_SCALE, TargetCPUPercent: 50, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, @@ -257,7 +257,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 5, TargetCPUPercent: 88, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectedResult: &fv1.InvokeStrategy{ @@ -267,7 +267,7 @@ func TestGetInvokeStrategy(t *testing.T) { MinScale: 2, MaxScale: 5, TargetCPUPercent: 20, - SpecializationTimeout: DEFAULT_SPECIALIZATION_TIMEOUT, + SpecializationTimeout: fv1.DefaultSpecializationTimeOut, }, }, expectError: false, diff --git a/pkg/fission-cli/httptrigger.go b/pkg/fission-cli/httptrigger.go index 91c8f39d..40a11f3f 100644 --- a/pkg/fission-cli/httptrigger.go +++ b/pkg/fission-cli/httptrigger.go @@ -29,6 +29,7 @@ import ( fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" ferror "github.com/fission/fission/pkg/error" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" ) @@ -106,7 +107,7 @@ func htCreate(c *cli.Context) error { triggerName := c.String("name") fnNamespace := c.String("fnNamespace") - spec := c.Bool("spec") + toSpec := c.Bool("spec") m := &metav1.ObjectMeta{ Name: triggerName, @@ -135,7 +136,7 @@ func htCreate(c *cli.Context) error { } // For Specs, the spec validate checks for function reference - if !spec { + if !toSpec { err = util.CheckFunctionExistence(client, functionList, fnNamespace) if err != nil { log.Warn(err.Error()) @@ -169,9 +170,9 @@ func htCreate(c *cli.Context) error { } // if we're writing a spec, don't call the API - if spec { + if toSpec { specFile := fmt.Sprintf("route-%v.yaml", triggerName) - err := specSave(*ht, specFile) + err := spec.SpecSave(*ht, specFile) util.CheckErr(err, "create HTTP trigger spec") return nil } diff --git a/pkg/fission-cli/main.go b/pkg/fission-cli/main.go index a20374ea..4427d704 100644 --- a/pkg/fission-cli/main.go +++ b/pkg/fission-cli/main.go @@ -27,6 +27,9 @@ import ( "github.com/urfave/cli" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/environment" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/plugin" "github.com/fission/fission/pkg/fission-cli/support" @@ -66,9 +69,9 @@ func NewCliApp() *cli.App { } app.Flags = []cli.Flag{ - cli.StringFlag{Name: "server", Value: "", Usage: "Fission server URL"}, - cli.IntFlag{Name: "verbosity", Value: 1, Usage: "CLI verbosity (0 is quiet, 1 is the default, 2 is verbose.)"}, - cli.BoolFlag{Name: "plugin", Hidden: true}, + cli.StringFlag{Name: cmd.FISSION_SERVER, Value: "", Usage: "Fission server URL"}, + cli.IntFlag{Name: cmd.GLOBAL_VERBOSITY, Value: 1, Usage: "CLI verbosity (0 is quiet, 1 is the default, 2 is verbose.)"}, + cli.BoolFlag{Name: cmd.GLOBAL_PLUGIN, Hidden: true}, } // all resource create commands accept --spec @@ -76,7 +79,7 @@ func NewCliApp() *cli.App { // namespace reference for all objects fnNamespaceFlag := cli.StringFlag{Name: "fnNamespace, fns", Value: metav1.NamespaceDefault, Usage: "Namespace for function object"} - envNamespaceFlag := cli.StringFlag{Name: "envNamespace, envns", Value: metav1.NamespaceDefault, Usage: "Namespace for environment object"} + envNamespaceFlag := cli.StringFlag{Name: cmd.GetCliFlagName(cmd.ENVIRONMENT_NAMESPACE, cmd.ENVIRONMENT_NAMESPACE_ALIAS), Value: metav1.NamespaceDefault, Usage: "Namespace for environment object"} pkgNamespaceFlag := cli.StringFlag{Name: "pkgNamespace, pkgns", Value: metav1.NamespaceDefault, Usage: "Namespace for package object"} triggerNamespaceFlag := cli.StringFlag{Name: "triggerNamespace, triggerns", Value: metav1.NamespaceDefault, Usage: "Namespace for trigger object"} recorderNamespaceFlag := cli.StringFlag{Name: "recorderNamespace, recorderns", Value: metav1.NamespaceDefault, Usage: "Namespace for recorder object"} @@ -87,13 +90,13 @@ func NewCliApp() *cli.App { htUrlFlag := cli.StringFlag{Name: "url", Usage: "URL pattern (See gorilla/mux supported patterns)"} // Resource & scale related flags (Used in env and function) - minCpu := cli.IntFlag{Name: "mincpu", Usage: "Minimum CPU to be assigned to pod (In millicore, minimum 1)"} - maxCpu := cli.IntFlag{Name: "maxcpu", Usage: "Maximum CPU to be assigned to pod (In millicore, minimum 1)"} - minMem := cli.IntFlag{Name: "minmemory", Usage: "Minimum memory to be assigned to pod (In megabyte)"} - maxMem := cli.IntFlag{Name: "maxmemory", Usage: "Maximum memory to be assigned to pod (In megabyte)"} - minScale := cli.IntFlag{Name: "minscale", Usage: "Minimum number of pods (Uses resource inputs to configure HPA)"} - maxScale := cli.IntFlag{Name: "maxscale", Usage: "Maximum number of pods (Uses resource inputs to configure HPA)"} - targetcpu := cli.IntFlag{Name: "targetcpu", Usage: "Target average CPU usage percentage across pods for scaling"} + minCpu := cli.IntFlag{Name: cmd.RUNTIME_MINCPU, Usage: "Minimum CPU to be assigned to pod (In millicore, minimum 1)"} + maxCpu := cli.IntFlag{Name: cmd.RUNTIME_MAXCPU, Usage: "Maximum CPU to be assigned to pod (In millicore, minimum 1)"} + minMem := cli.IntFlag{Name: cmd.RUNTIME_MINMEMORY, Usage: "Minimum memory to be assigned to pod (In megabyte)"} + maxMem := cli.IntFlag{Name: cmd.RUNTIME_MAXMEMORY, Usage: "Maximum memory to be assigned to pod (In megabyte)"} + minScale := cli.IntFlag{Name: cmd.RUNTIME_MINSCALE, Usage: "Minimum number of pods (Uses resource inputs to configure HPA)"} + maxScale := cli.IntFlag{Name: cmd.RUNTIME_MAXSCALE, Usage: "Maximum number of pods (Uses resource inputs to configure HPA)"} + targetcpu := cli.IntFlag{Name: cmd.RUNTIME_TARGETCPU, Usage: "Target average CPU usage percentage across pods for scaling"} specializationTimeoutFlag := cli.IntFlag{Name: "specializationtimeout, st", Usage: "Timeout for newdeploy to wait for function pod creation"} // functions @@ -212,21 +215,21 @@ func NewCliApp() *cli.App { reqIDFlag := cli.StringFlag{Name: "reqUID", Usage: "Replay a particular request by providing the reqUID (to view reqUIDs, do 'fission records view')"} // environments - envNameFlag := cli.StringFlag{Name: "name", Usage: "Environment name"} - envPoolsizeFlag := cli.IntFlag{Name: "poolsize", Value: 3, Usage: "Size of the pool"} - envImageFlag := cli.StringFlag{Name: "image", Usage: "Environment image URL"} - envBuilderImageFlag := cli.StringFlag{Name: "builder", Usage: "Environment builder image URL (optional)"} - envBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for environment builder to build source package (optional)"} - envKeepArchiveFlag := cli.BoolFlag{Name: "keeparchive", Usage: "Keep the archive instead of extracting it into a directory (optional, defaults to false)"} - envExternalNetworkFlag := cli.BoolFlag{Name: "externalnetwork", Usage: "Allow environment access external network when istio feature enabled (optional, defaults to false)"} - envTerminationGracePeriodFlag := cli.Int64Flag{Name: "graceperiod, period", Value: 360, Usage: "The grace time (in seconds) for pod to perform connection draining before termination (optional)"} - envVersionFlag := cli.IntFlag{Name: "version", Value: 1, Usage: "Environment API version (1 means v1 interface)"} + envNameFlag := cli.StringFlag{Name: cmd.RESOURCE_NAME, Usage: "Environment name"} + envPoolsizeFlag := cli.IntFlag{Name: cmd.ENVIRONMENT_POOLSIZE, Value: 3, Usage: "Size of the pool"} + envImageFlag := cli.StringFlag{Name: cmd.ENVIRONMENT_IMAGE, Usage: "Environment image URL"} + envBuilderImageFlag := cli.StringFlag{Name: cmd.ENVIRONMENT_BUILDER, Usage: "Environment builder image URL (optional)"} + envBuildCmdFlag := cli.StringFlag{Name: cmd.ENVIRONMENT_BUILDCOMMAND, Usage: "Build command for environment builder to build source package (optional)"} + envKeepArchiveFlag := cli.BoolFlag{Name: cmd.ENVIRONMENT_KEEPARCHIVE, Usage: "Keep the archive instead of extracting it into a directory (optional, defaults to false)"} + envExternalNetworkFlag := cli.BoolFlag{Name: cmd.ENVIRONMENT_EXTERNAL_NETWORK, Usage: "Allow environment access external network when istio feature enabled (optional, defaults to false)"} + envTerminationGracePeriodFlag := cli.Int64Flag{Name: cmd.GetCliFlagName(cmd.ENVIRONMENT_GRACE_PERIOD, cmd.ENVIRONMENT_GRACE_PERIOD_ALIAS), Value: 360, Usage: "The grace time (in seconds) for pod to perform connection draining before termination (optional)"} + envVersionFlag := cli.IntFlag{Name: cmd.ENVIRONMENT_VERSION, Value: 1, Usage: "Environment API version (1 means v1 interface)"} envSubcommands := []cli.Command{ - {Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envVersionFlag, envExternalNetworkFlag, envTerminationGracePeriodFlag, specSaveFlag}, Action: envCreate}, - {Name: "get", Usage: "Get environment details", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: envGet}, - {Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envExternalNetworkFlag, envTerminationGracePeriodFlag}, Action: envUpdate}, - {Name: "delete", Usage: "Delete environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: envDelete}, - {Name: "list", Usage: "List all environments", Flags: []cli.Flag{envNamespaceFlag}, Action: envList}, + {Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envVersionFlag, envExternalNetworkFlag, envTerminationGracePeriodFlag, specSaveFlag}, Action: urfavecli.Wrapper(environment.Create)}, + {Name: "get", Usage: "Get environment details", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: urfavecli.Wrapper(environment.Get)}, + {Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envExternalNetworkFlag, envTerminationGracePeriodFlag}, Action: urfavecli.Wrapper(environment.Update)}, + {Name: "delete", Usage: "Delete environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: urfavecli.Wrapper(environment.Delete)}, + {Name: "list", Usage: "List all environments", Flags: []cli.Flag{envNamespaceFlag}, Action: urfavecli.Wrapper(environment.List)}, } // watches @@ -263,13 +266,6 @@ func NewCliApp() *cli.App { {Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgForceFlag, pkgOrphanFlag}, Action: pkgDelete}, } - // upgrades, data migrations - upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"} - upgradeSubCommands := []cli.Command{ - {Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState}, - {Name: "restore", Usage: "Restore state dumped from a v0.1 install into a v0.2+ install", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeRestoreState}, - } - // specs specDirFlag := cli.StringFlag{Name: "specdir", Usage: "Directory to store specs, defaults to ./specs"} specNameFlag := cli.StringFlag{Name: "name", Usage: "(optional) Name for the app, applied to resources as a Kubernetes annotation"} @@ -320,7 +316,6 @@ func NewCliApp() *cli.App { {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, {Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands}, {Name: "spec", Aliases: []string{"specs"}, Usage: "Manage a declarative app specification", Subcommands: specSubCommands}, - {Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands}, {Name: "support", Usage: "Collect an archive of diagnostic information for support", Subcommands: supportSubCommands}, cmdPlugin, {Name: "canary-config", Aliases: []string{}, Usage: "Create, Update and manage Canary Configs", Subcommands: canarySubCommands}, diff --git a/pkg/fission-cli/mqtrigger.go b/pkg/fission-cli/mqtrigger.go index 8d949d8e..4a88c198 100644 --- a/pkg/fission-cli/mqtrigger.go +++ b/pkg/fission-cli/mqtrigger.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" "github.com/fission/fission/pkg/types" @@ -110,7 +111,7 @@ func mqtCreate(c *cli.Context) error { // if we're writing a spec, don't call the API if c.Bool("spec") { specFile := fmt.Sprintf("mqtrigger-%v.yaml", mqtName) - err := specSave(*mqt, specFile) + err := spec.SpecSave(*mqt, specFile) util.CheckErr(err, "create message queue trigger spec") return nil } diff --git a/pkg/fission-cli/package.go b/pkg/fission-cli/package.go index a70f9aee..3ce23f2a 100644 --- a/pkg/fission-cli/package.go +++ b/pkg/fission-cli/package.go @@ -43,6 +43,9 @@ import ( fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" + cmdutils "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" storageSvcClient "github.com/fission/fission/pkg/storagesvc/client" @@ -95,7 +98,7 @@ func pkgCreate(c *cli.Context) error { log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.") } - createPackage(client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false) + createPackage(c, client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false) return nil } @@ -511,7 +514,7 @@ func createArchive(client *client.Client, includeFiles []string, noZip bool, spe if len(specFile) > 0 { // create an ArchiveUploadSpec and reference it from the archive - aus := &ArchiveUploadSpec{ + aus := &spec.ArchiveUploadSpec{ Name: archiveName("", includeFiles), IncludeGlobs: includeFiles, } @@ -519,19 +522,19 @@ func createArchive(client *client.Client, includeFiles []string, noZip bool, spe // check if this AUS exists in the specs; if so, don't create a new one fr, err := readSpecs(specDir) util.CheckErr(err, "read specs") - if m := fr.specExists(aus, false, true); m != nil { + if m := fr.SpecExists(aus, false, true); m != nil { fmt.Printf("Re-using previously created archive %v\n", m.Name) aus.Name = m.Name } else { // save the uploadspec - err := specSave(*aus, specFile) + err := spec.SpecSave(*aus, specFile) util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile)) } // create the archive object ar := &fv1.Archive{ Type: fv1.ArchiveTypeUrl, - URL: fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name), + URL: fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name), } return ar } @@ -581,7 +584,7 @@ func uploadArchive(ctx context.Context, client *client.Client, fileName string) return &archive } -func createPackage(client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) *metav1.ObjectMeta { +func createPackage(c *cli.Context, client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) *metav1.ObjectMeta { pkgSpec := fv1.PackageSpec{ Environment: fv1.EnvironmentReference{ Namespace: envNamespace, @@ -624,14 +627,14 @@ func createPackage(client *client.Client, pkgNamespace string, envName string, e if len(specFile) > 0 { // if a package sith the same spec exists, don't create a new spec file - fr, err := readSpecs(getSpecDir(nil)) + fr, err := readSpecs(cmdutils.GetSpecDir(urfavecli.Parse(c))) util.CheckErr(err, "read specs") - if m := fr.specExists(pkg, false, true); m != nil { + if m := fr.SpecExists(pkg, false, true); m != nil { fmt.Printf("Re-using previously created package %v\n", m.Name) return m } - err = specSave(*pkg, specFile) + err = spec.SpecSave(*pkg, specFile) util.CheckErr(err, "save package spec") return &pkg.Metadata } else { diff --git a/pkg/fission-cli/recorder.go b/pkg/fission-cli/recorder.go index da3200a9..9cdb327c 100644 --- a/pkg/fission-cli/recorder.go +++ b/pkg/fission-cli/recorder.go @@ -27,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" ) @@ -82,7 +83,7 @@ func recorderCreate(c *cli.Context) error { // If we're writing a spec, don't call the API if c.Bool("spec") { specFile := fmt.Sprintf("recorder-%v.yaml", recName) - err := specSave(*recorder, specFile) + err := spec.SpecSave(*recorder, specFile) util.CheckErr(err, "create recorder spec") return nil } diff --git a/pkg/fission-cli/spec.go b/pkg/fission-cli/spec.go index 736aa729..4d835b14 100644 --- a/pkg/fission-cli/spec.go +++ b/pkg/fission-cli/spec.go @@ -34,109 +34,21 @@ import ( "github.com/pkg/errors" uuid "github.com/satori/go.uuid" "github.com/urfave/cli" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli" + "github.com/fission/fission/pkg/fission-cli/cmd" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" "github.com/fission/fission/pkg/types" ) -const SPEC_API_VERSION = "fission.io/v1" - -const ARCHIVE_URL_PREFIX string = "archive://" - -const SPEC_README = ` -Fission Specs -============= - -This is a set of specifications for a Fission app. This includes functions, -environments, and triggers; we collectively call these things "resources". - -How to use these specs ----------------------- - -These specs are handled with the 'fission spec' command. See 'fission spec --help'. - -'fission spec apply' will "apply" all resources specified in this directory to your -cluster. That means it checks what resources exist on your cluster, what resources are -specified in the specs directory, and reconciles the difference by creating, updating or -deleting resources on the cluster. - -'fission spec apply' will also package up your source code (or compiled binaries) and -upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in -this directory to figure out which files to archive. - -You can use 'fission spec apply --watch' to watch for file changes and continuously keep -the cluster updated. - -You can add YAMLs to this directory by writing them manually, but it's easier to generate -them. Use 'fission function create --spec' to generate a function spec, -'fission environment create --spec' to generate an environment spec, and so on. - -You can edit any of the files in this directory, except 'fission-deployment-config.yaml', -which contains a UID that you should never change. To apply your changes simply use -'fission spec apply'. - -fission-deployment-config.yaml ------------------------------- - -fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate -resources on the cluster to resources in this directory. - -All resources created by 'fission spec apply' are annotated with this UID. Resources on -the cluster that are _not_ annotated with this UID are never modified or deleted by -fission. - -` - -type ( - FissionResources struct { - deploymentConfig DeploymentConfig - packages []fv1.Package - functions []fv1.Function - environments []fv1.Environment - httpTriggers []fv1.HTTPTrigger - kubernetesWatchTriggers []fv1.KubernetesWatchTrigger - timeTriggers []fv1.TimeTrigger - messageQueueTriggers []fv1.MessageQueueTrigger - archiveUploadSpecs []ArchiveUploadSpec - - sourceMap sourceMap - } - - resourceApplyStatus struct { - created []*metav1.ObjectMeta - updated []*metav1.ObjectMeta - deleted []*metav1.ObjectMeta - } - - location struct { - path string - line int - } - sourceMap struct { - // kind -> namespace -> name -> location - locations map[string](map[string](map[string]location)) - } -) - -func getSpecDir(c *cli.Context) string { - specDir := "" - if c != nil { - specDir = c.String("specdir") - } - if len(specDir) == 0 { - specDir = "specs" - } - return specDir -} - // writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new // fission-config.yaml in specDir. -func writeDeploymentConfig(specDir string, dc *DeploymentConfig) error { +func writeDeploymentConfig(specDir string, dc *spec.DeploymentConfig) error { y, err := yaml.Marshal(dc) if err != nil { return err @@ -157,7 +69,7 @@ func writeDeploymentConfig(specDir string, dc *DeploymentConfig) error { // sample YAMLs in there that might be useful. func specInit(c *cli.Context) error { // Figure out spec directory - specDir := getSpecDir(c) + specDir := cmd.GetSpecDir(urfavecli.Parse(c)) name := c.String("name") if len(name) == 0 { @@ -179,15 +91,15 @@ func specInit(c *cli.Context) error { util.CheckErr(err, fmt.Sprintf("create spec directory '%v'", specDir)) // Add a bit of documentation to the spec dir here - err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644) + err = ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(spec.SPEC_README), 0644) if err != nil { return err } // Write the deployment config - dc := DeploymentConfig{ - TypeMeta: TypeMeta{ - APIVersion: SPEC_API_VERSION, + dc := spec.DeploymentConfig{ + TypeMeta: spec.TypeMeta{ + APIVersion: spec.SPEC_API_VERSION, Kind: "DeploymentConfig", }, Name: name, @@ -207,39 +119,16 @@ func specInit(c *cli.Context) error { return nil } -// validateFunctionReference checks a function reference -func (fr *FissionResources) validateFunctionReference(functions map[string]bool, kind string, meta *metav1.ObjectMeta, funcRef fv1.FunctionReference) error { - if funcRef.Type == fv1.FunctionReferenceTypeFunctionName { - // triggers only reference functions in their own namespace - namespace := meta.Namespace - name := funcRef.Name - m := &metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - } - if _, ok := functions[mapKey(m)]; !ok { - return fmt.Errorf("%v: %v '%v' references unknown function '%v'", - fr.sourceMap.locations[kind][meta.Namespace][meta.Name], - kind, - meta.Name, - name) - } else { - functions[mapKey(m)] = true - } - } - return nil -} - // specValidate parses a set of specs and checks for references to // resources that don't exist. func specValidate(c *cli.Context) error { // this will error on parse errors and on duplicates - specDir := getSpecDir(c) + specDir := cmd.GetSpecDir(urfavecli.Parse(c)) fr, err := readSpecs(specDir) util.CheckErr(err, "read specs") // this does the rest of the checks, like dangling refs - err = fr.validate(c) + err = fr.Validate(c) if err != nil { fmt.Printf("Error validating specs: %v", err) } @@ -247,348 +136,9 @@ func specValidate(c *cli.Context) error { return nil } -func (fr *FissionResources) validate(c *cli.Context) error { - var result *multierror.Error - - // check references: both dangling refs + garbage - // packages -> archives - // functions -> packages - // functions -> environments + shared environments between functions [TODO] - // functions -> secrets + configmaps (same ns) [TODO] - // triggers -> functions - - // index archives - archives := make(map[string]bool) - for _, a := range fr.archiveUploadSpecs { - archives[a.Name] = false - } - - // index packages, check outgoing refs, mark archives that are referenced - packages := make(map[string]bool) - for _, p := range fr.packages { - packages[mapKey(&p.Metadata)] = false - - // check archive refs from package - aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX) - if len(aname) > 0 { - if _, ok := archives[aname]; !ok { - result = multierror.Append(result, fmt.Errorf( - "%v: package '%v' references unknown source archive %v%v", - fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name], - p.Metadata.Name, - ARCHIVE_URL_PREFIX, - aname)) - } else { - archives[aname] = true - } - } - - aname = strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX) - if len(aname) > 0 { - if _, ok := archives[aname]; !ok { - result = multierror.Append(result, fmt.Errorf( - "%v: package '%v' references unknown deployment archive %v%v", - fr.sourceMap.locations["Package"][p.Metadata.Namespace][p.Metadata.Name], - p.Metadata.Name, - ARCHIVE_URL_PREFIX, - aname)) - } else { - archives[aname] = true - } - } - - result = multierror.Append(result, p.Validate()) - } - - // error on unreferenced archives - for name, referenced := range archives { - if !referenced { - result = multierror.Append(result, fmt.Errorf( - "%v: archive '%v' is not used in any package", - fr.sourceMap.locations["ArchiveUploadSpec"][""][name], - name)) - } - } - - // index functions, check function package refs, mark referenced packages - functions := make(map[string]bool) - for _, f := range fr.functions { - functions[mapKey(&f.Metadata)] = false - - pkgMeta := &metav1.ObjectMeta{ - Name: f.Spec.Package.PackageRef.Name, - Namespace: f.Spec.Package.PackageRef.Namespace, - } - - // check package ref from function - packageRefExists := func() bool { - _, ok := packages[mapKey(pkgMeta)] - return ok - } - - // check that the package referenced by each function is in the same ns as the function - packageRefInFuncNs := func(f *fv1.Function) bool { - return f.Spec.Package.PackageRef.Namespace == f.Metadata.Namespace - } - - if !packageRefInFuncNs(&f) { - result = multierror.Append(result, fmt.Errorf( - "%v: function '%v' references a package outside of its namespace %v/%v", - fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name], - f.Metadata.Name, - f.Spec.Package.PackageRef.Namespace, - f.Spec.Package.PackageRef.Name)) - } else if !packageRefExists() { - result = multierror.Append(result, fmt.Errorf( - "%v: function '%v' references unknown package %v/%v", - fr.sourceMap.locations["Function"][f.Metadata.Namespace][f.Metadata.Name], - f.Metadata.Name, - pkgMeta.Namespace, - pkgMeta.Name)) - } else { - packages[mapKey(pkgMeta)] = true - } - - client := util.GetApiClient(c.GlobalString("server")) - for _, cm := range f.Spec.ConfigMaps { - _, err := client.ConfigMapGet(&metav1.ObjectMeta{ - Name: cm.Name, - Namespace: cm.Namespace, - }) - if k8serrors.IsNotFound(err) { - log.Warn(fmt.Sprintf("Configmap %s is referred in the spec but not present in the cluster", cm.Name)) - } - } - - for _, s := range f.Spec.Secrets { - _, err := client.SecretGet(&metav1.ObjectMeta{ - Name: s.Name, - Namespace: s.Namespace, - }) - if k8serrors.IsNotFound(err) { - log.Warn(fmt.Sprintf("Secret %s is referred in the spec but not present in the cluster", s.Name)) - } - - } - - result = multierror.Append(result, f.Validate()) - } - - // error on unreferenced packages - for key, referenced := range packages { - ks := strings.Split(key, ":") - namespace, name := ks[0], ks[1] - if !referenced { - result = multierror.Append(result, fmt.Errorf( - "%v: package '%v' is not used in any function", - fr.sourceMap.locations["Package"][namespace][name], - name)) - } - } - - // check function refs from triggers - for _, t := range fr.httpTriggers { - err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) - if err != nil { - result = multierror.Append(result, err) - } - result = multierror.Append(result, t.Validate()) - } - for _, t := range fr.kubernetesWatchTriggers { - err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) - if err != nil { - result = multierror.Append(result, err) - } - result = multierror.Append(result, t.Validate()) - } - for _, t := range fr.timeTriggers { - err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) - if err != nil { - result = multierror.Append(result, err) - } - result = multierror.Append(result, t.Validate()) - } - for _, t := range fr.messageQueueTriggers { - err := fr.validateFunctionReference(functions, t.Kind, &t.Metadata, t.Spec.FunctionReference) - if err != nil { - result = multierror.Append(result, err) - } - result = multierror.Append(result, t.Validate()) - } - - // we do not error on unreferenced functions (you can call a function through workflows, - // `fission function test`, etc.) - - // Index envs, warn on functions referencing an environment for which spes does not exist - environments := make(map[string]struct{}) - for _, e := range fr.environments { - environments[fmt.Sprintf("%s:%s", e.Metadata.Name, e.Metadata.Namespace)] = struct{}{} - if (e.Spec.Runtime.Container != nil) && (e.Spec.Runtime.PodSpec != nil) { - log.Warn("You have provided both - container spec and pod spec and while merging the pod spec will take precedence.") - } - // Unlike CLI can change the environment version silently, - // we have to warn the user to modify spec file when this takes place. - if e.Spec.Version < 3 && e.Spec.Poolsize != 0 { - log.Warn("Poolsize can only be configured when environment version equals to 3, default poolsize 3 will be used for creating environment pool.") - } - } - - for _, f := range fr.functions { - if _, ok := environments[fmt.Sprintf("%s:%s", f.Spec.Environment.Name, f.Spec.Environment.Namespace)]; !ok { - log.Warn(fmt.Sprintf("Environment %s is referenced in function %s but not declared in specs", f.Spec.Environment.Name, f.Metadata.Name)) - } - strategy := f.Spec.InvokeStrategy.ExecutionStrategy - if strategy.ExecutorType == fv1.ExecutorTypeNewdeploy && strategy.SpecializationTimeout < DEFAULT_SPECIALIZATION_TIMEOUT { - log.Warn(fmt.Sprintf("SpecializationTimeout in function spec.InvokeStrategy.ExecutionStrategy should be a value equal to or greater than %v", DEFAULT_SPECIALIZATION_TIMEOUT)) - } - } - - // (ErrorOrNil returns nil if there were no errors appended.) - return result.ErrorOrNil() -} - -func (loc location) String() string { - return fmt.Sprintf("%v:%v", loc.path, loc.line) -} - -// Keep track of source location of resources, and track duplicates -func (fr *FissionResources) trackSourceMap(kind string, newobj *metav1.ObjectMeta, loc *location) error { - if _, exists := fr.sourceMap.locations[kind]; !exists { - fr.sourceMap.locations[kind] = make(map[string](map[string]location)) - } - if _, exists := fr.sourceMap.locations[kind][newobj.Namespace]; !exists { - fr.sourceMap.locations[kind][newobj.Namespace] = make(map[string]location) - } - - // check for duplicate resources - oldloc, exists := fr.sourceMap.locations[kind][newobj.Namespace][newobj.Name] - if exists { - return fmt.Errorf("%v: Duplicate %v '%v', first defined in %v", loc, kind, newobj.Name, oldloc) - } - - // track new resource - fr.sourceMap.locations[kind][newobj.Namespace][newobj.Name] = *loc - - return nil -} - -// parseYaml takes one yaml document, figures out its type, parses it, and puts it in -// the right list in the given fission resources set. -func (fr *FissionResources) parseYaml(b []byte, loc *location) error { - var m *metav1.ObjectMeta - - // Figure out the object type by unmarshaling into the TypeMeta struct; then - // unmarshal again into the "real" struct once we know the type. - var tm TypeMeta - err := yaml.Unmarshal(b, &tm) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b))) - } - - switch tm.Kind { - case "Package": - var v fv1.Package - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.packages = append(fr.packages, v) - case "Function": - var v fv1.Function - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.functions = append(fr.functions, v) - case "Environment": - var v fv1.Environment - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.environments = append(fr.environments, v) - case "HTTPTrigger": - var v fv1.HTTPTrigger - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - - // TODO move to validator - if !strings.HasPrefix(v.Spec.RelativeURL, "/") { - v.Spec.RelativeURL = fmt.Sprintf("/%s", v.Spec.RelativeURL) - } - - m = &v.Metadata - fr.httpTriggers = append(fr.httpTriggers, v) - case "KubernetesWatchTrigger": - var v fv1.KubernetesWatchTrigger - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.kubernetesWatchTriggers = append(fr.kubernetesWatchTriggers, v) - case "TimeTrigger": - var v fv1.TimeTrigger - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.timeTriggers = append(fr.timeTriggers, v) - case "MessageQueueTrigger": - var v fv1.MessageQueueTrigger - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &v.Metadata - fr.messageQueueTriggers = append(fr.messageQueueTriggers, v) - - // The following are not CRDs - - case "DeploymentConfig": - var v DeploymentConfig - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - fr.deploymentConfig = v - case "ArchiveUploadSpec": - var v ArchiveUploadSpec - err = yaml.Unmarshal(b, &v) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc)) - } - m = &metav1.ObjectMeta{ - Name: v.Name, - Namespace: "", - } - fr.archiveUploadSpecs = append(fr.archiveUploadSpecs, v) - default: - // no need to error out just because there's some extra files around; - // also good for compatibility. - log.Warn(fmt.Sprintf("Ignoring unknown type %v in %v", tm.Kind, loc)) - } - - // add to source map, check for duplicates - if m != nil { - err = fr.trackSourceMap(tm.Kind, m, loc) - if err != nil { - return err - } - } - - return nil -} - // readSpecs reads all specs in the specified directory and returns a parsed set of // fission resources. -func readSpecs(specDir string) (*FissionResources, error) { +func readSpecs(specDir string) (*spec.FissionResources, error) { // make sure spec directory exists before continue if _, err := os.Stat(specDir); os.IsNotExist(err) { @@ -596,17 +146,17 @@ func readSpecs(specDir string) (*FissionResources, error) { "Please check directory path or run \"fission spec init\" to create it.", specDir)) } - fr := FissionResources{ - packages: make([]fv1.Package, 0), - functions: make([]fv1.Function, 0), - environments: make([]fv1.Environment, 0), - httpTriggers: make([]fv1.HTTPTrigger, 0), - kubernetesWatchTriggers: make([]fv1.KubernetesWatchTrigger, 0), - timeTriggers: make([]fv1.TimeTrigger, 0), - messageQueueTriggers: make([]fv1.MessageQueueTrigger, 0), + fr := spec.FissionResources{ + Packages: make([]fv1.Package, 0), + Functions: make([]fv1.Function, 0), + Environments: make([]fv1.Environment, 0), + HttpTriggers: make([]fv1.HTTPTrigger, 0), + KubernetesWatchTriggers: make([]fv1.KubernetesWatchTrigger, 0), + TimeTriggers: make([]fv1.TimeTrigger, 0), + MessageQueueTriggers: make([]fv1.MessageQueueTrigger, 0), - sourceMap: sourceMap{ - locations: make(map[string](map[string](map[string]location))), + SourceMap: spec.SourceMap{ + Locations: make(map[string](map[string](map[string]spec.Location))), }, } @@ -637,9 +187,9 @@ func readSpecs(specDir string) (*FissionResources, error) { d := []byte(strings.TrimSpace(string(doc))) if len(d) != 0 { // parse this document and add whatever is in it to fr - err = fr.parseYaml(d, &location{ - path: path, - line: lines, + err = fr.ParseYaml(d, &spec.Location{ + Path: path, + Line: lines, }) if err != nil { // collect all errors so user can fix them all @@ -696,7 +246,7 @@ func waitForFileWatcherToSettleDown(watcher *fsnotify.Watcher) error { // they can retry their apply command once they're back online. func specApply(c *cli.Context) error { fclient := util.GetApiClient(c.GlobalString("server")) - specDir := getSpecDir(c) + specDir := cmd.GetSpecDir(urfavecli.Parse(c)) deleteResources := c.Bool("delete") watchResources := c.Bool("watch") @@ -736,7 +286,7 @@ func specApply(c *cli.Context) error { util.CheckErr(err, "read specs") // validate - err = fr.validate(c) + err = fr.Validate(c) util.CheckErr(err, "validate specs") // make changes to the cluster based on the specs @@ -795,23 +345,23 @@ func specApply(c *cli.Context) error { // printApplyStatus prints a summary of what changed on the cluster as the result of a spec apply // operation. -func printApplyStatus(applyStatus map[string]resourceApplyStatus) { +func printApplyStatus(applyStatus map[string]spec.ResourceApplyStatus) { changed := false for typ, ras := range applyStatus { - n := len(ras.created) + n := len(ras.Created) if n > 0 { changed = true - fmt.Printf("%v %v created: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.created), ", ")) + fmt.Printf("%v %v created: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.Created), ", ")) } - n = len(ras.updated) + n = len(ras.Updated) if n > 0 { changed = true - fmt.Printf("%v %v updated: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.updated), ", ")) + fmt.Printf("%v %v updated: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.Updated), ", ")) } - n = len(ras.deleted) + n = len(ras.Deleted) if n > 0 { changed = true - fmt.Printf("%v %v deleted: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.deleted), ", ")) + fmt.Printf("%v %v deleted: %v\n", n, pluralize(n, typ), strings.Join(metadataNames(ras.Deleted), ", ")) } } @@ -842,15 +392,15 @@ func specDestroy(c *cli.Context) error { fclient := util.GetApiClient(c.GlobalString("server")) // get specdir - specDir := getSpecDir(c) + specDir := cmd.GetSpecDir(urfavecli.Parse(c)) // read everything fr, err := readSpecs(specDir) util.CheckErr(err, "read specs") // set desired state to nothing, but keep the UID so "apply" can find it - emptyFr := FissionResources{} - emptyFr.deploymentConfig = fr.deploymentConfig + emptyFr := spec.FissionResources{} + emptyFr.DeploymentConfig = fr.DeploymentConfig // "apply" the empty state _, _, err = applyResources(fclient, specDir, &emptyFr, true) @@ -860,7 +410,7 @@ func specDestroy(c *cli.Context) error { } // applyArchives figures out the set of archives that need to be uploaded, and uploads them. -func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) error { +func applyArchives(fclient *client.Client, specDir string, fr *spec.FissionResources) error { // archive:// URL -> archive map. archiveFiles := make(map[string]fv1.Archive) @@ -869,12 +419,12 @@ func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) // point at archive URLs. // create archives locally and calculate checksums - for _, aus := range fr.archiveUploadSpecs { + for _, aus := range fr.ArchiveUploadSpecs { ar, err := localArchiveFromSpec(specDir, &aus) if err != nil { return err } - archiveUrl := fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name) + archiveUrl := fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name) archiveFiles[archiveUrl] = *ar } @@ -913,12 +463,12 @@ func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) } // resolve references to urls in packages to be applied - for i := range fr.packages { - for _, ar := range []*fv1.Archive{&fr.packages[i].Spec.Source, &fr.packages[i].Spec.Deployment} { - if strings.HasPrefix(ar.URL, ARCHIVE_URL_PREFIX) { + for i := range fr.Packages { + for _, ar := range []*fv1.Archive{&fr.Packages[i].Spec.Source, &fr.Packages[i].Spec.Deployment} { + if strings.HasPrefix(ar.URL, spec.ARCHIVE_URL_PREFIX) { availableAr, ok := archiveFiles[ar.URL] if !ok { - return fmt.Errorf("unknown archive name %v", strings.TrimPrefix(ar.URL, ARCHIVE_URL_PREFIX)) + return fmt.Errorf("unknown archive name %v", strings.TrimPrefix(ar.URL, spec.ARCHIVE_URL_PREFIX)) } ar.Type = availableAr.Type ar.Literal = availableAr.Literal @@ -931,11 +481,11 @@ func applyArchives(fclient *client.Client, specDir string, fr *FissionResources) } // applyResources applies the given set of fission resources. -func applyResources(fclient *client.Client, specDir string, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, map[string]resourceApplyStatus, error) { +func applyResources(fclient *client.Client, specDir string, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, map[string]spec.ResourceApplyStatus, error) { - applyStatus := make(map[string]resourceApplyStatus) + applyStatus := make(map[string]spec.ResourceApplyStatus) - // upload archives that need to be uploaded. Changes archive references in fr.packages. + // upload archives that need to be uploaded. Changes archive references in fr.Packages. err := applyArchives(fclient, specDir, fr) if err != nil { return nil, nil, err @@ -956,7 +506,7 @@ func applyResources(fclient *client.Client, specDir string, fr *FissionResources // Each reference to a package from a function must contain the resource version // of the package. This ensures that various caches can invalidate themselves // when the package changes. - for i, f := range fr.functions { + for i, f := range fr.Functions { k := mapKey(&metav1.ObjectMeta{ Namespace: f.Spec.Package.PackageRef.Namespace, Name: f.Spec.Package.PackageRef.Name, @@ -970,7 +520,7 @@ func applyResources(fclient *client.Client, specDir string, fr *FissionResources return nil, nil, fmt.Errorf("function %v/%v references package %v/%v, which doesn't exist in the specs", f.Metadata.Namespace, f.Metadata.Name, f.Spec.Package.PackageRef.Namespace, f.Spec.Package.PackageRef.Name) } - fr.functions[i].Spec.Package.PackageRef.ResourceVersion = m.ResourceVersion + fr.Functions[i].Spec.Package.PackageRef.ResourceVersion = m.ResourceVersion } _, ras, err = applyFunctions(fclient, fr, delete) @@ -1008,7 +558,7 @@ func applyResources(fclient *client.Client, specDir string, fr *FissionResources // localArchiveFromSpec creates an archive on the local filesystem from the given spec, // and returns its path and checksum. -func localArchiveFromSpec(specDir string, aus *ArchiveUploadSpec) (*fv1.Archive, error) { +func localArchiveFromSpec(specDir string, aus *spec.ArchiveUploadSpec) (*fv1.Archive, error) { // get root dir var rootDir string if len(aus.RootDir) == 0 { @@ -1106,20 +656,20 @@ func mapKey(m *metav1.ObjectMeta) string { return fmt.Sprintf("%v:%v", m.Namespace, m.Name) } -func applyDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) { +func applyDeploymentConfig(m *metav1.ObjectMeta, fr *spec.FissionResources) { if m.Annotations == nil { m.Annotations = make(map[string]string) } - m.Annotations[FISSION_DEPLOYMENT_NAME_KEY] = fr.deploymentConfig.Name - m.Annotations[FISSION_DEPLOYMENT_UID_KEY] = fr.deploymentConfig.UID + m.Annotations[spec.FISSION_DEPLOYMENT_NAME_KEY] = fr.DeploymentConfig.Name + m.Annotations[spec.FISSION_DEPLOYMENT_UID_KEY] = fr.DeploymentConfig.UID } -func hasDeploymentConfig(m *metav1.ObjectMeta, fr *FissionResources) bool { +func hasDeploymentConfig(m *metav1.ObjectMeta, fr *spec.FissionResources) bool { if m.Annotations == nil { return false } - uid, ok := m.Annotations[FISSION_DEPLOYMENT_UID_KEY] - if ok && uid == fr.deploymentConfig.UID { + uid, ok := m.Annotations[spec.FISSION_DEPLOYMENT_UID_KEY] + if ok && uid == fr.DeploymentConfig.UID { return true } return false @@ -1146,7 +696,7 @@ func waitForPackageBuild(fclient *client.Client, pkg *fv1.Package) (*fv1.Package } } -func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyPackages(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.PackageList(metav1.NamespaceAll) if err != nil { @@ -1171,10 +721,10 @@ func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (m // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.packages { + for _, o := range fr.Packages { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1217,7 +767,7 @@ func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (m return nil, nil, err // TODO check for resourceVersion conflict errors and retry } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1227,7 +777,7 @@ func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (m if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1242,7 +792,7 @@ func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (m if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1251,7 +801,7 @@ func applyPackages(fclient *client.Client, fr *FissionResources, delete bool) (m return metadataMap, &ras, nil } -func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyFunctions(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.FunctionList(metav1.NamespaceAll) if err != nil { @@ -1276,10 +826,10 @@ func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) ( // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.functions { + for _, o := range fr.Functions { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1300,7 +850,7 @@ func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) ( if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1310,7 +860,7 @@ func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) ( if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1325,7 +875,7 @@ func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) ( if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1334,7 +884,7 @@ func applyFunctions(fclient *client.Client, fr *FissionResources, delete bool) ( return metadataMap, &ras, nil } -func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyEnvironments(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.EnvironmentList(metav1.NamespaceAll) if err != nil { @@ -1359,10 +909,10 @@ func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.environments { + for _, o := range fr.Environments { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1383,7 +933,7 @@ func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1393,7 +943,7 @@ func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1408,7 +958,7 @@ func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1417,7 +967,7 @@ func applyEnvironments(fclient *client.Client, fr *FissionResources, delete bool return metadataMap, &ras, nil } -func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyHTTPTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.HTTPTriggerList(metav1.NamespaceAll) if err != nil { @@ -1442,10 +992,10 @@ func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.httpTriggers { + for _, o := range fr.HttpTriggers { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1466,7 +1016,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1476,7 +1026,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1491,7 +1041,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1500,7 +1050,7 @@ func applyHTTPTriggers(fclient *client.Client, fr *FissionResources, delete bool return metadataMap, &ras, nil } -func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyKubernetesWatchTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.WatchList(metav1.NamespaceAll) if err != nil { @@ -1525,10 +1075,10 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.kubernetesWatchTriggers { + for _, o := range fr.KubernetesWatchTriggers { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1549,7 +1099,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1559,7 +1109,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1574,7 +1124,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1583,7 +1133,7 @@ func applyKubernetesWatchTriggers(fclient *client.Client, fr *FissionResources, return metadataMap, &ras, nil } -func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyTimeTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.TimeTriggerList(metav1.NamespaceAll) if err != nil { @@ -1608,10 +1158,10 @@ func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.timeTriggers { + for _, o := range fr.TimeTriggers { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1632,7 +1182,7 @@ func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1642,7 +1192,7 @@ func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1657,7 +1207,7 @@ func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1666,7 +1216,7 @@ func applyTimeTriggers(fclient *client.Client, fr *FissionResources, delete bool return metadataMap, &ras, nil } -func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, delete bool) (map[string]metav1.ObjectMeta, *resourceApplyStatus, error) { +func applyMessageQueueTriggers(fclient *client.Client, fr *spec.FissionResources, delete bool) (map[string]metav1.ObjectMeta, *spec.ResourceApplyStatus, error) { // get list allObjs, err := fclient.MessageQueueTriggerList("", metav1.NamespaceAll) if err != nil { @@ -1691,10 +1241,10 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, del // desired set. used to compute the set to delete. desired := make(map[string]bool) - var ras resourceApplyStatus + var ras spec.ResourceApplyStatus // create or update desired state - for _, o := range fr.messageQueueTriggers { + for _, o := range fr.MessageQueueTriggers { // apply deploymentConfig so we can find our objects on future apply invocations applyDeploymentConfig(&o.Metadata, fr) @@ -1715,7 +1265,7 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, del if err != nil { return nil, nil, err } - ras.updated = append(ras.updated, newmeta) + ras.Updated = append(ras.Updated, newmeta) // keep track of metadata in case we need to create a reference to it metadataMap[mapKey(&o.Metadata)] = *newmeta } @@ -1725,7 +1275,7 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, del if err != nil { return nil, nil, err } - ras.created = append(ras.created, newmeta) + ras.Created = append(ras.Created, newmeta) metadataMap[mapKey(&o.Metadata)] = *newmeta } } @@ -1740,7 +1290,7 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, del if err != nil { return nil, nil, err } - ras.deleted = append(ras.deleted, &o.Metadata) + ras.Deleted = append(ras.Deleted, &o.Metadata) fmt.Printf("Deleted %v %v/%v\n", o.TypeMeta.Kind, o.Metadata.Namespace, o.Metadata.Name) } } @@ -1748,125 +1298,3 @@ func applyMessageQueueTriggers(fclient *client.Client, fr *FissionResources, del return metadataMap, &ras, nil } - -// called from `fission * create --spec` -func specSave(resource interface{}, specFile string) error { - specDir := "specs" - - // verify - if _, err := os.Stat(filepath.Join(specDir, "fission-deployment-config.yaml")); os.IsNotExist(err) { - return errors.Wrap(err, "Couldn't find specs, run `fission spec init` first") - } - - // make sure we're writing a known type - var data []byte - var err error - switch typedres := resource.(type) { - case ArchiveUploadSpec: - typedres.Kind = "ArchiveUploadSpec" - data, err = yaml.Marshal(typedres) - case fv1.Package: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "Package" - data, err = yaml.Marshal(typedres) - case fv1.Function: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "Function" - data, err = yaml.Marshal(typedres) - case fv1.Environment: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "Environment" - data, err = yaml.Marshal(typedres) - case fv1.HTTPTrigger: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "HTTPTrigger" - data, err = yaml.Marshal(typedres) - case fv1.KubernetesWatchTrigger: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "KubernetesWatchTrigger" - data, err = yaml.Marshal(typedres) - case fv1.MessageQueueTrigger: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "MessageQueueTrigger" - data, err = yaml.Marshal(typedres) - case fv1.TimeTrigger: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "TimeTrigger" - data, err = yaml.Marshal(typedres) - case fv1.Recorder: - typedres.TypeMeta.APIVersion = SPEC_API_VERSION - typedres.TypeMeta.Kind = "Recorder" - data, err = yaml.Marshal(typedres) - default: - return fmt.Errorf("can't save resource %#v", resource) - } - if err != nil { - return errors.Wrap(err, "Couldn't marshal YAML") - } - - filename := filepath.Join(specDir, specFile) - // check if the file is new - newFile := false - if _, err := os.Stat(filename); os.IsNotExist(err) { - newFile = true - } - - // open spec file to append or write - f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil { - return errors.Wrap(err, "couldn't create spec file") - } - defer f.Close() - - // if we're appending, add a yaml document separator - if !newFile { - _, err = f.Write([]byte("\n---\n")) - if err != nil { - return errors.Wrap(err, "couldn't write to spec file") - } - } - - // write our resource - _, err = f.Write(data) - if err != nil { - return errors.Wrap(err, "couldn't write to spec file") - } - return nil -} - -// Returns metadata if the given resource exists in the specs, nil -// otherwise. compareMetadata and compareSpec control how the -// equality check is performed. -func (fr *FissionResources) specExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta { - switch typedres := resource.(type) { - case *ArchiveUploadSpec: - for _, aus := range fr.archiveUploadSpecs { - if compareMetadata && aus.Name != typedres.Name { - continue - } - if compareSpec && - !(reflect.DeepEqual(aus.RootDir, typedres.RootDir) && - reflect.DeepEqual(aus.IncludeGlobs, typedres.IncludeGlobs) && - reflect.DeepEqual(aus.ExcludeGlobs, typedres.ExcludeGlobs)) { - continue - } - return &metav1.ObjectMeta{Name: aus.Name} - } - return nil - case *fv1.Package: - for _, p := range fr.packages { - if compareMetadata && !reflect.DeepEqual(p.Metadata, typedres.Metadata) { - continue - } - if compareSpec && !reflect.DeepEqual(p.Spec, typedres.Spec) { - continue - } - return &p.Metadata - } - return nil - - default: - // XXX not implemented - return nil - } -} diff --git a/pkg/fission-cli/timetrigger.go b/pkg/fission-cli/timetrigger.go index f58822e5..2f19bad0 100644 --- a/pkg/fission-cli/timetrigger.go +++ b/pkg/fission-cli/timetrigger.go @@ -29,6 +29,7 @@ import ( fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" "github.com/fission/fission/pkg/controller/client" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" ) @@ -93,7 +94,7 @@ func ttCreate(c *cli.Context) error { // if we're writing a spec, don't call the API if c.Bool("spec") { specFile := fmt.Sprintf("timetrigger-%v.yaml", name) - err := specSave(*tt, specFile) + err := spec.SpecSave(*tt, specFile) util.CheckErr(err, "create time trigger spec") return nil } diff --git a/pkg/fission-cli/types.go b/pkg/fission-cli/types.go deleted file mode 100644 index 7740bee9..00000000 --- a/pkg/fission-cli/types.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fission_cli - -const ( - FISSION_DEPLOYMENT_NAME_KEY = "fission-name" - FISSION_DEPLOYMENT_UID_KEY = "fission-uid" -) - -// CLI spec types -type ( - // DeploymentConfig is the global configuration for a set of Fission specs. - DeploymentConfig struct { - // TypeMeta describes the type of this object. It is inlined. The Kind - // field should always be "DeploymentConfig". - TypeMeta `json:",inline"` - - // Name is a user-friendly name for the deployment. It is also stored in - // all uploaded resources as an annotation. - Name string `json:"name"` - - // UID uniquely identifies the deployment. It is stored as a label and - // used to find resources to clean up when local specs are changed. - UID string `json:"uid"` - } - - // ArchiveUploadSpec specifies a set of files to be archived and uploaded. - // - // The resulting archive can be referenced as archive:// in PackageSpecs, - // using the name specified in the archive. The fission spec applier will - // replace the archive:// URL with a real HTTP URL after uploading the file. - ArchiveUploadSpec struct { - // TypeMeta describes the type of this object. It is inlined. The Kind - // field should always be "ArchiveUploadSpec". - TypeMeta `json:",inline"` - - // Name is a local name that can be used to reference this archive. It - // must be unique; duplicate names will cause an error while handling - // specs. - Name string `json:"name"` - - // RootDir specifies the root that the globs below are relative to. It - // is optional and defaults to the parent directory of the spec - // directory: for example, if the deployment config is at - // /path/to/project/specs/config.yaml, the RootDir is /path/to/project. - RootDir string `json:"rootdir,omitempty"` - - // IncludeGlobs is a list of Unix shell globs to include - IncludeGlobs []string `json:"include,omitempty"` - - // ExcludeGlobs is a list of globs to exclude from the set specified by - // IncludeGlobs. - ExcludeGlobs []string `json:"exclude,omitempty"` - } - - // TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and - // unmarshal local-only objects (like ArchiveUploadSpec) the same way that - // Kubernetes does. - TypeMeta struct { - Kind string `json:"kind,omitempty"` - APIVersion string `json:"apiVersion,omitempty"` - } -) diff --git a/pkg/fission-cli/upgrade.go b/pkg/fission-cli/upgrade.go deleted file mode 100644 index cf0ec0ff..00000000 --- a/pkg/fission-cli/upgrade.go +++ /dev/null @@ -1,416 +0,0 @@ -/* -Copyright 2016 The Fission Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fission_cli - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "regexp" - "strings" - - "github.com/dchest/uniuri" - "github.com/urfave/cli" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" - "github.com/fission/fission/pkg/fission-cli/log" - "github.com/fission/fission/pkg/fission-cli/util" - v1 "github.com/fission/fission/pkg/v1" -) - -type ( - V1FissionState struct { - Functions []v1.Function `json:"functions"` - Environments []v1.Environment `json:"environments"` - HTTPTriggers []v1.HTTPTrigger `json:"httptriggers"` - Mqtriggers []v1.MessageQueueTrigger `json:"mqtriggers"` - TimeTriggers []v1.TimeTrigger `json:"timetriggers"` - Watches []v1.Watch `json:"watches"` - NameChanges map[string]string `json:"namechanges"` - } - nameRemapper struct { - oldToNew map[string]string - newNames map[string]bool - } -) - -func getV1URL(serverUrl string) string { - if len(serverUrl) == 0 { - log.Fatal("Need --server or FISSION_URL set to your fission server.") - } - isHTTPS := strings.Index(serverUrl, "https://") == 0 - isHTTP := strings.Index(serverUrl, "http://") == 0 - if !(isHTTP || isHTTPS) { - serverUrl = "http://" + serverUrl - } - v1url := strings.TrimSuffix(serverUrl, "/") + "/v1" - return v1url -} - -func get(url string) []byte { - resp, err := http.Get(url) - util.CheckErr(err, "get fission v0.1 state") - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - util.CheckErr(err, "reading server response") - - if resp.StatusCode != 200 { - log.Fatal(fmt.Sprintf("Failed to fetch fission v0.1 state: %v", string(body))) - } - return body -} - -// track a name in the remapper, creating a new name if needed -func (nr *nameRemapper) trackName(old string) { - // all kubernetes names must match this regex - kubeNameRegex := "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" - maxLen := 63 - - ok, err := regexp.MatchString(kubeNameRegex, old) - util.CheckErr(err, "match name regexp") - if ok && len(old) < maxLen { - // no rename - nr.oldToNew[old] = old - nr.newNames[old] = true - return - } - - newName := strings.ToLower(old) - - // remove disallowed - inv, err := regexp.Compile("[^-a-z0-9]") - util.CheckErr(err, "compile regexp") - newName = string(inv.ReplaceAll([]byte(newName), []byte("-"))) - - // trim leading non-alphabetic - leadingnonalpha, err := regexp.Compile("^[^a-z]+") - util.CheckErr(err, "compile regexp") - newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{})) - - // trim trailing - trailing, err := regexp.Compile("[^a-z0-9]+$") - util.CheckErr(err, "compile regexp") - newName = string(trailing.ReplaceAll([]byte(newName), []byte{})) - - // truncate to length - if len(newName) > maxLen-4 { - newName = newName[0:(maxLen - 4)] - } - - // uniqueness - n := newName - i := 0 - for { - _, exists := nr.newNames[n] - if !exists { - break - } else { - i++ - n = fmt.Sprintf("%v-%v", newName, i) - } - } - newName = n - - // track - nr.oldToNew[old] = newName - nr.newNames[newName] = true -} - -func upgradeDumpV1State(v1url string, filename string) { - var v1state V1FissionState - - fmt.Println("Getting environments") - resp := get(v1url + "/environments") - err := json.Unmarshal(resp, &v1state.Environments) - util.CheckErr(err, "parse server response") - - fmt.Println("Getting watches") - resp = get(v1url + "/watches") - err = json.Unmarshal(resp, &v1state.Watches) - util.CheckErr(err, "parse server response") - - fmt.Println("Getting routes") - resp = get(v1url + "/triggers/http") - err = json.Unmarshal(resp, &v1state.HTTPTriggers) - util.CheckErr(err, "parse server response") - - fmt.Println("Getting message queue triggers") - resp = get(v1url + "/triggers/messagequeue") - err = json.Unmarshal(resp, &v1state.Mqtriggers) - util.CheckErr(err, "parse server response") - - fmt.Println("Getting time triggers") - resp = get(v1url + "/triggers/time") - err = json.Unmarshal(resp, &v1state.TimeTriggers) - util.CheckErr(err, "parse server response") - - fmt.Println("Getting function list") - resp = get(v1url + "/functions") - err = json.Unmarshal(resp, &v1state.Functions) - util.CheckErr(err, "parse server response") - - // we have to change names that are disallowed in kubernetes - nr := nameRemapper{ - oldToNew: make(map[string]string), - newNames: make(map[string]bool), - } - - // get all referenced function metadata - funcMetaSet := make(map[v1.Metadata]bool) - for _, f := range v1state.Functions { - funcMetaSet[f.Metadata] = true - nr.trackName(f.Metadata.Name) - } - for _, t := range v1state.HTTPTriggers { - funcMetaSet[t.Function] = true - nr.trackName(t.Metadata.Name) - } - for _, t := range v1state.Mqtriggers { - funcMetaSet[t.Function] = true - nr.trackName(t.Metadata.Name) - } - for _, t := range v1state.Watches { - funcMetaSet[t.Function] = true - nr.trackName(t.Metadata.Name) - } - for _, t := range v1state.TimeTriggers { - funcMetaSet[t.Function] = true - nr.trackName(t.Metadata.Name) - } - - for _, e := range v1state.Environments { - nr.trackName(e.Metadata.Name) - } - - fmt.Println("Getting functions") - // get each function - funcs := make(map[v1.Metadata]v1.Function) - for m := range funcMetaSet { - if len(m.Uid) != 0 { - resp = get(fmt.Sprintf("%v/functions/%v?uid=%v", v1url, m.Name, m.Uid)) - } else { - resp = get(fmt.Sprintf("%v/functions/%v", v1url, m.Name)) - } - - var f v1.Function - - // unmarshal - err = json.Unmarshal(resp, &f) - util.CheckErr(err, "parse server response") - - // load into a map to remove duplicates - funcs[f.Metadata] = f - } - - // add list of unique functions to v1state from map - v1state.Functions = make([]v1.Function, 0) - for _, f := range funcs { - v1state.Functions = append(v1state.Functions, f) - } - - // dump name changes - v1state.NameChanges = nr.oldToNew - - // serialize v1state - out, err := json.MarshalIndent(v1state, "", " ") - util.CheckErr(err, "serialize v0.1 state") - - // dump to file fission-v01-state.json - if len(filename) == 0 { - filename = "fission-v01-state.json" - } - err = ioutil.WriteFile(filename, out, 0644) - util.CheckErr(err, "write file") - - fmt.Printf("Done: Saved %v functions, %v HTTP triggers, %v watches, %v message queue triggers, %v time triggers.\n", - len(v1state.Functions), len(v1state.HTTPTriggers), len(v1state.Watches), len(v1state.Mqtriggers), len(v1state.TimeTriggers)) -} - -func functionRefFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *fv1.FunctionReference { - return &fv1.FunctionReference{ - Type: fv1.FunctionReferenceTypeFunctionName, - Name: nameRemap[m.Name], - } -} - -func crdMetadataFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *metav1.ObjectMeta { - return &metav1.ObjectMeta{ - Name: nameRemap[m.Name], - Namespace: metav1.NamespaceDefault, - } -} - -func upgradeDumpState(c *cli.Context) error { - u := getV1URL(c.GlobalString("server")) - filename := c.String("file") - - // check v1 - resp, err := http.Get(u + "/environments") - util.CheckErr(err, "reach fission server") - if resp.StatusCode == http.StatusNotFound { - msg := fmt.Sprintf("Server %v isn't a v1 Fission server. Use --server to point at a pre-0.2.x Fission server.", u) - log.Fatal(msg) - } - - upgradeDumpV1State(u, filename) - return nil -} - -func upgradeRestoreState(c *cli.Context) error { - filename := c.String("file") - if len(filename) == 0 { - filename = "fission-v01-state.json" - } - - contents, err := ioutil.ReadFile(filename) - util.CheckErr(err, fmt.Sprintf("open file %v", filename)) - - var v1state V1FissionState - err = json.Unmarshal(contents, &v1state) - util.CheckErr(err, "parse dumped v1 state") - - // create a regular v2 client - client := util.GetApiClient(c.GlobalString("server")) - - // create functions - for _, f := range v1state.Functions { - - // get post-rename function name, derive pkg name from it - fnName := v1state.NameChanges[f.Metadata.Name] - pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6))) - - // write function to file - tmpfile, err := ioutil.TempFile("", pkgName) - util.CheckErr(err, "create temporary file") - code, err := base64.StdEncoding.DecodeString(f.Code) - util.CheckErr(err, "decode base64 function contents") - tmpfile.Write(code) - tmpfile.Sync() - tmpfile.Close() - - // upload - ctx := context.Background() - archive := uploadArchive(ctx, client, tmpfile.Name()) - os.Remove(tmpfile.Name()) - - // create pkg - pkgSpec := fv1.PackageSpec{ - Environment: fv1.EnvironmentReference{ - Name: v1state.NameChanges[f.Environment.Name], - Namespace: metav1.NamespaceDefault, - }, - Deployment: *archive, - } - pkg, err := client.PackageCreate(&fv1.Package{ - Metadata: metav1.ObjectMeta{ - Name: pkgName, - Namespace: metav1.NamespaceDefault, - }, - Spec: pkgSpec, - }) - util.CheckErr(err, fmt.Sprintf("create package %v", pkgName)) - _, err = client.FunctionCreate(&fv1.Function{ - Metadata: *crdMetadataFromV1Metadata(&f.Metadata, v1state.NameChanges), - Spec: fv1.FunctionSpec{ - Environment: pkgSpec.Environment, - Package: fv1.FunctionPackageRef{ - PackageRef: fv1.PackageRef{ - Name: pkg.Name, - Namespace: pkg.Namespace, - ResourceVersion: pkg.ResourceVersion, - }, - }, - }, - }) - util.CheckErr(err, fmt.Sprintf("create function %v", v1state.NameChanges[f.Metadata.Name])) - - } - - // create envs - for _, e := range v1state.Environments { - _, err = client.EnvironmentCreate(&fv1.Environment{ - Metadata: *crdMetadataFromV1Metadata(&e.Metadata, v1state.NameChanges), - Spec: fv1.EnvironmentSpec{ - Version: 1, - Runtime: fv1.Runtime{ - Image: e.RunContainerImageUrl, - }, - }, - }) - util.CheckErr(err, fmt.Sprintf("create environment %v", e.Metadata.Name)) - } - - // create httptriggers - for _, t := range v1state.HTTPTriggers { - _, err = client.HTTPTriggerCreate(&fv1.HTTPTrigger{ - Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), - Spec: fv1.HTTPTriggerSpec{ - RelativeURL: t.UrlPattern, - Method: t.Method, - FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), - }, - }) - util.CheckErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name)) - } - - // create mqtriggers - for _, t := range v1state.Mqtriggers { - _, err = client.MessageQueueTriggerCreate(&fv1.MessageQueueTrigger{ - Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), - Spec: fv1.MessageQueueTriggerSpec{ - FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), - MessageQueueType: fv1.MessageQueueTypeNats, // only NATS is supported at that time (v1 types) - Topic: t.Topic, - ResponseTopic: t.ResponseTopic, - }, - }) - util.CheckErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name)) - } - - // create time triggers - for _, t := range v1state.TimeTriggers { - _, err = client.TimeTriggerCreate(&fv1.TimeTrigger{ - Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), - Spec: fv1.TimeTriggerSpec{ - FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), - Cron: t.Cron, - }, - }) - util.CheckErr(err, fmt.Sprintf("create time trigger %v", t.Metadata.Name)) - } - - // create watches - for _, t := range v1state.Watches { - _, err = client.WatchCreate(&fv1.KubernetesWatchTrigger{ - Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), - Spec: fv1.KubernetesWatchTriggerSpec{ - Namespace: t.Namespace, - Type: t.ObjType, - FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), - }, - }) - util.CheckErr(err, fmt.Sprintf("create kubernetes watch trigger %v", t.Metadata.Name)) - } - - return nil -} diff --git a/pkg/fission-cli/watch.go b/pkg/fission-cli/watch.go index b3c01c91..c5ee5f96 100644 --- a/pkg/fission-cli/watch.go +++ b/pkg/fission-cli/watch.go @@ -26,6 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/fission-cli/cmd/spec" "github.com/fission/fission/pkg/fission-cli/log" "github.com/fission/fission/pkg/fission-cli/util" ) @@ -82,7 +83,7 @@ func wCreate(c *cli.Context) error { // if we're writing a spec, don't call the API if c.Bool("spec") { specFile := fmt.Sprintf("kubewatch-%v.yaml", watchName) - err := specSave(*w, specFile) + err := spec.SpecSave(*w, specFile) util.CheckErr(err, "create kubernetes watch spec") return nil } diff --git a/pkg/generator/doc.go b/pkg/generator/doc.go new file mode 100644 index 00000000..8f2bcf09 --- /dev/null +++ b/pkg/generator/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package generator is a generator that helps to convert a resource object +// into target encoding format like JSON, YAML ...etc. +package generator diff --git a/pkg/generator/encoder/encoder.go b/pkg/generator/encoder/encoder.go new file mode 100644 index 00000000..6093f9bc --- /dev/null +++ b/pkg/generator/encoder/encoder.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package encoder + +import ( + "encoding/json" + "github.com/ghodss/yaml" +) + +type ( + EncodeCodec string + + Encoder interface { + Marshal(v interface{}) ([]byte, error) + Unmarshal(data []byte, v interface{}) error + } + + JSONEncoder struct{} + YAMLEncoder struct{} +) + +var _ Encoder = DefaultJSONEncoder() + +func DefaultJSONEncoder() Encoder { + return JSONEncoder{} +} + +func (encoder JSONEncoder) Marshal(v interface{}) ([]byte, error) { + return json.Marshal(v) +} + +func (encoder JSONEncoder) Unmarshal(data []byte, v interface{}) error { + return json.Unmarshal(data, v) +} + +var _ Encoder = DefaultYAMLEncoder() + +func DefaultYAMLEncoder() Encoder { + return YAMLEncoder{} +} + +func (encoder YAMLEncoder) Marshal(v interface{}) ([]byte, error) { + return yaml.Marshal(v) +} + +func (encoder YAMLEncoder) Unmarshal(data []byte, v interface{}) error { + return yaml.Unmarshal(data, v) +} diff --git a/pkg/generator/generate.go b/pkg/generator/generate.go new file mode 100644 index 00000000..6e5c07d5 --- /dev/null +++ b/pkg/generator/generate.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "github.com/fission/fission/pkg/generator/encoder" +) + +type ( + StructuredGenerator interface { + StructuredGenerate(enc encoder.Encoder) ([]byte, error) + } +) diff --git a/pkg/generator/v1/environment.go b/pkg/generator/v1/environment.go new file mode 100644 index 00000000..8eaf2c53 --- /dev/null +++ b/pkg/generator/v1/environment.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "github.com/pkg/errors" + + fv1 "github.com/fission/fission/pkg/apis/fission.io/v1" + "github.com/fission/fission/pkg/generator" + "github.com/fission/fission/pkg/generator/encoder" +) + +// ensure environment generator fits the generator interface +var _ generator.StructuredGenerator = &EnvironmentGenerator{} + +type ( + EnvironmentGenerator struct { + obj *fv1.Environment + } +) + +// CreateEnvironmentGeneratorFromObj creates environment generator initialized with pass-in environment object. +func CreateEnvironmentGeneratorFromObj(env *fv1.Environment) (*EnvironmentGenerator, error) { + if env == nil { + return nil, errors.New("cannot create environment generator with nil pointer") + } + + if len(env.TypeMeta.Kind) == 0 { + env.TypeMeta.Kind = fv1.CRD_NAME_ENVIRONMENT + } + + if len(env.TypeMeta.APIVersion) == 0 { + env.TypeMeta.APIVersion = fv1.CRD_VERSION + } + + err := env.Validate() + if err != nil { + return nil, fv1.AggregateValidationErrors("Environment", err) + } + + return &EnvironmentGenerator{ + obj: env, + }, nil +} + +func (generator EnvironmentGenerator) StructuredGenerate(enc encoder.Encoder) ([]byte, error) { + return enc.Marshal(generator.obj) +}