Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor package CLI command (#1345)

Browse Source
This commit is contained in:
Ta-Ching Chen
2019-10-13 01:30:20 +08:00
committed by GitHub
parent f30e7df43d
commit e42f81a7a1
22 changed files with 1769 additions and 1172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/fission-cli/cmd/spec/apply.go
+1132
View File
File diff suppressed because it is too large Load Diff
pkg/fission-cli/cmd/spec/buildwatch.go
+121
View File
@@ -0,0 +1,121 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"context"
"fmt"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/util"
"github.com/fission/fission/pkg/types"
)
type (
// packageBuildWatcher is used to watch a set of in-progress builds.
packageBuildWatcher struct {
// fission client
fclient *client.Client
// set of packages already printed, ensures we don't duplicate the notifications
finished map[string]bool
// set of metadata in the app spec. packages outside this set should be ignored.
pkgMeta map[string]metav1.ObjectMeta
}
)
func makePackageBuildWatcher(fclient *client.Client) *packageBuildWatcher {
return &packageBuildWatcher{
fclient: fclient,
finished: make(map[string]bool),
pkgMeta: make(map[string]metav1.ObjectMeta),
}
}
func (w *packageBuildWatcher) addPackages(pkgMeta map[string]metav1.ObjectMeta) {
for k, v := range pkgMeta {
w.pkgMeta[k] = v
}
}
func (w *packageBuildWatcher) watch(ctx context.Context) {
for {
// non-blocking check if we're cancelled
select {
case <-ctx.Done():
return
default:
}
// pull list of packages (TODO: convert to watch)
pkgs, err := w.fclient.PackageList(metav1.NamespaceAll)
util.CheckErr(err, "Getting list of packages")
// find packages that (a) are in the app spec and (b) have an interesting
// build status (either succeeded or failed; not "none")
keepWaiting := false
buildpkgs := make([]fv1.Package, 0)
for _, pkg := range pkgs {
_, ok := w.pkgMeta[mapKey(&pkg.Metadata)]
if !ok {
continue
}
if pkg.Status.BuildStatus == types.BuildStatusNone {
continue
}
if pkg.Status.BuildStatus == types.BuildStatusPending ||
pkg.Status.BuildStatus == types.BuildStatusRunning {
keepWaiting = true
}
buildpkgs = append(buildpkgs, pkg)
}
// print package status, and error logs if any
for _, pkg := range buildpkgs {
k := pkgKey(&pkg)
if _, printed := w.finished[k]; printed {
continue
}
if pkg.Status.BuildStatus == types.BuildStatusFailed {
w.finished[k] = true
fmt.Printf("--- Build FAILED: ---\n%v\n------\n", pkg.Status.BuildLog)
} else if pkg.Status.BuildStatus == types.BuildStatusSucceeded {
w.finished[k] = true
fmt.Printf("--- Build SUCCEEDED ---\n")
if len(pkg.Status.BuildLog) > 0 {
fmt.Printf("%v\n------\n", pkg.Status.BuildLog)
}
}
}
// if there are no builds running, we can stop polling
if !keepWaiting {
return
}
time.Sleep(time.Second)
}
}
func pkgKey(pkg *fv1.Package) string {
// packages are mutable so we want to keep track of them by resource version
return fmt.Sprintf("%v:%v:%v", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Metadata.ResourceVersion)
}
pkg/fission-cli/cmd/spec/destroy.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/util"
)
type DestroySubCommand struct {
client *client.Client
}
// Destroy destroys everything in the spec.
func Destroy(flags cli.Input) error {
opts := &DestroySubCommand{
client: cmd.GetServer(flags),
}
return opts.do(flags)
}
func (opts *DestroySubCommand) do(flags cli.Input) error {
return opts.run(flags)
}
func (opts *DestroySubCommand) run(flags cli.Input) error {
// get specdir
specDir := cmd.GetSpecDir(flags)
// read everything
fr, err := ReadSpecs(specDir)
util.CheckErr(err, "read specs")
// set desired state to nothing, but keep the UID so "apply" can find it
emptyFr := FissionResources{}
emptyFr.DeploymentConfig = fr.DeploymentConfig
// "apply" the empty state
_, _, err = applyResources(opts.client, specDir, &emptyFr, true)
util.CheckErr(err, "delete resources")
return nil
}
pkg/fission-cli/cmd/spec/init.go
+131
View File
@@ -0,0 +1,131 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/ghodss/yaml"
uuid "github.com/satori/go.uuid"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
spectypes "github.com/fission/fission/pkg/fission-cli/cmd/spec/types"
"github.com/fission/fission/pkg/fission-cli/util"
)
type InitSubCommand struct {
client *client.Client
deployConfig *spectypes.DeploymentConfig
}
func Init(flags cli.Input) error {
opts := InitSubCommand{
client: cmd.GetServer(flags),
}
return opts.do(flags)
}
func (opts *InitSubCommand) do(flags cli.Input) error {
err := opts.complete(flags)
if err != nil {
return err
}
return opts.run(flags)
}
func (opts *InitSubCommand) complete(flags cli.Input) error {
// Figure out spec directory
specDir := cmd.GetSpecDir(flags)
name := flags.String("name")
if len(name) == 0 {
// come up with a name using the current dir
dir, err := filepath.Abs(".")
util.CheckErr(err, "get current working directory")
basename := filepath.Base(dir)
name = util.KubifyName(basename)
}
deployID := flags.String("deployid")
if len(deployID) == 0 {
deployID = uuid.NewV4().String()
}
// Create spec dir
fmt.Printf("Creating fission spec directory '%v'\n", specDir)
err := os.MkdirAll(specDir, 0755)
util.CheckErr(err, fmt.Sprintf("create spec directory '%v'", specDir))
// Write the deployment config
opts.deployConfig = &spectypes.DeploymentConfig{
TypeMeta: spectypes.TypeMeta{
APIVersion: SPEC_API_VERSION,
Kind: "DeploymentConfig",
},
Name: name,
// All resources will be annotated with the UID when they're created. This allows
// us to be idempotent, as well as to delete resources when their specs are
// removed.
UID: deployID,
}
return nil
}
// run just initializes an empty spec directory and adds some
// sample YAMLs in there that might be useful.
func (opts *InitSubCommand) run(flags cli.Input) error {
specDir := cmd.GetSpecDir(flags)
// Add a bit of documentation to the spec dir here
err := ioutil.WriteFile(filepath.Join(specDir, "README"), []byte(SPEC_README), 0644)
if err != nil {
return err
}
err = writeDeploymentConfig(specDir, opts.deployConfig)
util.CheckErr(err, "write deployment config")
// Other possible things to do here:
// - add example specs to the dir to make it easy to manually
// add new ones
return nil
}
// writeDeploymentConfig serializes the DeploymentConfig to YAML and writes it to a new
// fission-config.yaml in specDir.
func writeDeploymentConfig(specDir string, dc *spectypes.DeploymentConfig) error {
y, err := yaml.Marshal(dc)
if err != nil {
return err
}
msg := []byte("# This file is generated by the 'fission spec init' command.\n" +
"# See the README in this directory for background and usage information.\n" +
"# Do not edit the UID below: that will break 'fission spec apply'\n")
err = ioutil.WriteFile(filepath.Join(specDir, "fission-deployment-config.yaml"), append(msg, y...), 0644)
if err != nil {
return err
}
return nil
}
pkg/fission-cli/cmd/spec/spec.go
+12 -63
View File
@@ -26,13 +26,14 @@ import (
"github.com/ghodss/yaml"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/urfave/cli"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/cmd/spec/types"
"github.com/fission/fission/pkg/fission-cli/log"
"github.com/fission/fission/pkg/fission-cli/util"
"github.com/fission/fission/pkg/generator/encoder"
v1generator "github.com/fission/fission/pkg/generator/v1"
)
@@ -92,60 +93,8 @@ fission.
// CLI spec types
type (
// DeploymentConfig is the global configuration for a set of Fission specs.
DeploymentConfig struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "DeploymentConfig".
TypeMeta `json:",inline"`
// Name is a user-friendly name for the deployment. It is also stored in
// all uploaded resources as an annotation.
Name string `json:"name"`
// UID uniquely identifies the deployment. It is stored as a label and
// used to find resources to clean up when local specs are changed.
UID string `json:"uid"`
}
// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
//
// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
// using the name specified in the archive. The fission spec applier will
// replace the archive:// URL with a real HTTP URL after uploading the file.
ArchiveUploadSpec struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "ArchiveUploadSpec".
TypeMeta `json:",inline"`
// Name is a local name that can be used to reference this archive. It
// must be unique; duplicate names will cause an error while handling
// specs.
Name string `json:"name"`
// RootDir specifies the root that the globs below are relative to. It
// is optional and defaults to the parent directory of the spec
// directory: for example, if the deployment config is at
// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
RootDir string `json:"rootdir,omitempty"`
// IncludeGlobs is a list of Unix shell globs to include
IncludeGlobs []string `json:"include,omitempty"`
// ExcludeGlobs is a list of globs to exclude from the set specified by
// IncludeGlobs.
ExcludeGlobs []string `json:"exclude,omitempty"`
}
// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
// Kubernetes does.
TypeMeta struct {
Kind string `json:"kind,omitempty"`
APIVersion string `json:"apiVersion,omitempty"`
}
FissionResources struct {
DeploymentConfig DeploymentConfig
DeploymentConfig types.DeploymentConfig
Packages []fv1.Package
Functions []fv1.Function
Environments []fv1.Environment
@@ -153,7 +102,7 @@ type (
KubernetesWatchTriggers []fv1.KubernetesWatchTrigger
TimeTriggers []fv1.TimeTrigger
MessageQueueTriggers []fv1.MessageQueueTrigger
ArchiveUploadSpecs []ArchiveUploadSpec
ArchiveUploadSpecs []types.ArchiveUploadSpec
SourceMap SourceMap
}
@@ -224,7 +173,7 @@ func SpecSave(resource interface{}, specFile string) error {
var data []byte
var err error
switch typedres := resource.(type) {
case ArchiveUploadSpec:
case types.ArchiveUploadSpec:
typedres.Kind = "ArchiveUploadSpec"
data, err = yaml.Marshal(typedres)
case fv1.Package:
@@ -296,7 +245,7 @@ func (fr *FissionResources) validateFunctionReference(functions map[string]bool,
return nil
}
func (fr *FissionResources) Validate(c *cli.Context) error {
func (fr *FissionResources) Validate(flags cli.Input) error {
result := &multierror.Error{}
// check references: both dangling refs + garbage
@@ -398,7 +347,7 @@ func (fr *FissionResources) Validate(c *cli.Context) error {
packages[MapKey(pkgMeta)] = true
}
client := util.GetApiClient(c.GlobalString("server"))
client := cmd.GetServer(flags)
for _, cm := range f.Spec.ConfigMaps {
_, err := client.ConfigMapGet(&metav1.ObjectMeta{
Name: cm.Name,
@@ -532,7 +481,7 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error {
// Figure out the object type by unmarshaling into the TypeMeta struct; then
// unmarshal again into the "real" struct once we know the type.
var tm TypeMeta
var tm types.TypeMeta
err := yaml.Unmarshal(b, &tm)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to decode yaml %v", string(b)))
@@ -605,14 +554,14 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error {
// The following are not CRDs
case "DeploymentConfig":
var v DeploymentConfig
var v types.DeploymentConfig
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
}
fr.DeploymentConfig = v
case "ArchiveUploadSpec":
var v ArchiveUploadSpec
var v types.ArchiveUploadSpec
err = yaml.Unmarshal(b, &v)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to parse %v in %v", tm.Kind, loc))
@@ -644,7 +593,7 @@ func (fr *FissionResources) ParseYaml(b []byte, loc *Location) error {
// equality check is performed.
func (fr *FissionResources) SpecExists(resource interface{}, compareMetadata bool, compareSpec bool) *metav1.ObjectMeta {
switch typedres := resource.(type) {
case *ArchiveUploadSpec:
case *types.ArchiveUploadSpec:
for _, aus := range fr.ArchiveUploadSpecs {
if compareMetadata && aus.Name != typedres.Name {
continue
pkg/fission-cli/cmd/spec/types/types.go
+72
View File
@@ -0,0 +1,72 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package types
// CLI spec types
type (
// DeploymentConfig is the global configuration for a set of Fission specs.
DeploymentConfig struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "DeploymentConfig".
TypeMeta `json:",inline"`
// Name is a user-friendly name for the deployment. It is also stored in
// all uploaded resources as an annotation.
Name string `json:"name"`
// UID uniquely identifies the deployment. It is stored as a label and
// used to find resources to clean up when local specs are changed.
UID string `json:"uid"`
}
// ArchiveUploadSpec specifies a set of files to be archived and uploaded.
//
// The resulting archive can be referenced as archive://<Name> in PackageSpecs,
// using the name specified in the archive. The fission spec applier will
// replace the archive:// URL with a real HTTP URL after uploading the file.
ArchiveUploadSpec struct {
// TypeMeta describes the type of this object. It is inlined. The Kind
// field should always be "ArchiveUploadSpec".
TypeMeta `json:",inline"`
// Name is a local name that can be used to reference this archive. It
// must be unique; duplicate names will cause an error while handling
// specs.
Name string `json:"name"`
// RootDir specifies the root that the globs below are relative to. It
// is optional and defaults to the parent directory of the spec
// directory: for example, if the deployment config is at
// /path/to/project/specs/config.yaml, the RootDir is /path/to/project.
RootDir string `json:"rootdir,omitempty"`
// IncludeGlobs is a list of Unix shell globs to include
IncludeGlobs []string `json:"include,omitempty"`
// ExcludeGlobs is a list of globs to exclude from the set specified by
// IncludeGlobs.
ExcludeGlobs []string `json:"exclude,omitempty"`
}
// TypeMeta is the same as Kubernetes' TypeMeta, and allows us to version and
// unmarshal local-only objects (like ArchiveUploadSpec) the same way that
// Kubernetes does.
TypeMeta struct {
Kind string `json:"kind,omitempty"`
APIVersion string `json:"apiVersion,omitempty"`
}
)
pkg/fission-cli/cmd/spec/validate.go
+146
View File
@@ -0,0 +1,146 @@
/*
Copyright 2019 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/hashicorp/go-multierror"
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
"github.com/fission/fission/pkg/controller/client"
"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
"github.com/fission/fission/pkg/fission-cli/cmd"
"github.com/fission/fission/pkg/fission-cli/log"
"github.com/fission/fission/pkg/fission-cli/util"
)
type ValidateSubCommand struct {
client *client.Client
}
// Validate parses a set of specs and checks for references to
// resources that don't exist.
func Validate(flags cli.Input) error {
opts := &ValidateSubCommand{
client: cmd.GetServer(flags),
}
return opts.do(flags)
}
func (opts *ValidateSubCommand) do(flags cli.Input) error {
return opts.run(flags)
}
func (opts *ValidateSubCommand) run(flags cli.Input) error {
// this will error on parse errors and on duplicates
specDir := cmd.GetSpecDir(flags)
fr, err := ReadSpecs(specDir)
util.CheckErr(err, "read specs")
// this does the rest of the checks, like dangling refs
err = fr.Validate(flags)
if err != nil {
fmt.Printf("Error validating specs: %v", err)
return nil
}
return nil
}
// ReadSpecs reads all specs in the specified directory and returns a parsed set of
// fission resources.
func ReadSpecs(specDir string) (*FissionResources, error) {
// make sure spec directory exists before continue
if _, err := os.Stat(specDir); os.IsNotExist(err) {
log.Fatal(fmt.Sprintf("Spec directory %v doesn't exist. "+
"Please check directory path or run \"fission spec init\" to create it.", specDir))
}
fr := FissionResources{
Packages: make([]fv1.Package, 0),
Functions: make([]fv1.Function, 0),
Environments: make([]fv1.Environment, 0),
HttpTriggers: make([]fv1.HTTPTrigger, 0),
KubernetesWatchTriggers: make([]fv1.KubernetesWatchTrigger, 0),
TimeTriggers: make([]fv1.TimeTrigger, 0),
MessageQueueTriggers: make([]fv1.MessageQueueTrigger, 0),
SourceMap: SourceMap{
Locations: make(map[string](map[string](map[string]Location))),
},
}
var result *multierror.Error
// Users can organize the specdir into subdirs if they want to.
err := filepath.Walk(specDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// For now just read YAML files. We'll add jsonnet at some point. Skip
// unsupported files.
if !(strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) {
return nil
}
// read
b, err := ioutil.ReadFile(path)
if err != nil {
result = multierror.Append(result, err)
return nil
}
// handle the case where there are multiple YAML docs per file. go-yaml
// doesn't support this directly, yet.
docs := bytes.Split(b, []byte("\n---"))
lines := 1
for _, doc := range docs {
d := []byte(strings.TrimSpace(string(doc)))
if len(d) != 0 {
// parse this document and add whatever is in it to fr
err = fr.ParseYaml(d, &Location{
Path: path,
Line: lines,
})
if err != nil {
// collect all errors so user can fix them all
result = multierror.Append(result, err)
}
}
// the separator occupies one line, hence the +1
lines += strings.Count(string(doc), "\n") + 1
}
return nil
})
if err != nil {
return nil, err
}
if err = result.ErrorOrNil(); err != nil {
return nil, err
}
return &fr, nil
}