Refactor support dump CLI (#1301)

charts/fission-all/templates/deployment.yaml

@@ -118,6 +118,8 @@ metadata:
   name: controller
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: controller
+    application: fission-api
 spec:
   replicas: 1
   template:
@@ -178,13 +180,15 @@ metadata:
   name: router
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: router
+    application: fission-router
 spec:
   replicas: 1
   template:
     metadata:
       labels:
-        application: fission-router
         svc: router
+        application: fission-router
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/path: "/metrics"
@@ -272,6 +276,7 @@ metadata:
   name: executor
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: executor
 spec:
   replicas: 1
   template:
@@ -342,6 +347,7 @@ metadata:
   name: buildermgr
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: buildermgr
 spec:
   replicas: 1
   template:
@@ -390,6 +396,7 @@ metadata:
   name: kubewatcher
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: kubewatcher
 spec:
   replicas: 1
   template:
@@ -436,6 +443,7 @@ metadata:
   name: influxdb
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: influxdb
 spec:
   replicas: 1
   template:
@@ -491,6 +499,7 @@ metadata:
   namespace: kube-system
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: heapster
 spec:
   replicas: 1
   template:
@@ -514,6 +523,7 @@ metadata:
   name: timer
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: timer
 spec:
   replicas: 1
   template:
@@ -618,6 +628,8 @@ metadata:
   name: mqtrigger-nats-streaming
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: mqtrigger
+    messagequeue: nats-streaming
 spec:
   replicas: 1
   template:
@@ -655,6 +667,8 @@ metadata:
   name: mqtrigger-kafka
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: mqtrigger
+    messagequeue: kafka
 spec:
   replicas: 1
   template:
@@ -694,6 +708,8 @@ metadata:
   name: mqtrigger-azure-storage-queue
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: mqtrigger
+    messagequeue: azure-storage-queue
 spec:
   replicas: 1
   template:
@@ -734,6 +750,8 @@ metadata:
   name: storagesvc
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: storagesvc
+    application: fission-storage
 spec:
   replicas: 1
   template:

charts/fission-all/templates/fluentbit.yaml

@@ -26,6 +26,7 @@ metadata:
   name: logger
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: logger
 spec:
   template:
     metadata:

charts/fission-core/templates/deployment.yaml

@@ -119,6 +119,8 @@ metadata:
   name: controller
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: controller
+    application: fission-api
 spec:
   replicas: 1
   template:
@@ -178,6 +180,8 @@ metadata:
   name: router
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: router
+    application: fission-router
 spec:
   replicas: 1
   template:
@@ -272,6 +276,7 @@ metadata:
   name: executor
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: executor
 spec:
   replicas: 1
   template:
@@ -338,6 +343,7 @@ metadata:
   name: buildermgr
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: buildermgr
 spec:
   replicas: 1
   template:
@@ -384,6 +390,7 @@ metadata:
   name: kubewatcher
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: kubewatcher
 spec:
   replicas: 1
   template:
@@ -412,6 +419,7 @@ metadata:
   name: timer
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: timer
 spec:
   replicas: 1
   template:
@@ -440,6 +448,8 @@ metadata:
   name: storagesvc
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    svc: storagesvc
+    application: fission-storage
 spec:
   replicas: 1
   template:

pkg/fission-cli/cmd/support/dump.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Fission Authors.
+Copyright 2019 The Fission Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,9 +24,11 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/urfave/cli"
 
-	"github.com/fission/fission/pkg/fission-cli/support/resources"
+	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/fission-cli/cliwrapper/cli"
+	"github.com/fission/fission/pkg/fission-cli/cmd"
+	"github.com/fission/fission/pkg/fission-cli/cmd/support/resources"
 	"github.com/fission/fission/pkg/fission-cli/util"
 	"github.com/fission/fission/pkg/utils"
 )
@@ -36,12 +38,22 @@ const (
 	DEFAULT_OUTPUT_DIR  = "fission-dump"
 )
 
-func DumpInfo(c *cli.Context) error {
+type DumpSubCommand struct {
+	client *client.Client
+}
 
+func Dump(flags cli.Input) error {
+	opts := &DumpSubCommand{
+		client: cmd.GetServer(flags),
+	}
+	return opts.do(flags)
+}
+
+func (opts *DumpSubCommand) do(flags cli.Input) error {
 	fmt.Println("Start dumping process...")
 
-	nozip := c.Bool("nozip")
+	nozip := flags.Bool("nozip")
-	outputDir := c.String("output")
+	outputDir := flags.String("output")
 
 	// check whether the dump directory exists.
 	_, err := os.Stat(outputDir)
@@ -59,7 +71,6 @@ func DumpInfo(c *cli.Context) error {
 		panic(errors.Wrap(err, "Error creating dump directory for dumping files"))
 	}
 
-	client := util.GetApiClient(util.GetServerUrl())
 	_, k8sClient := util.GetKubernetesClient()
 
 	ress := map[string]resources.Resource{
@@ -68,13 +79,15 @@ func DumpInfo(c *cli.Context) error {
 		"kubernetes-nodes":   resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesNode, ""),
 
 		// fission info
-		"fission-version": resources.NewFissionVersion(client),
+		"fission-version": resources.NewFissionVersion(opts.client),
 
 		// fission component logs & spec
 		"fission-components-svc-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesService,
 			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
 		"fission-components-deployment-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesDeployment,
 			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
+		"fission-components-daemonset-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesDaemonSet,
+			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
 		"fission-components-pod-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesPod,
 			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
 		"fission-components-pod-log": resources.NewKubernetesPodLogDumper(k8sClient,
@@ -93,13 +106,13 @@ func DumpInfo(c *cli.Context) error {
 		"fission-function-pod-log":         resources.NewKubernetesPodLogDumper(k8sClient, "executorType in (poolmgr, newdeploy)"),
 
 		// CRD resources
-		"fission-crd-packages":     resources.NewCrdDumper(client, resources.CrdPackage),
+		"fission-crd-packages":     resources.NewCrdDumper(opts.client, resources.CrdPackage),
-		"fission-crd-environments": resources.NewCrdDumper(client, resources.CrdEnvironment),
+		"fission-crd-environments": resources.NewCrdDumper(opts.client, resources.CrdEnvironment),
-		"fission-crd-functions":    resources.NewCrdDumper(client, resources.CrdFunction),
+		"fission-crd-functions":    resources.NewCrdDumper(opts.client, resources.CrdFunction),
-		"fission-crd-httptriggers": resources.NewCrdDumper(client, resources.CrdHttpTrigger),
+		"fission-crd-httptriggers": resources.NewCrdDumper(opts.client, resources.CrdHttpTrigger),
-		"fission-crd-kubewatchers": resources.NewCrdDumper(client, resources.CrdKubeWatcher),
+		"fission-crd-kubewatchers": resources.NewCrdDumper(opts.client, resources.CrdKubeWatcher),
-		"fission-crd-mqtriggers":   resources.NewCrdDumper(client, resources.CrdMessageQueueTrigger),
+		"fission-crd-mqtriggers":   resources.NewCrdDumper(opts.client, resources.CrdMessageQueueTrigger),
-		"fission-crd-timetriggers": resources.NewCrdDumper(client, resources.CrdTimeTrigger),
+		"fission-crd-timetriggers": resources.NewCrdDumper(opts.client, resources.CrdTimeTrigger),
 	}
 
 	dumpName := fmt.Sprintf("%v_%v", DUMP_ARCHIVE_PREFIX, time.Now().Unix())
@@ -107,8 +120,14 @@ func DumpInfo(c *cli.Context) error {
 
 	wg := &sync.WaitGroup{}
 
+	tempDir, err := utils.GetTempDir()
+	if err != nil {
+		fmt.Printf("Error creating temporary directory: %v\n", err.Error())
+		return err
+	}
+
 	for key, res := range ress {
-		dir := fmt.Sprintf("%v/%v/", dumpDir, key)
+		dir := fmt.Sprintf("%v/%v/", tempDir, key)
 		if _, err := os.Stat(dir); os.IsNotExist(err) {
 			err = os.MkdirAll(dir, 0755)
 			if err != nil {
@@ -125,15 +144,20 @@ func DumpInfo(c *cli.Context) error {
 	wg.Wait()
 
 	if !nozip {
-		defer os.Remove(dumpDir)
+		defer os.RemoveAll(tempDir)
 		path := filepath.Join(outputDir, fmt.Sprintf("%v.zip", dumpName))
-		_, err := utils.MakeArchive(path, dumpDir)
+		_, err := utils.MakeArchive(path, tempDir)
 		if err != nil {
 			fmt.Printf("Error creating archive for dump files: %v", err)
-			return nil
+			return err
 		}
 		fmt.Printf("The archive dump file is %v\n", path)
 	} else {
+		err = os.Rename(tempDir, dumpDir)
+		if err != nil {
+			fmt.Printf("Error creating dump directory: %v\n", err.Error())
+			return err
+		}
 		fmt.Printf("The dump files are placed at %v\n", dumpDir)
 	}
 

pkg/fission-cli/cmd/support/resources/kubernetes.go

@@ -29,7 +29,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/fission/fission/pkg/fission-cli/log"
-	"github.com/fission/fission/pkg/utils"
 )
 
 const (
@@ -38,6 +37,7 @@ const (
 	KubernetesPod        = "Pod"
 	KubernetesHPA        = "HPA"
 	KubernetesNode       = "Node"
+	KubernetesDaemonSet  = "DaemonSet"
 )
 
 // Kubernetes Version
@@ -126,6 +126,18 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
 			writeToFile(f, item)
 		}
 
+	case KubernetesDaemonSet:
+		objs, err := res.client.AppsV1().DaemonSets(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
+			return
+		}
+
+		for _, item := range objs.Items {
+			f := getFileName(dumpDir, item.ObjectMeta)
+			writeToFile(f, item)
+		}
+
 	case KubernetesNode:
 		objs, err := res.client.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: res.selector})
 		if err != nil {
@@ -137,12 +149,12 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
 			item = nodeClean(item)
 			// Node doesn't have namespace value, use name here
 			f := filepath.Clean(fmt.Sprintf("%v/%v", dumpDir, item.Name))
-			getFileName(dumpDir, item.ObjectMeta)
 			writeToFile(f, item)
 		}
 
 	default:
 		log.Info(fmt.Sprintf("Unknown type: %v", res.objType))
+		return
 	}
 }
 
@@ -199,11 +211,6 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
 		go func(pod corev1.Pod) {
 			defer wg.Done()
 
-			if !utils.IsReadyPod(&pod) {
-				log.Info(fmt.Sprintf("Pod %v is not in ready state, ignore it\n", pod.Name))
-				return
-			}
-
 			// dump logs from each containers
 			for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) {
 				req := res.client.CoreV1().Pods(pod.Namespace).
@@ -235,7 +242,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
 					}
 				}
 
-				f := getFileName(dumpDir, pod.ObjectMeta)
+				f := getPodFileName(dumpDir, pod.ObjectMeta, container.Name)
 				writeToFile(f, buffer.String())
 
 				stream.Close()

pkg/fission-cli/cmd/support/resources/resource.go

@@ -37,6 +37,11 @@ func getFileName(dumpdir string, meta metav1.ObjectMeta) string {
 	return filepath.Clean(f)
 }
 
+func getPodFileName(dumpdir string, pod metav1.ObjectMeta, containerName string) string {
+	f := fmt.Sprintf("%v/%v_%v_%v_%v.txt", dumpdir, pod.Namespace, pod.Name, pod.ResourceVersion, containerName)
+	return filepath.Clean(f)
+}
+
 func writeToFile(file string, obj interface{}) {
 	bs, err := yaml.Marshal(obj)
 	if err != nil {

pkg/fission-cli/main.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -30,9 +31,9 @@ import (
 	"github.com/fission/fission/pkg/fission-cli/cliwrapper/driver/urfavecli"
 	"github.com/fission/fission/pkg/fission-cli/cmd"
 	"github.com/fission/fission/pkg/fission-cli/cmd/environment"
+	"github.com/fission/fission/pkg/fission-cli/cmd/support"
 	"github.com/fission/fission/pkg/fission-cli/log"
 	"github.com/fission/fission/pkg/fission-cli/plugin"
-	"github.com/fission/fission/pkg/fission-cli/support"
 	"github.com/fission/fission/pkg/fission-cli/util"
 	"github.com/fission/fission/pkg/info"
 	"github.com/fission/fission/pkg/types"
@@ -286,7 +287,7 @@ func NewCliApp() *cli.App {
 	supportOutputFlag := cli.StringFlag{Name: "output, o", Value: support.DEFAULT_OUTPUT_DIR, Usage: "Output directory to save dump archive/files"}
 	supportNoZipFlag := cli.BoolFlag{Name: "nozip", Usage: "Save dump information into multiple files instead of single zip file"}
 	supportSubCommands := []cli.Command{
-		{Name: "dump", Usage: "Collect & dump all necessary for troubleshooting", Flags: []cli.Flag{supportOutputFlag, supportNoZipFlag}, Action: support.DumpInfo},
+		{Name: "dump", Usage: "Collect & dump all necessary for troubleshooting", Flags: []cli.Flag{supportOutputFlag, supportNoZipFlag}, Action: urfavecli.Wrapper(support.Dump)},
 	}
 
 	// canary configs
@@ -395,7 +396,11 @@ To install it for your local Fission CLI:
 func versionPrinter(_ *cli.Context) {
 	client := util.GetApiClient(util.GetServerUrl())
 	ver := util.GetVersion(client)
-	fmt.Print(string(ver))
+	bs, err := yaml.Marshal(ver)
+	if err != nil {
+		log.Fatal("Error formatting versions: " + err.Error())
+	}
+	fmt.Print(string(bs))
 }
 
 func flagValueParser(args []string) error {

pkg/fission-cli/util/version.go

@@ -3,8 +3,6 @@ package util
 import (
 	"fmt"
 
-	yaml "gopkg.in/yaml.v2"
-
 	"github.com/fission/fission/pkg/controller/client"
 	"github.com/fission/fission/pkg/fission-cli/log"
 	"github.com/fission/fission/pkg/fission-cli/plugin"
@@ -17,7 +15,7 @@ type Versions struct {
 	Server map[string]info.BuildMeta `json:"server"`
 }
 
-func GetVersion(client *client.Client) []byte {
+func GetVersion(client *client.Client) Versions {
 	// Fetch client versions
 	versions := Versions{
 		Client: map[string]info.BuildMeta{
@@ -43,10 +41,6 @@ func GetVersion(client *client.Client) []byte {
 	}
 
 	// FUTURE: fetch versions of plugins server-side
-	bs, err := yaml.Marshal(versions)
-	if err != nil {
-		log.Fatal("Failed to format versions: " + err.Error())
-	}
 
-	return bs
+	return versions
 }

test/test_utils.sh

@@ -294,7 +294,7 @@ check_gitcommit_version() {
     while true
     do
         # ensure we run tests against with the same git commit version of CLI & server
-	      ip=$(fission --version|grep "gitcommit"|tr -d ' '|uniq -c|grep "2 gitcommit")
+	      ip=$(fission --version|grep "GitCommit"|tr -d ' '|uniq -c|grep "2 GitCommit")
 	      if [ $? -eq 0 ]; then
 	        break
 	      fi