Canary deployments for fission functions. (#892)

fission/canaryconfig.go

@@ -0,0 +1,237 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/crd"
+	"github.com/fission/fission/fission/log"
+	"github.com/fission/fission/fission/util"
+)
+
+func canaryConfigCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	// canary configs can be created for functions in the same namespace
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	trigger := c.String("httptrigger")
+	funcN := c.String("funcN")
+	funcNminus1 := c.String("funcN-1")
+	ns := c.String("fnNamespace")
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// check that the trigger exists in the same namespace.
+	m := &metav1.ObjectMeta{
+		Name:      trigger,
+		Namespace: ns,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if err != nil {
+		util.CheckErr(err, "Trigger referenced in the canary config is not created")
+	}
+
+	// check that the trigger has function reference type function weights
+	if htTrigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionWeights {
+		log.Fatal("Canary config cannot be created for http triggers that do not reference functions by weights")
+	}
+
+	// check that the trigger references same functions in the function weights
+	_, ok := htTrigger.Spec.FunctionReference.FunctionWeights[funcN]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", funcN))
+	}
+
+	_, ok = htTrigger.Spec.FunctionReference.FunctionWeights[funcNminus1]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", funcNminus1))
+	}
+
+	// check that the functions exist in the same namespace
+	fnList := []string{funcN, funcNminus1}
+	err = util.CheckFunctionExistence(client, fnList, ns)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("checkFunctionExistence err : %v", err))
+	}
+
+	// finally create canaryCfg in the same namespace as the functions referenced
+	canaryCfg := &crd.CanaryConfig{
+		Metadata: metav1.ObjectMeta{
+			Name:      canaryConfigName,
+			Namespace: ns,
+		},
+		Spec: fission.CanaryConfigSpec{
+			Trigger:                 trigger,
+			FunctionN:               funcN,
+			FunctionNminus1:         funcNminus1,
+			WeightIncrement:         incrementStep,
+			WeightIncrementDuration: incrementInterval,
+			FailureThreshold:        failureThreshold,
+			FailureType:             fission.FailureTypeStatusCode,
+		},
+		Status: fission.CanaryConfigStatus{
+			Status: fission.CanaryConfigStatusPending,
+		},
+	}
+
+	fmt.Printf("Canary config name : %s, ns : %s, trigger : %s", canaryConfigName, ns, trigger)
+
+	_, err = client.CanaryConfigCreate(canaryCfg)
+	util.CheckErr(err, "create canary config")
+
+	fmt.Printf("canary config '%v' created\n", canaryConfigName)
+	return err
+}
+
+func canaryConfigGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	if len(name) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	ns := c.String("canaryNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+		canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.FunctionN, canaryCfg.Spec.FunctionNminus1, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+		canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType)
+
+	w.Flush()
+	return nil
+}
+
+func canaryConfigUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	var updateNeeded bool
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	if incrementStep != canaryCfg.Spec.WeightIncrement {
+		canaryCfg.Spec.WeightIncrement = incrementStep
+		updateNeeded = true
+	}
+
+	if failureThreshold != canaryCfg.Spec.FailureThreshold {
+		canaryCfg.Spec.FailureThreshold = failureThreshold
+		updateNeeded = true
+	}
+
+	if incrementInterval != canaryCfg.Spec.WeightIncrementDuration {
+		canaryCfg.Spec.WeightIncrementDuration = incrementInterval
+		updateNeeded = true
+	}
+
+	if updateNeeded {
+		canaryCfg.Status.Status = fission.CanaryConfigStatusPending
+
+		_, err = client.CanaryConfigUpdate(canaryCfg)
+		util.CheckErr(err, "update canary config")
+	}
+
+	return nil
+}
+
+func canaryConfigDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	err := client.CanaryConfigDelete(m)
+	util.CheckErr(err, fmt.Sprintf("delete function '%v.%v'", canaryConfigName, ns))
+
+	fmt.Printf("canaryconfig '%v.%v' deleted\n", canaryConfigName, ns)
+	return err
+}
+
+func canaryConfigList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	ns := c.String("canaryNamespace")
+
+	canaryCfgs, err := client.CanaryConfigList(ns)
+	util.CheckErr(err, "list canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE")
+	for _, canaryCfg := range canaryCfgs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.FunctionN, canaryCfg.Spec.FunctionNminus1, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+			canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType)
+	}
+
+	w.Flush()
+	return nil
+}

fission/httptrigger.go

@@ -29,7 +29,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
-	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/crd"
 	"github.com/fission/fission/fission/log"
 )
@@ -60,28 +59,65 @@ func getMethod(method string) string {
 	return ""
 }
 
-func checkFunctionExistence(fissionClient *client.Client, fnName string, fnNamespace string) {
-	meta := &metav1.ObjectMeta{
-		Name:      fnName,
-		Namespace: fnNamespace,
+func setHtFunctionRef(functionList []string, functionWeightsList []int) (*fission.FunctionReference, error) {
+	if len(functionList) == 1 {
+		return &fission.FunctionReference{
+			Type: fission.FunctionReferenceTypeFunctionName,
+			Name: functionList[0],
+		}, nil
+	} else if len(functionList) == 2 {
+		if len(functionWeightsList) != 2 {
+			return nil, fmt.Errorf("weights of the function need to be specified when 2 functions are supplied")
+		}
+
+		totalWeight := functionWeightsList[0] + functionWeightsList[1]
+		if totalWeight != 100 {
+			log.Fatal("The function weights should add up to 100")
+		}
+
+		functionWeights := make(map[string]int, 0)
+		for index := range functionList {
+			functionWeights[functionList[index]] = functionWeightsList[index]
+		}
+
+		return &fission.FunctionReference{
+			Type:            fission.FunctionReferenceTypeFunctionWeights,
+			FunctionWeights: functionWeights,
+		}, nil
 	}
 
-	_, err := fissionClient.FunctionGet(meta)
-	if err != nil {
-		fmt.Printf("function '%v' does not exist, use 'fission function create --name %v ...' to create the function\n", fnName, fnName)
-	}
+	return nil, fmt.Errorf("the number of functions in a trigger can be 1 or 2(for canary feature along with their weights)")
 }
 
 func htCreate(c *cli.Context) error {
 	client := util.GetApiClient(c.GlobalString("server"))
 
-	fnName := c.String("function")
-	if len(fnName) == 0 {
+	functionList := c.StringSlice("function")
+	functionWeightsList := c.IntSlice("weight")
+
+	if len(functionList) == 0 {
 		log.Fatal("Need a function name to create a trigger, use --function")
 	}
+
+	functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	triggerName := c.String("name")
 	fnNamespace := c.String("fnNamespace")
 	spec := c.Bool("spec")
 
+	m := &metav1.ObjectMeta{
+		Name:      triggerName,
+		Namespace: fnNamespace,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if htTrigger != nil {
+		util.CheckErr(fmt.Errorf("duplicate trigger exists"), "choose a different name or leave it empty for fission to auto-generate it")
+	}
+
 	triggerUrl := c.String("url")
 	if len(triggerUrl) == 0 {
 		log.Fatal("Need a trigger URL, use --url")
@@ -97,7 +133,10 @@ func htCreate(c *cli.Context) error {
 
 	// For Specs, the spec validate checks for function reference
 	if !spec {
-		checkFunctionExistence(client, fnName, fnNamespace)
+		err = util.CheckFunctionExistence(client, functionList, fnNamespace)
+		if err != nil {
+			log.Warn(err.Error())
+		}
 	}
 
 	createIngress := false
@@ -108,7 +147,9 @@ func htCreate(c *cli.Context) error {
 	host := c.String("host")
 
 	// just name triggers by uuid.
-	triggerName := uuid.NewV4().String()
+	if triggerName == "" {
+		triggerName = uuid.NewV4().String()
+	}
 
 	ht := &crd.HTTPTrigger{
 		Metadata: metav1.ObjectMeta{
@@ -116,14 +157,11 @@ func htCreate(c *cli.Context) error {
 			Namespace: fnNamespace,
 		},
 		Spec: fission.HTTPTriggerSpec{
-			Host:        host,
-			RelativeURL: triggerUrl,
-			Method:      getMethod(method),
-			FunctionReference: fission.FunctionReference{
-				Type: fission.FunctionReferenceTypeFunctionName,
-				Name: fnName,
-			},
-			CreateIngress: createIngress,
+			Host:              host,
+			RelativeURL:       triggerUrl,
+			Method:            getMethod(method),
+			FunctionReference: *functionRef,
+			CreateIngress:     createIngress,
 		},
 	}
 
@@ -135,7 +173,7 @@ func htCreate(c *cli.Context) error {
 		return nil
 	}
 
-	_, err := client.HTTPTriggerCreate(ht)
+	_, err = client.HTTPTriggerCreate(ht)
 	util.CheckErr(err, "create HTTP trigger")
 
 	fmt.Printf("trigger '%v' created\n", triggerName)
@@ -143,7 +181,39 @@ func htCreate(c *cli.Context) error {
 }
 
 func htGet(c *cli.Context) error {
-	return nil
+	cliClient := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	ns := c.String("fnNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	htTrigger, err := cliClient.HTTPTriggerGet(m)
+	util.CheckErr(err, "get http trigger")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "METHOD", "RELATIVE-URL", "FUNCTION-REFERENCE-TYPE", "FUNCTION(s)")
+
+	function := ""
+	if htTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionName {
+		function = htTrigger.Spec.FunctionReference.Name
+	} else {
+		for k, v := range htTrigger.Spec.FunctionReference.FunctionWeights {
+			function += fmt.Sprintf("%s:%v ", k, v)
+		}
+	}
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		htTrigger.Metadata.Name, htTrigger.Metadata.UID, htTrigger.Spec.Method, htTrigger.Spec.RelativeURL,
+		htTrigger.Spec.FunctionReference.Type, function)
+
+	w.Flush()
+
+	return err
 }
 
 func htUpdate(c *cli.Context) error {
@@ -161,9 +231,28 @@ func htUpdate(c *cli.Context) error {
 	util.CheckErr(err, "get HTTP trigger")
 
 	if c.IsSet("function") {
-		ht.Spec.FunctionReference.Name = c.String("function")
+		// get the functions and their weights if specified
+		functionList := c.StringSlice("function")
+		err := util.CheckFunctionExistence(client, functionList, triggerNamespace)
+		if err != nil {
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+		var functionWeightsList []int
+		if c.IsSet("weight") {
+			functionWeightsList = c.IntSlice("weight")
+		}
+
+		// set function reference
+		functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+
+		ht.Spec.FunctionReference = *functionRef
 	}
-	checkFunctionExistence(client, ht.Spec.FunctionReference.Name, triggerNamespace)
 
 	if c.IsSet("createingress") {
 		ht.Spec.CreateIngress = c.Bool("createingress")

fission/main.go

@@ -67,6 +67,7 @@ func main() {
 	pkgNamespaceFlag := cli.StringFlag{Name: "pkgNamespace, pkgns", Value: metav1.NamespaceDefault, Usage: "Namespace for package object"}
 	triggerNamespaceFlag := cli.StringFlag{Name: "triggerNamespace, triggerns", Value: metav1.NamespaceDefault, Usage: "Namespace for trigger object"}
 	recorderNamespaceFlag := cli.StringFlag{Name: "recorderNamespace, recorderns", Value: metav1.NamespaceDefault, Usage: "Namespace for recorder object"}
+	canaryNamespaceFlag := cli.StringFlag{Name: "canaryNamespace, canaryns", Value: metav1.NamespaceDefault, Usage: "Namespace for canary config object"}
 
 	// trigger method and url flags (used in function and route CLIs)
 	htMethodFlag := cli.StringFlag{Name: "method", Value: "GET", Usage: "HTTP Method: GET|POST|PUT|DELETE|HEAD"}
@@ -118,14 +119,16 @@ func main() {
 
 	// httptriggers
 	htNameFlag := cli.StringFlag{Name: "name", Usage: "HTTP Trigger name"}
-	htFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"}
 	htHostFlag := cli.StringFlag{Name: "host", Usage: "FQDN of the network host for route"}
 	htIngressFlag := cli.BoolFlag{Name: "createingress", Usage: "Creates ingress with same URL, defaults to false"}
+	htFnNameFlag := cli.StringSliceFlag{Name: "function", Usage: "Name(s) of the function for this trigger. If 2 functions are supplied with this flag, traffic gets routed to them based on weights supplied with --weight flag."}
+	htFnWeightFlag := cli.IntSliceFlag{Name: "weight", Usage: "Weight for each function supplied with --function flag, in the same order. Used for canary deployment"}
+
 	htSubcommands := []cli.Command{
 
-		{Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag, htHostFlag, htIngressFlag, fnNamespaceFlag, specSaveFlag}, Action: htCreate},
-		{Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag}, Action: htGet},
-		{Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag, htFnNameFlag, htHostFlag, htIngressFlag}, Action: htUpdate},
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htNameFlag, htMethodFlag, htUrlFlag, htFnNameFlag, htHostFlag, htIngressFlag, fnNamespaceFlag, specSaveFlag, htFnWeightFlag}, Action: htCreate},
+		{Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htNameFlag}, Action: htGet},
+		{Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag, htFnNameFlag, htHostFlag, htIngressFlag, htFnWeightFlag}, Action: htUpdate},
 		{Name: "delete", Usage: "Delete HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag}, Action: htDelete},
 		{Name: "list", Usage: "List HTTP triggers", Flags: []cli.Flag{triggerNamespaceFlag}, Action: htList},
 	}
@@ -271,6 +274,22 @@ func main() {
 		{Name: "dump", Usage: "Collect & dump all necessary for troubleshooting", Flags: []cli.Flag{supportOutputFlag, supportNoZipFlag}, Action: support.DumpInfo},
 	}
 
+	// canary configs
+	canaryConfigNameFlag := cli.StringFlag{Name: "name", Usage: "Name for the canary config"}
+	triggerNameFlag := cli.StringFlag{Name: "httptrigger", Usage: "Http trigger that this config references"}
+	funcNFlag := cli.StringFlag{Name: "funcN", Usage: "New version of the function"}
+	funcNminus1Flag := cli.StringFlag{Name: "funcN-1", Usage: "Old stable version of the function"}
+	weightIncrementFlag := cli.IntFlag{Name: "increment-step", Value: 20, Usage: "Weight increment step for function"}
+	incrementIntervalFlag := cli.StringFlag{Name: "increment-interval", Value: "2m", Usage: "Weight increment interval, string representation of time.Duration, ex : 1m, 2h, 2d"}
+	failureThresholdFlag := cli.IntFlag{Name: "failure-threshold", Value: 10, Usage: "Threshold in percentage beyond which the new version of the function is considered unstable"}
+	canarySubCommands := []cli.Command{
+		{Name: "create", Usage: "Create a canary config", Flags: []cli.Flag{canaryConfigNameFlag, triggerNameFlag, funcNFlag, funcNminus1Flag, fnNamespaceFlag, weightIncrementFlag, incrementIntervalFlag, failureThresholdFlag}, Action: canaryConfigCreate},
+		{Name: "get", Usage: "View parameters in a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigGet},
+		{Name: "update", Usage: "Update parameters of a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag, incrementIntervalFlag, weightIncrementFlag, failureThresholdFlag}, Action: canaryConfigUpdate},
+		{Name: "delete", Usage: "Delete a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigDelete},
+		{Name: "list", Usage: "List all canary configs in a namespace", Flags: []cli.Flag{canaryNamespaceFlag}, Action: canaryConfigList},
+	}
+
 	app.Commands = []cli.Command{
 		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
 		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -286,6 +305,7 @@ func main() {
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "support", Usage: "Collect an archive of diagnostic information for support", Subcommands: supportSubCommands},
 		cmdPlugin,
+		{Name: "canary-config", Aliases: []string{}, Usage: "Create, Update and manage Canary Configs", Subcommands: canarySubCommands},
 	}
 	app.Before = cliHook
 	app.CommandNotFound = handleCommandNotFound

fission/util/util.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -140,3 +141,25 @@ func GetKubernetesClient(kubeConfig string) (*restclient.Config, *kubernetes.Cli
 
 	return config, clientset
 }
+
+// given a list of functions, this checks if the functions actually exist on the cluster
+func CheckFunctionExistence(fissionClient *client.Client, functions []string, fnNamespace string) (err error) {
+	fnMissing := make([]string, 0)
+	for _, fnName := range functions {
+		meta := &metav1.ObjectMeta{
+			Name:      fnName,
+			Namespace: fnNamespace,
+		}
+
+		_, err := fissionClient.FunctionGet(meta)
+		if err != nil {
+			fnMissing = append(fnMissing, fnName)
+		}
+	}
+
+	if len(fnMissing) > 0 {
+		return fmt.Errorf("function(s) %s, not present in namespace : %s", fnMissing, fnNamespace)
+	}
+
+	return nil
+}