Send the error message to user when enabling canary feature fails. (#990)

* Send the error message to user when enabling canary feature fails.
* Fix controller doesn’t reply canary error when promClient tries connect to invalid prometheus server

canaryconfigmgr/canaryConfigMgr.go

@@ -62,11 +62,16 @@ func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernete
 		}
 	}
 
+	promClient, err := MakePrometheusClient(prometheusSvc)
+	if err != nil {
+		return nil, err
+	}
+
 	configMgr := &canaryConfigMgr{
 		fissionClient:          fissionClient,
 		kubeClient:             kubeClient,
 		crdClient:              crdClient,
-		promClient:             MakePrometheusClient(prometheusSvc),
+		promClient:             promClient,
 		canaryCfgCancelFuncMap: makecanaryConfigCancelFuncMap(),
 	}
 

canaryconfigmgr/prometheusClient.go

@@ -18,9 +18,10 @@ package canaryconfigmgr
 
 import (
 	"fmt"
-	"golang.org/x/net/context"
 	"time"
 
+	"golang.org/x/net/context"
+
 	promClient "github.com/prometheus/client_golang/api/prometheus"
 	"github.com/prometheus/common/model"
 	log "github.com/sirupsen/logrus"
@@ -30,7 +31,7 @@ type PrometheusApiClient struct {
 	client promClient.QueryAPI
 }
 
-func MakePrometheusClient(prometheusSvc string) *PrometheusApiClient {
+func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) {
 	promApiConfig := promClient.Config{
 		Address: prometheusSvc,
 	}
@@ -38,14 +39,31 @@ func MakePrometheusClient(prometheusSvc string) *PrometheusApiClient {
 	promApiClient, err := promClient.New(promApiConfig)
 	if err != nil {
 		log.Errorf("Error creating prometheus api client for svc : %s, err : %v", prometheusSvc, err)
+		return nil, err
 	}
 
 	apiQueryClient := promClient.NewQueryAPI(promApiClient)
 
+	// By default, the prometheus client library doesn't test server connectivity when creating
+	// prometheus client. As a workaround, here we send out a test query string to ensure that
+	// prometheus server is running.
+	for i := 0; i < 15; i++ {
+		_, err = apiQueryClient.Query(context.Background(), "http_requests_total", time.Now())
+		if err == nil {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+
+	if err != nil {
+		log.Printf("Error sending test query to prometheus server: %v", err)
+		return nil, err
+	}
+
 	log.Printf("Successfully made prometheus client with service : %s", prometheusSvc)
 	return &PrometheusApiClient{
 		client: apiQueryClient,
-	}
+	}, nil
 }
 
 func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, method, funcName, funcNs string, window string) (float64, error) {