Canary deployments for fission functions. (#892)

.travis.yml

@@ -22,6 +22,7 @@ services:
 before_install:
   - sudo apt-get update
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+  - sudo apt-get -y install apache2-utils
   - sudo sysctl net.ipv6.conf.all.disable_ipv6=0
 
 install:

canaryconfigmgr/canaryConfigCache.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/cache"
+)
+
+type (
+	canaryConfigCancelFuncMap struct {
+		cache *cache.Cache // map[metadataKey]*context.Context
+	}
+
+	// metav1.ObjectMeta is not hashable, so we make a hashable copy
+	// of the subset of its fields that are identifiable.
+	metadataKey struct {
+		Name      string
+		Namespace string
+	}
+)
+
+func makecanaryConfigCancelFuncMap() *canaryConfigCancelFuncMap {
+	return &canaryConfigCancelFuncMap{
+		cache: cache.MakeCache(0, 0),
+	}
+}
+
+func keyFromMetadata(m *metav1.ObjectMeta) metadataKey {
+	return metadataKey{
+		Name:      m.Name,
+		Namespace: m.Namespace,
+	}
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) lookup(f *metav1.ObjectMeta) (*context.CancelFunc, error) {
+	mk := keyFromMetadata(f)
+	item, err := cancelFuncMap.cache.Get(mk)
+	if err != nil {
+		return nil, err
+	}
+	cancelFunc := item.(*context.CancelFunc)
+	return cancelFunc, nil
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) assign(f *metav1.ObjectMeta, cancelFunc *context.CancelFunc) error {
+	mk := keyFromMetadata(f)
+	err, _ := cancelFuncMap.cache.Set(mk, cancelFunc)
+	return err
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) remove(f *metav1.ObjectMeta) error {
+	mk := keyFromMetadata(f)
+	return cancelFuncMap.cache.Delete(mk)
+}

canaryconfigmgr/canaryConfigMgr.go

@@ -0,0 +1,355 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"context"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"time"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/crd"
+)
+
+type canaryConfigMgr struct {
+	fissionClient          *crd.FissionClient
+	kubeClient             *kubernetes.Clientset
+	canaryConfigStore      k8sCache.Store
+	canaryConfigController k8sCache.Controller
+	promClient             *PrometheusApiClient
+	crdClient              *rest.RESTClient
+	canaryCfgCancelFuncMap *canaryConfigCancelFuncMap
+}
+
+func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) {
+	if prometheusSvc == "" {
+		return nil, fmt.Errorf("prometheus service not found, cant create canary config manager")
+	}
+
+	configMgr := &canaryConfigMgr{
+		fissionClient:          fissionClient,
+		kubeClient:             kubeClient,
+		crdClient:              crdClient,
+		promClient:             MakePrometheusClient(prometheusSvc),
+		canaryCfgCancelFuncMap: makecanaryConfigCancelFuncMap(),
+	}
+
+	store, controller := configMgr.initCanaryConfigController()
+	configMgr.canaryConfigStore = store
+	configMgr.canaryConfigController = controller
+
+	return configMgr, nil
+}
+
+func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(canaryCfgMgr.crdClient, "canaryconfigs", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &crd.CanaryConfig{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				canaryConfig := obj.(*crd.CanaryConfig)
+				if canaryConfig.Status.Status == fission.CanaryConfigStatusPending {
+					go canaryCfgMgr.addCanaryConfig(canaryConfig)
+				}
+			},
+			DeleteFunc: func(obj interface{}) {
+				canaryConfig := obj.(*crd.CanaryConfig)
+				go canaryCfgMgr.deleteCanaryConfig(canaryConfig)
+			},
+			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+				oldConfig := oldObj.(*crd.CanaryConfig)
+				newConfig := newObj.(*crd.CanaryConfig)
+				if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion &&
+					newConfig.Status.Status == fission.CanaryConfigStatusPending {
+					log.Printf("update canary config invoked for : %s.%s, newConfig.Status.Status=%s", newConfig.Metadata.Name, newConfig.Metadata.Namespace, newConfig.Status.Status)
+					go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig)
+				}
+				go canaryCfgMgr.reSyncCanaryConfigs()
+
+			},
+		})
+
+	return store, controller
+}
+
+func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) {
+	go canaryCfgMgr.canaryConfigController.Run(ctx.Done())
+	log.Printf("started Canary configmgr controller")
+}
+
+func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryConfig) {
+	log.Printf("addCanaryConfig called for %s", canaryConfig.Metadata.Name)
+	ctx, cancel := context.WithCancel(context.Background())
+	err := canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, &cancel)
+	if err != nil {
+		log.Printf("Error caching canary config : %s.%s. err : %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, err)
+		return
+	}
+	canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig)
+}
+
+func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, canaryConfig *crd.CanaryConfig) {
+	interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration)
+	if err != nil {
+		log.Printf("Error parsing duration: %v, cant proceed with this canaryConfig : %v.%v", err,
+			canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
+		return
+	}
+
+	ticker := time.NewTicker(interval)
+	quit := make(chan struct{})
+
+	for i := 0; i < fission.MaxIterationsForCanaryConfig; i++ {
+		select {
+		case <-(*ctx).Done():
+			// this case when someone deleted their canary config in the middle of it being processed
+			log.Printf("Cancel Func called for canary config : %s", canaryConfig.Metadata.Name)
+			ticker.Stop()
+			return
+
+		case <-ticker.C:
+			// every weightIncrementDuration, check if failureThreshold has reached.
+			// if yes, rollback.
+			// else, increment the weight of funcN and decrement funcN-1 by `weightIncrement`
+			log.Printf("Processing canary config : %s.%s", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
+			canaryCfgMgr.IncrementWeightOrRollback(canaryConfig, quit)
+
+		case <-quit:
+			// we're done processing this canary config either because the new function receives 100% of the traffic
+			// or we rolled back to send all 100% traffic to old function
+			log.Printf("Quit processing canaryConfig : %s", canaryConfig.Metadata.Name)
+			ticker.Stop()
+			err = canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
+			if err != nil {
+				log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err)
+			}
+			return
+		}
+	}
+
+	// This is to prevent infinitely processing a canary config
+	log.Printf("Reached max iterations for CanaryConfig %s.%s, quitting", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
+	close(quit)
+	err = canaryCfgMgr.updateCanaryConfigStatusWithRetries(canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+		fission.CanaryConfigStatusAborted)
+	if err != nil {
+		log.Printf("Error updating the status of canary config : %s.%s to aborted after max retries. err : %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+			err)
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) IncrementWeightOrRollback(canaryConfig *crd.CanaryConfig, quit chan struct{}) {
+	// get the http trigger object associated with this canary config
+	triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(canaryConfig.Metadata.Namespace).Get(canaryConfig.Spec.Trigger)
+	if err != nil {
+		// if the http trigger is not found, then give up processing this config.
+		if k8serrors.IsNotFound(err) {
+			log.Printf("Http trigger object : %v.%v missing", canaryConfig.Spec.Trigger, canaryConfig.Metadata.Namespace)
+			close(quit)
+			return
+		}
+
+		// just silently ignore. wait for next window to increment weight
+		log.Printf("Error fetching http trigger object, err : %v", err)
+		return
+	}
+
+	if triggerObj.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights &&
+		triggerObj.Spec.FunctionReference.FunctionWeights[canaryConfig.Spec.FunctionN] != 0 {
+		failurePercent, err := canaryCfgMgr.promClient.GetFunctionFailurePercentage(triggerObj.Spec.RelativeURL, triggerObj.Spec.Method,
+			canaryConfig.Spec.FunctionN, canaryConfig.Metadata.Namespace, canaryConfig.Spec.WeightIncrementDuration)
+
+		if err != nil {
+			// silently ignore. wait for next window to increment weight
+			log.Printf("Error calculating failure percentage, err : %v", err)
+			return
+		}
+
+		log.Printf("Failure percentage calculated : %v for canaryConfig %s", failurePercent, canaryConfig.Metadata.Name)
+		if failurePercent == -1 {
+			// this means there were no requests triggered to this url during this window. return here and check back
+			// during next iteration
+			log.Printf("Total requests received for url : %v is 0", triggerObj.Spec.RelativeURL)
+			return
+		}
+
+		if int(failurePercent) > canaryConfig.Spec.FailureThreshold {
+			log.Printf("Failure percent %v crossed the threshold %v, so rolling back", failurePercent, canaryConfig.Spec.FailureThreshold)
+			canaryCfgMgr.rollback(canaryConfig, triggerObj)
+			close(quit)
+			return
+		}
+	}
+
+	doneProcessingCanaryConfig, err := canaryCfgMgr.incrementWeights(canaryConfig, triggerObj)
+	if err != nil {
+		// just log the error and hope that next iteration will succeed
+		log.Printf("Error incrementing weights for triggerObj : %v, err : %v", triggerObj.Metadata.Name, err)
+		return
+	}
+
+	if doneProcessingCanaryConfig {
+		// update the status of canary config as done processing, we dont care if we arent able to update because
+		// resync takes care of the update
+		err = canaryCfgMgr.updateCanaryConfigStatusWithRetries(canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+			fission.CanaryConfigStatusSucceeded)
+		if err != nil {
+			// cant do much after max retries other than logging it.
+			log.Printf("Error updating canary config : %s.%s after max retries, err :%v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+				err)
+		}
+
+		log.Printf("We're done processing canary config : %s. The new function is receiving all the traffic", canaryConfig.Metadata.Name)
+		close(quit)
+		return
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, triggerNamespace string, fnWeights map[string]int) (err error) {
+	for i := 0; i < fission.MaxRetries; i++ {
+		triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName)
+		if err != nil {
+			log.Printf("Error getting http trigger object : %v", err)
+			return err
+		}
+
+		triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights
+
+		_, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj)
+		switch {
+		case err == nil:
+			log.Printf("Updated Http trigger : %s.%s", triggerName, triggerNamespace)
+			return nil
+		case k8serrors.IsConflict(err):
+			log.Printf("Conflict in updating http trigger : %s.%s, retrying", triggerName, triggerNamespace)
+			continue
+		default:
+			log.Printf("Error updating trigger : %s.%s = %v", triggerName, triggerNamespace, err)
+			return err
+		}
+	}
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfigStatusWithRetries(cfgName, cfgNamespace string, status string) (err error) {
+	for i := 0; i < fission.MaxRetries; i++ {
+		canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName)
+		if err != nil {
+			log.Printf("Error getting http Canary Config object : %v", err)
+			return err
+		}
+
+		log.Printf("Updating status of canaryCfg : %s.%s to %s", cfgName, cfgNamespace, status)
+		canaryCfgObj.Status.Status = status
+
+		_, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj)
+		switch {
+		case err == nil:
+			log.Printf("Updated Canary Config : %s.%s", cfgName, cfgNamespace)
+			return nil
+		case k8serrors.IsConflict(err):
+			log.Printf("Conflict in updating Canary Config : %s.%s, retrying", cfgName, cfgNamespace)
+			continue
+		default:
+			log.Printf("Error updating Canary Config : %s.%s = %v", cfgName, cfgNamespace, err)
+			return err
+		}
+	}
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) rollback(canaryConfig *crd.CanaryConfig, trigger *crd.HTTPTrigger) error {
+	functionWeights := trigger.Spec.FunctionReference.FunctionWeights
+	functionWeights[canaryConfig.Spec.FunctionN] = 0
+	functionWeights[canaryConfig.Spec.FunctionNminus1] = 100
+
+	err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
+
+	err = canaryCfgMgr.updateCanaryConfigStatusWithRetries(canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+		fission.CanaryConfigStatusFailed)
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) incrementWeights(canaryConfig *crd.CanaryConfig, trigger *crd.HTTPTrigger) (bool, error) {
+	doneProcessingCanaryConfig := false
+
+	functionWeights := trigger.Spec.FunctionReference.FunctionWeights
+	if functionWeights[canaryConfig.Spec.FunctionN]+canaryConfig.Spec.WeightIncrement >= 100 {
+		doneProcessingCanaryConfig = true
+		functionWeights[canaryConfig.Spec.FunctionN] = 100
+		functionWeights[canaryConfig.Spec.FunctionNminus1] = 0
+	} else {
+		functionWeights[canaryConfig.Spec.FunctionN] += canaryConfig.Spec.WeightIncrement
+		if functionWeights[canaryConfig.Spec.FunctionNminus1]-canaryConfig.Spec.WeightIncrement < 0 {
+			functionWeights[canaryConfig.Spec.FunctionNminus1] = 0
+		} else {
+			functionWeights[canaryConfig.Spec.FunctionNminus1] -= canaryConfig.Spec.WeightIncrement
+		}
+	}
+
+	log.Printf("Incremented functionWeights : %v", functionWeights)
+
+	err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
+	return doneProcessingCanaryConfig, err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
+	for _, obj := range canaryCfgMgr.canaryConfigStore.List() {
+		canaryConfig := obj.(*crd.CanaryConfig)
+		cancelFunc, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
+		if err != nil || cancelFunc == nil || canaryConfig.Status.Status == fission.CanaryConfigStatusPending {
+			log.Printf("Adding canary config : %s.%s from resync loop", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
+
+			// new canaryConfig detected, add it to our cache and start processing it
+			go canaryCfgMgr.addCanaryConfig(canaryConfig)
+		}
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *crd.CanaryConfig) {
+	log.Printf("Delete event received for canary config : %v, %v, %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, canaryConfig.Metadata.ResourceVersion)
+	cancelFunc, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
+	if err != nil {
+		log.Printf("lookup of canaryConfig failed, err : %v", err)
+		return
+	}
+	// when this is called, the ctx.Done returns inside processCanaryConfig function and processing gets stopped
+	(*cancelFunc)()
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfig(oldCanaryConfig *crd.CanaryConfig, newCanaryConfig *crd.CanaryConfig) {
+	// before removing the object from cache, we need to get it's cancel func and cancel it
+	canaryCfgMgr.deleteCanaryConfig(oldCanaryConfig)
+
+	err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata)
+	if err != nil {
+		log.Printf("error removing canary config: %s from map, err : %v", oldCanaryConfig.Metadata.Name, err)
+		return
+	}
+	canaryCfgMgr.addCanaryConfig(newCanaryConfig)
+}

canaryconfigmgr/prometheusClient.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"fmt"
+	"golang.org/x/net/context"
+	"time"
+
+	promClient "github.com/prometheus/client_golang/api/prometheus"
+	"github.com/prometheus/common/model"
+	log "github.com/sirupsen/logrus"
+)
+
+type PrometheusApiClient struct {
+	client promClient.QueryAPI
+}
+
+func MakePrometheusClient(prometheusSvc string) *PrometheusApiClient {
+	promApiConfig := promClient.Config{
+		Address: prometheusSvc,
+	}
+
+	promApiClient, err := promClient.New(promApiConfig)
+	if err != nil {
+		log.Errorf("Error creating prometheus api client for svc : %s, err : %v", prometheusSvc, err)
+	}
+
+	apiQueryClient := promClient.NewQueryAPI(promApiClient)
+
+	log.Printf("Successfully made prometheus client with service : %s", prometheusSvc)
+	return &PrometheusApiClient{
+		client: apiQueryClient,
+	}
+}
+
+func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, method, funcName, funcNs string, window string) (float64, error) {
+	// first get a total count of requests to this url in a time window
+	reqs, err := promApiClient.GetRequestsToFuncInWindow(path, method, funcName, funcNs, window)
+	if err != nil {
+		return 0, err
+	}
+
+	if reqs <= 0 {
+		return -1, fmt.Errorf("no requests to this url %v and method %v in the window : %v", path, method, window)
+	}
+
+	// next, get a total count of errored out requests to this function in the same window
+	failedReqs, err := promApiClient.GetTotalFailedRequestsToFuncInWindow(funcName, funcNs, path, method, window)
+	if err != nil {
+		return 0, err
+	}
+
+	// calculate the failure percentage of the function
+	failurePercentForFunc := (failedReqs / reqs) * 100
+	log.Printf("failurePercentForFunc for func: %v.%v is %v", funcName, funcNs, failurePercentForFunc)
+
+	return failurePercentForFunc, nil
+}
+
+func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, method string, funcName string, funcNs string, window string) (float64, error) {
+	queryString := fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"}[%v]", path, method, funcName, funcNs, window)
+	log.Printf("Querying total function calls for : %s ", queryString)
+
+	reqs, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		log.Printf("Error executing query : %s, err : %v", queryString, err)
+		return 0, err
+	}
+
+	queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window)
+	log.Printf("Querying total function calls for : %s ", queryString)
+
+	reqsInPrevWindow, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		log.Printf("Error executing query : %s, err : %v", queryString, err)
+		return 0, err
+	}
+
+	log.Printf("reqs : %v, reqsInPrevWindow : %v", reqs, reqsInPrevWindow)
+	reqsInCurrentWindow := reqs - reqsInPrevWindow
+	log.Printf("reqsInCurrentWindow to this function %v : %v", funcName, reqsInCurrentWindow)
+
+	return reqsInCurrentWindow, nil
+}
+
+func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(funcName string, funcNs string, path string, method string, window string) (float64, error) {
+	queryString := fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"}[%v]", funcName, funcNs, path, method, window)
+	log.Printf("Querying fission_function_errors_total qs : %s", queryString)
+
+	failedRequests, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		log.Printf("Error executing query : %s, err : %v", queryString, err)
+		return 0, err
+	}
+
+	queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window)
+	log.Printf("Querying fission_function_errors_total qs : %s", queryString)
+
+	failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		log.Printf("Error executing query : %s, err : %v", queryString, err)
+		return 0, err
+	}
+
+	log.Printf("failedReqs : %v, failedReqsInPrevWindow : %v", failedRequests, failedReqsInPrevWindow)
+	failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow
+	log.Printf("failedReqsInCurrentWindow to function: %v.%v : %v", funcName, funcNs, failedReqsInCurrentWindow)
+
+	return failedReqsInCurrentWindow, nil
+}
+
+func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) {
+	val, err := promApiClient.client.Query(context.Background(), queryString, time.Now())
+	if err != nil {
+		log.Errorf("Error querying prometheus qs : %v, err : %v", queryString, err)
+		return 0, err
+	}
+
+	switch {
+	case val.Type() == model.ValScalar:
+		scalarVal := val.(*model.Scalar)
+		return float64(scalarVal.Value), nil
+
+	case val.Type() == model.ValVector:
+		vectorVal := val.(model.Vector)
+		total := float64(0)
+		for _, elem := range vectorVal {
+			total = total + float64(elem.Value)
+		}
+		return total, nil
+
+	case val.Type() == model.ValMatrix:
+		matrixVal := val.(model.Matrix)
+		total := float64(0)
+		for _, elem := range matrixVal {
+			//log.Printf("Only one value, so taking the 0th elem")
+			total += float64(elem.Values[len(elem.Values)-1].Value)
+		}
+		return total, nil
+
+	default:
+		log.Printf("type unrecognized")
+		return 0, nil
+	}
+}
+
+func addInterval(window string) string {
+	timeDuration, _ := time.ParseDuration(window)
+	log.Println("window in seconds", int64(timeDuration/time.Second))
+
+	timeInStr := fmt.Sprintf("%ds", int64((timeDuration+timeDuration)/time.Second))
+	fmt.Println(timeInStr)
+
+	return timeInStr
+}

charts/fission-all/requirements.yaml

@@ -0,0 +1,4 @@
+dependencies:
+  - name: prometheus
+    version: 7.1.0
+    repository: https://kubernetes-charts.storage.googleapis.com

charts/fission-all/templates/deployment.yaml

@@ -104,6 +104,7 @@ metadata:
   namespace: {{ .Values.builderNamespace }}
 
 ---
+# TODO : Configure controller with prometheus endpoint
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -123,7 +124,7 @@ spec:
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--controllerPort", "8888"]
+        args: ["--controllerPort", "8888", "--prometheusSvc", "http://{{ .Release.Name }}-prometheus-server.{{ .Release.Namespace }}"]
         env:
           - name: FISSION_FUNCTION_NAMESPACE
             value: "{{ .Values.functionNamespace }}"

charts/fission-core/requirements.yaml

@@ -0,0 +1,4 @@
+dependencies:
+  - name: prometheus
+    version: 7.1.0
+    repository: https://kubernetes-charts.storage.googleapis.com

controller/api.go

@@ -253,6 +253,12 @@ func (api *API) Serve(port int) {
 	r.HandleFunc("/v2/secrets/{secret}", api.SecretGet).Methods("GET")
 	r.HandleFunc("/v2/configmaps/{configmap}", api.ConfigMapGet).Methods("GET")
 
+	r.HandleFunc("/v2/canaryconfigs", api.CanaryConfigApiCreate).Methods("POST")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiGet).Methods("GET")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiDelete).Methods("DELETE")
+	r.HandleFunc("/v2/canaryconfigs", api.CanaryConfigApiList).Methods("GET")
+
 	r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST")
 	r.HandleFunc("/proxy/storage/v1/archive", api.StorageServiceProxy)
 	r.HandleFunc("/proxy/logs/{function}", api.FunctionPodLogs).Methods("POST")

controller/api_test.go

@@ -171,7 +171,10 @@ func TestHTTPTriggerApi(t *testing.T) {
 
 	tr, err := g.client.HTTPTriggerGet(m)
 	panicIf(err)
-	assert(testTrigger.Spec == tr.Spec, "trigger should match after reading")
+	assert(testTrigger.Spec.Method == tr.Spec.Method &&
+		testTrigger.Spec.RelativeURL == tr.Spec.RelativeURL &&
+		testTrigger.Spec.FunctionReference.Type == tr.Spec.FunctionReference.Type &&
+		testTrigger.Spec.FunctionReference.Name == tr.Spec.FunctionReference.Name, "trigger should match after reading")
 
 	testTrigger.Metadata.ResourceVersion = m.ResourceVersion
 	testTrigger.Spec.RelativeURL = "/hi"
@@ -272,7 +275,8 @@ func TestWatchApi(t *testing.T) {
 	panicIf(err)
 	assert(testWatch.Spec.Namespace == w.Spec.Namespace &&
 		testWatch.Spec.Type == w.Spec.Type &&
-		testWatch.Spec.FunctionReference == w.Spec.FunctionReference, "watch should match after reading")
+		testWatch.Spec.FunctionReference.Type == w.Spec.FunctionReference.Type &&
+		testWatch.Spec.FunctionReference.Name == w.Spec.FunctionReference.Name, "watch should match after reading")
 
 	testWatch.Metadata.Name = "yyy"
 	m2, err := g.client.WatchCreate(testWatch)
@@ -310,7 +314,9 @@ func TestTimeTriggerApi(t *testing.T) {
 
 	tr, err := g.client.TimeTriggerGet(m)
 	panicIf(err)
-	assert(testTrigger.Spec == tr.Spec, "trigger should match after reading")
+	assert(testTrigger.Spec.Cron == tr.Spec.Cron &&
+		testTrigger.Spec.FunctionReference.Type == tr.Spec.FunctionReference.Type &&
+		testTrigger.Spec.FunctionReference.Name == tr.Spec.FunctionReference.Name, "trigger should match after reading")
 
 	testTrigger.Metadata.ResourceVersion = m.ResourceVersion
 	testTrigger.Spec.Cron = "@hourly"
@@ -338,7 +344,7 @@ func TestMain(m *testing.M) {
 		return
 	}
 
-	go Start(8888)
+	go Start(8888, "http://localhost:9090")
 
 	time.Sleep(5 * time.Second)
 	g.client = client.MakeClient("http://localhost:8888")

controller/canaryConfigApi.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/crd"
+)
+
+func (a *API) CanaryConfigApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var canaryCfg crd.CanaryConfig
+	err = json.Unmarshal(body, &canaryCfg)
+	if err != nil {
+		log.Printf("Failed to unmarshal request body: [%v]", body)
+		a.respondWithError(w, err)
+		return
+	}
+
+	canaryCfgNew, err := a.fissionClient.CanaryConfigs(canaryCfg.Metadata.Namespace).Create(&canaryCfg)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfgNew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["canaryConfig"]
+
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	canaryCfg, err := a.fissionClient.CanaryConfigs(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfg)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	canaryCfgs, err := a.fissionClient.CanaryConfigs(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfgs.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiUpdate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var c crd.CanaryConfig
+	err = json.Unmarshal(body, &c)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	canayCfgNew, err := a.fissionClient.CanaryConfigs(c.Metadata.Namespace).Update(&c)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canayCfgNew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["canaryConfig"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.CanaryConfigs(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

controller/client/canaryconfig.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/crd"
+)
+
+func (c *Client) CanaryConfigCreate(canaryConf *crd.CanaryConfig) (*metav1.ObjectMeta, error) {
+	reqbody, err := json.Marshal(canaryConf)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("canaryconfigs"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) CanaryConfigGet(m *metav1.ObjectMeta) (*crd.CanaryConfig, error) {
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var canaryCfg crd.CanaryConfig
+	err = json.Unmarshal(body, &canaryCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &canaryCfg, nil
+}
+
+func (c *Client) CanaryConfigUpdate(canaryConf *crd.CanaryConfig) (*metav1.ObjectMeta, error) {
+	reqbody, err := json.Marshal(canaryConf)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", canaryConf.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) CanaryConfigDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) CanaryConfigList(ns string) ([]crd.CanaryConfig, error) {
+	relativeUrl := fmt.Sprintf("canaryconfigs?namespace=%v", ns)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	canaryCfgs := make([]crd.CanaryConfig, 0)
+	err = json.Unmarshal(body, &canaryCfgs)
+	if err != nil {
+		return nil, err
+	}
+
+	return canaryCfgs, nil
+}

controller/controller.go

@@ -17,17 +17,19 @@ limitations under the License.
 package controller
 
 import (
+	"context"
 	"log"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/canaryconfigmgr"
 	"github.com/fission/fission/crd"
 )
 
-func Start(port int) {
+func Start(port int, prometheusSvc string) {
 	// setup a signal handler for SIGTERM
 	fission.SetupStackTraceHandler()
 
-	fc, _, apiExtClient, err := crd.MakeFissionClient()
+	fc, kc, apiExtClient, err := crd.MakeFissionClient()
 	if err != nil {
 		log.Fatalf("Failed to connect to K8s API: %v", err)
 	}
@@ -42,6 +44,15 @@ func Start(port int) {
 		log.Fatalf("Error waiting for CRDs: %v", err)
 	}
 
+	// create canary config manager
+	canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(fc, kc, fc.GetCrdClient(), prometheusSvc)
+	if err != nil {
+		log.Fatalf("Failed to start canary config manager: %v", err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	canaryCfgMgr.Run(ctx)
+
 	api, err := MakeAPI()
 	if err != nil {
 		log.Fatalf("Failed to start controller: %v", err)

crd/canaryConfig.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+)
+
+type (
+	CanaryConfigInterface interface {
+		Create(*CanaryConfig) (*CanaryConfig, error)
+		Get(name string) (*CanaryConfig, error)
+		Update(*CanaryConfig) (*CanaryConfig, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*CanaryConfigList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	canaryConfigClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeCanaryConfigInterface(crdClient *rest.RESTClient, namespace string) CanaryConfigInterface {
+	return &canaryConfigClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (c *canaryConfigClient) Create(f *CanaryConfig) (*CanaryConfig, error) {
+	var result CanaryConfig
+	err := c.client.Post().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Get(name string) (*CanaryConfig, error) {
+	var result CanaryConfig
+	err := c.client.Get().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Update(f *CanaryConfig) (*CanaryConfig, error) {
+	var result CanaryConfig
+	err := c.client.Put().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (c *canaryConfigClient) List(opts metav1.ListOptions) (*CanaryConfigList, error) {
+	var result CanaryConfigList
+	err := c.client.Get().
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return c.client.Get().
+		Prefix("watch").
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

crd/client.go

@@ -160,6 +160,13 @@ func configureClient(config *rest.Config) {
 				&metav1.ListOptions{},
 				&metav1.DeleteOptions{},
 			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&CanaryConfig{},
+				&CanaryConfigList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
 			return nil
 		})
 	schemeBuilder.AddToScheme(scheme.Scheme)
@@ -221,6 +228,9 @@ func (fc *FissionClient) Recorders(ns string) RecorderInterface {
 func (fc *FissionClient) Packages(ns string) PackageInterface {
 	return MakePackageInterface(fc.crdClient, ns)
 }
+func (fc *FissionClient) CanaryConfigs(ns string) CanaryConfigInterface {
+	return MakeCanaryConfigInterface(fc.crdClient, ns)
+}
 func (fc *FissionClient) WaitForCRDs() error {
 	return waitForCRDs(fc.crdClient)
 }

crd/crd.go

@@ -189,6 +189,22 @@ func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error {
 				},
 			},
 		},
+		// CanaryConfig: configuration for canary deployment of functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "canaryconfigs.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "CanaryConfig",
+					Plural:   "canaryconfigs",
+					Singular: "canaryconfig",
+				},
+			},
+		},
 	}
 	for _, crd := range crds {
 		err := ensureCRD(clientset, &crd)

crd/types.go

@@ -37,4 +37,6 @@ type (
 	MessageQueueTriggerList    = fv1.MessageQueueTriggerList
 	Recorder                   = fv1.Recorder
 	RecorderList               = fv1.RecorderList
+	CanaryConfig               = fv1.CanaryConfig
+	CanaryConfigList           = fv1.CanaryConfigList
 )

demos/canary-rollback-scenario/cleanup.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+kubectl delete canaryconfig canary-2
+kubectl delete httptrigger route-fail

demos/canary-rollback-scenario/run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# this script is useful to demo canary deployment when the latest function starts receiving 100% of the traffic
+
+DEMO_RUN_FAST=1
+ROOT_DIR=$(dirname $0)/..
+. $ROOT_DIR/util.sh
+
+
+desc "Function version-1"
+run "fission function get --name fn1-v6"
+
+desc "Function version-2"
+run "fission function get --name fn1-v7"
+
+desc "Create a route \(HTTP trigger\) the version-1 of the function with weight 100% and version-2 with weight 0%"
+run "fission route create --name route-fail --method GET --url /fail --function fn1-v6 --weight 100 --function fn1-v7 --weight 0"
+
+desc "Create a canary config to gradually increment the weight of version-2 by a step of 20 every 1 minute"
+run "fission canary-config create --name canary-2 --funcN fn1-v7 --funcN-1 fn1-v6 --trigger route-fail --increment-step 30 --increment-interval 30s --failure-threshold 10"
+
+desc "Fire requests to the route"
+run "ab -n 10000 -c 1 http://$FISSION_ROUTER/fail"

demos/canary-successful-scenario/cleanup.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+kubectl delete canaryconfig canary-1
+kubectl delete httptrigger route-hello

demos/canary-successful-scenario/run.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# this script is useful to demo canary deployment when the latest function starts receiving 100% of the traffic
+
+DEMO_RUN_FAST=1
+ROOT_DIR=$(dirname $0)/..
+. $ROOT_DIR/util.sh
+
+desc "Kubernetes cluster"
+run "kubectl get nodes"
+
+desc "Fission installed"
+run "kubectl --namespace default get deployment"
+
+clear
+
+desc "NodeJS environment pods"
+run "kubectl --namespace fission-function get pod -l environmentName=nodejs"
+
+desc "Function version-1"
+run "fission function get --name fn1-v4"
+
+desc "Function version-2"
+run "fission function get --name fn1-v5"
+
+desc "Create a route \(HTTP trigger\) the version-1 of the function with weight 100% and version-2 with weight 0%"
+run "fission route create --name route-hello --method GET --url /hello --function fn1-v5 --weight 0 --function fn1-v4 --weight 100"
+
+desc "Create a canary config to gradually increment the weight of version-2 by a step of 20 every 1 minute"
+run "fission canary-config create --name canary-1 --funcN fn1-v5 --funcN-1 fn1-v4 --trigger route-hello --increment-step 30 --increment-interval 30s --failure-threshold 10"
+
+desc "Fire requests to the route"
+run "ab -n 10000 -c 1 http://$FISSION_ROUTER/hello"

examples/nodejs/hello.js

@@ -2,6 +2,6 @@
 module.exports = async function(context) {
     return {
         status: 200,
-        body: "Hello, world!\n"
+        body: "hello, world!\n"
     };
 }

fission-bundle/main.go

@@ -19,8 +19,8 @@ import (
 	"github.com/fission/fission/timer"
 )
 
-func runController(port int) {
-	controller.Start(port)
+func runController(port int, prometheusSvc string) {
+	controller.Start(port, prometheusSvc)
 	log.Fatalf("Error: Controller exited.")
 }
 
@@ -114,7 +114,7 @@ Use it to start one or more of the fission servers:
  backends.
 
 Usage:
-  fission-bundle --controllerPort=<port>
+  fission-bundle --controllerPort=<port> --prometheusSvc=<url>
   fission-bundle --routerPort=<port> [--executorUrl=<url>]
   fission-bundle --executorPort=<port> [--namespace=<namespace>] [--fission-namespace=<namespace>]
   fission-bundle --kubewatcher [--routerUrl=<url>]
@@ -125,6 +125,7 @@ Usage:
   fission-bundle --version
 Options:
   --controllerPort=<port>         Port that the controller should listen on.
+  --prometheusSvc=<url>           Service endpoint of prometheus server
   --routerPort=<port>             Port that the router should listen on.
   --executorPort=<port>           Port that the executor should listen on.
   --storageServicePort=<port>     Port that the storage service should listen on.
@@ -153,10 +154,11 @@ Options:
 	executorUrl := getStringArgWithDefault(arguments["--executorUrl"], "http://executor.fission")
 	routerUrl := getStringArgWithDefault(arguments["--routerUrl"], "http://router.fission")
 	storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission")
+	prometheusSvcUrl := getStringArgWithDefault(arguments["--prometheusSvc"], "")
 
 	if arguments["--controllerPort"] != nil {
 		port := getPort(arguments["--controllerPort"])
-		runController(port)
+		runController(port, prometheusSvcUrl)
 	}
 
 	if arguments["--routerPort"] != nil {

fission/canaryconfig.go

@@ -0,0 +1,237 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/crd"
+	"github.com/fission/fission/fission/log"
+	"github.com/fission/fission/fission/util"
+)
+
+func canaryConfigCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	// canary configs can be created for functions in the same namespace
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	trigger := c.String("httptrigger")
+	funcN := c.String("funcN")
+	funcNminus1 := c.String("funcN-1")
+	ns := c.String("fnNamespace")
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// check that the trigger exists in the same namespace.
+	m := &metav1.ObjectMeta{
+		Name:      trigger,
+		Namespace: ns,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if err != nil {
+		util.CheckErr(err, "Trigger referenced in the canary config is not created")
+	}
+
+	// check that the trigger has function reference type function weights
+	if htTrigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionWeights {
+		log.Fatal("Canary config cannot be created for http triggers that do not reference functions by weights")
+	}
+
+	// check that the trigger references same functions in the function weights
+	_, ok := htTrigger.Spec.FunctionReference.FunctionWeights[funcN]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", funcN))
+	}
+
+	_, ok = htTrigger.Spec.FunctionReference.FunctionWeights[funcNminus1]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", funcNminus1))
+	}
+
+	// check that the functions exist in the same namespace
+	fnList := []string{funcN, funcNminus1}
+	err = util.CheckFunctionExistence(client, fnList, ns)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("checkFunctionExistence err : %v", err))
+	}
+
+	// finally create canaryCfg in the same namespace as the functions referenced
+	canaryCfg := &crd.CanaryConfig{
+		Metadata: metav1.ObjectMeta{
+			Name:      canaryConfigName,
+			Namespace: ns,
+		},
+		Spec: fission.CanaryConfigSpec{
+			Trigger:                 trigger,
+			FunctionN:               funcN,
+			FunctionNminus1:         funcNminus1,
+			WeightIncrement:         incrementStep,
+			WeightIncrementDuration: incrementInterval,
+			FailureThreshold:        failureThreshold,
+			FailureType:             fission.FailureTypeStatusCode,
+		},
+		Status: fission.CanaryConfigStatus{
+			Status: fission.CanaryConfigStatusPending,
+		},
+	}
+
+	fmt.Printf("Canary config name : %s, ns : %s, trigger : %s", canaryConfigName, ns, trigger)
+
+	_, err = client.CanaryConfigCreate(canaryCfg)
+	util.CheckErr(err, "create canary config")
+
+	fmt.Printf("canary config '%v' created\n", canaryConfigName)
+	return err
+}
+
+func canaryConfigGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	if len(name) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	ns := c.String("canaryNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+		canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.FunctionN, canaryCfg.Spec.FunctionNminus1, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+		canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType)
+
+	w.Flush()
+	return nil
+}
+
+func canaryConfigUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	var updateNeeded bool
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	if incrementStep != canaryCfg.Spec.WeightIncrement {
+		canaryCfg.Spec.WeightIncrement = incrementStep
+		updateNeeded = true
+	}
+
+	if failureThreshold != canaryCfg.Spec.FailureThreshold {
+		canaryCfg.Spec.FailureThreshold = failureThreshold
+		updateNeeded = true
+	}
+
+	if incrementInterval != canaryCfg.Spec.WeightIncrementDuration {
+		canaryCfg.Spec.WeightIncrementDuration = incrementInterval
+		updateNeeded = true
+	}
+
+	if updateNeeded {
+		canaryCfg.Status.Status = fission.CanaryConfigStatusPending
+
+		_, err = client.CanaryConfigUpdate(canaryCfg)
+		util.CheckErr(err, "update canary config")
+	}
+
+	return nil
+}
+
+func canaryConfigDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	err := client.CanaryConfigDelete(m)
+	util.CheckErr(err, fmt.Sprintf("delete function '%v.%v'", canaryConfigName, ns))
+
+	fmt.Printf("canaryconfig '%v.%v' deleted\n", canaryConfigName, ns)
+	return err
+}
+
+func canaryConfigList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	ns := c.String("canaryNamespace")
+
+	canaryCfgs, err := client.CanaryConfigList(ns)
+	util.CheckErr(err, "list canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE")
+	for _, canaryCfg := range canaryCfgs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.FunctionN, canaryCfg.Spec.FunctionNminus1, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+			canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType)
+	}
+
+	w.Flush()
+	return nil
+}

fission/httptrigger.go

@@ -29,7 +29,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
-	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/crd"
 	"github.com/fission/fission/fission/log"
 )
@@ -60,28 +59,65 @@ func getMethod(method string) string {
 	return ""
 }
 
-func checkFunctionExistence(fissionClient *client.Client, fnName string, fnNamespace string) {
-	meta := &metav1.ObjectMeta{
-		Name:      fnName,
-		Namespace: fnNamespace,
+func setHtFunctionRef(functionList []string, functionWeightsList []int) (*fission.FunctionReference, error) {
+	if len(functionList) == 1 {
+		return &fission.FunctionReference{
+			Type: fission.FunctionReferenceTypeFunctionName,
+			Name: functionList[0],
+		}, nil
+	} else if len(functionList) == 2 {
+		if len(functionWeightsList) != 2 {
+			return nil, fmt.Errorf("weights of the function need to be specified when 2 functions are supplied")
+		}
+
+		totalWeight := functionWeightsList[0] + functionWeightsList[1]
+		if totalWeight != 100 {
+			log.Fatal("The function weights should add up to 100")
+		}
+
+		functionWeights := make(map[string]int, 0)
+		for index := range functionList {
+			functionWeights[functionList[index]] = functionWeightsList[index]
+		}
+
+		return &fission.FunctionReference{
+			Type:            fission.FunctionReferenceTypeFunctionWeights,
+			FunctionWeights: functionWeights,
+		}, nil
 	}
 
-	_, err := fissionClient.FunctionGet(meta)
-	if err != nil {
-		fmt.Printf("function '%v' does not exist, use 'fission function create --name %v ...' to create the function\n", fnName, fnName)
-	}
+	return nil, fmt.Errorf("the number of functions in a trigger can be 1 or 2(for canary feature along with their weights)")
 }
 
 func htCreate(c *cli.Context) error {
 	client := util.GetApiClient(c.GlobalString("server"))
 
-	fnName := c.String("function")
-	if len(fnName) == 0 {
+	functionList := c.StringSlice("function")
+	functionWeightsList := c.IntSlice("weight")
+
+	if len(functionList) == 0 {
 		log.Fatal("Need a function name to create a trigger, use --function")
 	}
+
+	functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	triggerName := c.String("name")
 	fnNamespace := c.String("fnNamespace")
 	spec := c.Bool("spec")
 
+	m := &metav1.ObjectMeta{
+		Name:      triggerName,
+		Namespace: fnNamespace,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if htTrigger != nil {
+		util.CheckErr(fmt.Errorf("duplicate trigger exists"), "choose a different name or leave it empty for fission to auto-generate it")
+	}
+
 	triggerUrl := c.String("url")
 	if len(triggerUrl) == 0 {
 		log.Fatal("Need a trigger URL, use --url")
@@ -97,7 +133,10 @@ func htCreate(c *cli.Context) error {
 
 	// For Specs, the spec validate checks for function reference
 	if !spec {
-		checkFunctionExistence(client, fnName, fnNamespace)
+		err = util.CheckFunctionExistence(client, functionList, fnNamespace)
+		if err != nil {
+			log.Warn(err.Error())
+		}
 	}
 
 	createIngress := false
@@ -108,7 +147,9 @@ func htCreate(c *cli.Context) error {
 	host := c.String("host")
 
 	// just name triggers by uuid.
-	triggerName := uuid.NewV4().String()
+	if triggerName == "" {
+		triggerName = uuid.NewV4().String()
+	}
 
 	ht := &crd.HTTPTrigger{
 		Metadata: metav1.ObjectMeta{
@@ -116,14 +157,11 @@ func htCreate(c *cli.Context) error {
 			Namespace: fnNamespace,
 		},
 		Spec: fission.HTTPTriggerSpec{
-			Host:        host,
-			RelativeURL: triggerUrl,
-			Method:      getMethod(method),
-			FunctionReference: fission.FunctionReference{
-				Type: fission.FunctionReferenceTypeFunctionName,
-				Name: fnName,
-			},
-			CreateIngress: createIngress,
+			Host:              host,
+			RelativeURL:       triggerUrl,
+			Method:            getMethod(method),
+			FunctionReference: *functionRef,
+			CreateIngress:     createIngress,
 		},
 	}
 
@@ -135,7 +173,7 @@ func htCreate(c *cli.Context) error {
 		return nil
 	}
 
-	_, err := client.HTTPTriggerCreate(ht)
+	_, err = client.HTTPTriggerCreate(ht)
 	util.CheckErr(err, "create HTTP trigger")
 
 	fmt.Printf("trigger '%v' created\n", triggerName)
@@ -143,7 +181,39 @@ func htCreate(c *cli.Context) error {
 }
 
 func htGet(c *cli.Context) error {
-	return nil
+	cliClient := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	ns := c.String("fnNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	htTrigger, err := cliClient.HTTPTriggerGet(m)
+	util.CheckErr(err, "get http trigger")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "METHOD", "RELATIVE-URL", "FUNCTION-REFERENCE-TYPE", "FUNCTION(s)")
+
+	function := ""
+	if htTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionName {
+		function = htTrigger.Spec.FunctionReference.Name
+	} else {
+		for k, v := range htTrigger.Spec.FunctionReference.FunctionWeights {
+			function += fmt.Sprintf("%s:%v ", k, v)
+		}
+	}
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		htTrigger.Metadata.Name, htTrigger.Metadata.UID, htTrigger.Spec.Method, htTrigger.Spec.RelativeURL,
+		htTrigger.Spec.FunctionReference.Type, function)
+
+	w.Flush()
+
+	return err
 }
 
 func htUpdate(c *cli.Context) error {
@@ -161,9 +231,28 @@ func htUpdate(c *cli.Context) error {
 	util.CheckErr(err, "get HTTP trigger")
 
 	if c.IsSet("function") {
-		ht.Spec.FunctionReference.Name = c.String("function")
+		// get the functions and their weights if specified
+		functionList := c.StringSlice("function")
+		err := util.CheckFunctionExistence(client, functionList, triggerNamespace)
+		if err != nil {
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+		var functionWeightsList []int
+		if c.IsSet("weight") {
+			functionWeightsList = c.IntSlice("weight")
+		}
+
+		// set function reference
+		functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+
+		ht.Spec.FunctionReference = *functionRef
 	}
-	checkFunctionExistence(client, ht.Spec.FunctionReference.Name, triggerNamespace)
 
 	if c.IsSet("createingress") {
 		ht.Spec.CreateIngress = c.Bool("createingress")

fission/main.go

@@ -67,6 +67,7 @@ func main() {
 	pkgNamespaceFlag := cli.StringFlag{Name: "pkgNamespace, pkgns", Value: metav1.NamespaceDefault, Usage: "Namespace for package object"}
 	triggerNamespaceFlag := cli.StringFlag{Name: "triggerNamespace, triggerns", Value: metav1.NamespaceDefault, Usage: "Namespace for trigger object"}
 	recorderNamespaceFlag := cli.StringFlag{Name: "recorderNamespace, recorderns", Value: metav1.NamespaceDefault, Usage: "Namespace for recorder object"}
+	canaryNamespaceFlag := cli.StringFlag{Name: "canaryNamespace, canaryns", Value: metav1.NamespaceDefault, Usage: "Namespace for canary config object"}
 
 	// trigger method and url flags (used in function and route CLIs)
 	htMethodFlag := cli.StringFlag{Name: "method", Value: "GET", Usage: "HTTP Method: GET|POST|PUT|DELETE|HEAD"}
@@ -118,14 +119,16 @@ func main() {
 
 	// httptriggers
 	htNameFlag := cli.StringFlag{Name: "name", Usage: "HTTP Trigger name"}
-	htFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"}
 	htHostFlag := cli.StringFlag{Name: "host", Usage: "FQDN of the network host for route"}
 	htIngressFlag := cli.BoolFlag{Name: "createingress", Usage: "Creates ingress with same URL, defaults to false"}
+	htFnNameFlag := cli.StringSliceFlag{Name: "function", Usage: "Name(s) of the function for this trigger. If 2 functions are supplied with this flag, traffic gets routed to them based on weights supplied with --weight flag."}
+	htFnWeightFlag := cli.IntSliceFlag{Name: "weight", Usage: "Weight for each function supplied with --function flag, in the same order. Used for canary deployment"}
+
 	htSubcommands := []cli.Command{
 
-		{Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag, htFnNameFlag, htHostFlag, htIngressFlag, fnNamespaceFlag, specSaveFlag}, Action: htCreate},
-		{Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htMethodFlag, htUrlFlag}, Action: htGet},
-		{Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag, htFnNameFlag, htHostFlag, htIngressFlag}, Action: htUpdate},
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htNameFlag, htMethodFlag, htUrlFlag, htFnNameFlag, htHostFlag, htIngressFlag, fnNamespaceFlag, specSaveFlag, htFnWeightFlag}, Action: htCreate},
+		{Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htNameFlag}, Action: htGet},
+		{Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag, htFnNameFlag, htHostFlag, htIngressFlag, htFnWeightFlag}, Action: htUpdate},
 		{Name: "delete", Usage: "Delete HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag}, Action: htDelete},
 		{Name: "list", Usage: "List HTTP triggers", Flags: []cli.Flag{triggerNamespaceFlag}, Action: htList},
 	}
@@ -271,6 +274,22 @@ func main() {
 		{Name: "dump", Usage: "Collect & dump all necessary for troubleshooting", Flags: []cli.Flag{supportOutputFlag, supportNoZipFlag}, Action: support.DumpInfo},
 	}
 
+	// canary configs
+	canaryConfigNameFlag := cli.StringFlag{Name: "name", Usage: "Name for the canary config"}
+	triggerNameFlag := cli.StringFlag{Name: "httptrigger", Usage: "Http trigger that this config references"}
+	funcNFlag := cli.StringFlag{Name: "funcN", Usage: "New version of the function"}
+	funcNminus1Flag := cli.StringFlag{Name: "funcN-1", Usage: "Old stable version of the function"}
+	weightIncrementFlag := cli.IntFlag{Name: "increment-step", Value: 20, Usage: "Weight increment step for function"}
+	incrementIntervalFlag := cli.StringFlag{Name: "increment-interval", Value: "2m", Usage: "Weight increment interval, string representation of time.Duration, ex : 1m, 2h, 2d"}
+	failureThresholdFlag := cli.IntFlag{Name: "failure-threshold", Value: 10, Usage: "Threshold in percentage beyond which the new version of the function is considered unstable"}
+	canarySubCommands := []cli.Command{
+		{Name: "create", Usage: "Create a canary config", Flags: []cli.Flag{canaryConfigNameFlag, triggerNameFlag, funcNFlag, funcNminus1Flag, fnNamespaceFlag, weightIncrementFlag, incrementIntervalFlag, failureThresholdFlag}, Action: canaryConfigCreate},
+		{Name: "get", Usage: "View parameters in a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigGet},
+		{Name: "update", Usage: "Update parameters of a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag, incrementIntervalFlag, weightIncrementFlag, failureThresholdFlag}, Action: canaryConfigUpdate},
+		{Name: "delete", Usage: "Delete a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigDelete},
+		{Name: "list", Usage: "List all canary configs in a namespace", Flags: []cli.Flag{canaryNamespaceFlag}, Action: canaryConfigList},
+	}
+
 	app.Commands = []cli.Command{
 		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
 		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -286,6 +305,7 @@ func main() {
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "support", Usage: "Collect an archive of diagnostic information for support", Subcommands: supportSubCommands},
 		cmdPlugin,
+		{Name: "canary-config", Aliases: []string{}, Usage: "Create, Update and manage Canary Configs", Subcommands: canarySubCommands},
 	}
 	app.Before = cliHook
 	app.CommandNotFound = handleCommandNotFound

fission/util/util.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -140,3 +141,25 @@ func GetKubernetesClient(kubeConfig string) (*restclient.Config, *kubernetes.Cli
 
 	return config, clientset
 }
+
+// given a list of functions, this checks if the functions actually exist on the cluster
+func CheckFunctionExistence(fissionClient *client.Client, functions []string, fnNamespace string) (err error) {
+	fnMissing := make([]string, 0)
+	for _, fnName := range functions {
+		meta := &metav1.ObjectMeta{
+			Name:      fnName,
+			Namespace: fnNamespace,
+		}
+
+		_, err := fissionClient.FunctionGet(meta)
+		if err != nil {
+			fnMissing = append(fnMissing, fnName)
+		}
+	}
+
+	if len(fnMissing) > 0 {
+		return fmt.Errorf("function(s) %s, not present in namespace : %s", fnMissing, fnNamespace)
+	}
+
+	return nil
+}

pkg/apis/fission.io/v1/const.go

@@ -73,8 +73,25 @@ const (
 	// reference is simply by function name.
 	FunctionReferenceTypeFunctionName = "name"
 
+	FunctionReferenceTypeFunctionWeights = "function-weights"
+
 	// Other function reference types we'd like to support:
 	//   Versioned function, latest version
 	//   Versioned function. by semver "latest compatible"
 	//   Set of function references (recursively), by percentage of traffic
 )
+
+const (
+	// failure type currently supported is http status code. This could be extended
+	// in the future.
+	FailureTypeStatusCode FailureType = "status-code"
+
+	// Status of canary config can be one of the following
+	CanaryConfigStatusPending   = "pending"
+	CanaryConfigStatusSucceeded = "succeeded"
+	CanaryConfigStatusFailed    = "failed"
+	CanaryConfigStatusAborted   = "aborted"
+
+	// set a max number for iterations to prevent infinite processing of canary config
+	MaxIterationsForCanaryConfig = 10
+)

pkg/apis/fission.io/v1/typefields.go

@@ -185,6 +185,10 @@ type (
 
 		// Name of the function.
 		Name string `json:"name"`
+
+		// Function Reference by weight. this map contains function name as key and its weight
+		// as the value.
+		FunctionWeights map[string]int `json:"functionweights"`
 	}
 
 	//
@@ -328,4 +332,22 @@ type (
 		Cron              string `json:"cron"`
 		FunctionReference `json:"functionref"`
 	}
+
+	FailureType string
+
+	// Canary Config Spec
+	CanaryConfigSpec struct {
+		Trigger                 string      `json:"trigger"`
+		FunctionN               string      `json:"funcn"`
+		FunctionNminus1         string      `json:"funcn-1"`
+		WeightIncrement         int         `json:"weightincrement"`
+		WeightIncrementDuration string      `json:"duration"`
+		FailureThreshold        int         `json:"failurethreshold"`
+		FailureType             FailureType `json:"failureType"`
+	}
+
+	// CanaryConfig Status
+	CanaryConfigStatus struct {
+		Status string `json:"status"`
+	}
 )

pkg/apis/fission.io/v1/types.go

@@ -167,6 +167,22 @@ type (
 
 		Items []Recorder `json:"items"`
 	}
+
+	// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+	CanaryConfig struct {
+		metav1.TypeMeta `json:",inline"`
+		Metadata        metav1.ObjectMeta  `json:"metadata"`
+		Spec            CanaryConfigSpec   `json:"spec"`
+		Status          CanaryConfigStatus `json:"status"`
+	}
+
+	// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+	CanaryConfigList struct {
+		metav1.TypeMeta `json:",inline"`
+		Metadata        metav1.ListMeta `json:"metadata"`
+
+		Items []CanaryConfig `json:"items"`
+	}
 )
 
 // Each CRD type needs:
@@ -199,6 +215,9 @@ func (m *MessageQueueTrigger) GetObjectKind() schema.ObjectKind {
 func (p *Package) GetObjectKind() schema.ObjectKind {
 	return &p.TypeMeta
 }
+func (c *CanaryConfig) GetObjectKind() schema.ObjectKind {
+	return &c.TypeMeta
+}
 
 func (r *Recorder) GetObjectKind() schema.ObjectKind {
 	return &r.TypeMeta
@@ -225,6 +244,9 @@ func (m *MessageQueueTrigger) GetObjectMeta() metav1.Object {
 func (p *Package) GetObjectMeta() metav1.Object {
 	return &p.Metadata
 }
+func (c *CanaryConfig) GetObjectMeta() metav1.Object {
+	return &c.Metadata
+}
 
 func (r *Recorder) GetObjectMeta() metav1.Object {
 	return &r.Metadata
@@ -255,6 +277,10 @@ func (rl *RecorderList) GetObjectKind() schema.ObjectKind {
 	return &rl.TypeMeta
 }
 
+func (cl *CanaryConfigList) GetObjectKind() schema.ObjectKind {
+	return &cl.TypeMeta
+}
+
 func (fl *FunctionList) GetListMeta() metav1.ListInterface {
 	return &fl.Metadata
 }
@@ -276,10 +302,15 @@ func (ml *MessageQueueTriggerList) GetListMeta() metav1.ListInterface {
 func (pl *PackageList) GetListMeta() metav1.ListInterface {
 	return &pl.Metadata
 }
+
 func (rl *RecorderList) GetListMeta() metav1.ListInterface {
 	return &rl.Metadata
 }
 
+func (cl *CanaryConfigList) GetListMeta() metav1.ListInterface {
+	return &cl.Metadata
+}
+
 func validateMetadata(field string, m metav1.ObjectMeta) error {
 	return ValidateKubeReference(field, m.Name, m.Namespace)
 }

pkg/apis/fission.io/v1/validation.go

@@ -329,11 +329,14 @@ func (ref FunctionReference) Validate() error {
 
 	switch ref.Type {
 	case FunctionReferenceTypeFunctionName: // no op
+	case FunctionReferenceTypeFunctionWeights: // no op
 	default:
 		result = multierror.Append(result, MakeValidationErr(ErrorUnsupportedType, "FunctionReference.Type", ref.Type, "not a valid function reference type"))
 	}
 
-	result = multierror.Append(result, ValidateKubeName("FunctionReference.Name", ref.Name))
+	if ref.Type == FunctionReferenceTypeFunctionName {
+		result = multierror.Append(result, ValidateKubeName("FunctionReference.Name", ref.Name))
+	}
 
 	return result.ErrorOrNil()
 }

pkg/apis/fission.io/v1/zz_generated.deepcopy.go

@@ -21,7 +21,7 @@ limitations under the License.
 package v1
 
 import (
-	core_v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -52,12 +52,8 @@ func (in *Builder) DeepCopyInto(out *Builder) {
 	*out = *in
 	if in.Container != nil {
 		in, out := &in.Container, &out.Container
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(core_v1.Container)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(corev1.Container)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -72,6 +68,99 @@ func (in *Builder) DeepCopy() *Builder {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryConfig) DeepCopyInto(out *CanaryConfig) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.Metadata.DeepCopyInto(&out.Metadata)
+	out.Spec = in.Spec
+	out.Status = in.Status
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryConfig.
+func (in *CanaryConfig) DeepCopy() *CanaryConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CanaryConfig) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryConfigList) DeepCopyInto(out *CanaryConfigList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	out.Metadata = in.Metadata
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]CanaryConfig, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryConfigList.
+func (in *CanaryConfigList) DeepCopy() *CanaryConfigList {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryConfigList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CanaryConfigList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryConfigSpec) DeepCopyInto(out *CanaryConfigSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryConfigSpec.
+func (in *CanaryConfigSpec) DeepCopy() *CanaryConfigSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryConfigSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryConfigStatus) DeepCopyInto(out *CanaryConfigStatus) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryConfigStatus.
+func (in *CanaryConfigStatus) DeepCopy() *CanaryConfigStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryConfigStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Checksum) DeepCopyInto(out *Checksum) {
 	*out = *in
@@ -295,6 +384,13 @@ func (in *FunctionPackageRef) DeepCopy() *FunctionPackageRef {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FunctionReference) DeepCopyInto(out *FunctionReference) {
 	*out = *in
+	if in.FunctionWeights != nil {
+		in, out := &in.FunctionWeights, &out.FunctionWeights
+		*out = make(map[string]int, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 
@@ -343,7 +439,7 @@ func (in *HTTPTrigger) DeepCopyInto(out *HTTPTrigger) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.Metadata.DeepCopyInto(&out.Metadata)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	return
 }
 
@@ -401,7 +497,7 @@ func (in *HTTPTriggerList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HTTPTriggerSpec) DeepCopyInto(out *HTTPTriggerSpec) {
 	*out = *in
-	out.FunctionReference = in.FunctionReference
+	in.FunctionReference.DeepCopyInto(&out.FunctionReference)
 	return
 }
 
@@ -502,7 +598,7 @@ func (in *KubernetesWatchTriggerSpec) DeepCopyInto(out *KubernetesWatchTriggerSp
 			(*out)[key] = val
 		}
 	}
-	out.FunctionReference = in.FunctionReference
+	in.FunctionReference.DeepCopyInto(&out.FunctionReference)
 	return
 }
 
@@ -521,7 +617,7 @@ func (in *MessageQueueTrigger) DeepCopyInto(out *MessageQueueTrigger) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.Metadata.DeepCopyInto(&out.Metadata)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	return
 }
 
@@ -579,7 +675,7 @@ func (in *MessageQueueTriggerList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MessageQueueTriggerSpec) DeepCopyInto(out *MessageQueueTriggerSpec) {
 	*out = *in
-	out.FunctionReference = in.FunctionReference
+	in.FunctionReference.DeepCopyInto(&out.FunctionReference)
 	return
 }
 
@@ -791,12 +887,8 @@ func (in *Runtime) DeepCopyInto(out *Runtime) {
 	*out = *in
 	if in.Container != nil {
 		in, out := &in.Container, &out.Container
-		if *in == nil {
-			*out = nil
-		} else {
-			*out = new(core_v1.Container)
-			(*in).DeepCopyInto(*out)
-		}
+		*out = new(corev1.Container)
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -832,7 +924,7 @@ func (in *TimeTrigger) DeepCopyInto(out *TimeTrigger) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.Metadata.DeepCopyInto(&out.Metadata)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	return
 }
 
@@ -890,7 +982,7 @@ func (in *TimeTriggerList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TimeTriggerSpec) DeepCopyInto(out *TimeTriggerSpec) {
 	*out = *in
-	out.FunctionReference = in.FunctionReference
+	in.FunctionReference.DeepCopyInto(&out.FunctionReference)
 	return
 }
 

router/functionHandler.go

@@ -19,8 +19,10 @@ package router
 import (
 	"bytes"
 	"fmt"
+	"github.com/gorilla/mux"
+	"github.com/satori/go.uuid"
 	"io/ioutil"
-	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -28,9 +30,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gorilla/mux"
-	"github.com/satori/go.uuid"
-	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
@@ -47,14 +47,16 @@ type tsRoundTripperParams struct {
 }
 
 type functionHandler struct {
-	fmap                 *functionServiceMap
-	frmap                *functionRecorderMap
-	trmap                *triggerRecorderMap
-	executor             *executorClient.Client
-	function             *metav1.ObjectMeta
-	httpTrigger          *crd.HTTPTrigger
-	tsRoundTripperParams *tsRoundTripperParams
-	recorderName         string
+	fmap                     *functionServiceMap
+	frmap                    *functionRecorderMap
+	trmap                    *triggerRecorderMap
+	executor                 *executorClient.Client
+	function                 *metav1.ObjectMeta
+	httpTrigger              *crd.HTTPTrigger
+	functionMetadataMap      map[string]*metav1.ObjectMeta
+	fnWeightDistributionList []FunctionWeightDistribution
+	tsRoundTripperParams     *tsRoundTripperParams
+	recorderName             string
 }
 
 // A layer on top of http.DefaultTransport, with retries.
@@ -62,6 +64,11 @@ type RetryingRoundTripper struct {
 	funcHandler *functionHandler
 }
 
+func init() {
+	// just seeding the random number for getting the canary function
+	rand.Seed(time.Now().UnixNano())
+}
+
 // RoundTrip is a custom transport with retries for http requests that forwards the request to the right serviceUrl, obtained
 // from router's cache or from executor if router entry is stale.
 //
@@ -108,7 +115,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 
 			rdr1.Read(p)
 			postedBody = string(p)
-			logrus.Info(fmt.Sprintf("%v", postedBody))
+			log.Info(fmt.Sprintf("%v", postedBody))
 			req.Body = rdr2
 		}
 	}
@@ -136,6 +143,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 	serviceUrl, err = roundTripper.funcHandler.fmap.lookup(roundTripper.funcHandler.function)
 	if err != nil || serviceUrl == nil {
 		// cache miss or nil entry in cache
+		log.Printf("Setting needExecutor to true for function : %s", roundTripper.funcHandler.function.Name)
 		needExecutor = true
 	}
 
@@ -149,6 +157,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 			service, err := roundTripper.funcHandler.executor.GetServiceForFunction(
 				roundTripper.funcHandler.function)
 			if err != nil {
+				log.Printf("Err from GetServiceForFunction : %v", err)
 				// We might want a specific error code or header for fission failures as opposed to
 				// user function bugs.
 				return nil, err
@@ -161,6 +170,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 			}
 
 			// add the address in router's cache
+			log.Printf("assigning serviceUrl : %s for function : %s", service, roundTripper.funcHandler.function.Name)
 			roundTripper.funcHandler.fmap.assign(roundTripper.funcHandler.function, serviceUrl)
 
 			// flag denotes that service was not obtained from cache, instead, created just now by executor
@@ -271,7 +281,7 @@ func (fh *functionHandler) tapService(serviceUrl *url.URL) {
 	fh.executor.TapService(serviceUrl)
 }
 
-func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *http.Request) {
+func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *http.Request) {
 	// retrieve url params and add them to request header
 	vars := mux.Vars(request)
 	for k, v := range vars {
@@ -286,6 +296,18 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 		log.Print("Record request with ReqUID: ", reqUID)
 	}
 
+	if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights {
+		// canary deployment. need to determine the function to send request to now
+		fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList)
+		if fnMetadata == nil {
+			log.Printf("Error getting canary backend ")
+			// TODO : write error to responseWrite and return response
+			return
+		}
+		fh.function = fnMetadata
+		log.Debugf("chosen fnBackend's metadata : %+v", fh.function)
+	}
+
 	// system params
 	MetadataToHeaders(HEADERS_FISSION_FUNCTION_PREFIX, fh.function, request)
 
@@ -299,9 +321,44 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 	proxy := &httputil.ReverseProxy{
 		Director: director,
 		Transport: &RetryingRoundTripper{
-			funcHandler: fh,
+			funcHandler: &fh,
 		},
 	}
 
 	proxy.ServeHTTP(responseWriter, request)
 }
+
+// findCeil picks a function from the functionWeightDistribution list based on the
+// random number generated. It uses the prefix calculated for the function weights.
+func findCeil(randomNumber int, wtDistrList []FunctionWeightDistribution) string {
+	low := 0
+	high := len(wtDistrList) - 1
+
+	for {
+		if low >= high {
+			break
+		}
+
+		mid := low + high/2
+		if randomNumber >= wtDistrList[mid].sumPrefix {
+			low = mid + 1
+		} else {
+			high = mid
+		}
+	}
+
+	if wtDistrList[low].sumPrefix >= randomNumber {
+		return wtDistrList[low].name
+	} else {
+		return ""
+	}
+}
+
+// picks a function to route to based on a random number generated
+func getCanaryBackend(fnMetadatamap map[string]*metav1.ObjectMeta, fnWtDistributionList []FunctionWeightDistribution) *metav1.ObjectMeta {
+	randomNumber := rand.Intn(fnWtDistributionList[len(fnWtDistributionList)-1].sumPrefix + 1)
+
+	fnName := findCeil(randomNumber, fnWtDistributionList)
+
+	return fnMetadatamap[fnName]
+}

router/functionHandler_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/fission/fission"
+	"github.com/fission/fission/crd"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -55,6 +57,19 @@ func TestFunctionProxying(t *testing.T) {
 	fmap := makeFunctionServiceMap(0)
 	fmap.assign(fn, backendURL)
 
+	httpTrigger := &crd.HTTPTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:            "xxx",
+			Namespace:       metav1.NamespaceDefault,
+			ResourceVersion: "1234",
+		},
+		Spec: fission.HTTPTriggerSpec{
+			FunctionReference: fission.FunctionReference{
+				Type: fission.FunctionReferenceTypeFunctionName,
+			},
+		},
+	}
+
 	fh := &functionHandler{fmap: fmap,
 		function: fn,
 		tsRoundTripperParams: &tsRoundTripperParams{
@@ -63,6 +78,7 @@ func TestFunctionProxying(t *testing.T) {
 			keepAlive:       30 * time.Second,
 			maxRetries:      10,
 		},
+		httpTrigger: httpTrigger,
 	}
 	functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler))
 	fhURL := functionHandlerServer.URL

router/functionReferenceResolver.go

@@ -45,25 +45,33 @@ type (
 
 	resolveResultType int
 
-	// resolveResult is the result of resolving a function reference; for now
-	// it's just the metadata of one function, but in the future could support
+	FunctionWeightDistribution struct {
+		name      string
+		weight    int
+		sumPrefix int
+	}
+
+	// resolveResult is the result of resolving a function reference;
+	// it could be the metadata of one function or
 	// a distribution of requests across two functions.
 	resolveResult struct {
 		resolveResultType
-		functionMetadata *metav1.ObjectMeta
+		functionMetadataMap        map[string]*metav1.ObjectMeta
+		functionWtDistributionList []FunctionWeightDistribution
 	}
 
-	// namespacedFunctionReference is just a function reference plus a
-	// namespace. Since a function reference works on names, it's only
-	// meaningful within a namespace.
-	namespacedFunctionReference struct {
-		namespace         string
-		functionReference fission.FunctionReference
+	// namespacedTriggerReference is just a trigger reference plus a
+	// namespace.
+	namespacedTriggerReference struct {
+		namespace              string
+		triggerName            string
+		triggerResourceVersion string
 	}
 )
 
 const (
 	resolveResultSingleFunction = iota
+	resolveResultMultipleFunctions
 )
 
 func makeFunctionReferenceResolver(store k8sCache.Store) *functionReferenceResolver {
@@ -89,15 +97,12 @@ func makeK8SCache(crdClient *rest.RESTClient) (k8sCache.Store, k8sCache.Controll
 		k8sCache.ResourceEventHandlerFuncs{})
 }
 
-// resolve translates a namespace and a function reference to resolveResult.
-// The resolveResult for now is just a function's metadata. In the future, some
-// function ref types may resolve to two functions rather than just one
-// (e.g. for incremental deployment), which will make the resolveResult a bit
-// more complex.
-func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.FunctionReference) (*resolveResult, error) {
-	nfr := namespacedFunctionReference{
-		namespace:         namespace,
-		functionReference: *fr,
+// resolve translates a trigger's function reference to a resolveResult.
+func (frr *functionReferenceResolver) resolve(trigger crd.HTTPTrigger) (*resolveResult, error) {
+	nfr := namespacedTriggerReference{
+		namespace:              trigger.Metadata.Namespace,
+		triggerName:            trigger.Metadata.Name,
+		triggerResourceVersion: trigger.Metadata.ResourceVersion,
 	}
 
 	// check cache
@@ -110,14 +115,21 @@ func (frr *functionReferenceResolver) resolve(namespace string, fr *fission.Func
 	// resolve on cache miss
 	var rr *resolveResult
 
-	switch fr.Type {
+	switch trigger.Spec.FunctionReference.Type {
 	case fission.FunctionReferenceTypeFunctionName:
-		rr, err = frr.resolveByName(namespace, fr.Name)
+		rr, err = frr.resolveByName(nfr.namespace, trigger.Spec.FunctionReference.Name)
 		if err != nil {
 			return nil, err
 		}
+
+	case fission.FunctionReferenceTypeFunctionWeights:
+		rr, err = frr.resolveByFunctionWeights(nfr.namespace, &trigger.Spec.FunctionReference)
+		if err != nil {
+			return nil, err
+		}
+
 	default:
-		return nil, fmt.Errorf("Unrecognized function reference type %v", fr.Type)
+		return nil, fmt.Errorf("Unrecognized function reference type %v", trigger.Spec.FunctionReference.Type)
 	}
 
 	// cache resolve result
@@ -143,25 +155,71 @@ func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*re
 	}
 
 	f := obj.(*crd.Function)
+	functionMetadataMap := make(map[string]*metav1.ObjectMeta, 1)
+	functionMetadataMap[f.Metadata.Name] = &f.Metadata
+
 	rr := resolveResult{
-		resolveResultType: resolveResultSingleFunction,
-		functionMetadata:  &f.Metadata,
+		resolveResultType:   resolveResultSingleFunction,
+		functionMetadataMap: functionMetadataMap,
 	}
+
 	return &rr, nil
 }
 
-func (frr *functionReferenceResolver) delete(namespace string, fr *fission.FunctionReference) error {
-	nfr := namespacedFunctionReference{
-		namespace:         namespace,
-		functionReference: *fr,
+func (frr *functionReferenceResolver) resolveByFunctionWeights(namespace string, fr *fission.FunctionReference) (*resolveResult, error) {
+
+	functionMetadataMap := make(map[string]*metav1.ObjectMeta, 0)
+	fnWtDistrList := make([]FunctionWeightDistribution, 0)
+	sumPrefix := 0
+
+	for functionName, functionWeight := range fr.FunctionWeights {
+		// get function from cache
+		obj, isExist, err := frr.store.Get(&crd.Function{
+			Metadata: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      functionName,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		if !isExist {
+			return nil, fmt.Errorf("function %v does not exist", functionName)
+		}
+
+		f := obj.(*crd.Function)
+		functionMetadataMap[f.Metadata.Name] = &f.Metadata
+		sumPrefix = sumPrefix + functionWeight
+		fnWtDistrList = append(fnWtDistrList, FunctionWeightDistribution{
+			name:      functionName,
+			weight:    functionWeight,
+			sumPrefix: sumPrefix,
+		})
+
+	}
+
+	rr := resolveResult{
+		resolveResultType:          resolveResultMultipleFunctions,
+		functionMetadataMap:        functionMetadataMap,
+		functionWtDistributionList: fnWtDistrList,
+	}
+
+	return &rr, nil
+}
+
+func (frr *functionReferenceResolver) delete(namespace string, triggerName, triggerRV string) error {
+	nfr := namespacedTriggerReference{
+		namespace:              namespace,
+		triggerName:            triggerName,
+		triggerResourceVersion: triggerRV,
 	}
 	return frr.refCache.Delete(nfr)
 }
 
-func (frr *functionReferenceResolver) copy() map[namespacedFunctionReference]resolveResult {
-	cache := make(map[namespacedFunctionReference]resolveResult)
+func (frr *functionReferenceResolver) copy() map[namespacedTriggerReference]resolveResult {
+	cache := make(map[namespacedTriggerReference]resolveResult)
 	for k, v := range frr.refCache.Copy() {
-		key := k.(namespacedFunctionReference)
+		key := k.(namespacedTriggerReference)
 		val := v.(resolveResult)
 		cache[key] = val
 	}

router/httpTriggers.go

@@ -119,7 +119,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 		trigger := ts.triggers[i]
 
 		// resolve function reference
-		rr, err := ts.resolver.resolve(trigger.Metadata.Namespace, &trigger.Spec.FunctionReference)
+		rr, err := ts.resolver.resolve(trigger)
 		if err != nil {
 			// Unresolvable function reference. Report the error via
 			// the trigger's status.
@@ -135,22 +135,27 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 			recorderName = recorder.Spec.Name
 		}
 
-		//log.Printf("The trigger %v should be recorded: %v", trigger.Metadata.Name, doRecord)
-
-		if rr.resolveResultType != resolveResultSingleFunction {
+		if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions {
 			// not implemented yet
 			log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType)
 		}
 
 		fh := &functionHandler{
-			fmap:                 ts.functionServiceMap,
-			frmap:                ts.recorderSet.functionRecorderMap,
-			trmap:                ts.recorderSet.triggerRecorderMap,
-			function:             rr.functionMetadata,
-			executor:             ts.executor,
-			httpTrigger:          &trigger,
-			tsRoundTripperParams: ts.tsRoundTripperParams,
-			recorderName:         recorderName,
+			fmap:                     ts.functionServiceMap,
+			frmap:                    ts.recorderSet.functionRecorderMap,
+			trmap:                    ts.recorderSet.triggerRecorderMap,
+			executor:                 ts.executor,
+			httpTrigger:              &trigger,
+			functionMetadataMap:      rr.functionMetadataMap,
+			fnWeightDistributionList: rr.functionWtDistributionList,
+			tsRoundTripperParams:     ts.tsRoundTripperParams,
+			recorderName:             recorderName,
+		}
+
+		if rr.resolveResultType == resolveResultSingleFunction {
+			for _, metadata := range fh.functionMetadataMap {
+				fh.function = metadata
+			}
 		}
 
 		ht := muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
@@ -270,12 +275,16 @@ func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Con
 
 				// update resolver function reference cache
 				for key, rr := range ts.resolver.copy() {
-					if key.functionReference.Name == fn.Metadata.Name &&
-						rr.functionMetadata.ResourceVersion != fn.Metadata.ResourceVersion {
-						err := ts.resolver.delete(key.namespace, &key.functionReference)
+					if key.namespace == fn.Metadata.Namespace &&
+						rr.functionMetadataMap[fn.Metadata.Name] != nil &&
+						rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion {
+						// invalidate resolver cache
+						log.Printf("Invalidating resolver cache")
+						err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
 						if err != nil {
 							log.Printf("Error deleting functionReferenceResolver cache: %v", err)
 						}
+
 						break
 					}
 				}

router/router.go

@@ -124,13 +124,12 @@ func Start(port int, executorUrl string) {
 		log.Fatalf("Failed to parse max retry times: %v", err)
 	}
 
-	triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient,
-		&tsRoundTripperParams{
-			timeout:         timeout,
-			timeoutExponent: timeoutExponent,
-			keepAlive:       keepAlive,
-			maxRetries:      maxRetries,
-		})
+	triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
+		timeout:         timeout,
+		timeoutExponent: timeoutExponent,
+		keepAlive:       keepAlive,
+		maxRetries:      maxRetries,
+	})
 
 	resolver := makeFunctionReferenceResolver(fnStore)
 

router/router_test.go

@@ -46,18 +46,6 @@ func TestRouter(t *testing.T) {
 	fmap := makeFunctionServiceMap(0)
 	fmap.assign(fn, testServiceUrl)
 
-	// set up the resolver's cache for this function
-	frr := makeFunctionReferenceResolver(nil)
-	nfr := namespacedFunctionReference{
-		namespace:         metav1.NamespaceDefault,
-		functionReference: fr,
-	}
-	rr := resolveResult{
-		resolveResultType: resolveResultSingleFunction,
-		functionMetadata:  fn,
-	}
-	frr.refCache.Set(nfr, rr)
-
 	frmap := makeFunctionRecorderMap(time.Minute)
 
 	trmap := makeTriggerRecorderMap(time.Minute)
@@ -74,8 +62,9 @@ func TestRouter(t *testing.T) {
 	triggers.triggers = append(triggers.triggers,
 		crd.HTTPTrigger{
 			Metadata: metav1.ObjectMeta{
-				Name:      "xxx",
-				Namespace: metav1.NamespaceDefault,
+				Name:            "xxx",
+				Namespace:       metav1.NamespaceDefault,
+				ResourceVersion: "1234",
 			},
 			Spec: fission.HTTPTriggerSpec{
 				RelativeURL:       triggerUrl,
@@ -84,6 +73,23 @@ func TestRouter(t *testing.T) {
 			},
 		})
 
+	// set up the resolver's cache for this function
+	frr := makeFunctionReferenceResolver(nil)
+	nfr := namespacedTriggerReference{
+		namespace:              metav1.NamespaceDefault,
+		triggerName:            "xxx",
+		triggerResourceVersion: "1234",
+	}
+
+	fnMetaMap := make(map[string]*metav1.ObjectMeta, 1)
+	fnMetaMap[fn.Name] = fn
+
+	rr := resolveResult{
+		resolveResultType:   resolveResultSingleFunction,
+		functionMetadataMap: fnMetaMap,
+	}
+	frr.refCache.Set(nfr, rr)
+
 	// run the router
 	port := 4242
 	ctx, cancel := context.WithCancel(context.Background())

test/charts/prometheus/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

test/charts/prometheus/Chart.yaml

@@ -0,0 +1,19 @@
+appVersion: 2.3.2
+description: Prometheus is a monitoring system and time series database.
+engine: gotpl
+home: https://prometheus.io/
+icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png
+maintainers:
+- email: mgoodness@gmail.com
+  name: mgoodness
+- email: gianrubio@gmail.com
+  name: gianrubio
+name: prometheus
+sources:
+- https://github.com/prometheus/alertmanager
+- https://github.com/prometheus/prometheus
+- https://github.com/prometheus/pushgateway
+- https://github.com/prometheus/node_exporter
+- https://github.com/kubernetes/kube-state-metrics
+tillerVersion: '>=2.8.0'
+version: 7.0.3

test/charts/prometheus/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+- mgoodness
+- gianrubio
+reviewers:
+- mgoodness
+- gianrubio

test/charts/prometheus/README.md

@@ -0,0 +1,347 @@
+# Prometheus
+
+[Prometheus](https://prometheus.io/), a [Cloud Native Computing Foundation](https://cncf.io/) project, is a systems and service monitoring system. It collects metrics from configured targets at given intervals, evaluates rule expressions, displays the results, and can trigger alerts if some condition is observed to be true.
+
+## TL;DR;
+
+```console
+$ helm install stable/prometheus
+```
+
+## Introduction
+
+This chart bootstraps a [Prometheus](https://prometheus.io/) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
+
+## Prerequisites
+
+- Kubernetes 1.3+ with Beta APIs enabled
+
+## Installing the Chart
+
+To install the chart with the release name `my-release`:
+
+```console
+$ helm install --name my-release stable/prometheus
+```
+
+The command deploys Prometheus on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+
+> **Tip**: List all releases using `helm list`
+
+## Uninstalling the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```console
+$ helm delete my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Prometheus 2.x
+
+Prometheus version 2.x has made changes to alertmanager, storage and recording rules. Check out the migration guide [here](https://prometheus.io/docs/prometheus/2.0/migration/)
+
+Users of this chart will need to update their alerting rules to the new format before they can upgrade.
+
+## Upgrading from previous chart versions.
+
+As of version 5.0, this chart uses Prometheus 2.1. This version of prometheus introduces a new data format and is not compatible with prometheus 1.x. It is recommended to install this as a new release, as updating existing releases will not work. See the [prometheus docs](https://prometheus.io/docs/prometheus/latest/migration/#storage) for instructions on retaining your old data.
+
+### Example migration
+
+Assuming you have an existing release of the prometheus chart, named `prometheus-old`. In order to update to prometheus 2.1 while keeping your old data do the following:
+
+1. Update the `prometheus-old` release. Disable scraping on every component besides the prometheus server, similar to the configuration below:
+
+	```
+	alertmanager:
+	  enabled: false
+	alertmanagerFiles:
+	  alertmanager.yml: ""
+	kubeStateMetrics:
+	  enabled: false
+	nodeExporter:
+	  enabled: false
+	pushgateway:
+	  enabled: false
+	server:
+	  extraArgs:
+	    storage.local.retention: 720h
+	serverFiles:
+	  alerts: ""
+	  prometheus.yml: ""
+	  rules: ""
+	```
+
+1. Deploy a new release of the chart with version 5.0+ using prometheus 2.x. In the values.yaml set the scrape config as usual, and also add the `prometheus-old` instance as a remote-read target.
+
+   ```
+	  prometheus.yml:
+	    ...
+	    remote_read:
+	    - url: http://prometheus-old/api/v1/read
+	    ...
+   ```
+
+   Old data will be available when you query the new prometheus instance.
+
+## Configuration
+
+The following table lists the configurable parameters of the Prometheus chart and their default values.
+
+Parameter | Description | Default
+--------- | ----------- | -------
+`alertmanager.enabled` | If true, create alertmanager | `true`
+`alertmanager.name` | alertmanager container name | `alertmanager`
+`alertmanager.image.repository` | alertmanager container image repository | `prom/alertmanager`
+`alertmanager.image.tag` | alertmanager container image tag | `v0.15.2`
+`alertmanager.image.pullPolicy` | alertmanager container image pull policy | `IfNotPresent`
+`alertmanager.prefixURL` | The prefix slug at which the server can be accessed | ``
+`alertmanager.baseURL` | The external url at which the server can be accessed | `/`
+`alertmanager.extraArgs` | Additional alertmanager container arguments | `{}`
+`alertmanager.configMapOverrideName` | Prometheus alertmanager ConfigMap override where full-name is `{{.Release.Name}}-{{.Values.alertmanager.configMapOverrideName}}` and setting this value will prevent the default alertmanager ConfigMap from being generated | `""`
+`alertmanager.ingress.enabled` | If true, alertmanager Ingress will be created | `false`
+`alertmanager.ingress.annotations` | alertmanager Ingress annotations | `{}`
+`alertmanager.ingress.extraLabels` | alertmanager Ingress additional labels | `{}`
+`alertmanager.ingress.hosts` | alertmanager Ingress hostnames | `[]`
+`alertmanager.ingress.tls` | alertmanager Ingress TLS configuration (YAML) | `[]`
+`alertmanager.nodeSelector` | node labels for alertmanager pod assignment | `{}`
+`alertmanager.tolerations` | node taints to tolerate (requires Kubernetes >=1.6) | `[]`
+`alertmanager.affinity` | pod affinity | `{}`
+`alertmanager.schedulerName` | alertmanager alternate scheduler name | `nil`
+`alertmanager.persistentVolume.enabled` | If true, alertmanager will create a Persistent Volume Claim | `true`
+`alertmanager.persistentVolume.accessModes` | alertmanager data Persistent Volume access modes | `[ReadWriteOnce]`
+`alertmanager.persistentVolume.annotations` | Annotations for alertmanager Persistent Volume Claim | `{}`
+`alertmanager.persistentVolume.existingClaim` | alertmanager data Persistent Volume existing claim name | `""`
+`alertmanager.persistentVolume.mountPath` | alertmanager data Persistent Volume mount root path | `/data`
+`alertmanager.persistentVolume.size` | alertmanager data Persistent Volume size | `2Gi`
+`alertmanager.persistentVolume.storageClass` | alertmanager data Persistent Volume Storage Class | `unset`
+`alertmanager.persistentVolume.subPath` | Subdirectory of alertmanager data Persistent Volume to mount | `""`
+`alertmanager.podAnnotations` | annotations to be added to alertmanager pods | `{}`
+`alertmanager.replicaCount` | desired number of alertmanager pods | `1`
+`alertmanager.priorityClassName` | alertmanager priorityClassName | `nil`
+`alertmanager.resources` | alertmanager pod resource requests & limits | `{}`
+`alertmanager.securityContext` | Custom [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for Alert Manager containers | `{}`
+`alertmanager.service.annotations` | annotations for alertmanager service | `{}`
+`alertmanager.service.clusterIP` | internal alertmanager cluster service IP | `""`
+`alertmanager.service.externalIPs` | alertmanager service external IP addresses | `[]`
+`alertmanager.service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""`
+`alertmanager.service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to load balancer (if supported) | `[]`
+`alertmanager.service.servicePort` | alertmanager service port | `80`
+`alertmanager.service.type` | type of alertmanager service to create | `ClusterIP`
+`alertmanagerFiles.alertmanager.yml` | Prometheus alertmanager configuration | example configuration
+`configmapReload.name` | configmap-reload container name | `configmap-reload`
+`configmapReload.image.repository` | configmap-reload container image repository | `jimmidyson/configmap-reload`
+`configmapReload.image.tag` | configmap-reload container image tag | `v0.2.2`
+`configmapReload.image.pullPolicy` | configmap-reload container image pull policy | `IfNotPresent`
+`configmapReload.extraArgs` | Additional configmap-reload container arguments | `{}`
+`configmapReload.extraConfigmapMounts` | Additional configmap-reload configMap mounts | `[]`
+`configmapReload.resources` | configmap-reload pod resource requests & limits | `{}`
+`initChownData.enabled`  | If false, don't reset data ownership at startup | true
+`initChownData.name` | init-chown-data container name | `init-chown-data`
+`initChownData.image.repository` | init-chown-data container image repository | `busybox`
+`initChownData.image.tag` | init-chown-data container image tag | `latest`
+`initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent`
+`initChownData.resources` | init-chown-data pod resource requests & limits | `{}`
+`kubeStateMetrics.enabled` | If true, create kube-state-metrics | `true`
+`kubeStateMetrics.name` | kube-state-metrics container name | `kube-state-metrics`
+`kubeStateMetrics.image.repository` | kube-state-metrics container image repository| `quay.io/coreos/kube-state-metrics`
+`kubeStateMetrics.image.tag` | kube-state-metrics container image tag | `v1.4.0`
+`kubeStateMetrics.image.pullPolicy` | kube-state-metrics container image pull policy | `IfNotPresent`
+`kubeStateMetrics.args` | kube-state-metrics container arguments | `{}`
+`kubeStateMetrics.nodeSelector` | node labels for kube-state-metrics pod assignment | `{}`
+`kubeStateMetrics.podAnnotations` | annotations to be added to kube-state-metrics pods | `{}`
+`kubeStateMetrics.deploymentAnnotations` | annotations to be added to kube-state-metrics deployment | `{}`
+`kubeStateMetrics.tolerations` | node taints to tolerate (requires Kubernetes >=1.6) | `[]`
+`kubeStateMetrics.replicaCount` | desired number of kube-state-metrics pods | `1`
+`kubeStateMetrics.priorityClassName` | kube-state-metrics priorityClassName | `nil`
+`kubeStateMetrics.resources` | kube-state-metrics resource requests and limits (YAML) | `{}`
+`kubeStateMetrics.securityContext` | Custom [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for kube-state-metrics containers | `{}`
+`kubeStateMetrics.service.annotations` | annotations for kube-state-metrics service | `{prometheus.io/scrape: "true"}`
+`kubeStateMetrics.service.clusterIP` | internal kube-state-metrics cluster service IP | `None`
+`kubeStateMetrics.service.externalIPs` | kube-state-metrics service external IP addresses | `[]`
+`kubeStateMetrics.service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""`
+`kubeStateMetrics.service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to load balancer (if supported) | `[]`
+`kubeStateMetrics.service.servicePort` | kube-state-metrics service port | `80`
+`kubeStateMetrics.service.type` | type of kube-state-metrics service to create | `ClusterIP`
+`nodeExporter.enabled` | If true, create node-exporter | `true`
+`nodeExporter.name` | node-exporter container name | `node-exporter`
+`nodeExporter.image.repository` | node-exporter container image repository| `prom/node-exporter`
+`nodeExporter.image.tag` | node-exporter container image tag | `v0.16.0`
+`nodeExporter.image.pullPolicy` | node-exporter container image pull policy | `IfNotPresent`
+`nodeExporter.extraArgs` | Additional node-exporter container arguments | `{}`
+`nodeExporter.extraHostPathMounts` | Additional node-exporter hostPath mounts | `[]`
+`nodeExporter.extraConfigmapMounts` | Additional node-exporter configMap mounts | `[]`
+`nodeExporter.nodeSelector` | node labels for node-exporter pod assignment | `{}`
+`nodeExporter.podAnnotations` | annotations to be added to node-exporter pods | `{}`
+`nodeExporter.pod.labels` | labels to be added to node-exporter pods | `{}`
+`nodeExporter.tolerations` | node taints to tolerate (requires Kubernetes >=1.6) | `[]`
+`nodeExporter.priorityClassName` | node-exporter priorityClassName | `nil`
+`nodeExporter.resources` | node-exporter resource requests and limits (YAML) | `{}`
+`nodeExporter.securityContext` | securityContext for containers in pod | `{}`
+`nodeExporter.service.annotations` | annotations for node-exporter service | `{prometheus.io/scrape: "true"}`
+`nodeExporter.service.clusterIP` | internal node-exporter cluster service IP | `None`
+`nodeExporter.service.externalIPs` | node-exporter service external IP addresses | `[]`
+`nodeExporter.service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""`
+`nodeExporter.service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to load balancer (if supported) | `[]`
+`nodeExporter.service.servicePort` | node-exporter service port | `9100`
+`nodeExporter.service.type` | type of node-exporter service to create | `ClusterIP`
+`pushgateway.enabled` | If true, create pushgateway | `true`
+`pushgateway.name` | pushgateway container name | `pushgateway`
+`pushgateway.image.repository` | pushgateway container image repository | `prom/pushgateway`
+`pushgateway.image.tag` | pushgateway container image tag | `v0.5.2`
+`pushgateway.image.pullPolicy` | pushgateway container image pull policy | `IfNotPresent`
+`pushgateway.extraArgs` | Additional pushgateway container arguments | `{}`
+`pushgateway.ingress.enabled` | If true, pushgateway Ingress will be created | `false`
+`pushgateway.ingress.annotations` | pushgateway Ingress annotations | `{}`
+`pushgateway.ingress.hosts` | pushgateway Ingress hostnames | `[]`
+`pushgateway.ingress.tls` | pushgateway Ingress TLS configuration (YAML) | `[]`
+`pushgateway.nodeSelector` | node labels for pushgateway pod assignment | `{}`
+`pushgateway.podAnnotations` | annotations to be added to pushgateway pods | `{}`
+`pushgateway.tolerations` | node taints to tolerate (requires Kubernetes >=1.6) | `[]`
+`pushgateway.replicaCount` | desired number of pushgateway pods | `1`
+`pushgateway.priorityClassName` | pushgateway priorityClassName | `nil`
+`pushgateway.resources` | pushgateway pod resource requests & limits | `{}`
+`pushgateway.service.annotations` | annotations for pushgateway service | `{}`
+`pushgateway.service.clusterIP` | internal pushgateway cluster service IP | `""`
+`pushgateway.service.externalIPs` | pushgateway service external IP addresses | `[]`
+`pushgateway.service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""`
+`pushgateway.service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to load balancer (if supported) | `[]`
+`pushgateway.service.servicePort` | pushgateway service port | `9091`
+`pushgateway.service.type` | type of pushgateway service to create | `ClusterIP`
+`rbac.create` | If true, create & use RBAC resources | `true`
+`server.name` | Prometheus server container name | `server`
+`server.image.repository` | Prometheus server container image repository | `prom/prometheus`
+`server.image.tag` | Prometheus server container image tag | `v2.3.2`
+`server.image.pullPolicy` | Prometheus server container image pull policy | `IfNotPresent`
+`server.enableAdminApi` |  If true, Prometheus administrative HTTP API will be enabled. Please note, that you should take care of administrative API access protection (ingress or some frontend Nginx with auth) before enabling it. | `false`
+`server.global.scrape_interval` | How frequently to scrape targets by default | `1m`
+`server.global.scrape_timeout` | How long until a scrape request times out | `10s`
+`server.global.evaluation_interval` | How frequently to evaluate rules | `1m`
+`server.extraArgs` | Additional Prometheus server container arguments | `{}`
+`server.prefixURL` | The prefix slug at which the server can be accessed | ``
+`server.baseURL` | The external url at which the server can be accessed | ``
+`server.extraHostPathMounts` | Additional Prometheus server hostPath mounts | `[]`
+`server.extraConfigmapMounts` | Additional Prometheus server configMap mounts | `[]`
+`server.extraSecretMounts` | Additional Prometheus server Secret mounts | `[]`
+`server.configMapOverrideName` | Prometheus server ConfigMap override where full-name is `{{.Release.Name}}-{{.Values.server.configMapOverrideName}}` and setting this value will prevent the default server ConfigMap from being generated | `""`
+`server.ingress.enabled` | If true, Prometheus server Ingress will be created | `false`
+`server.ingress.annotations` | Prometheus server Ingress annotations | `[]`
+`server.ingress.extraLabels` | Prometheus server Ingress additional labels | `{}`
+`server.ingress.hosts` | Prometheus server Ingress hostnames | `[]`
+`server.ingress.tls` | Prometheus server Ingress TLS configuration (YAML) | `[]`
+`server.nodeSelector` | node labels for Prometheus server pod assignment | `{}`
+`server.tolerations` | node taints to tolerate (requires Kubernetes >=1.6) | `[]`
+`server.affinity` | pod affinity | `{}`
+`server.priorityClassName` | Prometheus server priorityClassName | `nil`
+`server.schedulerName` | Prometheus server alternate scheduler name | `nil`
+`server.persistentVolume.enabled` | If true, Prometheus server will create a Persistent Volume Claim | `true`
+`server.persistentVolume.accessModes` | Prometheus server data Persistent Volume access modes | `[ReadWriteOnce]`
+`server.persistentVolume.annotations` | Prometheus server data Persistent Volume annotations | `{}`
+`server.persistentVolume.existingClaim` | Prometheus server data Persistent Volume existing claim name | `""`
+`server.persistentVolume.mountPath` | Prometheus server data Persistent Volume mount root path | `/data`
+`server.persistentVolume.size` | Prometheus server data Persistent Volume size | `8Gi`
+`server.persistentVolume.storageClass` | Prometheus server data Persistent Volume Storage Class |  `unset`
+`server.persistentVolume.subPath` | Subdirectory of Prometheus server data Persistent Volume to mount | `""`
+`server.podAnnotations` | annotations to be added to Prometheus server pods | `{}`
+`server.deploymentAnnotations` | annotations to be added to Prometheus server deployment | `{}'
+`server.replicaCount` | desired number of Prometheus server pods | `1`
+`server.resources` | Prometheus server resource requests and limits | `{}`
+`server.securityContext` | Custom [security context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for server containers | `{}`
+`server.service.annotations` | annotations for Prometheus server service | `{}`
+`server.service.clusterIP` | internal Prometheus server cluster service IP | `""`
+`server.service.externalIPs` | Prometheus server service external IP addresses | `[]`
+`server.service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `""`
+`server.service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to load balancer (if supported) | `[]`
+`server.service.nodePort` | Port to be used as the service NodePort (ignored if `server.service.type` is not `NodePort`) | `0`
+`server.service.servicePort` | Prometheus server service port | `80`
+`server.service.type` | type of Prometheus server service to create | `ClusterIP`
+`serviceAccounts.alertmanager.create` | If true, create the alertmanager service account | `true`
+`serviceAccounts.alertmanager.name` | name of the alertmanager service account to use or create | `{{ prometheus.alertmanager.fullname }}`
+`serviceAccounts.kubeStateMetrics.create` | If true, create the kubeStateMetrics service account | `true`
+`serviceAccounts.kubeStateMetrics.name` | name of the kubeStateMetrics service account to use or create | `{{ prometheus.kubeStateMetrics.fullname }}`
+`serviceAccounts.nodeExporter.create` | If true, create the nodeExporter service account | `true`
+`serviceAccounts.nodeExporter.name` | name of the nodeExporter service account to use or create | `{{ prometheus.nodeExporter.fullname }}`
+`serviceAccounts.pushgateway.create` | If true, create the pushgateway service account | `true`
+`serviceAccounts.pushgateway.name` | name of the pushgateway service account to use or create | `{{ prometheus.pushgateway.fullname }}`
+`serviceAccounts.server.create` | If true, create the server service account | `true`
+`serviceAccounts.server.name` | name of the server service account to use or create | `{{ prometheus.server.fullname }}`
+`server.terminationGracePeriodSeconds` | Prometheus server Pod termination grace period | `300`
+`server.retention` | (optional) Prometheus data retention | `""`
+`serverFiles.alerts` | Prometheus server alerts configuration | `{}`
+`serverFiles.rules` | Prometheus server rules configuration | `{}`
+`serverFiles.prometheus.yml` | Prometheus server scrape configuration | example configuration
+`networkPolicy.enabled` | Enable NetworkPolicy | `false` |
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
+
+```console
+$ helm install stable/prometheus --name my-release \
+    --set server.terminationGracePeriodSeconds=360
+```
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+```console
+$ helm install stable/prometheus --name my-release -f values.yaml
+```
+
+> **Tip**: You can use the default [values.yaml](values.yaml)
+
+### RBAC Configuration
+Roles and RoleBindings resources will be created automatically for `server` and `kubeStateMetrics` services.
+
+To manually setup RBAC you need to set the parameter `rbac.create=false` and specify the service account to be used for each service by setting the parameters: `serviceAccounts.{{ component }}.create` to `false` and `serviceAccounts.{{ component }}.name` to the name of a pre-existing service account.
+
+> **Tip**: You can refer to the default `*-clusterrole.yaml` and `*-clusterrolebinding.yaml` files in [templates](templates/) to customize your own.
+
+### ConfigMap Files
+AlertManager is configured through [alertmanager.yml](https://prometheus.io/docs/alerting/configuration/). This file (and any others listed in `alertmanagerFiles`) will be mounted into the `alertmanager` pod.
+
+Prometheus is configured through [prometheus.yml](https://prometheus.io/docs/operating/configuration/). This file (and any others listed in `serverFiles`) will be mounted into the `server` pod.
+
+### Ingress TLS
+If your cluster allows automatic creation/retrieval of TLS certificates (e.g. [kube-lego](https://github.com/jetstack/kube-lego)), please refer to the documentation for that mechanism.
+
+To manually configure TLS, first create/retrieve a key & certificate pair for the address(es) you wish to protect. Then create a TLS secret in the namespace:
+
+```console
+kubectl create secret tls prometheus-server-tls --cert=path/to/tls.cert --key=path/to/tls.key
+```
+
+Include the secret's name, along with the desired hostnames, in the alertmanager/server Ingress TLS section of your custom `values.yaml` file:
+
+```yaml
+server:
+  ingress:
+    ## If true, Prometheus server Ingress will be created
+    ##
+    enabled: true
+
+    ## Prometheus server Ingress hostnames
+    ## Must be provided if Ingress is enabled
+    ##
+    hosts:
+      - prometheus.domain.com
+
+    ## Prometheus server Ingress TLS configuration
+    ## Secrets must be manually created in the namespace
+    ##
+    tls:
+      - secretName: prometheus-server-tls
+        hosts:
+          - prometheus.domain.com
+```
+
+### NetworkPolicy
+
+Enabling Network Policy for Prometheus will secure connections to Alert Manager
+and Kube State Metrics by only accepting connections from Prometheus Server.
+All inbound connections to Prometheus Server are still allowed.
+
+To enable network policy for Prometheus, install a networking plugin that
+implements the Kubernetes NetworkPolicy spec, and set `networkPolicy.enabled` to true.
+
+If NetworkPolicy is enabled for Prometheus' scrape targets, you may also need
+to manually create a networkpolicy which allows it.

test/charts/prometheus/templates/NOTES.txt

@@ -0,0 +1,100 @@
+The Prometheus server can be accessed via port {{ .Values.server.service.servicePort }} on the following DNS name from within your cluster:
+{{ template "prometheus.server.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+
+{{ if .Values.server.ingress.enabled -}}
+From outside the cluster, the server URL(s) are:
+{{- range .Values.server.ingress.hosts }}
+http://{{ . }}
+{{- end }}
+{{- else }}
+Get the Prometheus server URL by running these commands in the same shell:
+{{- if contains "NodePort" .Values.server.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus.server.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.server.service.type }}
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "prometheus.server.fullname" . }}'
+
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "prometheus.server.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.server.service.servicePort }}
+{{- else if contains "ClusterIP"  .Values.server.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "prometheus.name" . }},component={{ .Values.server.name }}" -o jsonpath="{.items[0].metadata.name}")
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 9090
+{{- end }}
+{{- end }}
+
+{{- if .Values.server.persistentVolume.enabled }}
+{{- else }}
+#################################################################################
+######   WARNING: Persistence is disabled!!! You will lose your data when   #####
+######            the Server pod is terminated.                             #####
+#################################################################################
+{{- end }}
+
+{{ if .Values.alertmanager.enabled }}
+The Prometheus alertmanager can be accessed via port {{ .Values.alertmanager.service.servicePort }} on the following DNS name from within your cluster:
+{{ template "prometheus.alertmanager.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+
+{{ if .Values.alertmanager.ingress.enabled -}}
+From outside the cluster, the alertmanager URL(s) are:
+{{- range .Values.alertmanager.ingress.hosts }}
+http://{{ . }}
+{{- end }}
+{{- else }}
+Get the Alertmanager URL by running these commands in the same shell:
+{{- if contains "NodePort" .Values.alertmanager.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus.alertmanager.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.alertmanager.service.type }}
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "prometheus.alertmanager.fullname" . }}'
+
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "prometheus.alertmanager.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.alertmanager.service.servicePort }}
+{{- else if contains "ClusterIP"  .Values.alertmanager.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "prometheus.name" . }},component={{ .Values.alertmanager.name }}" -o jsonpath="{.items[0].metadata.name}")
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 9093
+{{- end }}
+{{- end }}
+
+{{- if .Values.alertmanager.persistentVolume.enabled }}
+{{- else }}
+#################################################################################
+######   WARNING: Persistence is disabled!!! You will lose your data when   #####
+######            the AlertManager pod is terminated.                       #####
+#################################################################################
+{{- end }}
+{{- end }}
+
+{{ if .Values.pushgateway.enabled }}
+The Prometheus PushGateway can be accessed via port {{ .Values.pushgateway.service.servicePort }} on the following DNS name from within your cluster:
+{{ template "prometheus.pushgateway.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+
+{{ if .Values.pushgateway.ingress.enabled -}}
+From outside the cluster, the pushgateway URL(s) are:
+{{- range .Values.pushgateway.ingress.hosts }}
+http://{{ . }}
+{{- end }}
+{{- else }}
+Get the PushGateway URL by running these commands in the same shell:
+{{- if contains "NodePort" .Values.pushgateway.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus.pushgateway.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.pushgateway.service.type }}
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+        You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "prometheus.pushgateway.fullname" . }}'
+
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "prometheus.pushgateway.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.pushgateway.service.servicePort }}
+{{- else if contains "ClusterIP"  .Values.pushgateway.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "prometheus.name" . }},component={{ .Values.pushgateway.name }}" -o jsonpath="{.items[0].metadata.name}")
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 9091
+{{- end }}
+{{- end }}
+{{- end }}
+
+For more information on running Prometheus, visit:
+https://prometheus.io/

test/charts/prometheus/templates/_helpers.tpl

@@ -0,0 +1,176 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "prometheus.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "prometheus.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a fully qualified alertmanager name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+
+{{- define "prometheus.alertmanager.fullname" -}}
+{{- if .Values.alertmanager.fullnameOverride -}}
+{{- .Values.alertmanager.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.alertmanager.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.alertmanager.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a fully qualified kube-state-metrics name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "prometheus.kubeStateMetrics.fullname" -}}
+{{- if .Values.kubeStateMetrics.fullnameOverride -}}
+{{- .Values.kubeStateMetrics.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.kubeStateMetrics.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.kubeStateMetrics.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a fully qualified node-exporter name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "prometheus.nodeExporter.fullname" -}}
+{{- if .Values.nodeExporter.fullnameOverride -}}
+{{- .Values.nodeExporter.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.nodeExporter.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeExporter.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a fully qualified Prometheus server name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "prometheus.server.fullname" -}}
+{{- if .Values.server.fullnameOverride -}}
+{{- .Values.server.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.server.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.server.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create a fully qualified pushgateway name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "prometheus.pushgateway.fullname" -}}
+{{- if .Values.pushgateway.fullnameOverride -}}
+{{- .Values.pushgateway.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- printf "%s-%s" .Release.Name .Values.pushgateway.name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s-%s" .Release.Name $name .Values.pushgateway.name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the appropriate apiVersion for networkpolicy.
+*/}}
+{{- define "prometheus.networkPolicy.apiVersion" -}}
+{{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- print "extensions/v1beta1" -}}
+{{- else if semverCompare "^1.7-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- print "networking.k8s.io/v1" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use for the alertmanager component
+*/}}
+{{- define "prometheus.serviceAccountName.alertmanager" -}}
+{{- if .Values.serviceAccounts.alertmanager.create -}}
+    {{ default (include "prometheus.alertmanager.fullname" .) .Values.serviceAccounts.alertmanager.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.alertmanager.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use for the kubeStateMetrics component
+*/}}
+{{- define "prometheus.serviceAccountName.kubeStateMetrics" -}}
+{{- if .Values.serviceAccounts.kubeStateMetrics.create -}}
+    {{ default (include "prometheus.kubeStateMetrics.fullname" .) .Values.serviceAccounts.kubeStateMetrics.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.kubeStateMetrics.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use for the nodeExporter component
+*/}}
+{{- define "prometheus.serviceAccountName.nodeExporter" -}}
+{{- if .Values.serviceAccounts.nodeExporter.create -}}
+    {{ default (include "prometheus.nodeExporter.fullname" .) .Values.serviceAccounts.nodeExporter.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.nodeExporter.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use for the pushgateway component
+*/}}
+{{- define "prometheus.serviceAccountName.pushgateway" -}}
+{{- if .Values.serviceAccounts.pushgateway.create -}}
+    {{ default (include "prometheus.pushgateway.fullname" .) .Values.serviceAccounts.pushgateway.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.pushgateway.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use for the server component
+*/}}
+{{- define "prometheus.serviceAccountName.server" -}}
+{{- if .Values.serviceAccounts.server.create -}}
+    {{ default (include "prometheus.server.fullname" .) .Values.serviceAccounts.server.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccounts.server.name }}
+{{- end -}}
+{{- end -}}

test/charts/prometheus/templates/alertmanager-configmap.yaml

@@ -0,0 +1,18 @@
+{{- if and .Values.alertmanager.enabled (empty .Values.alertmanager.configMapOverrideName) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+data:
+{{- $root := . -}}
+{{- range $key, $value := .Values.alertmanagerFiles }}
+  {{ $key }}: |
+{{ toYaml $value | default "{}" | indent 4 }}
+{{- end -}}
+{{- end -}}

test/charts/prometheus/templates/alertmanager-deployment.yaml

@@ -0,0 +1,115 @@
+{{- if .Values.alertmanager.enabled -}}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+spec:
+  replicas: {{ .Values.alertmanager.replicaCount }}
+  {{- if .Values.server.strategy }}
+  strategy:
+{{ toYaml .Values.server.strategy | indent 4 }}
+  {{- end }}
+  template:
+    metadata:
+    {{- if .Values.alertmanager.podAnnotations }}
+      annotations:
+{{ toYaml .Values.alertmanager.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        app: {{ template "prometheus.name" . }}
+        component: "{{ .Values.alertmanager.name }}"
+        release: {{ .Release.Name }}
+    spec:
+{{- if .Values.alertmanager.affinity }}
+      affinity:
+{{ toYaml .Values.alertmanager.affinity | indent 8 }}
+{{- end }}
+{{- if .Values.alertmanager.schedulerName }}
+      schedulerName: "{{ .Values.alertmanager.schedulerName }}"
+{{- end }}
+      serviceAccountName: {{ template "prometheus.serviceAccountName.alertmanager" . }}
+{{- if .Values.alertmanager.priorityClassName }}
+      priorityClassName: "{{ .Values.alertmanager.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: {{ template "prometheus.name" . }}-{{ .Values.alertmanager.name }}
+          image: "{{ .Values.alertmanager.image.repository }}:{{ .Values.alertmanager.image.tag }}"
+          imagePullPolicy: "{{ .Values.alertmanager.image.pullPolicy }}"
+          env:
+            {{- range $key, $value := .Values.alertmanager.extraEnv }}
+            - name: {{ $key }}
+              value: {{ $value }}
+            {{- end }}
+          args:
+            - --config.file=/etc/config/alertmanager.yml
+            - --storage.path={{ .Values.alertmanager.persistentVolume.mountPath }}
+          {{- range $key, $value := .Values.alertmanager.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          {{- if .Values.alertmanager.baseURL }}
+            - --web.external-url={{ .Values.alertmanager.baseURL }}
+          {{- end }}
+
+          ports:
+            - containerPort: 9093
+          readinessProbe:
+            httpGet:
+              path: {{ .Values.alertmanager.prefixURL }}/#/status
+              port: 9093
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          resources:
+{{ toYaml .Values.alertmanager.resources | indent 12 }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+            - name: storage-volume
+              mountPath: "{{ .Values.alertmanager.persistentVolume.mountPath }}"
+              subPath: "{{ .Values.alertmanager.persistentVolume.subPath }}"
+
+        - name: {{ template "prometheus.name" . }}-{{ .Values.alertmanager.name }}-{{ .Values.configmapReload.name }}
+          image: "{{ .Values.configmapReload.image.repository }}:{{ .Values.configmapReload.image.tag }}"
+          imagePullPolicy: "{{ .Values.configmapReload.image.pullPolicy }}"
+          args:
+            - --volume-dir=/etc/config
+            - --webhook-url=http://localhost:9093{{ .Values.alertmanager.prefixURL }}/-/reload
+          resources:
+{{ toYaml .Values.configmapReload.resources | indent 12 }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+              readOnly: true
+    {{- if .Values.alertmanager.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.alertmanager.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.alertmanager.securityContext }}
+      securityContext:
+{{ toYaml .Values.alertmanager.securityContext | indent 8 }}
+    {{- end }}
+    {{- if .Values.alertmanager.tolerations }}
+      tolerations:
+{{ toYaml .Values.alertmanager.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.alertmanager.affinity }}
+      affinity:
+{{ toYaml .Values.alertmanager.affinity | indent 8 }}
+    {{- end }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ if .Values.alertmanager.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.alertmanager.configMapOverrideName }}{{- else }}{{ template "prometheus.alertmanager.fullname" . }}{{- end }}
+        - name: storage-volume
+        {{- if .Values.alertmanager.persistentVolume.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.alertmanager.persistentVolume.existingClaim }}{{ .Values.alertmanager.persistentVolume.existingClaim }}{{- else }}{{ template "prometheus.alertmanager.fullname" . }}{{- end }}
+        {{- else }}
+          emptyDir: {}
+        {{- end -}}
+{{- end }}

test/charts/prometheus/templates/alertmanager-ingress.yaml

@@ -0,0 +1,38 @@
+{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled -}}
+{{- $releaseName := .Release.Name -}}
+{{- $serviceName := include "prometheus.alertmanager.fullname" . }}
+{{- $servicePort := .Values.alertmanager.service.servicePort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+{{- if .Values.alertmanager.ingress.annotations }}
+  annotations:
+{{ toYaml .Values.alertmanager.ingress.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- range $key, $value := .Values.alertmanager.ingress.extraLabels }}
+    {{ $key }}: {{ $value }}
+{{- end }}
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+spec:
+  rules:
+  {{- range .Values.alertmanager.ingress.hosts }}
+    {{- $url := splitList "/" . }}
+    - host: {{ first $url }}
+      http:
+        paths:
+          - path: /{{ rest $url | join "/" }}
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+  {{- end -}}
+{{- if .Values.alertmanager.ingress.tls }}
+  tls:
+{{ toYaml .Values.alertmanager.ingress.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}

test/charts/prometheus/templates/alertmanager-networkpolicy.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: {{ template "prometheus.networkPolicy.apiVersion" . }}
+kind: NetworkPolicy
+metadata:
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ template "prometheus.name" . }}
+      component: "{{ .Values.alertmanager.name }}"
+      release: {{ .Release.Name }}
+  ingress:
+    - from:
+      - podSelector:
+          matchLabels:
+            release: {{ .Release.Name }}
+            component: "{{ .Values.server.name }}"
+    - ports:
+      - port: 9093
+{{- end }}

test/charts/prometheus/templates/alertmanager-pvc.yaml

@@ -0,0 +1,31 @@
+{{- if and .Values.alertmanager.enabled .Values.alertmanager.persistentVolume.enabled -}}
+{{- if not .Values.alertmanager.persistentVolume.existingClaim -}}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  {{- if .Values.alertmanager.persistentVolume.annotations }}
+  annotations:
+{{ toYaml .Values.alertmanager.persistentVolume.annotations | indent 4 }}
+  {{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+spec:
+  accessModes:
+{{ toYaml .Values.alertmanager.persistentVolume.accessModes | indent 4 }}
+{{- if .Values.alertmanager.persistentVolume.storageClass }}
+{{- if (eq "-" .Values.alertmanager.persistentVolume.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.alertmanager.persistentVolume.storageClass }}"
+{{- end }}
+{{- end }}
+  resources:
+    requests:
+      storage: "{{ .Values.alertmanager.persistentVolume.size }}"
+{{- end -}}
+{{- end -}}

test/charts/prometheus/templates/alertmanager-service.yaml

@@ -0,0 +1,55 @@
+{{- if .Values.alertmanager.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.alertmanager.service.annotations }}
+  annotations:
+{{ toYaml .Values.alertmanager.service.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.alertmanager.service.labels }}
+{{ toYaml .Values.alertmanager.service.labels | indent 4 }}
+{{- end }}
+  name: {{ template "prometheus.alertmanager.fullname" . }}
+spec:
+{{- if .Values.alertmanager.service.clusterIP }}
+  clusterIP: {{ .Values.alertmanager.service.clusterIP }}
+{{- end }}
+{{- if .Values.alertmanager.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.alertmanager.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.alertmanager.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: http
+      port: {{ .Values.alertmanager.service.servicePort }}
+      protocol: TCP
+      targetPort: 9093
+    {{- if .Values.alertmanager.service.nodePort }}
+      nodePort: {{ .Values.alertmanager.service.nodePort }}
+    {{- end }}
+{{- if .Values.alertmanager.service.enableMeshPeer }}
+    - name: meshpeer
+      port: 6783
+      protocol: TCP
+      targetPort: 6783
+{{- end }}
+  selector:
+    app: {{ template "prometheus.name" . }}
+    component: "{{ .Values.alertmanager.name }}"
+    release: {{ .Release.Name }}
+  type: "{{ .Values.alertmanager.service.type }}"
+{{- end }}

test/charts/prometheus/templates/alertmanager-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccounts.alertmanager.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.alertmanager.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.serviceAccountName.alertmanager" . }}
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-clusterrole.yaml

@@ -0,0 +1,64 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - nodes
+      - persistentvolumeclaims
+      - pods
+      - services
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - endpoints
+      - secrets
+      - configmaps
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+      - jobs
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - list
+      - watch
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-clusterrolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "prometheus.serviceAccountName.kubeStateMetrics" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-deployment.yaml

@@ -0,0 +1,67 @@
+{{- if .Values.kubeStateMetrics.enabled -}}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+{{- if .Values.kubeStateMetrics.deploymentAnnotations }}
+  annotations:
+{{ toYaml .Values.kubeStateMetrics.deploymentAnnotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+spec:
+  replicas: {{ .Values.kubeStateMetrics.replicaCount }}
+  template:
+    metadata:
+    {{- if .Values.kubeStateMetrics.podAnnotations }}
+      annotations:
+{{ toYaml .Values.kubeStateMetrics.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        app: {{ template "prometheus.name" . }}
+        component: "{{ .Values.kubeStateMetrics.name }}"
+        release: {{ .Release.Name }}
+{{- if .Values.kubeStateMetrics.pod.labels }}
+{{ toYaml .Values.kubeStateMetrics.pod.labels | indent 8 }}
+{{- end }}
+    spec:
+      serviceAccountName: {{ template "prometheus.serviceAccountName.kubeStateMetrics" . }}
+{{- if .Values.kubeStateMetrics.priorityClassName }}
+      priorityClassName: "{{ .Values.kubeStateMetrics.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: {{ template "prometheus.name" . }}-{{ .Values.kubeStateMetrics.name }}
+          image: "{{ .Values.kubeStateMetrics.image.repository }}:{{ .Values.kubeStateMetrics.image.tag }}"
+          imagePullPolicy: "{{ .Values.kubeStateMetrics.image.pullPolicy }}"
+        {{- if .Values.kubeStateMetrics.args }}
+          args:
+          {{- range $key, $value := .Values.kubeStateMetrics.args }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+        {{- end }}
+          ports:
+            - name: metrics
+              containerPort: 8080
+          resources:
+{{ toYaml .Values.kubeStateMetrics.resources | indent 12 }}
+    {{- if .Values.kubeStateMetrics.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.kubeStateMetrics.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.kubeStateMetrics.securityContext }}
+      securityContext:
+{{ toYaml .Values.kubeStateMetrics.securityContext | indent 8 }}
+    {{- end }}
+    {{- if .Values.kubeStateMetrics.tolerations }}
+      tolerations:
+{{ toYaml .Values.kubeStateMetrics.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.kubeStateMetrics.affinity }}
+      affinity:
+{{ toYaml .Values.kubeStateMetrics.affinity | indent 8 }}
+    {{- end }}
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-networkpolicy.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: {{ template "prometheus.networkPolicy.apiVersion" . }}
+kind: NetworkPolicy
+metadata:
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ template "prometheus.name" . }}
+      component: "{{ .Values.kubeStateMetrics.name }}"
+      release: {{ .Release.Name }}
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          release: {{ .Release.Name }}
+          component: "{{ .Values.server.name }}"
+  - ports:
+    - port: 8080
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccounts.kubeStateMetrics.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.serviceAccountName.kubeStateMetrics" . }}
+{{- end }}

test/charts/prometheus/templates/kube-state-metrics-svc.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.kubeStateMetrics.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.kubeStateMetrics.service.annotations }}
+  annotations:
+{{ toYaml .Values.kubeStateMetrics.service.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.kubeStateMetrics.service.labels }}
+{{ toYaml .Values.kubeStateMetrics.service.labels | indent 4 }}
+{{- end }}
+  name: {{ template "prometheus.kubeStateMetrics.fullname" . }}
+spec:
+{{- if .Values.kubeStateMetrics.service.clusterIP }}
+  clusterIP: {{ .Values.kubeStateMetrics.service.clusterIP }}
+{{- end }}
+{{- if .Values.kubeStateMetrics.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.kubeStateMetrics.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.kubeStateMetrics.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.kubeStateMetrics.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.kubeStateMetrics.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.kubeStateMetrics.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: http
+      port: {{ .Values.kubeStateMetrics.service.servicePort }}
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: {{ template "prometheus.name" . }}
+    component: "{{ .Values.kubeStateMetrics.name }}"
+    release: {{ .Release.Name }}
+  type: "{{ .Values.kubeStateMetrics.service.type }}"
+{{- end }}

test/charts/prometheus/templates/node-exporter-daemonset.yaml

@@ -0,0 +1,108 @@
+{{- if .Values.nodeExporter.enabled -}}
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+{{- if .Values.nodeExporter.deploymentAnnotations }}
+  annotations:
+{{ toYaml .Values.nodeExporter.deploymentAnnotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.nodeExporter.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.nodeExporter.fullname" . }}
+spec:
+  {{- if .Values.nodeExporter.updateStrategy }}
+  updateStrategy:
+{{ toYaml .Values.nodeExporter.updateStrategy | indent 4 }}
+  {{- end }}
+  template:
+    metadata:
+    {{- if .Values.nodeExporter.podAnnotations }}
+      annotations:
+{{ toYaml .Values.nodeExporter.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        app: {{ template "prometheus.name" . }}
+        component: "{{ .Values.nodeExporter.name }}"
+        release: {{ .Release.Name }}
+{{- if .Values.nodeExporter.pod.labels }}
+{{ toYaml .Values.nodeExporter.pod.labels | indent 8 }}
+{{- end }}
+    spec:
+      serviceAccountName: {{ template "prometheus.serviceAccountName.nodeExporter" . }}
+{{- if .Values.nodeExporter.priorityClassName }}
+      priorityClassName: "{{ .Values.nodeExporter.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: {{ template "prometheus.name" . }}-{{ .Values.nodeExporter.name }}
+          image: "{{ .Values.nodeExporter.image.repository }}:{{ .Values.nodeExporter.image.tag }}"
+          imagePullPolicy: "{{ .Values.nodeExporter.image.pullPolicy }}"
+          args:
+            - --path.procfs=/host/proc
+            - --path.sysfs=/host/sys
+          {{- range $key, $value := .Values.nodeExporter.extraArgs }}
+          {{- if $value }}
+            - --{{ $key }}={{ $value }}
+          {{- else }}
+            - --{{ $key }}
+          {{- end }}
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: 9100
+              hostPort: {{ .Values.nodeExporter.service.hostPort }}
+          resources:
+{{ toYaml .Values.nodeExporter.resources | indent 12 }}
+          volumeMounts:
+            - name: proc
+              mountPath: /host/proc
+              readOnly:  true
+            - name: sys
+              mountPath: /host/sys
+              readOnly: true
+          {{- range .Values.nodeExporter.extraHostPathMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+          {{- range .Values.nodeExporter.extraConfigmapMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+      hostNetwork: true
+      hostPID: true
+    {{- if .Values.nodeExporter.tolerations }}
+      tolerations:
+{{ toYaml .Values.nodeExporter.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.nodeExporter.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeExporter.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.nodeExporter.securityContext }}
+      securityContext:
+{{ toYaml .Values.nodeExporter.securityContext | indent 8 }}
+    {{- end }}
+      volumes:
+        - name: proc
+          hostPath:
+            path: /proc
+        - name: sys
+          hostPath:
+            path: /sys
+      {{- range .Values.nodeExporter.extraHostPathMounts }}
+        - name: {{ .name }}
+          hostPath:
+            path: {{ .hostPath }}
+      {{- end }}
+      {{- range .Values.nodeExporter.extraConfigmapMounts }}
+        - name: {{ .name }}
+          configMap:
+            name: {{ .configMap }}
+      {{- end }}
+
+{{- end -}}

test/charts/prometheus/templates/node-exporter-service.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.nodeExporter.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.nodeExporter.service.annotations }}
+  annotations:
+{{ toYaml .Values.nodeExporter.service.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.nodeExporter.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.nodeExporter.service.labels }}
+{{ toYaml .Values.nodeExporter.service.labels | indent 4 }}
+{{- end }}
+  name: {{ template "prometheus.nodeExporter.fullname" . }}
+spec:
+{{- if .Values.nodeExporter.service.clusterIP }}
+  clusterIP: {{ .Values.nodeExporter.service.clusterIP }}
+{{- end }}
+{{- if .Values.nodeExporter.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.nodeExporter.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.nodeExporter.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.nodeExporter.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.nodeExporter.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.nodeExporter.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: metrics
+      port: {{ .Values.nodeExporter.service.servicePort }}
+      protocol: TCP
+      targetPort: 9100
+  selector:
+    app: {{ template "prometheus.name" . }}
+    component: "{{ .Values.nodeExporter.name }}"
+    release: {{ .Release.Name }}
+  type: "{{ .Values.nodeExporter.service.type }}"
+{{- end -}}

test/charts/prometheus/templates/node-exporter-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccounts.nodeExporter.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.nodeExporter.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.serviceAccountName.nodeExporter" . }}
+{{- end }}

test/charts/prometheus/templates/pushgateway-deployment.yaml

@@ -0,0 +1,67 @@
+{{- if .Values.pushgateway.enabled -}}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.pushgateway.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.pushgateway.fullname" . }}
+spec:
+  replicas: {{ .Values.pushgateway.replicaCount }}
+  template:
+    metadata:
+    {{- if .Values.pushgateway.podAnnotations }}
+      annotations:
+{{ toYaml .Values.pushgateway.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        app: {{ template "prometheus.name" . }}
+        component: "{{ .Values.pushgateway.name }}"
+        release: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ template "prometheus.serviceAccountName.pushgateway" . }}
+{{- if .Values.pushgateway.priorityClassName }}
+      priorityClassName: "{{ .Values.pushgateway.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: {{ template "prometheus.name" . }}-{{ .Values.pushgateway.name }}
+          image: "{{ .Values.pushgateway.image.repository }}:{{ .Values.pushgateway.image.tag }}"
+          imagePullPolicy: "{{ .Values.pushgateway.image.pullPolicy }}"
+          args:
+          {{- range $key, $value := .Values.pushgateway.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          ports:
+            - containerPort: 9091
+          readinessProbe:
+            httpGet:
+            {{- if (index .Values "pushgateway" "extraArgs" "web.route-prefix") }}
+              path: /{{ index .Values "pushgateway" "extraArgs" "web.route-prefix" }}/#/status
+            {{- else }}
+              path: /#/status
+            {{- end }}
+              port: 9091
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+          resources:
+{{ toYaml .Values.pushgateway.resources | indent 12 }}
+    {{- if .Values.pushgateway.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.pushgateway.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.pushgateway.securityContext }}
+      securityContext:
+{{ toYaml .Values.pushgateway.securityContext | indent 8 }}
+    {{- end }}
+    {{- if .Values.pushgateway.tolerations }}
+      tolerations:
+{{ toYaml .Values.pushgateway.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.pushgateway.affinity }}
+      affinity:
+{{ toYaml .Values.pushgateway.affinity | indent 8 }}
+    {{- end }}
+{{- end }}

test/charts/prometheus/templates/pushgateway-ingress.yaml

@@ -0,0 +1,35 @@
+{{- if and .Values.pushgateway.enabled .Values.pushgateway.ingress.enabled -}}
+{{- $releaseName := .Release.Name -}}
+{{- $serviceName := include "prometheus.pushgateway.fullname" . }}
+{{- $servicePort := .Values.pushgateway.service.servicePort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+{{- if .Values.pushgateway.ingress.annotations }}
+  annotations:
+{{ toYaml .Values.pushgateway.ingress.annotations | indent 4}}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.pushgateway.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.pushgateway.fullname" . }}
+spec:
+  rules:
+  {{- range .Values.pushgateway.ingress.hosts }}
+    {{- $url := splitList "/" . }}
+    - host: {{ first $url }}
+      http:
+        paths:
+          - path: /{{ rest $url | join "/" }}
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+  {{- end -}}
+{{- if .Values.pushgateway.ingress.tls }}
+  tls:
+{{ toYaml .Values.pushgateway.ingress.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}

test/charts/prometheus/templates/pushgateway-service.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.pushgateway.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.pushgateway.service.annotations }}
+  annotations:
+{{ toYaml .Values.pushgateway.service.annotations | indent 4}}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.pushgateway.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.pushgateway.service.labels }}
+{{ toYaml .Values.pushgateway.service.labels | indent 4}}
+{{- end }}
+  name: {{ template "prometheus.pushgateway.fullname" . }}
+spec:
+{{- if .Values.pushgateway.service.clusterIP }}
+  clusterIP: {{ .Values.pushgateway.service.clusterIP }}
+{{- end }}
+{{- if .Values.pushgateway.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.pushgateway.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.pushgateway.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.pushgateway.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.pushgateway.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.pushgateway.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: http
+      port: {{ .Values.pushgateway.service.servicePort }}
+      protocol: TCP
+      targetPort: 9091
+  selector:
+    app: {{ template "prometheus.name" . }}
+    component: "{{ .Values.pushgateway.name }}"
+    release: {{ .Release.Name }}
+  type: "{{ .Values.pushgateway.service.type }}"
+{{- end }}

test/charts/prometheus/templates/pushgateway-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccounts.pushgateway.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.pushgateway.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.serviceAccountName.pushgateway" . }}
+{{- end }}

test/charts/prometheus/templates/server-clusterrole.yaml

@@ -0,0 +1,45 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.server.fullname" . }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+  - apiGroups:
+      - "extensions"
+    resources:
+      - ingresses/status
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - nonResourceURLs:
+      - "/metrics"
+    verbs:
+      - get
+{{- end }}

test/charts/prometheus/templates/server-clusterrolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.server.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "prometheus.serviceAccountName.server" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "prometheus.server.fullname" . }}
+{{- end }}

test/charts/prometheus/templates/server-configmap.yaml

@@ -0,0 +1,49 @@
+{{- if (empty .Values.server.configMapOverrideName) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.server.fullname" . }}
+data:
+{{- $root := . -}}
+{{- range $key, $value := .Values.serverFiles }}
+  {{ $key }}: |
+{{- if eq $key "prometheus.yml" }}
+    global:
+{{ $root.Values.server.global | toYaml | indent 6 }}
+{{- end }}
+{{ toYaml $value | default "{}" | indent 4 }}
+{{- if eq $key "prometheus.yml" -}}
+{{- if $root.Values.alertmanager.enabled }}
+    alerting:
+      alertmanagers:
+      - kubernetes_sd_configs:
+          - role: pod
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        {{- if $root.Values.alertmanager.prefixURL }}
+        path_prefix: {{ $root.Values.alertmanager.prefixURL }}
+        {{- end }}
+        relabel_configs:
+        - source_labels: [__meta_kubernetes_namespace]
+          regex: {{ $root.Release.Namespace }}
+          action: keep
+        - source_labels: [__meta_kubernetes_pod_label_app]
+          regex: prometheus
+          action: keep
+        - source_labels: [__meta_kubernetes_pod_label_component]
+          regex: alertmanager
+          action: keep
+        - source_labels: [__meta_kubernetes_pod_container_port_number]
+          regex:
+          action: drop
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}

test/charts/prometheus/templates/server-deployment.yaml

@@ -0,0 +1,189 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+{{- if .Values.server.deploymentAnnotations }}
+  annotations:
+{{ toYaml .Values.server.deploymentAnnotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.server.fullname" . }}
+spec:
+  replicas: {{ .Values.server.replicaCount }}
+  {{- if .Values.server.strategy }}
+  strategy:
+{{ toYaml .Values.server.strategy | indent 4 }}
+  {{- end }}
+  template:
+    metadata:
+    {{- if .Values.server.podAnnotations }}
+      annotations:
+{{ toYaml .Values.server.podAnnotations | indent 8 }}
+    {{- end }}
+      labels:
+        app: {{ template "prometheus.name" . }}
+        component: "{{ .Values.server.name }}"
+        release: {{ .Release.Name }}
+    spec:
+{{- if .Values.server.affinity }}
+      affinity:
+{{ toYaml .Values.server.affinity | indent 8 }}
+{{- end }}
+{{- if .Values.server.priorityClassName }}
+      priorityClassName: "{{ .Values.server.priorityClassName }}"
+{{- end }}
+{{- if .Values.server.schedulerName }}
+      schedulerName: "{{ .Values.server.schedulerName }}"
+{{- end }}
+      serviceAccountName: {{ template "prometheus.serviceAccountName.server" . }}
+      {{- if .Values.initChownData.enabled }}
+      initContainers:
+      - name: "{{ .Values.initChownData.name }}"
+        image: "{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}"
+        imagePullPolicy: "{{ .Values.initChownData.image.pullPolicy }}"
+        resources:
+{{ toYaml .Values.initChownData.resources | indent 12 }}
+        # 65534 is the nobody user that prometheus uses.
+        command: ["chown", "-R", "65534:65534", "{{ .Values.server.persistentVolume.mountPath }}"]
+        volumeMounts:
+        - name: storage-volume
+          mountPath: {{ .Values.server.persistentVolume.mountPath }}
+          subPath: "{{ .Values.server.persistentVolume.subPath }}"
+      {{- end }}
+      containers:
+        - name: {{ template "prometheus.name" . }}-{{ .Values.server.name }}-{{ .Values.configmapReload.name }}
+          image: "{{ .Values.configmapReload.image.repository }}:{{ .Values.configmapReload.image.tag }}"
+          imagePullPolicy: "{{ .Values.configmapReload.image.pullPolicy }}"
+          args:
+            - --volume-dir=/etc/config
+            - --webhook-url=http://127.0.0.1:9090{{ .Values.server.prefixURL }}/-/reload
+          {{- range $key, $value := .Values.configmapReload.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          resources:
+{{ toYaml .Values.configmapReload.resources | indent 12 }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+              readOnly: true
+          {{- range .Values.configmapReload.extraConfigmapMounts }}
+            - name: {{ $.Values.configmapReload.name }}-{{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+
+        - name: {{ template "prometheus.name" . }}-{{ .Values.server.name }}
+          image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag }}"
+          imagePullPolicy: "{{ .Values.server.image.pullPolicy }}"
+          args:
+          {{- if .Values.server.retention }}
+            - --storage.tsdb.retention={{ .Values.server.retention }}
+          {{- end }}
+            - --config.file=/etc/config/prometheus.yml
+            - --storage.tsdb.path={{ .Values.server.persistentVolume.mountPath }}
+            - --web.console.libraries=/etc/prometheus/console_libraries
+            - --web.console.templates=/etc/prometheus/consoles
+            - --web.enable-lifecycle
+          {{- range $key, $value := .Values.server.extraArgs }}
+            - --{{ $key }}={{ $value }}
+          {{- end }}
+          {{- if .Values.server.baseURL }}
+            - --web.external-url={{ .Values.server.baseURL }}
+          {{- end }}
+          {{- if .Values.server.enableAdminApi }}
+            - --web.enable-admin-api
+          {{- end }}
+          ports:
+            - containerPort: 9090
+          readinessProbe:
+            httpGet:
+              path: {{ .Values.server.prefixURL }}/-/ready
+              port: 9090
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          livenessProbe:
+            httpGet:
+              path: {{ .Values.server.prefixURL }}/-/healthy
+              port: 9090
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
+          resources:
+{{ toYaml .Values.server.resources | indent 12 }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config
+            - name: storage-volume
+              mountPath: {{ .Values.server.persistentVolume.mountPath }}
+              subPath: "{{ .Values.server.persistentVolume.subPath }}"
+          {{- range .Values.server.extraHostPathMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+          {{- range .Values.server.extraConfigmapMounts }}
+            - name: {{ $.Values.server.name }}-{{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+          {{- range .Values.server.extraSecretMounts }}
+            - name: {{ .name }}
+              mountPath: {{ .mountPath }}
+              readOnly: {{ .readOnly }}
+          {{- end }}
+    {{- if .Values.server.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.server.nodeSelector | indent 8 }}
+    {{- end }}
+    {{- if .Values.server.securityContext }}
+      securityContext:
+{{ toYaml .Values.server.securityContext | indent 8 }}
+    {{- end }}
+    {{- if .Values.server.tolerations }}
+      tolerations:
+{{ toYaml .Values.server.tolerations | indent 8 }}
+    {{- end }}
+    {{- if .Values.server.affinity }}
+      affinity:
+{{ toYaml .Values.server.affinity | indent 8 }}
+    {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.server.terminationGracePeriodSeconds }}
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ if .Values.server.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.server.configMapOverrideName }}{{- else }}{{ template "prometheus.server.fullname" . }}{{- end }}
+        - name: storage-volume
+        {{- if .Values.server.persistentVolume.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ if .Values.server.persistentVolume.existingClaim }}{{ .Values.server.persistentVolume.existingClaim }}{{- else }}{{ template "prometheus.server.fullname" . }}{{- end }}
+        {{- else }}
+          emptyDir: {}
+        {{- end -}}
+      {{- range .Values.server.extraHostPathMounts }}
+        - name: {{ .name }}
+          hostPath:
+            path: {{ .hostPath }}
+      {{- end }}
+      {{- range .Values.configmapReload.extraConfigmapMounts }}
+        - name: {{ $.Values.configmapReload.name }}-{{ .name }}
+          configMap:
+            name: {{ .configMap }}
+      {{- end }}
+      {{- range .Values.server.extraConfigmapMounts }}
+        - name: {{ $.Values.server.name }}-{{ .name }}
+          configMap:
+            name: {{ .configMap }}
+      {{- end }}
+      {{- range .Values.server.extraSecretMounts }}
+        - name: {{ .name }}
+          secret:
+            secretName: {{ .secretName }}
+      {{- end }}
+      {{- range .Values.configmapReload.extraConfigmapMounts }}
+        - name: {{ .name }}
+          configMap:
+            name: {{ .configMap }}
+      {{- end }}

test/charts/prometheus/templates/server-ingress.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.server.ingress.enabled -}}
+{{- $releaseName := .Release.Name -}}
+{{- $serviceName := include "prometheus.server.fullname" . }}
+{{- $servicePort := .Values.server.service.servicePort -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+{{- if .Values.server.ingress.annotations }}
+  annotations:
+{{ toYaml .Values.server.ingress.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- range $key, $value := .Values.server.ingress.extraLabels }}
+    {{ $key }}: {{ $value }}
+{{- end }}
+  name: {{ template "prometheus.server.fullname" . }}
+spec:
+  rules:
+  {{- range .Values.server.ingress.hosts }}
+    {{- $url := splitList "/" . }}
+    - host: {{ first $url }}
+      http:
+        paths:
+          - path: /{{ rest $url | join "/" }}
+            backend:
+              serviceName: {{ $serviceName }}
+              servicePort: {{ $servicePort }}
+  {{- end -}}
+{{- if .Values.server.ingress.tls }}
+  tls:
+{{ toYaml .Values.server.ingress.tls | indent 4 }}
+  {{- end -}}
+{{- end -}}

test/charts/prometheus/templates/server-networkpolicy.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: {{ template "prometheus.networkPolicy.apiVersion" . }}
+kind: NetworkPolicy
+metadata:
+  name: {{ template "prometheus.server.fullname" . }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ template "prometheus.name" . }}
+      component: "{{ .Values.server.name }}"
+      release: {{ .Release.Name }}
+  ingress:
+    - ports:
+      - port: 9090
+{{- end }}

test/charts/prometheus/templates/server-pvc.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.server.persistentVolume.enabled -}}
+{{- if not .Values.server.persistentVolume.existingClaim -}}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  {{- if .Values.server.persistentVolume.annotations }}
+  annotations:
+{{ toYaml .Values.server.persistentVolume.annotations | indent 4 }}
+  {{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.server.fullname" . }}
+spec:
+  accessModes:
+{{ toYaml .Values.server.persistentVolume.accessModes | indent 4 }}
+{{- if .Values.server.persistentVolume.storageClass }}
+{{- if (eq "-" .Values.server.persistentVolume.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.server.persistentVolume.storageClass }}"
+{{- end }}
+{{- end }}
+  resources:
+    requests:
+      storage: "{{ .Values.server.persistentVolume.size }}"
+{{- end -}}
+{{- end -}}

test/charts/prometheus/templates/server-service.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Service
+metadata:
+{{- if .Values.server.service.annotations }}
+  annotations:
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.server.service.labels }}
+{{ toYaml .Values.server.service.labels | indent 4 }}
+{{- end }}
+  name: {{ template "prometheus.server.fullname" . }}
+spec:
+{{- if .Values.server.service.clusterIP }}
+  clusterIP: {{ .Values.server.service.clusterIP }}
+{{- end }}
+{{- if .Values.server.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.server.service.externalIPs | indent 4 }}
+{{- end }}
+{{- if .Values.server.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.server.service.loadBalancerIP }}
+{{- end }}
+{{- if .Values.server.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+  {{- range $cidr := .Values.server.service.loadBalancerSourceRanges }}
+    - {{ $cidr }}
+  {{- end }}
+{{- end }}
+  ports:
+    - name: http
+      port: {{ .Values.server.service.servicePort }}
+      protocol: TCP
+      targetPort: 9090
+    {{- if .Values.server.service.nodePort }}
+      nodePort: {{ .Values.server.service.nodePort }}
+    {{- end }}
+  selector:
+    app: {{ template "prometheus.name" . }}
+    component: "{{ .Values.server.name }}"
+    release: {{ .Release.Name }}
+  type: "{{ .Values.server.service.type }}"

test/charts/prometheus/templates/server-serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccounts.server.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "prometheus.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: "{{ .Values.server.name }}"
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  name: {{ template "prometheus.serviceAccountName.server" . }}
+{{- end }}

test/test_utils.sh

@@ -171,7 +171,7 @@ set_environment() {
 }
 
 generate_test_id() {
-    echo $(date|md5sum|cut -c1-6)
+    echo $(cat /dev/urandom | tr -dc 'a-z' | fold -w 6 | head -n 1)
 }
 
 helm_install_fission() {
@@ -192,7 +192,7 @@ helm_install_fission() {
     ns=f-$id
     fns=f-func-$id
 
-    helmVars=image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage
+    helmVars=image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false
 
     timeout 30 bash -c "helm_setup"
 
@@ -205,6 +205,9 @@ helm_install_fission() {
         sleep 5
     done
 
+    # only for tests, mv the prefetched prometheus chart to fission-all so helm install fission will install prometheus too
+    mv $ROOT/test/charts $ROOT/charts/fission-all/
+
     echo "Installing fission"
     helm install		\
 	 --wait			\

test/tests/test_canary.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# has 2 tests to verify the canary deployments - success scenario and a failure scenario
+
+set -euo pipefail
+
+id=""
+ROOT=$(dirname $0)/../..
+
+cleanup() {
+    fission env delete --name nodejs || true
+    fission fn delete --name fn-v1 || true
+    fission fn delete --name fn-v2 || true
+    fission fn delete --name fn-v3 || true
+    fission ht delete --name route-success || true
+    fission ht delete --name route-fail || true
+    fission canary-config delete --name canary-1 || true
+    fission canary-config delete --name canary-2 || true
+}
+
+success_scenario() {
+    log "Creating nodejs env"
+    fission env create --name nodejs --image fission/node-env --graceperiod 1
+
+    log "Creating function version-1"
+    fission fn create --name fn-v1 --env nodejs --code $ROOT/examples/nodejs/hello.js
+
+    log "Creating function version-1"
+    fission fn create --name fn-v2 --env nodejs --code $ROOT/examples/nodejs/hello.js
+
+    log "Create a route for the version-1 of the function with weight 100% and version-2 with weight 0%"
+    fission route create --name route-success --method GET --url /success --function fn-v1 --weight 100 --function fn-v2 --weight 0
+
+    log "Create a canary config to gradually increment the weight of version-2 by a step of 50 every 1m"
+    fission canary-config create --name canary-1 --funcN fn-v2 --funcN-1 fn-v1 --httptrigger route-success --increment-step 50 --increment-interval 1m --failure-threshold 10
+
+    sleep 60
+
+    log "Fire requests to the route"
+    ab -n 300 -c 1 http://$FISSION_ROUTER/success
+
+    sleep 60
+
+    log "verify that version-2 of the function is receiving 100% traffic"
+    weight=`kubectl get httptrigger route-success -o jsonpath='{.spec.functionref.functionweights.fn-v2}'`
+
+    if [ "$weight" != "100" ]; then
+        log "weight of fn-v2 at the end of the test is $weight"
+        cleanup
+        exit 1
+    else
+        log "canary success scenario test passed"
+    fi
+}
+
+failure_scenario() {
+    cp $ROOT/examples/nodejs/hello.js hello_400.js
+    sed -i 's/200/400/' hello_400.js
+
+    log "Creating function version-3"
+    fission fn create --name fn-v3 --env nodejs --code hello_400.js
+
+    log "Create a route for the version-1 of the function with weight 100% and version-3 with weight 0%"
+    fission route create --name route-fail --method GET --url /fail --function fn-v1 --weight 100 --function fn-v3 --weight 0
+    sleep 5
+
+    log "Create a canary config to gradually increment the weight of version-2 by a step of 50 every 1m"
+    fission canary-config create --name canary-2 --funcN fn-v3 --funcN-1 fn-v1 --httptrigger route-fail --increment-step 50 --increment-interval 1m --failure-threshold 10
+
+    sleep 60
+
+    log "Fire requests to the route"
+    ab -n 300 -c 1 http://$FISSION_ROUTER/fail
+
+    sleep 60
+
+    log "verify that version-3 of the function is receiving 0% traffic because of rollback"
+    weight=`kubectl get httptrigger route-fail -o jsonpath='{.spec.functionref.functionweights.fn-v3}'`
+
+    if [ "$weight" != "0" ]; then
+        log "weight of fn-v3 at the end of the test is $weight"
+        cleanup
+        exit 1
+    else
+        log "canary failure scenario test passed"
+    fi
+}
+
+main() {
+    # v2 of a function starts with receiving 0% of the traffic with a gradual increase all the way up to 100% of the traffic
+    success_scenario
+
+    # v3 of a function starts with receiving 0% of the traffic, but because of failure rates crossing the threshold,
+    # this test rollbacks the canary deployment to ensure v1 receives 100% of the traffic.
+    failure_scenario
+
+    cleanup
+}
+
+main

test/tests/test_router_cache_invalidation.sh

@@ -43,12 +43,12 @@ fission fn create --name $fn --env python --code testDir-$fn/hello.py
 log "rm testDir-$fn"
 rm -rf testDir-$fn
 
-log "Waiting for router to update cache"
-sleep 3
-
 log "Creating route"
 fission route create --function $fn --url /$fn --method GET
 
+log "Waiting for router to update cache"
+sleep 5
+
 http_status=`curl -sw "%{http_code}" http://$FISSION_ROUTER/$fn -o /dev/null`
 log "http_status: $http_status"
 if [ "$http_status" -ne "200" ]; then

types.go

@@ -52,6 +52,9 @@ type (
 	MessageQueueTriggerSpec      = fv1.MessageQueueTriggerSpec
 	TimeTriggerSpec              = fv1.TimeTriggerSpec
 	RecorderSpec                 = fv1.RecorderSpec
+	CanaryConfigSpec             = fv1.CanaryConfigSpec
+	CanaryConfigStatus           = fv1.CanaryConfigStatus
+	FailureType                  = fv1.FailureType
 )
 
 type (
@@ -146,10 +149,13 @@ const (
 	// reference is simply by function name.
 	FunctionReferenceTypeFunctionName = fv1.FunctionReferenceTypeFunctionName
 
+	//   Set of function references (recursively), by percentage of traffic
+	FunctionReferenceTypeFunctionWeights = fv1.FunctionReferenceTypeFunctionWeights
+
 	// Other function reference types we'd like to support:
 	//   Versioned function, latest version
 	//   Versioned function. by semver "latest compatible"
-	//   Set of function references (recursively), by percentage of traffic
+
 )
 
 const (
@@ -194,3 +200,12 @@ const (
 
 	ClusterRole = "ClusterRole"
 )
+
+const (
+	FailureTypeStatusCode        = fv1.FailureTypeStatusCode
+	CanaryConfigStatusPending    = fv1.CanaryConfigStatusPending
+	CanaryConfigStatusSucceeded  = fv1.CanaryConfigStatusSucceeded
+	CanaryConfigStatusFailed     = fv1.CanaryConfigStatusFailed
+	CanaryConfigStatusAborted    = fv1.CanaryConfigStatusAborted
+	MaxIterationsForCanaryConfig = fv1.MaxIterationsForCanaryConfig
+)