Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/router/analytics.go

@@ -0,0 +1,65 @@
+package router
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"github.com/dchest/uniuri"
+)
+
+type (
+	Analytics struct {
+		id  string
+		url string
+	}
+	AnalyticsData struct {
+		id                string
+		FunctionCallCount uint64 `json:"FunctionCallCount"`
+	}
+)
+
+func MakeAnalytics(url string) *Analytics {
+
+	if len(url) == 0 {
+		url = os.Getenv("ANALYTICS_URL")
+		if len(url) == 0 {
+			return nil
+		}
+	}
+
+	a := &Analytics{
+		url: url,
+		id:  uniuri.NewLen(8),
+	}
+	go a.run()
+	return a
+}
+
+func (a *Analytics) gatherData() *AnalyticsData {
+	return &AnalyticsData{
+		FunctionCallCount: atomic.LoadUint64(&globalFunctionCallCount),
+	}
+}
+
+func (a *Analytics) run() {
+	ticker := time.NewTicker(24 * time.Hour)
+	for range ticker.C {
+		msg := a.gatherData()
+		msg.id = a.id
+
+		msgbytes, err := json.Marshal(*msg)
+		if err != nil {
+			continue
+		}
+
+		resp, err := http.Post(a.url, "application/json", bytes.NewReader(msgbytes))
+		if resp != nil {
+			// close response body to prevent resources leak
+			resp.Body.Close()
+		}
+	}
+}

pkg/router/functionHandler.go

@@ -0,0 +1,638 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"github.com/satori/go.uuid"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+	executorClient "github.com/fission/fission/pkg/executor/client"
+	"github.com/fission/fission/pkg/redis"
+	"github.com/fission/fission/pkg/throttler"
+)
+
+const (
+	FORWARDED        = "Forwarded"
+	X_FORWARDED_HOST = "X-Forwarded-Host"
+)
+
+type (
+	functionHandler struct {
+		logger                   *zap.Logger
+		fmap                     *functionServiceMap
+		frmap                    *functionRecorderMap
+		trmap                    *triggerRecorderMap
+		executor                 *executorClient.Client
+		function                 *metav1.ObjectMeta
+		httpTrigger              *fv1.HTTPTrigger
+		functionMetadataMap      map[string]*metav1.ObjectMeta
+		fnWeightDistributionList []FunctionWeightDistribution
+		tsRoundTripperParams     *tsRoundTripperParams
+		recorderName             string
+		isDebugEnv               bool
+		svcAddrUpdateThrottler   *throttler.Throttler
+	}
+
+	tsRoundTripperParams struct {
+		timeout         time.Duration
+		timeoutExponent int
+		keepAlive       time.Duration
+
+		// maxRetires is the max times for RetryingRoundTripper to retry a request.
+		// Default maxRetries is 10, which means router will retry for
+		// up to 10 times and abort it if still not succeeded.
+		maxRetries int
+
+		// svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address
+		// Router sends requests to a specific service address for each function.
+		// A service address is considered as an invalid one if amount of non-network
+		// errors router received is higher than svcAddrRetryCount. In this situation,
+		// remove it from cache and try to get a new one from executor.
+		// Default svcAddrRetryCount is 5.
+		svcAddrRetryCount int
+	}
+
+	// A layer on top of http.DefaultTransport, with retries.
+	RetryingRoundTripper struct {
+		logger      *zap.Logger
+		funcHandler *functionHandler
+		base        http.RoundTripper
+	}
+
+	// To keep the request body open during retries, we create an interface with Close operation being a no-op.
+	// Details : https://github.com/flynn/flynn/pull/875
+	fakeCloseReadCloser struct {
+		io.ReadCloser
+	}
+
+	svcEntryRecord struct {
+		svcUrl    *url.URL
+		fromCache bool
+	}
+)
+
+func init() {
+	// just seeding the random number for getting the canary function
+	rand.Seed(time.Now().UnixNano())
+}
+
+func (w *fakeCloseReadCloser) Close() error {
+	return nil
+}
+
+func (w *fakeCloseReadCloser) RealClose() error {
+	if w.ReadCloser == nil {
+		return nil
+	}
+	return w.ReadCloser.Close()
+}
+
+// RoundTrip is a custom transport with retries for http requests that forwards the request to the right serviceUrl, obtained
+// from router's cache or from executor if router entry is stale.
+//
+// It first checks if the service address for this function came from router's cache.
+// If it didn't, it makes a request to executor to get a new service for function. If that succeeds, it adds the address
+// to it's cache and makes a request to that address with transport.RoundTrip call.
+// Initial requests to new k8s services sometimes seem to fail, but retries work. So, it retries with an exponential
+// back-off for maxRetries times.
+//
+// Else if it came from the cache, it makes a transport.RoundTrip with that cached address. If the response received is
+// a network dial error (which means that the pod doesn't exist anymore), it removes the cache entry and makes a request
+// to executor to get a new service for function. It then retries transport.RoundTrip with the new address.
+//
+// At any point in time, if the response received from transport.RoundTrip is other than dial network error, it is
+// relayed as-is to the user, without any retries.
+//
+// While this RoundTripper handles the case where a previously cached address of the function pod isn't valid anymore
+// (probably because the pod got deleted somehow), by making a request to executor to get a new service for this function,
+// it doesn't handle a case where a newly specialized pod gets deleted just after the GetServiceForFunction succeeds.
+// In such a case, the RoundTripper will retry requests against the new address and give up after maxRetries.
+// However, the subsequent http call for this function will ensure the cache is invalidated.
+//
+// If GetServiceForFunction returns an error or if RoundTripper exits with an error, it get's translated into 502
+// inside ServeHttp function of the reverseProxy.
+// Earlier, GetServiceForFunction was called inside handler function and fission explicitly set http status code to 500
+// if it returned an error.
+func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	// Set forwarded host header if not exists
+	roundTripper.addForwardedHostHeader(req)
+
+	// TODO: Keep? --> Needed for queries encoded in URL before they're stripped by the proxy
+	var originalUrl url.URL
+	originalUrl = *req.URL
+
+	// Iff this request needs to be recorded, we save the body
+	var postedBody string
+	if len(roundTripper.funcHandler.recorderName) > 0 {
+		if req.ContentLength > 0 {
+			p := make([]byte, req.ContentLength)
+			buf, _ := ioutil.ReadAll(req.Body)
+			// We need two io readers because a single reader will drain the buffer, hence we keep a replacement copy
+			rdr1 := ioutil.NopCloser(bytes.NewBuffer(buf))
+			rdr2 := ioutil.NopCloser(bytes.NewBuffer(buf))
+
+			rdr1.Read(p)
+			postedBody = string(p)
+			roundTripper.logger.Info("roundtripper posted body", zap.String("body", postedBody))
+			req.Body = rdr2
+		}
+	}
+
+	fnMeta := roundTripper.funcHandler.function
+
+	// Metrics stuff
+	startTime := time.Now()
+	funcMetricLabels := &functionLabels{
+		namespace: fnMeta.Namespace,
+		name:      fnMeta.Name,
+	}
+	httpMetricLabels := &httpLabels{
+		method: req.Method,
+	}
+	if roundTripper.funcHandler.httpTrigger != nil {
+		httpMetricLabels.host = roundTripper.funcHandler.httpTrigger.Spec.Host
+		httpMetricLabels.path = roundTripper.funcHandler.httpTrigger.Spec.RelativeURL
+	}
+
+	// set the timeout for transport context
+	transport := roundTripper.getDefaultTransport()
+
+	executingTimeout := roundTripper.funcHandler.tsRoundTripperParams.timeout
+
+	// wrap the req.Body with another ReadCloser interface.
+	if req.Body != nil {
+		req.Body = &fakeCloseReadCloser{req.Body}
+	}
+
+	// close req body
+	defer func() {
+		if req.Body != nil {
+			req.Body.(*fakeCloseReadCloser).RealClose()
+		}
+	}()
+
+	// The reason for request failure may vary from case to case.
+	// After some investigation, found most of the failure are due to
+	// network timeout or target function is under heavy workload. In
+	// such cases, if router keeps trying to get new function service
+	// will increase executor burden and cause 502 error.
+	//
+	// The "retryCounter" was introduced to solve this problem by retrying
+	// requests for "limited threshold". Once a request's retryCounter higher
+	// than the predefined threshold, reset retryCounter and remove service
+	// cache, then retry to get new svc record from executor again.
+	retryCounter := 0
+
+	for i := 0; i < roundTripper.funcHandler.tsRoundTripperParams.maxRetries-1; i++ {
+		// get function service url from cache or executor
+		serviceUrl, serviceUrlFromCache, err := roundTripper.funcHandler.getServiceEntry(req.Context())
+		if err != nil {
+			// We might want a specific error code or header for fission failures as opposed to
+			// user function bugs.
+			statusCode, errMsg := ferror.GetHTTPError(err)
+			if roundTripper.funcHandler.isDebugEnv {
+				return &http.Response{
+					StatusCode:    statusCode,
+					Proto:         req.Proto,
+					ProtoMajor:    req.ProtoMajor,
+					ProtoMinor:    req.ProtoMinor,
+					Body:          ioutil.NopCloser(bytes.NewBufferString(errMsg)),
+					ContentLength: int64(len(errMsg)),
+					Request:       req,
+					Header:        make(http.Header, 0),
+				}, nil
+			}
+			return nil, ferror.MakeError(http.StatusInternalServerError, err.Error())
+		}
+
+		// service url maybe nil if router cannot find one in cache,
+		// so here we retry to get service url again
+		if serviceUrl == nil {
+			time.Sleep(executingTimeout)
+			continue
+		}
+
+		// tapService before invoking roundTrip for the serviceUrl
+		if serviceUrlFromCache {
+			go roundTripper.funcHandler.tapService(serviceUrl)
+		}
+
+		// modify the request to reflect the service url
+		// this service url may have come from the cache lookup or from executor response
+		req.URL.Scheme = serviceUrl.Scheme
+		req.URL.Host = serviceUrl.Host
+
+		// To keep the function run container simple, it
+		// doesn't do any routing.  In the future if we have
+		// multiple functions per container, we could use the
+		// function metadata here.
+		// leave the query string intact (req.URL.RawQuery)
+		req.URL.Path = "/"
+
+		// Overwrite request host with internal host,
+		// or request will be blocked in some situations
+		// (e.g. istio-proxy)
+		req.Host = serviceUrl.Host
+
+		// over-riding default settings.
+		transport.DialContext = (&net.Dialer{
+			Timeout:   executingTimeout,
+			KeepAlive: roundTripper.funcHandler.tsRoundTripperParams.keepAlive,
+		}).DialContext
+
+		overhead := time.Since(startTime)
+
+		// forward the request to the function service
+		resp, err = roundTripper.base.RoundTrip(req)
+		if err == nil {
+			// Track metrics
+			httpMetricLabels.code = resp.StatusCode
+			funcMetricLabels.cached = serviceUrlFromCache
+
+			functionCallCompleted(funcMetricLabels, httpMetricLabels,
+				overhead, time.Since(startTime), resp.ContentLength)
+
+			if len(roundTripper.funcHandler.recorderName) > 0 {
+				if roundTripper.funcHandler.httpTrigger != nil {
+					trigger := roundTripper.funcHandler.httpTrigger.Metadata.Name
+					redis.Record(
+						roundTripper.logger,
+						trigger,
+						roundTripper.funcHandler.recorderName,
+						req.Header.Get("X-Fission-ReqUID"), req, originalUrl, postedBody, resp, fnMeta.Namespace,
+						time.Now().UnixNano(),
+					)
+				} else {
+					roundTripper.logger.Error("no http trigger attached for recorder",
+						zap.String("recorder", roundTripper.funcHandler.recorderName))
+				}
+			}
+
+			// return response back to user
+			return resp, nil
+		}
+
+		// if transport.RoundTrip returns a non-network dial error, then relay it back to user
+		if !utils.IsNetworkDialError(err) {
+			err = errors.Wrapf(err, "error sending request to function %v", fnMeta.Name)
+			return resp, err
+		}
+
+		// dial timeout or dial network errors goes here
+
+		if retryCounter < roundTripper.funcHandler.tsRoundTripperParams.svcAddrRetryCount {
+			executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent)
+			retryCounter++
+
+			roundTripper.logger.Info("request errored out - backing off before retrying",
+				zap.String("url", req.URL.Host),
+				zap.Duration("backoff_timeout", executingTimeout))
+
+			time.Sleep(executingTimeout)
+
+			if serviceUrlFromCache {
+				continue
+			}
+		} else {
+			// if transport.RoundTrip returns a network dial error and serviceUrl was from cache,
+			// it means, the entry in router cache is stale, so invalidate it.
+			roundTripper.logger.Error("request errored out - removing function from router's cache and requesting a new service for function",
+				zap.String("url", req.URL.Host),
+				zap.String("function_name", fnMeta.Name))
+			roundTripper.funcHandler.fmap.remove(fnMeta)
+			retryCounter = 0
+		}
+
+		// break directly if we still fail at the last round
+		if i >= roundTripper.funcHandler.tsRoundTripperParams.maxRetries-1 {
+			break
+		}
+	}
+
+	// finally, one more retry with the default timeout
+	resp, err = http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		roundTripper.logger.Error("error getting response from function",
+			zap.Error(err),
+			zap.String("function_name", fnMeta.Name))
+	}
+
+	return resp, err
+}
+
+// getDefaultTransport returns a pointer to new copy of http.Transport object to prevent
+// the value of http.DefaultTransport from being changed by goroutines.
+func (roundTripper RetryingRoundTripper) getDefaultTransport() *http.Transport {
+	// The transport setup here follows the configurations of http.DefaultTransport
+	// but without Dialer since we will change it later.
+	transport := http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	// Disables caching, Please refer to issue and specifically
+	// comment: https://github.com/fission/fission/issues/723#issuecomment-398781995
+	transport.DisableKeepAlives = true
+
+	return &transport
+}
+
+func (fh *functionHandler) tapService(serviceUrl *url.URL) {
+	if fh.executor == nil {
+		return
+	}
+	fh.executor.TapService(serviceUrl)
+}
+
+func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *http.Request) {
+	// retrieve url params and add them to request header
+	vars := mux.Vars(request)
+	for k, v := range vars {
+		request.Header.Set(fmt.Sprintf("X-Fission-Params-%v", k), v)
+	}
+
+	var reqUID string
+	if len(fh.recorderName) > 0 {
+		UID := strings.ToLower(uuid.NewV4().String())
+		reqUID = "REQ" + UID
+		request.Header.Set("X-Fission-ReqUID", reqUID)
+		fh.logger.Info("record request", zap.String("request_id", reqUID))
+	}
+
+	if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == types.FunctionReferenceTypeFunctionWeights {
+		// canary deployment. need to determine the function to send request to now
+		fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList)
+		if fnMetadata == nil {
+			fh.logger.Error("could not get canary backend", zap.String("request_id", reqUID))
+			// TODO : write error to responseWrite and return response
+			return
+		}
+		fh.function = fnMetadata
+		fh.logger.Debug("chosen function backend's metadata", zap.Any("metadata", fh.function))
+	}
+
+	// system params
+	MetadataToHeaders(HEADERS_FISSION_FUNCTION_PREFIX, fh.function, request)
+
+	director := func(req *http.Request) {
+		if _, ok := req.Header["User-Agent"]; !ok {
+			// explicitly disable User-Agent so it's not set to default value
+			req.Header.Set("User-Agent", "")
+		}
+	}
+
+	proxy := &httputil.ReverseProxy{
+		Director: director,
+		Transport: &RetryingRoundTripper{
+			logger:      fh.logger.Named("roundtripper"),
+			funcHandler: &fh,
+			base: &ochttp.Transport{
+				Base: &http.Transport{
+					Proxy: http.ProxyFromEnvironment,
+					DialContext: (&net.Dialer{
+						Timeout:   fh.tsRoundTripperParams.timeout,
+						KeepAlive: fh.tsRoundTripperParams.keepAlive,
+					}).DialContext,
+					MaxIdleConns:          100,
+					IdleConnTimeout:       90 * time.Second,
+					TLSHandshakeTimeout:   10 * time.Second,
+					ExpectContinueTimeout: 1 * time.Second,
+					// Disables caching, Please refer to issue and specifically comment: https://github.com/fission/fission/issues/723#issuecomment-398781995
+					DisableKeepAlives: true,
+				},
+			},
+		},
+	}
+
+	proxy.ServeHTTP(responseWriter, request)
+}
+
+// findCeil picks a function from the functionWeightDistribution list based on the
+// random number generated. It uses the prefix calculated for the function weights.
+func findCeil(randomNumber int, wtDistrList []FunctionWeightDistribution) string {
+	low := 0
+	high := len(wtDistrList) - 1
+
+	for {
+		if low >= high {
+			break
+		}
+
+		mid := low + high/2
+		if randomNumber >= wtDistrList[mid].sumPrefix {
+			low = mid + 1
+		} else {
+			high = mid
+		}
+	}
+
+	if wtDistrList[low].sumPrefix >= randomNumber {
+		return wtDistrList[low].name
+	} else {
+		return ""
+	}
+}
+
+// picks a function to route to based on a random number generated
+func getCanaryBackend(fnMetadatamap map[string]*metav1.ObjectMeta, fnWtDistributionList []FunctionWeightDistribution) *metav1.ObjectMeta {
+	randomNumber := rand.Intn(fnWtDistributionList[len(fnWtDistributionList)-1].sumPrefix + 1)
+
+	fnName := findCeil(randomNumber, fnWtDistributionList)
+
+	return fnMetadatamap[fnName]
+}
+
+// addForwardedHostHeader add "forwarded host" to request header
+func (roundTripper RetryingRoundTripper) addForwardedHostHeader(req *http.Request) {
+	// for more detailed information, please visit:
+	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
+
+	if len(req.Header.Get(FORWARDED)) > 0 || len(req.Header.Get(X_FORWARDED_HOST)) > 0 {
+		// forwarded headers were set by external proxy, leave them intact
+		return
+	}
+
+	// Format of req.Host is <host>:<port>
+	// We need to extract hostname from it, than
+	// check whether a host is ipv4 or ipv6 or FQDN
+	reqUrl := fmt.Sprintf("%s://%s", req.Proto, req.Host)
+	u, err := url.Parse(reqUrl)
+	if err != nil {
+		roundTripper.logger.Error("error parsing request url while adding forwarded host headers",
+			zap.Error(err),
+			zap.String("url", reqUrl))
+		return
+	}
+
+	var host string
+
+	// ip will be nil if the Hostname is a FQDN string
+	ip := net.ParseIP(u.Hostname())
+
+	// ip == nil -> hostname is FQDN instead of ip address
+	// The order of To4() and To16() here matters, To16() will
+	// converts an IPv4 address to IPv6 format address and may
+	// cause router append wrong host value to header. To prevent
+	// this we need to check whether To4() is nil first.
+	if ip == nil || (ip != nil && ip.To4() != nil) {
+		host = fmt.Sprintf(`host=%s;`, req.Host)
+	} else if ip != nil && ip.To16() != nil {
+		// For the "Forwarded" header, if a host is an IPv6 address it should be quoted
+		host = fmt.Sprintf(`host="%s";`, req.Host)
+	}
+
+	req.Header.Set(FORWARDED, host)
+	req.Header.Set(X_FORWARDED_HOST, req.Host)
+}
+
+// getServiceEntry is a short-hand for developers to get service url entry that may returns from executor or cache
+func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url.URL, serviceUrlFromCache bool, err error) {
+	// try to find service url from cache first
+	serviceUrl, err = fh.getServiceEntryFromCache()
+	if err == nil && serviceUrl != nil {
+		return serviceUrl, true, nil
+	} else if err != nil {
+		return nil, false, err
+	}
+
+	// cache miss or nil entry in cache
+
+	// Use throttle to limit the total amount of requests sent
+	// to the executor to prevent it from overloaded.
+	recordObj, err := fh.svcAddrUpdateThrottler.RunOnce(
+		crd.CacheKey(fh.function),
+		func(firstToTheLock bool) (interface{}, error) {
+			var u *url.URL
+			// Get service entry from executor and update cache if its the first goroutine
+			if firstToTheLock { // first to the service url
+				fh.logger.Info("calling getServiceForFunction",
+					zap.String("function_name", fh.function.Name))
+				u, err = fh.getServiceEntryFromExecutor(ctx)
+				if err != nil {
+					fh.logger.Error("error getting service url from executor",
+						zap.Error(err),
+						zap.String("function_name", fh.function.Name))
+					return nil, err
+				}
+				// add the address in router's cache
+				fh.logger.Info("assigning service url for function",
+					zap.String("url", u.String()),
+					zap.String("function_name", fh.function.Name))
+				fh.fmap.assign(fh.function, u)
+			} else {
+				u, err = fh.getServiceEntryFromCache()
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			return svcEntryRecord{
+				svcUrl:    u,
+				fromCache: firstToTheLock,
+			}, err
+		},
+	)
+	if err != nil {
+		e := "error updating service address entry for function"
+		fh.logger.Error(e,
+			zap.Error(err),
+			zap.String("function_name", fh.function.Name),
+			zap.String("function_namespace", fh.function.Namespace))
+		return nil, false, errors.Wrapf(err, "%s %s_%s", e, fh.function.Name, fh.function.Namespace)
+	}
+
+	record, ok := recordObj.(svcEntryRecord)
+	if !ok {
+		return nil, false, errors.Errorf("Received unknown service record type")
+	}
+
+	return record.svcUrl, record.fromCache, nil
+}
+
+// getServiceEntryFromCache returns service url entry returns from cache
+func (fh *functionHandler) getServiceEntryFromCache() (serviceUrl *url.URL, err error) {
+	// cache lookup to get serviceUrl
+	serviceUrl, err = fh.fmap.lookup(fh.function)
+	if err != nil {
+		var errMsg string
+
+		e, ok := err.(ferror.Error)
+		if !ok {
+			errMsg = fmt.Sprintf("Unknown error when looking up service entry: %v", err)
+		} else {
+			// Ignore ErrorNotFound error here, it's an expected error,
+			// roundTripper will try to get service url later.
+			if e.Code == ferror.ErrorNotFound {
+				return nil, nil
+			}
+			errMsg = fmt.Sprintf("Error getting function %v;s service entry from cache: %v", fh.function.Name, err)
+		}
+		return nil, ferror.MakeError(http.StatusInternalServerError, errMsg)
+	}
+	return serviceUrl, nil
+}
+
+// getServiceEntryFromExecutor returns service url entry returns from executor
+func (fh *functionHandler) getServiceEntryFromExecutor(ctx context.Context) (*url.URL, error) {
+	// send a request to executor to specialize a new pod
+	service, err := fh.executor.GetServiceForFunction(ctx, fh.function)
+	if err != nil {
+		statusCode, errMsg := ferror.GetHTTPError(err)
+		fh.logger.Error("error from GetServiceForFunction",
+			zap.Error(err),
+			zap.String("error_message", errMsg),
+			zap.Any("function", fh.function),
+			zap.Int("status_code", statusCode))
+		return nil, err
+	}
+
+	// parse the address into url
+	serviceUrl, err := url.Parse(fmt.Sprintf("http://%v", service))
+	if err != nil {
+		fh.logger.Error("error parsing service url",
+			zap.Error(err),
+			zap.String("service_url", serviceUrl.String()))
+		return nil, err
+	}
+
+	return serviceUrl, nil
+}

pkg/router/functionHandler_test.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/types"
+)
+
+func createBackendService(testResponseString string) *url.URL {
+	backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(testResponseString))
+	}))
+
+	backendURL, err := url.Parse(backendServer.URL)
+	if err != nil {
+		panic("error parsing url")
+	}
+	return backendURL
+}
+
+/*
+   1. Create a service at some URL
+   2. Add it to the function service map
+   3. Create a http server with some trigger url pointed at function handler
+   4. Send a request to that server, ensure it reaches the first service.
+*/
+func TestFunctionProxying(t *testing.T) {
+	testResponseString := "hi"
+	backendURL := createBackendService(testResponseString)
+	log.Printf("Created backend svc at %v", backendURL)
+
+	fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	fmap := makeFunctionServiceMap(logger, 0)
+	fmap.assign(fn, backendURL)
+
+	httpTrigger := &fv1.HTTPTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:            "xxx",
+			Namespace:       metav1.NamespaceDefault,
+			ResourceVersion: "1234",
+		},
+		Spec: fv1.HTTPTriggerSpec{
+			FunctionReference: fv1.FunctionReference{
+				Type: types.FunctionReferenceTypeFunctionName,
+			},
+		},
+	}
+
+	fh := &functionHandler{
+		logger:   logger,
+		fmap:     fmap,
+		function: fn,
+		tsRoundTripperParams: &tsRoundTripperParams{
+			timeout:         50 * time.Millisecond,
+			timeoutExponent: 2,
+			keepAlive:       30 * time.Second,
+			maxRetries:      10,
+		},
+		httpTrigger: httpTrigger,
+	}
+	functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler))
+	fhURL := functionHandlerServer.URL
+
+	testRequest(fhURL, testResponseString)
+}

pkg/router/functionRecorderMap.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"time"
+
+	"go.uber.org/zap"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type (
+	functionRecorderMap struct {
+		logger *zap.Logger
+		cache  *cache.Cache // map[string]*fv1.Recorder
+	}
+)
+
+// Why do we need an expiry?
+func makeFunctionRecorderMap(logger *zap.Logger, expiry time.Duration) *functionRecorderMap {
+	return &functionRecorderMap{
+		logger: logger.Named("function_recorder_map"),
+		cache:  cache.MakeCache(expiry, 0),
+	}
+}
+
+func (frmap *functionRecorderMap) lookup(function string) (*fv1.Recorder, error) {
+	item, err := frmap.cache.Get(function)
+	if err != nil {
+		return nil, err
+	}
+	u := item.(*fv1.Recorder)
+	return u, nil
+}
+
+func (frmap *functionRecorderMap) assign(function string, recorder *fv1.Recorder) {
+	err, _ := frmap.cache.Set(function, recorder)
+	if err != nil {
+		if e, ok := err.(ferror.Error); ok && e.Code == ferror.ErrorNameExists {
+			return
+		}
+		frmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
+	}
+
+}
+
+func (frmap *functionRecorderMap) remove(function string) error {
+	return frmap.cache.Delete(function)
+}

pkg/router/functionReferenceResolver.go

@@ -0,0 +1,226 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+)
+
+type (
+	// functionReferenceResolver provides a resolver to turn a function
+	// reference into a resolveResult
+	functionReferenceResolver struct {
+		// FunctionReference -> function metadata
+		refCache *cache.Cache
+
+		stopCh chan struct{}
+		store  k8sCache.Store
+	}
+
+	resolveResultType int
+
+	FunctionWeightDistribution struct {
+		name      string
+		weight    int
+		sumPrefix int
+	}
+
+	// resolveResult is the result of resolving a function reference;
+	// it could be the metadata of one function or
+	// a distribution of requests across two functions.
+	resolveResult struct {
+		resolveResultType
+		functionMetadataMap        map[string]*metav1.ObjectMeta
+		functionWtDistributionList []FunctionWeightDistribution
+	}
+
+	// namespacedTriggerReference is just a trigger reference plus a
+	// namespace.
+	namespacedTriggerReference struct {
+		namespace              string
+		triggerName            string
+		triggerResourceVersion string
+	}
+)
+
+const (
+	resolveResultSingleFunction = iota
+	resolveResultMultipleFunctions
+)
+
+func makeFunctionReferenceResolver(store k8sCache.Store) *functionReferenceResolver {
+	frr := &functionReferenceResolver{
+		refCache: cache.MakeCache(time.Minute, 0),
+		store:    store,
+	}
+	return frr
+}
+
+func makeK8SCache(crdClient *rest.RESTClient) (k8sCache.Store, k8sCache.Controller) {
+	watchlist := k8sCache.NewListWatchFromClient(crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
+	listWatch := &k8sCache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			return watchlist.List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			return watchlist.Watch(options)
+		},
+	}
+	resyncPeriod := 30 * time.Second
+	return k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{})
+}
+
+// resolve translates a trigger's function reference to a resolveResult.
+func (frr *functionReferenceResolver) resolve(trigger fv1.HTTPTrigger) (*resolveResult, error) {
+	nfr := namespacedTriggerReference{
+		namespace:              trigger.Metadata.Namespace,
+		triggerName:            trigger.Metadata.Name,
+		triggerResourceVersion: trigger.Metadata.ResourceVersion,
+	}
+
+	// check cache
+	rrInt, err := frr.refCache.Get(nfr)
+	if err == nil {
+		result := rrInt.(resolveResult)
+		return &result, nil
+	}
+
+	// resolve on cache miss
+	var rr *resolveResult
+
+	switch trigger.Spec.FunctionReference.Type {
+	case fv1.FunctionReferenceTypeFunctionName:
+		rr, err = frr.resolveByName(nfr.namespace, trigger.Spec.FunctionReference.Name)
+		if err != nil {
+			return nil, err
+		}
+
+	case fv1.FunctionReferenceTypeFunctionWeights:
+		rr, err = frr.resolveByFunctionWeights(nfr.namespace, &trigger.Spec.FunctionReference)
+		if err != nil {
+			return nil, err
+		}
+
+	default:
+		return nil, fmt.Errorf("Unrecognized function reference type %v", trigger.Spec.FunctionReference.Type)
+	}
+
+	// cache resolve result
+	frr.refCache.Set(nfr, *rr)
+
+	return rr, nil
+}
+
+// resolveByName simply looks up function by name in a namespace.
+func (frr *functionReferenceResolver) resolveByName(namespace, name string) (*resolveResult, error) {
+	// get function from cache
+	obj, isExist, err := frr.store.Get(&fv1.Function{
+		Metadata: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if !isExist {
+		return nil, fmt.Errorf("function %v does not exist", name)
+	}
+
+	f := obj.(*fv1.Function)
+	functionMetadataMap := make(map[string]*metav1.ObjectMeta, 1)
+	functionMetadataMap[f.Metadata.Name] = &f.Metadata
+
+	rr := resolveResult{
+		resolveResultType:   resolveResultSingleFunction,
+		functionMetadataMap: functionMetadataMap,
+	}
+
+	return &rr, nil
+}
+
+func (frr *functionReferenceResolver) resolveByFunctionWeights(namespace string, fr *fv1.FunctionReference) (*resolveResult, error) {
+
+	functionMetadataMap := make(map[string]*metav1.ObjectMeta, 0)
+	fnWtDistrList := make([]FunctionWeightDistribution, 0)
+	sumPrefix := 0
+
+	for functionName, functionWeight := range fr.FunctionWeights {
+		// get function from cache
+		obj, isExist, err := frr.store.Get(&fv1.Function{
+			Metadata: metav1.ObjectMeta{
+				Namespace: namespace,
+				Name:      functionName,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		if !isExist {
+			return nil, fmt.Errorf("function %v does not exist", functionName)
+		}
+
+		f := obj.(*fv1.Function)
+		functionMetadataMap[f.Metadata.Name] = &f.Metadata
+		sumPrefix = sumPrefix + functionWeight
+		fnWtDistrList = append(fnWtDistrList, FunctionWeightDistribution{
+			name:      functionName,
+			weight:    functionWeight,
+			sumPrefix: sumPrefix,
+		})
+
+	}
+
+	rr := resolveResult{
+		resolveResultType:          resolveResultMultipleFunctions,
+		functionMetadataMap:        functionMetadataMap,
+		functionWtDistributionList: fnWtDistrList,
+	}
+
+	return &rr, nil
+}
+
+func (frr *functionReferenceResolver) delete(namespace string, triggerName, triggerRV string) error {
+	nfr := namespacedTriggerReference{
+		namespace:              namespace,
+		triggerName:            triggerName,
+		triggerResourceVersion: triggerRV,
+	}
+	return frr.refCache.Delete(nfr)
+}
+
+func (frr *functionReferenceResolver) copy() map[namespacedTriggerReference]resolveResult {
+	cache := make(map[namespacedTriggerReference]resolveResult)
+	for k, v := range frr.refCache.Copy() {
+		key := k.(namespacedTriggerReference)
+		val := v.(resolveResult)
+		cache[key] = val
+	}
+	return cache
+}

pkg/router/functionServiceMap.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"net/url"
+	"time"
+
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/pkg/cache"
+)
+
+type (
+	functionServiceMap struct {
+		logger *zap.Logger
+		cache  *cache.Cache // map[metadataKey]*url.URL
+	}
+
+	// metav1.ObjectMeta is not hashable, so we make a hashable copy
+	// of the subset of its fields that are identifiable.
+	metadataKey struct {
+		Name            string
+		Namespace       string
+		ResourceVersion string
+	}
+)
+
+func makeFunctionServiceMap(logger *zap.Logger, expiry time.Duration) *functionServiceMap {
+	return &functionServiceMap{
+		logger: logger.Named("function_service_map"),
+		cache:  cache.MakeCache(expiry, 0),
+	}
+}
+
+func keyFromMetadata(m *metav1.ObjectMeta) *metadataKey {
+	return &metadataKey{
+		Name:            m.Name,
+		Namespace:       m.Namespace,
+		ResourceVersion: m.ResourceVersion,
+	}
+}
+
+func (fmap *functionServiceMap) lookup(f *metav1.ObjectMeta) (*url.URL, error) {
+	mk := keyFromMetadata(f)
+	item, err := fmap.cache.Get(*mk)
+	if err != nil {
+		return nil, err
+	}
+	u := item.(*url.URL)
+	return u, nil
+}
+
+func (fmap *functionServiceMap) assign(f *metav1.ObjectMeta, serviceUrl *url.URL) {
+	mk := keyFromMetadata(f)
+	err, old := fmap.cache.Set(*mk, serviceUrl)
+	if err != nil {
+		if *serviceUrl == *(old.(*url.URL)) {
+			return
+		}
+		fmap.logger.Error("error caching service url for function with a different value", zap.Error(err))
+		// ignore error
+	}
+}
+
+func (fmap *functionServiceMap) remove(f *metav1.ObjectMeta) error {
+	mk := keyFromMetadata(f)
+	return fmap.cache.Delete(*mk)
+}

pkg/router/functionServiceMap_test.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"net/url"
+	"testing"
+
+	"go.uber.org/zap"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestFunctionServiceMap(t *testing.T) {
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	m := makeFunctionServiceMap(logger, 0)
+	fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
+	u, err := url.Parse("/foo012")
+	if err != nil {
+		t.Errorf("can't parse url")
+	}
+
+	m.assign(fn, u)
+
+	v, err := m.lookup(fn)
+	if err != nil {
+		t.Errorf("Lookup error: %v", err)
+	}
+	if *v != *u {
+		t.Errorf("Expected %#v, got %#v", u, v)
+	}
+
+	fn.Name = "bar"
+	_, err2 := m.lookup(fn)
+	if err2 == nil {
+		t.Errorf("No error on missing entry")
+	}
+}

pkg/router/httpTriggers.go

@@ -0,0 +1,376 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	executorClient "github.com/fission/fission/pkg/executor/client"
+	"github.com/fission/fission/pkg/throttler"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type HTTPTriggerSet struct {
+	*functionServiceMap
+	*mutableRouter
+
+	logger                     *zap.Logger
+	fissionClient              *crd.FissionClient
+	kubeClient                 *kubernetes.Clientset
+	executor                   *executorClient.Client
+	resolver                   *functionReferenceResolver
+	crdClient                  *rest.RESTClient
+	triggers                   []fv1.HTTPTrigger
+	triggerStore               k8sCache.Store
+	triggerController          k8sCache.Controller
+	functions                  []fv1.Function
+	funcStore                  k8sCache.Store
+	funcController             k8sCache.Controller
+	recorderSet                *RecorderSet
+	updateRouterRequestChannel chan struct{}
+	tsRoundTripperParams       *tsRoundTripperParams
+	isDebugEnv                 bool
+	svcAddrUpdateThrottler     *throttler.Throttler
+}
+
+func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient,
+	kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
+
+	httpTriggerSet := &HTTPTriggerSet{
+		logger:                     logger.Named("http_trigger_set"),
+		functionServiceMap:         fmap,
+		triggers:                   []fv1.HTTPTrigger{},
+		fissionClient:              fissionClient,
+		kubeClient:                 kubeClient,
+		executor:                   executor,
+		crdClient:                  crdClient,
+		updateRouterRequestChannel: make(chan struct{}),
+		tsRoundTripperParams:       params,
+		isDebugEnv:                 isDebugEnv,
+		svcAddrUpdateThrottler:     actionThrottler,
+	}
+	var tStore, fnStore, rStore k8sCache.Store
+	var tController, fnController k8sCache.Controller
+	var recorderSet *RecorderSet
+	if httpTriggerSet.crdClient != nil {
+		tStore, tController = httpTriggerSet.initTriggerController()
+		httpTriggerSet.triggerStore = tStore
+		httpTriggerSet.triggerController = tController
+		fnStore, fnController = httpTriggerSet.initFunctionController()
+		httpTriggerSet.funcStore = fnStore
+		httpTriggerSet.funcController = fnController
+	}
+	recorderSet = MakeRecorderSet(logger, httpTriggerSet, crdClient, rStore, frmap, trmap)
+	httpTriggerSet.recorderSet = recorderSet
+	return httpTriggerSet, tStore, fnStore
+}
+
+func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter, resolver *functionReferenceResolver) {
+	ts.resolver = resolver
+	ts.mutableRouter = mr
+	mr.updateRouter(ts.getRouter())
+
+	if ts.fissionClient == nil {
+		// Used in tests only.
+		ts.logger.Info("skipping continuous trigger updates")
+		return
+	}
+	go ts.updateRouter()
+	go ts.runWatcher(ctx, ts.funcController)
+	go ts.runWatcher(ctx, ts.triggerController)
+	if ts.recorderSet.recController != nil {
+		go ts.runWatcher(ctx, ts.recorderSet.recController)
+	} else {
+		ts.logger.Fatal("failed to run recorder controller")
+	}
+}
+
+func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func routerHealthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func (ts *HTTPTriggerSet) getRouter() *mux.Router {
+	muxRouter := mux.NewRouter()
+
+	// HTTP triggers setup by the user
+	homeHandled := false
+	for i := range ts.triggers {
+		trigger := ts.triggers[i]
+
+		// resolve function reference
+		rr, err := ts.resolver.resolve(trigger)
+		if err != nil {
+			// Unresolvable function reference. Report the error via
+			// the trigger's status.
+			go ts.updateTriggerStatusFailed(&trigger, err)
+
+			// Ignore this route and let it 404.
+			continue
+		}
+
+		var recorderName string
+		recorder, err := ts.recorderSet.triggerRecorderMap.lookup(trigger.Metadata.Name)
+		if err == nil && recorder != nil {
+			recorderName = recorder.Spec.Name
+		}
+
+		if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions {
+			// not implemented yet
+			ts.logger.Panic("resolve result type not implemented", zap.Any("type", rr.resolveResultType))
+		}
+
+		fh := &functionHandler{
+			logger:                   ts.logger.Named(trigger.Metadata.Name),
+			fmap:                     ts.functionServiceMap,
+			frmap:                    ts.recorderSet.functionRecorderMap,
+			trmap:                    ts.recorderSet.triggerRecorderMap,
+			executor:                 ts.executor,
+			httpTrigger:              &trigger,
+			functionMetadataMap:      rr.functionMetadataMap,
+			fnWeightDistributionList: rr.functionWtDistributionList,
+			tsRoundTripperParams:     ts.tsRoundTripperParams,
+			recorderName:             recorderName,
+			isDebugEnv:               ts.isDebugEnv,
+			svcAddrUpdateThrottler:   ts.svcAddrUpdateThrottler,
+		}
+
+		// The functionHandler for HTTP trigger with fn reference type "FunctionReferenceTypeFunctionName",
+		// it's function metadata is set here.
+
+		// The functionHandler For HTTP trigger with fn reference type "FunctionReferenceTypeFunctionWeights",
+		// it's function metadata is decided dynamically before proxying the request in order to support canary
+		// deployment. For more details, please check "handler" function of functionHandler.
+
+		if rr.resolveResultType == resolveResultSingleFunction {
+			for _, metadata := range fh.functionMetadataMap {
+				fh.function = metadata
+			}
+		}
+
+		ht := muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
+		ht.Methods(trigger.Spec.Method)
+		if trigger.Spec.Host != "" {
+			ht.Host(trigger.Spec.Host)
+		}
+		if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" {
+			homeHandled = true
+		}
+	}
+	if !homeHandled {
+		//
+		// This adds a no-op handler that returns 200-OK to make sure that the
+		// "GET /" request succeeds.  This route is used by GKE Ingress (and
+		// perhaps other ingress implementations) as a health check, so we don't
+		// want it to be a 404 even if the user doesn't have a function mapped to
+		// this route.
+		//
+		muxRouter.HandleFunc("/", defaultHomeHandler).Methods("GET")
+	}
+
+	// Internal triggers for each function by name. Non-http
+	// triggers route into these.
+	for _, function := range ts.functions {
+		m := function.Metadata
+
+		var recorderName string
+		recorder, err := ts.recorderSet.functionRecorderMap.lookup(m.Name)
+		if err == nil && recorder != nil {
+			recorderName = recorder.Spec.Name
+		}
+
+		fh := &functionHandler{
+			logger:                 ts.logger.Named(m.Name),
+			fmap:                   ts.functionServiceMap,
+			frmap:                  ts.recorderSet.functionRecorderMap,
+			trmap:                  ts.recorderSet.triggerRecorderMap,
+			function:               &m,
+			executor:               ts.executor,
+			tsRoundTripperParams:   ts.tsRoundTripperParams,
+			recorderName:           recorderName,
+			isDebugEnv:             ts.isDebugEnv,
+			svcAddrUpdateThrottler: ts.svcAddrUpdateThrottler,
+		}
+		muxRouter.HandleFunc(utils.UrlForFunction(function.Metadata.Name, function.Metadata.Namespace), fh.handler)
+	}
+
+	// Healthz endpoint for the router.
+	muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET")
+
+	return muxRouter
+}
+
+func (ts *HTTPTriggerSet) updateTriggerStatusFailed(ht *fv1.HTTPTrigger, err error) {
+	// TODO
+}
+
+func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(ts.crdClient, "httptriggers", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.HTTPTrigger{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				trigger := obj.(*fv1.HTTPTrigger)
+				go createIngress(ts.logger, trigger, ts.kubeClient)
+				ts.syncTriggers()
+				// Check if this trigger's function needs to be recorded
+				fnRef := trigger.Spec.FunctionReference.Name
+				recorder, err := ts.recorderSet.functionRecorderMap.lookup(fnRef)
+				if err == nil && recorder != nil {
+					if len(recorder.Spec.Triggers) == 0 {
+						ts.recorderSet.triggerRecorderMap.assign(trigger.Metadata.Name, recorder)
+					}
+				} else if err != nil {
+					ts.logger.Error("unable to lookup function in functionRecorderMap", zap.Error(err))
+				} else {
+					ts.logger.Error("unable to lookup function in functionRecorderMap")
+
+				}
+			},
+			DeleteFunc: func(obj interface{}) {
+				ts.syncTriggers()
+				trigger := obj.(*fv1.HTTPTrigger)
+				go deleteIngress(ts.logger, trigger, ts.kubeClient)
+				go ts.recorderSet.DeleteTriggerFromRecorderMap(trigger)
+			},
+			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+				oldTrigger := oldObj.(*fv1.HTTPTrigger)
+				newTrigger := newObj.(*fv1.HTTPTrigger)
+
+				if oldTrigger.Metadata.ResourceVersion == newTrigger.Metadata.ResourceVersion {
+					return
+				}
+
+				go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient)
+				ts.syncTriggers()
+			},
+		})
+	return store, controller
+}
+
+func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(ts.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				ts.syncTriggers()
+			},
+			DeleteFunc: func(obj interface{}) {
+				function := obj.(*fv1.Function)
+				ts.syncTriggers()
+				go ts.recorderSet.DeleteFunctionFromRecorderMap(function)
+			},
+			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+				oldFn := oldObj.(*fv1.Function)
+				fn := newObj.(*fv1.Function)
+
+				if oldFn.Metadata.ResourceVersion == fn.Metadata.ResourceVersion {
+					return
+				}
+
+				// update resolver function reference cache
+				for key, rr := range ts.resolver.copy() {
+					if key.namespace == fn.Metadata.Namespace &&
+						rr.functionMetadataMap[fn.Metadata.Name] != nil &&
+						rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion {
+						// invalidate resolver cache
+						ts.logger.Info("invalidating resolver cache")
+						err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
+						if err != nil {
+							ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
+						}
+
+						break
+					}
+				}
+				ts.syncTriggers()
+			},
+		})
+	return store, controller
+}
+
+func (ts *HTTPTriggerSet) initRecorderController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(ts.crdClient, "recorders", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.Recorder{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				recorder := obj.(*fv1.Recorder)
+				ts.recorderSet.newRecorder(recorder)
+			},
+			DeleteFunc: func(obj interface{}) {
+				recorder := obj.(*fv1.Recorder)
+				ts.recorderSet.disableRecorder(recorder)
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldRecorder := oldObj.(*fv1.Recorder)
+				newRecorder := newObj.(*fv1.Recorder)
+				ts.recorderSet.updateRecorder(oldRecorder, newRecorder)
+			},
+		},
+	)
+	return store, controller
+}
+
+func (ts *HTTPTriggerSet) runWatcher(ctx context.Context, controller k8sCache.Controller) {
+	go func() {
+		controller.Run(ctx.Done())
+	}()
+}
+
+func (ts *HTTPTriggerSet) syncTriggers() {
+	ts.updateRouterRequestChannel <- struct{}{}
+}
+
+func (ts *HTTPTriggerSet) updateRouter() {
+	for range ts.updateRouterRequestChannel {
+		// get triggers
+		latestTriggers := ts.triggerStore.List()
+		triggers := make([]fv1.HTTPTrigger, len(latestTriggers))
+		for _, t := range latestTriggers {
+			triggers = append(triggers, *t.(*fv1.HTTPTrigger))
+		}
+		ts.triggers = triggers
+
+		// get functions
+		latestFunctions := ts.funcStore.List()
+		functions := make([]fv1.Function, len(latestFunctions))
+		for _, f := range latestFunctions {
+			functions = append(functions, *f.(*fv1.Function))
+		}
+		ts.functions = functions
+
+		// make a new router and use it
+		ts.mutableRouter.updateRouter(ts.getRouter())
+	}
+}

pkg/router/ingress.go

@@ -0,0 +1,160 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"os"
+
+	"go.uber.org/zap"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+var podNamespace string
+
+func init() {
+	podNamespace = os.Getenv("POD_NAMESPACE")
+	if podNamespace == "" {
+		podNamespace = "fission"
+	}
+}
+
+func createIngress(logger *zap.Logger, trigger *fv1.HTTPTrigger, kubeClient *kubernetes.Clientset) {
+
+	if !trigger.Spec.CreateIngress {
+		logger.Info("skipping creation of ingress for trigger", zap.String("trigger", trigger.Metadata.Name))
+		return
+	}
+
+	_, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
+	if err == nil {
+		logger.Info("ingress for trigger exists already", zap.String("trigger", trigger.Metadata.Name))
+		return
+	}
+
+	ing := &v1beta1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: getDeployLabels(trigger),
+			Name:   trigger.Metadata.Name,
+			// The Ingress NS MUST be same as Router NS, check long discussion:
+			// https://github.com/kubernetes/kubernetes/issues/17088
+			// We need to revisit this in future, once Kubernetes supports cross namespace ingress
+			Namespace: podNamespace,
+		},
+		Spec: v1beta1.IngressSpec{
+			Rules: []v1beta1.IngressRule{
+				{
+					Host: trigger.Spec.Host,
+					IngressRuleValue: v1beta1.IngressRuleValue{
+						HTTP: &v1beta1.HTTPIngressRuleValue{
+							Paths: []v1beta1.HTTPIngressPath{
+								{
+									Backend: v1beta1.IngressBackend{
+										ServiceName: "router",
+										ServicePort: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 80,
+										},
+									},
+									Path: trigger.Spec.RelativeURL,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Create(ing)
+	if err != nil {
+		logger.Error("failed to create ingress", zap.Error(err))
+		return
+	}
+	logger.Info("created ingress successfully for trigger", zap.String("trigger", trigger.Metadata.Name))
+}
+
+func getDeployLabels(trigger *fv1.HTTPTrigger) map[string]string {
+	return map[string]string{
+		"triggerName":      trigger.Metadata.Name,
+		"functionName":     trigger.Spec.FunctionReference.Name,
+		"triggerNamespace": trigger.Metadata.Namespace,
+	}
+}
+
+func deleteIngress(logger *zap.Logger, trigger *fv1.HTTPTrigger, kubeClient *kubernetes.Clientset) {
+	if !trigger.Spec.CreateIngress {
+		return
+	}
+
+	ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
+	if err != nil {
+		logger.Error("failed to get ingress when deleting trigger", zap.Error(err), zap.String("trigger", trigger.Metadata.Name))
+	}
+
+	err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Delete(ingress.Name, &v1.DeleteOptions{})
+
+	if err != nil {
+		logger.Error("failed to delete ingress for trigger",
+			zap.Error(err),
+			zap.Any("ingress", ingress),
+			zap.String("trigger", trigger.Metadata.Name))
+	}
+
+}
+
+func updateIngress(logger *zap.Logger, oldT *fv1.HTTPTrigger, newT *fv1.HTTPTrigger, kubeClient *kubernetes.Clientset) {
+
+	if oldT.Spec.CreateIngress == false && newT.Spec.CreateIngress == true {
+		createIngress(logger, newT, kubeClient)
+		return
+	}
+
+	if newT.Spec.CreateIngress == false && oldT.Spec.CreateIngress == true {
+		deleteIngress(logger, oldT, kubeClient)
+		return
+	}
+
+	if newT.Spec.Host != oldT.Spec.Host || newT.Spec.RelativeURL != oldT.Spec.RelativeURL {
+		logger.Info("updating ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
+		ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(oldT.Metadata.Name, v1.GetOptions{})
+		if err != nil {
+			logger.Error("failed to get ingress when updating trigger",
+				zap.Error(err),
+				zap.String("trigger", oldT.Metadata.Name))
+		}
+
+		if newT.Spec.Host != oldT.Spec.Host {
+			ingress.Spec.Rules[0].Host = newT.Spec.Host
+		}
+
+		if newT.Spec.RelativeURL != oldT.Spec.RelativeURL {
+			ingress.Spec.Rules[0].HTTP.Paths[0].Path = newT.Spec.RelativeURL
+		}
+
+		_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Update(ingress)
+		if err != nil {
+			logger.Error("failed to update ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
+		}
+	}
+
+}

pkg/router/metrics.go

@@ -0,0 +1,144 @@
+package router
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var globalFunctionCallCount uint64
+
+type (
+	// functionLabels is the set of metrics labels that relate to
+	// functions.
+	//
+	// cached indicates whether or not the function call hit the
+	// cache in this service.
+	//
+	// namespace and name are the metadata of the function.
+	functionLabels struct {
+		cached    bool
+		namespace string
+		name      string
+	}
+
+	// httpLabels is the set of metrics labels that relate to HTTP
+	// requests.
+	//
+	// host is the host that the HTTP request was made to
+	// path is the relative URL of the request
+	// method is the HTTP method ("GET", "POST", ...)
+	// code is the HTTP status code
+	httpLabels struct {
+		host   string
+		path   string
+		method string
+		code   int
+	}
+)
+
+var (
+	metricAddr = ":8080"
+
+	// function + http labels as strings
+	labelsStrings = []string{"cached", "namespace", "name", "host", "path", "method", "code"}
+
+	// Function http calls count
+	// cached: true | false, is this function service address cached locally
+	// namespace: function namespace
+	// name: function name
+	// code: http status code
+	// path: the client call the function on which http path
+	// method: the function's http method
+	functionCalls = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "fission_function_calls_total",
+			Help: "Count of Fission function calls",
+		},
+		labelsStrings,
+	)
+	functionCallErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "fission_function_errors_total",
+			Help: "Count of Fission function errors",
+		},
+		labelsStrings,
+	)
+	functionCallDuration = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_function_duration_seconds",
+			Help:       "Runtime duration of the Fission function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labelsStrings,
+	)
+	functionCallOverhead = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_function_overhead_seconds",
+			Help:       "The function call delay caused by fission.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labelsStrings,
+	)
+	functionCallResponseSize = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_function_response_size_bytes",
+			Help:       "The response size of the http call to target function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		labelsStrings,
+	)
+)
+
+func init() {
+	prometheus.MustRegister(functionCalls)
+	prometheus.MustRegister(functionCallErrors)
+	prometheus.MustRegister(functionCallDuration)
+	prometheus.MustRegister(functionCallOverhead)
+	prometheus.MustRegister(functionCallResponseSize)
+}
+
+func labelsToStrings(f *functionLabels, h *httpLabels) []string {
+	var cached string
+	if f.cached {
+		cached = "true"
+	} else {
+		cached = "false"
+	}
+	return []string{
+		cached,
+		f.namespace,
+		f.name,
+		h.host,
+		h.path,
+		h.method,
+		fmt.Sprint(h.code),
+	}
+}
+
+func functionCallCompleted(f *functionLabels, h *httpLabels, overhead, duration time.Duration, respSize int64) {
+	atomic.AddUint64(&globalFunctionCallCount, 1)
+
+	l := labelsToStrings(f, h)
+
+	// overhead: time from request ingress into router upto proxing into function pod
+	functionCallOverhead.WithLabelValues(l...).Observe(float64(overhead.Nanoseconds()) / 1e9)
+
+	// total function call counter
+	functionCalls.WithLabelValues(l...).Inc()
+
+	// error counter
+	if h.code >= 400 {
+		functionCallErrors.WithLabelValues(l...).Inc()
+	}
+
+	// duration summary
+	functionCallDuration.WithLabelValues(l...).Observe(float64(duration.Nanoseconds()) / 1e9)
+
+	// Response size.  -1 means the size unknown, in which case we don't report it.
+	if respSize != -1 {
+		functionCallResponseSize.WithLabelValues(l...).Observe(float64(respSize))
+	}
+}

pkg/router/mutablemux.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"net/http"
+	"sync/atomic"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+//
+// mutableRouter wraps the mux router, and allows the router to be
+// atomically changed.
+//
+
+type mutableRouter struct {
+	logger *zap.Logger
+	router atomic.Value // mux.Router
+}
+
+func NewMutableRouter(logger *zap.Logger, handler *mux.Router) *mutableRouter {
+	mr := mutableRouter{
+		logger: logger.Named("mutable_router"),
+	}
+	mr.router.Store(handler)
+	return &mr
+}
+
+func (mr *mutableRouter) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
+	// Atomically grab the underlying mux router and call it.
+	routerValue := mr.router.Load()
+	router, ok := routerValue.(*mux.Router)
+	if !ok {
+		mr.logger.Panic("invalid router type")
+	}
+	router.ServeHTTP(responseWriter, request)
+}
+
+func (mr *mutableRouter) updateRouter(newHandler *mux.Router) {
+	mr.router.Store(newHandler)
+}

pkg/router/mutablemux_test.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"log"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+)
+
+func OldHandler(responseWriter http.ResponseWriter, request *http.Request) {
+	responseWriter.Write([]byte("old handler"))
+}
+func NewHandler(responseWriter http.ResponseWriter, request *http.Request) {
+	responseWriter.Write([]byte("new handler"))
+}
+
+func verifyRequest(expectedResponse string) {
+	targetUrl := "http://localhost:3333"
+	testRequest(targetUrl, expectedResponse)
+}
+
+func startServer(mr *mutableRouter) {
+	http.ListenAndServe(":3333", mr)
+}
+
+func spamServer(quit chan bool) {
+	i := 0
+	for {
+		select {
+		case <-quit:
+			break
+		default:
+			i = i + 1
+			resp, err := http.Get("http://localhost:3333")
+			if err != nil {
+				log.Panicf("failed to make get request %v: %v", i, err)
+			}
+			resp.Body.Close()
+		}
+	}
+}
+
+func TestMutableMux(t *testing.T) {
+	// make a simple mutable router
+	log.Print("Create mutable router")
+	muxRouter := mux.NewRouter()
+	muxRouter.HandleFunc("/", OldHandler)
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	mr := NewMutableRouter(logger, muxRouter)
+
+	// start http server
+	log.Print("Start http server")
+	go startServer(mr)
+
+	// continuously make requests, panic if any fails
+	time.Sleep(100 * time.Millisecond)
+	q := make(chan bool)
+	go spamServer(q)
+
+	time.Sleep(5 * time.Millisecond)
+
+	// connect and verify old handler
+	log.Print("Verify old handler")
+	verifyRequest("old handler")
+
+	// change the muxer
+	log.Print("Change mux router")
+	newMuxRouter := mux.NewRouter()
+	newMuxRouter.HandleFunc("/", NewHandler)
+	mr.updateRouter(newMuxRouter)
+
+	// connect and verify the new handler
+	log.Print("Verify new handler")
+	verifyRequest("new handler")
+
+	q <- true
+	time.Sleep(100 * time.Millisecond)
+}

pkg/router/recorderController.go

@@ -0,0 +1,135 @@
+package router
+
+import (
+	"go.uber.org/zap"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type RecorderSet struct {
+	logger *zap.Logger
+
+	httpTriggerSet *HTTPTriggerSet
+
+	crdClient *rest.RESTClient
+
+	recStore      k8sCache.Store
+	recController k8sCache.Controller
+
+	functionRecorderMap *functionRecorderMap
+	triggerRecorderMap  *triggerRecorderMap
+}
+
+func MakeRecorderSet(logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet {
+	recorderSet := &RecorderSet{
+		logger:              logger.Named("recorder_set"),
+		httpTriggerSet:      httpTriggerSet,
+		crdClient:           crdClient,
+		recStore:            rStore,
+		functionRecorderMap: frmap,
+		triggerRecorderMap:  trmap,
+	}
+	recorderSet.recStore, recorderSet.recController = httpTriggerSet.initRecorderController()
+	return recorderSet
+}
+
+// All new recorders are by default enabled
+func (rs *RecorderSet) newRecorder(r *fv1.Recorder) {
+	function := r.Spec.Function
+	triggers := r.Spec.Triggers
+
+	// If triggers are not explicitly specified during the creation of this recorder,
+	// keep track of those associated with the function specified [implicitly added triggers]
+	needTrackByFunction := len(triggers) == 0
+
+	rs.functionRecorderMap.assign(function, r)
+
+	if needTrackByFunction {
+		for _, t := range rs.httpTriggerSet.triggerStore.List() {
+			trigger := *t.(*fv1.HTTPTrigger)
+			if trigger.Spec.FunctionReference.Name == function {
+				rs.triggerRecorderMap.assign(trigger.Metadata.Name, r)
+			}
+		}
+	} else {
+		for _, trigger := range triggers {
+			rs.triggerRecorderMap.assign(trigger, r)
+		}
+	}
+
+	rs.httpTriggerSet.syncTriggers()
+}
+
+// TODO: Delete or disable?
+func (rs *RecorderSet) disableRecorder(r *fv1.Recorder) {
+	function := r.Spec.Function
+	triggers := r.Spec.Triggers
+
+	rs.logger.Info("disabling recorder",
+		zap.String("recorder", r.Metadata.Name),
+		zap.String("function", function))
+
+	// Account for function
+	err := rs.functionRecorderMap.remove(function)
+	if err != nil {
+		rs.logger.Error("error disabling recorder (failed to remove function from functionRecorderMap)",
+			zap.Error(err),
+			zap.String("recorder", r.Metadata.Name),
+			zap.String("function", function))
+	}
+
+	// Account for explicitly added triggers
+	if len(triggers) != 0 {
+		for _, trigger := range triggers {
+			err := rs.triggerRecorderMap.remove(trigger)
+			if err != nil {
+				rs.logger.Error("error disabling recorder (failed to remove triggers from triggerRecorderMap)",
+					zap.Error(err),
+					zap.String("recorder", r.Metadata.Name),
+					zap.String("function", function),
+					zap.String("trigger", trigger))
+			}
+		}
+	} else {
+		// Account for implicitly added triggers
+		for _, t := range rs.httpTriggerSet.triggerStore.List() {
+			trigger := *t.(*fv1.HTTPTrigger)
+			if trigger.Spec.FunctionReference.Name == function {
+				err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
+				if err != nil {
+					rs.logger.Error("failed to remove trigger from triggerRecorderMap",
+						zap.Error(err),
+						zap.String("recorder", r.Metadata.Name),
+						zap.String("function", function),
+						zap.String("trigger", trigger.Metadata.Name))
+				}
+			}
+		}
+	}
+
+	rs.httpTriggerSet.syncTriggers()
+}
+
+func (rs *RecorderSet) updateRecorder(old *fv1.Recorder, newer *fv1.Recorder) {
+	if newer.Spec.Enabled == true {
+		rs.newRecorder(newer) // TODO: Test this
+	} else {
+		rs.disableRecorder(old)
+	}
+}
+
+func (rs *RecorderSet) DeleteTriggerFromRecorderMap(trigger *fv1.HTTPTrigger) {
+	err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
+	if err != nil {
+		rs.logger.Error("failed to remove trigger from triggerRecorderMap", zap.Error(err))
+	}
+}
+
+func (rs *RecorderSet) DeleteFunctionFromRecorderMap(function *fv1.Function) {
+	err := rs.functionRecorderMap.remove(function.Metadata.Name)
+	if err != nil {
+		rs.logger.Error("failed to remove function from functionRecorderMap", zap.Error(err))
+	}
+}

pkg/router/router.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+
+This is the Fission Router package.
+
+Its job is to:
+
+  1. Keep track of HTTP triggers and their mappings to functions
+
+     Use the controller API to get and watch this state.
+
+  2. Given a function, get a reference to a routable function run service
+
+     Use the ContainerPoolManager API to get a service backed by one
+     or more function run containers.  The container(s) backing the
+     service may be newly created, or they might be reused.  The only
+     requirement is that one or more containers backs the service.
+
+  3. Forward the request to the service, and send the response back.
+
+     Plain ol HTTP.
+
+*/
+
+package router
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opencensus.io/plugin/ochttp"
+	"go.opencensus.io/trace"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/crd"
+	executorClient "github.com/fission/fission/pkg/executor/client"
+	"github.com/fission/fission/pkg/throttler"
+	"github.com/fission/fission/pkg/utils"
+)
+
+// request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url
+
+// request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url
+
+func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter {
+	muxRouter := mux.NewRouter()
+	mr := NewMutableRouter(logger, muxRouter)
+	muxRouter.Use(utils.LoggingMiddleware(logger))
+	httpTriggerSet.subscribeRouter(ctx, mr, resolver)
+	return mr
+}
+
+func serve(ctx context.Context, logger *zap.Logger, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) {
+	mr := router(ctx, logger, httpTriggerSet, resolver)
+	url := fmt.Sprintf(":%v", port)
+	http.ListenAndServe(url, &ochttp.Handler{
+		Handler: mr,
+		StartOptions: trace.StartOptions{
+			Sampler: trace.AlwaysSample(),
+		},
+	})
+}
+
+func serveMetric(logger *zap.Logger) {
+	// Expose the registered metrics via HTTP.
+	http.Handle("/metrics", promhttp.Handler())
+	err := http.ListenAndServe(metricAddr, nil)
+
+	logger.Fatal("done listening on metrics endpoint", zap.Error(err))
+}
+
+func Start(logger *zap.Logger, port int, executorUrl string) {
+	// setup a signal handler for SIGTERM
+	utils.SetupStackTraceHandler()
+
+	_ = MakeAnalytics("")
+
+	fmap := makeFunctionServiceMap(logger, time.Minute)
+
+	frmap := makeFunctionRecorderMap(logger, time.Minute)
+
+	trmap := makeTriggerRecorderMap(logger, time.Minute)
+
+	fissionClient, kubeClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		logger.Fatal("error connecting to kubernetes API", zap.Error(err))
+	}
+
+	err = fissionClient.WaitForCRDs()
+	if err != nil {
+		logger.Fatal("error waiting for CRDs", zap.Error(err))
+	}
+
+	restClient := fissionClient.GetCrdClient()
+
+	executor := executorClient.MakeClient(logger, executorUrl)
+
+	timeoutStr := os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT")
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		logger.Fatal("failed to parse timeout duration from 'ROUTER_ROUND_TRIP_TIMEOUT'",
+			zap.Error(err),
+			zap.String("value", timeoutStr))
+	}
+
+	timeoutExponentStr := os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT")
+	timeoutExponent, err := strconv.Atoi(timeoutExponentStr)
+	if err != nil {
+		logger.Fatal("failed to parse timeout exponent from 'ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT'",
+			zap.Error(err),
+			zap.String("value", timeoutExponentStr))
+	}
+
+	keepAliveStr := os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME")
+	keepAlive, err := time.ParseDuration(keepAliveStr)
+	if err != nil {
+		logger.Fatal("failed to parse keep alive duration from 'ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME'",
+			zap.Error(err),
+			zap.String("value", keepAliveStr))
+	}
+
+	maxRetriesStr := os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES")
+	maxRetries, err := strconv.Atoi(maxRetriesStr)
+	if err != nil {
+		logger.Fatal("failed to parse max retries from 'ROUTER_ROUND_TRIP_MAX_RETRIES'",
+			zap.Error(err),
+			zap.String("value", maxRetriesStr))
+	}
+
+	isDebugEnvStr := os.Getenv("DEBUG_ENV")
+	isDebugEnv, err := strconv.ParseBool(isDebugEnvStr)
+	if err != nil {
+		logger.Fatal("failed to parse debug env from 'DEBUG_ENV'",
+			zap.Error(err),
+			zap.String("value", isDebugEnvStr))
+	}
+
+	// svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address
+	svcAddrRetryCountStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES")
+	svcAddrRetryCount, err := strconv.Atoi(svcAddrRetryCountStr)
+	if err != nil {
+		svcAddrRetryCount = 5
+		logger.Info("failed to parse service address retry count from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES' - set to the default value",
+			zap.Error(err),
+			zap.String("value", svcAddrRetryCountStr),
+			zap.Int("default", svcAddrRetryCount))
+	}
+
+	// svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry.
+	// If the update process cannot be done within the timeout window, consider it failed.
+	svcAddrUpdateTimeoutStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT")
+	svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT"))
+	if err != nil {
+		svcAddrUpdateTimeout = 30 * time.Second
+		logger.Info("failed to parse service address update timeout duration from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT' - set to the default value",
+			zap.Error(err),
+			zap.String("value", svcAddrUpdateTimeoutStr),
+			zap.Duration("default", svcAddrUpdateTimeout))
+	}
+
+	triggers, _, fnStore := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
+		timeout:           timeout,
+		timeoutExponent:   timeoutExponent,
+		keepAlive:         keepAlive,
+		maxRetries:        maxRetries,
+		svcAddrRetryCount: svcAddrRetryCount,
+	}, isDebugEnv, throttler.MakeThrottler(svcAddrUpdateTimeout))
+
+	resolver := makeFunctionReferenceResolver(fnStore)
+
+	go serveMetric(logger)
+
+	logger.Info("starting router", zap.Int("port", port))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	serve(ctx, logger, port, triggers, resolver)
+}

pkg/router/router_test.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/throttler"
+)
+
+func TestRouter(t *testing.T) {
+	// metadata for a fake function
+	fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
+
+	// and a reference to it
+	fr := fv1.FunctionReference{
+		Type: types.FunctionReferenceTypeFunctionName,
+		Name: fn.Name,
+	}
+
+	// start a fake service
+	testResponseString := "hi"
+	testServiceUrl := createBackendService(testResponseString)
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	// set up the cache with this fake service
+	fmap := makeFunctionServiceMap(logger, 0)
+	fmap.assign(fn, testServiceUrl)
+
+	frmap := makeFunctionRecorderMap(logger, time.Minute)
+
+	trmap := makeTriggerRecorderMap(logger, time.Minute)
+
+	// HTTP trigger set with a trigger for this function
+	triggers, _, _ := makeHTTPTriggerSet(logger, fmap, frmap, trmap, nil, nil, nil, nil,
+		&tsRoundTripperParams{
+			timeout:         50 * time.Millisecond,
+			timeoutExponent: 2,
+			keepAlive:       30 * time.Second,
+			maxRetries:      10,
+		}, false, throttler.MakeThrottler(30*time.Second))
+	triggerUrl := "/foo"
+	triggers.triggers = append(triggers.triggers,
+		fv1.HTTPTrigger{
+			Metadata: metav1.ObjectMeta{
+				Name:            "xxx",
+				Namespace:       metav1.NamespaceDefault,
+				ResourceVersion: "1234",
+			},
+			Spec: fv1.HTTPTriggerSpec{
+				RelativeURL:       triggerUrl,
+				FunctionReference: fr,
+				Method:            "GET",
+			},
+		})
+
+	// set up the resolver's cache for this function
+	frr := makeFunctionReferenceResolver(nil)
+	nfr := namespacedTriggerReference{
+		namespace:              metav1.NamespaceDefault,
+		triggerName:            "xxx",
+		triggerResourceVersion: "1234",
+	}
+
+	fnMetaMap := make(map[string]*metav1.ObjectMeta, 1)
+	fnMetaMap[fn.Name] = fn
+
+	rr := resolveResult{
+		resolveResultType:   resolveResultSingleFunction,
+		functionMetadataMap: fnMetaMap,
+	}
+	frr.refCache.Set(nfr, rr)
+
+	// run the router
+	port := 4242
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go serve(ctx, logger, port, triggers, frr)
+	time.Sleep(100 * time.Millisecond)
+
+	// hit the router
+	testUrl := fmt.Sprintf("http://localhost:%v%v", port, triggerUrl)
+	testRequest(testUrl, testResponseString)
+}

pkg/router/triggerRecorderMap.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package router
+
+import (
+	"time"
+
+	"go.uber.org/zap"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type (
+	triggerRecorderMap struct {
+		logger *zap.Logger
+		cache  *cache.Cache // map[string]*fv1.Recorder
+	}
+)
+
+func makeTriggerRecorderMap(logger *zap.Logger, expiry time.Duration) *triggerRecorderMap {
+	return &triggerRecorderMap{
+		logger: logger.Named("trigger_recorder_map"),
+		cache:  cache.MakeCache(expiry, 0),
+	}
+}
+
+func (trmap *triggerRecorderMap) lookup(trigger string) (*fv1.Recorder, error) {
+	item, err := trmap.cache.Get(trigger)
+	if err != nil {
+		return nil, err
+	}
+	u := item.(*fv1.Recorder)
+	return u, nil
+}
+
+func (trmap *triggerRecorderMap) assign(trigger string, recorder *fv1.Recorder) {
+	err, _ := trmap.cache.Set(trigger, recorder)
+	if err != nil {
+		if e, ok := err.(ferror.Error); ok && e.Code == ferror.ErrorNameExists {
+			return
+		}
+		trmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
+	}
+
+}
+
+func (trmap *triggerRecorderMap) remove(trigger string) error {
+	return trmap.cache.Delete(trigger)
+}

pkg/router/util.go

@@ -0,0 +1,29 @@
+package router
+
+import (
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	HEADERS_FISSION_FUNCTION_PREFIX = "Fission-Function"
+)
+
+func MetadataToHeaders(prefix string, meta *metav1.ObjectMeta, request *http.Request) {
+	request.Header.Set(fmt.Sprintf("X-%s-Uid", prefix), string(meta.UID))
+	request.Header.Set(fmt.Sprintf("X-%s-Name", prefix), meta.Name)
+	request.Header.Set(fmt.Sprintf("X-%s-Namespace", prefix), meta.Namespace)
+	request.Header.Set(fmt.Sprintf("X-%s-ResourceVersion", prefix), meta.ResourceVersion)
+}
+
+func HeadersToMetadata(prefix string, headers http.Header) *metav1.ObjectMeta {
+	return &metav1.ObjectMeta{
+		Name:            headers.Get(fmt.Sprintf("X-%s-Name", prefix)),
+		UID:             types.UID(headers.Get(fmt.Sprintf("X-%s-Uid", prefix))),
+		Namespace:       headers.Get(fmt.Sprintf("X-%s-Namespace", prefix)),
+		ResourceVersion: headers.Get(fmt.Sprintf("X-%s-ResourceVersion", prefix)),
+	}
+}

pkg/router/util_test.go

@@ -0,0 +1,36 @@
+package router
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+func testRequest(targetUrl string, expectedResponse string) {
+	resp, err := http.Get(targetUrl)
+	if err != nil {
+		log.Panicf("failed to make get request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		log.Panicf("response status: %v", resp.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Panic("failed to read response")
+	}
+
+	bodyStr := string(body)
+	log.Printf("Server responded with %v", bodyStr)
+	if bodyStr != expectedResponse {
+		log.Panic("Unexpected response")
+	}
+}
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("Error: %v", err)
+	}
+}