Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Round Tripper of Fission Router: parameters be configurable (#713)

Browse Source 
The Round Tripper parameters of timeout, keep alive time and Max retry
times is configured in router system environment.

And they are set to Round Tripper when router service initializing.

This setup new nested structure `tsRoundTripperParams` to transfer them.
This commit is contained in:
xiekeyang
2018-07-12 12:08:52 -07:00
committed by Soam Vasani
parent 2982efebb8
commit 3707b95edb
7 changed files with 103 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/fission-all/templates/deployment.yaml
+8
View File
@@ -175,6 +175,14 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: ROUTER_ROUND_TRIP_TIMEOUT
value: {{ .Values.routerRoundTripTimeout | default "50ms" | quote }}
- name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT
value: {{ .Values.routerRoundTripTimeoutExponent | default 2 | quote }}
- name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME
value: {{ .Values.routerRoundTripKeepAliveTime | default "30s" | quote }}
- name: ROUTER_ROUND_TRIP_MAX_RETRIES
value: {{ .Values.routerRoundTripMaxRetries | default 10 | quote }}
readinessProbe:
httpGet:
path: "/router-healthz"
charts/fission-core/templates/deployment.yaml
+8
View File
@@ -172,6 +172,14 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: ROUTER_ROUND_TRIP_TIMEOUT
value: {{ .Values.routerRoundTripTimeout | default "50ms" | quote }}
- name: ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT
value: {{ .Values.routerRoundTripTimeoutExponent | default 2 | quote }}
- name: ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME
value: {{ .Values.routerRoundTripKeepAliveTime | default "30s" | quote }}
- name: ROUTER_ROUND_TRIP_MAX_RETRIES
value: {{ .Values.routerRoundTripMaxRetries | default 10 | quote }}
readinessProbe:
httpGet:
path: "/router-healthz"
router/functionHandler.go
+24 -17
View File
@@ -33,18 +33,24 @@ import (
executorClient "github.com/fission/fission/executor/client"
)
type tsRoundTripperParams struct {
timeout time.Duration
timeoutExponent int
keepAlive time.Duration
maxRetries int
}
type functionHandler struct {
fmap *functionServiceMap
executor *executorClient.Client
function *metav1.ObjectMeta
httpTrigger *crd.HTTPTrigger
fmap *functionServiceMap
executor *executorClient.Client
function *metav1.ObjectMeta
httpTrigger *crd.HTTPTrigger
tsRoundTripperParams *tsRoundTripperParams
}
// A layer on top of http.DefaultTransport, with retries.
type RetryingRoundTripper struct {
maxRetries int
initialTimeout time.Duration
funcHandler *functionHandler
funcHandler *functionHandler
}
// RoundTrip is a custom transport with retries for http requests that forwards the request to the right serviceUrl, obtained
@@ -92,7 +98,6 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
}
// set the timeout for transport context
timeout := roundTripper.initialTimeout
transport := http.DefaultTransport.(*http.Transport)
// Disables caching, Please refer to issue and specifically comment: https://github.com/fission/fission/issues/723#issuecomment-398781995
transport.DisableKeepAlives = true
@@ -104,7 +109,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
needExecutor = true
}
for i := 0; i < roundTripper.maxRetries-1; i++ {
executingTimeout := roundTripper.funcHandler.tsRoundTripperParams.timeout
for i := 0; i < roundTripper.funcHandler.tsRoundTripperParams.maxRetries-1; i++ {
if needExecutor {
log.Printf("Calling getServiceForFunction for function: %s", roundTripper.funcHandler.function.Name)
@@ -149,8 +156,8 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// over-riding default settings.
transport.DialContext = (&net.Dialer{
Timeout: timeout,
KeepAlive: 30 * time.Second,
Timeout: executingTimeout,
KeepAlive: roundTripper.funcHandler.tsRoundTripperParams.keepAlive,
}).DialContext
overhead := time.Since(startTime)
@@ -183,9 +190,11 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// just retry after backing off for timeout period.
if serviceUrlFromExecutor {
log.Printf("request to %s errored out. backing off for %v before retrying",
req.URL.Host, timeout)
timeout *= time.Duration(2)
time.Sleep(timeout)
req.URL.Host, executingTimeout)
executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent)
time.Sleep(executingTimeout)
needExecutor = false
continue
} else {
@@ -231,9 +240,7 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
proxy := &httputil.ReverseProxy{
Director: director,
Transport: &RetryingRoundTripper{
initialTimeout: 50 * time.Millisecond,
maxRetries: 10,
funcHandler: fh,
funcHandler: fh,
},
}
router/functionHandler_test.go
+7
View File
@@ -22,6 +22,7 @@ import (
"net/http/httptest"
"net/url"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -56,6 +57,12 @@ func TestFunctionProxying(t *testing.T) {
fh := &functionHandler{fmap: fmap,
function: fn,
tsRoundTripperParams: &tsRoundTripperParams{
timeout: 50 * time.Millisecond,
timeoutExponent: 2,
keepAlive: 30 * time.Second,
maxRetries: 10,
},
}
functionHandlerServer := httptest.NewServer(http.HandlerFunc(fh.handler))
fhURL := functionHandlerServer.URL
router/httpTriggers.go
+19 -14
View File
@@ -49,17 +49,20 @@ type HTTPTriggerSet struct {
functions []crd.Function
funcStore k8sCache.Store
funcController k8sCache.Controller
tsRoundTripperParams *tsRoundTripperParams
}
func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *crd.FissionClient,
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
httpTriggerSet := &HTTPTriggerSet{
functionServiceMap: fmap,
triggers: []crd.HTTPTrigger{},
fissionClient: fissionClient,
kubeClient: kubeClient,
executor: executor,
crdClient: crdClient,
functionServiceMap: fmap,
triggers: []crd.HTTPTrigger{},
fissionClient: fissionClient,
kubeClient: kubeClient,
executor: executor,
crdClient: crdClient,
tsRoundTripperParams: params,
}
var tStore, fnStore k8sCache.Store
var tController, fnController k8sCache.Controller
@@ -121,10 +124,11 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
}
fh := &functionHandler{
fmap: ts.functionServiceMap,
function: rr.functionMetadata,
executor: ts.executor,
httpTrigger: &trigger,
fmap: ts.functionServiceMap,
function: rr.functionMetadata,
executor: ts.executor,
httpTrigger: &trigger,
tsRoundTripperParams: ts.tsRoundTripperParams,
}
ht := muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
@@ -152,9 +156,10 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
for _, function := range ts.functions {
m := function.Metadata
fh := &functionHandler{
fmap: ts.functionServiceMap,
function: &m,
executor: ts.executor,
fmap: ts.functionServiceMap,
function: &m,
executor: ts.executor,
tsRoundTripperParams: ts.tsRoundTripperParams,
}
muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name, function.Metadata.Namespace), fh.handler)
}
router/router.go
+30 -1
View File
@@ -44,6 +44,8 @@ import (
"fmt"
"log"
"net/http"
"os"
"strconv"
"time"
"github.com/gorilla/mux"
@@ -97,7 +99,34 @@ func Start(port int, executorUrl string) {
restClient := fissionClient.GetCrdClient()
executor := executorClient.MakeClient(executorUrl)
triggers, _, fnStore := makeHTTPTriggerSet(fmap, fissionClient, kubeClient, executor, restClient)
timeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT"))
if err != nil {
log.Fatalf("Failed to parse timeout: %v", err)
}
timeoutExponent, err := strconv.Atoi(os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT"))
if err != nil {
log.Fatalf("Failed to parse timeout exponent: %v", err)
}
keepAlive, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME"))
if err != nil {
log.Fatalf("Failed to parse keep alive time: %v", err)
}
maxRetries, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES"))
if err != nil {
log.Fatalf("Failed to parse max retry times: %v", err)
}
triggers, _, fnStore := makeHTTPTriggerSet(fmap, fissionClient, kubeClient, executor, restClient,
&tsRoundTripperParams{
timeout: timeout,
timeoutExponent: timeoutExponent,
keepAlive: keepAlive,
maxRetries: maxRetries,
})
resolver := makeFunctionReferenceResolver(fnStore)
go serveMetric()
router/router_test.go
+7 -1
View File
@@ -59,7 +59,13 @@ func TestRouter(t *testing.T) {
frr.refCache.Set(nfr, rr)
// HTTP trigger set with a trigger for this function
triggers, _, _ := makeHTTPTriggerSet(fmap, nil, nil, nil, nil)
triggers, _, _ := makeHTTPTriggerSet(fmap, nil, nil, nil, nil,
&tsRoundTripperParams{
timeout: 50 * time.Millisecond,
timeoutExponent: 2,
keepAlive: 30 * time.Second,
maxRetries: 10,
})
triggerUrl := "/foo"
triggers.triggers = append(triggers.triggers,
crd.HTTPTrigger{