Optimize function latency when cache expired/invalid under high concurrency (#856)

* Optimize router response time by adding update lock

In 0.9.2, the router sends multiple GetServiceForFunction requests to executor
to get the service URL. However, the response time of executor will increase
under high-concurrency situation due to too many requests are waiting for
processing.

To solve the problem, an update lock was added to the router. All of goroutines belongs
to the same function need to grab the update lock before sending the request.
Only the first goroutine which gets the update lock is allowed to send request.
In this way, we reduce the burden of executor and lower the failure rate.

router/router.go

@@ -129,12 +129,28 @@ func Start(port int, executorUrl string) {
 		log.Fatalf("Failed to parse DEBUG_ENV: %v", err)
 	}
 
+	// svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address
+	svcAddrRetryCount, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES"))
+	if err != nil {
+		svcAddrRetryCount = 5
+		log.Printf("Failed to parse svc address retry conunt, set it to default value(5): %v", err)
+	}
+
+	// svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry.
+	// If the update process cannot be done within the timeout window, consider it failed.
+	svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT"))
+	if err != nil {
+		svcAddrUpdateTimeout = 30 * time.Second
+		log.Printf("Failed to parse svc address update timeout, set it to default value(30): %v", err)
+	}
+
 	triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
-		timeout:         timeout,
-		timeoutExponent: timeoutExponent,
-		keepAlive:       keepAlive,
-		maxRetries:      maxRetries,
-	}, isDebugEnv)
+		timeout:           timeout,
+		timeoutExponent:   timeoutExponent,
+		keepAlive:         keepAlive,
+		maxRetries:        maxRetries,
+		svcAddrRetryCount: svcAddrRetryCount,
+	}, isDebugEnv, MakeUpdateLocks(svcAddrUpdateTimeout))
 
 	resolver := makeFunctionReferenceResolver(fnStore)