Fission meets OpenTelemetry (#2157)

* add opentracing section and otelCollectorEndpoint
* initialize OTLP exporter
* pkg/controller: changes for context propagation
* pkg/executor: changes for context propagation
* pkg/fetcher: changes for context propagation
* pkg/router: changes for context propagation
* pkg/storagesvc: changes for context propagation
* set no default value for otel collector endpoint
* update readme and add notes to charts
* move common code to pkg/utils/otel
* adding fn and env as attributes
* don't use otelhttp transport for websocket
* URL ignore with common filter UrlsToIgnore

Note: The web socket example does not work when using OTEL HTTP. Here is an issue related to that on open-telemetry/opentelemetry-js-contrib.

Signed-off-by: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/router/functionHandler.go

@@ -32,6 +32,8 @@ import (
 
 	"github.com/pkg/errors"
 	"go.opencensus.io/plugin/ochttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 
@@ -40,8 +42,10 @@ import (
 	ferror "github.com/fission/fission/pkg/error"
 	"github.com/fission/fission/pkg/error/network"
 	executorClient "github.com/fission/fission/pkg/executor/client"
+	"github.com/fission/fission/pkg/router/util"
 	"github.com/fission/fission/pkg/throttler"
 	"github.com/fission/fission/pkg/utils"
+	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
 
 const (
@@ -66,6 +70,7 @@ type (
 		svcAddrUpdateThrottler   *throttler.Throttler
 		functionTimeoutMap       map[k8stypes.UID]int
 		unTapServiceTimeout      time.Duration
+		openTracingEnabled       bool
 	}
 
 	tsRoundTripperParams struct {
@@ -154,10 +159,11 @@ func (w *fakeCloseReadCloser) RealClose() error {
 // Earlier, GetServiceForFunction was called inside handler function and fission explicitly set http status code to 500
 // if it returned an error.
 func (roundTripper *RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	ctx := req.Context()
+
 	// set the timeout for transport context
 	roundTripper.addForwardedHostHeader(req)
 	transport := roundTripper.getDefaultTransport()
-	ocRoundTripper := &ochttp.Transport{Base: transport}
 
 	executingTimeout := roundTripper.funcHandler.tsRoundTripperParams.timeout
 
@@ -220,7 +226,7 @@ func (roundTripper *RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Re
 		// trying to get new service url from cache/executor.
 		if retryCounter == 0 {
 			// get function service url from cache or executor
-			roundTripper.serviceURL, roundTripper.urlFromCache, err = roundTripper.funcHandler.getServiceEntry()
+			roundTripper.serviceURL, roundTripper.urlFromCache, err = roundTripper.funcHandler.getServiceEntry(ctx)
 			if err != nil {
 				// We might want a specific error code or header for fission failures as opposed to
 				// user function bugs.
@@ -249,9 +255,9 @@ func (roundTripper *RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Re
 				continue
 			}
 			if roundTripper.funcHandler.function.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypePoolmgr {
-				defer func(fn *fv1.Function, serviceURL *url.URL) {
+				defer func(ctx context.Context, fn *fv1.Function, serviceURL *url.URL) {
 					go roundTripper.funcHandler.unTapService(fn, serviceURL) //nolint errcheck
-				}(roundTripper.funcHandler.function, roundTripper.serviceURL)
+				}(ctx, roundTripper.funcHandler.function, roundTripper.serviceURL)
 			}
 
 			// modify the request to reflect the service url
@@ -309,8 +315,23 @@ func (roundTripper *RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Re
 			dumpReqFunc(newReq)
 		}
 
+		// The otelhttp.NewTransport() does not work with WebSocket.
+		// This is probably because it modifies the response body.
+		// Until we find a better solution to handle websocket requests, we will continue to
+		// use ochttp.Transport(). We check if the request isWebsocketRequest() and use the
+		// ochttp.Transport() irrespective of open telemetry is enabled or not.
+		// Related issue: https://github.com/open-telemetry/opentelemetry-js-contrib/issues/12
+
 		// forward the request to the function service
-		resp, err := ocRoundTripper.RoundTrip(newReq)
+		var resp *http.Response
+		if roundTripper.funcHandler.openTracingEnabled || util.IsWebsocketRequest(newReq) {
+			ocRoundTripper := &ochttp.Transport{Base: transport}
+			resp, err = ocRoundTripper.RoundTrip(newReq)
+		} else {
+			otelRoundTripper := otelhttp.NewTransport(transport)
+			resp, err = otelRoundTripper.RoundTrip(newReq)
+		}
+
 		if err == nil {
 			// return response back to user
 			if roundTripper.funcHandler.isDebugEnv {
@@ -494,6 +515,10 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h
 		rrt.closeContext()
 	}()
 
+	// add attributes to current span
+	span := trace.SpanFromContext(request.Context())
+	span.SetAttributes(otelUtils.GetAttributesForFunction(fh.function)...)
+
 	proxy.ServeHTTP(responseWriter, request)
 }
 
@@ -657,7 +682,7 @@ func (fh functionHandler) getServiceEntryFromExecutor() (serviceUrl *url.URL, er
 }
 
 // getServiceEntryFromExecutor returns service url entry returns from executor
-func (fh functionHandler) getServiceEntry() (svcURL *url.URL, cacheHit bool, err error) {
+func (fh functionHandler) getServiceEntry(ctx context.Context) (svcURL *url.URL, cacheHit bool, err error) {
 	if fh.function.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypePoolmgr {
 		svcURL, err = fh.getServiceEntryFromExecutor()
 		return svcURL, false, err

pkg/router/httpTriggers.go

@@ -19,6 +19,8 @@ package router
 import (
 	"context"
 	"net/http"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -33,6 +35,7 @@ import (
 	genInformer "github.com/fission/fission/pkg/generated/informers/externalversions"
 	"github.com/fission/fission/pkg/throttler"
 	"github.com/fission/fission/pkg/utils"
+	"github.com/fission/fission/pkg/utils/otel"
 )
 
 // HTTPTriggerSet represents an HTTP trigger set
@@ -110,6 +113,11 @@ func routerHealthHandler(w http.ResponseWriter, r *http.Request) {
 func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) *mux.Router {
 	muxRouter := mux.NewRouter()
 
+	openTracingEnabled, err := strconv.ParseBool(os.Getenv("OPENTRACING_ENABLED"))
+	if err != nil {
+		ts.logger.Fatal("error parsing OPENTRACING_ENABLED", zap.Error(err))
+	}
+
 	// HTTP triggers setup by the user
 	homeHandled := false
 	for i := range ts.triggers {
@@ -143,6 +151,7 @@ func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) *mux.Router
 			svcAddrUpdateThrottler:   ts.svcAddrUpdateThrottler,
 			functionTimeoutMap:       fnTimeoutMap,
 			unTapServiceTimeout:      ts.unTapServiceTimeout,
+			openTracingEnabled:       openTracingEnabled,
 		}
 
 		// The functionHandler for HTTP trigger with fn reference type "FunctionReferenceTypeFunctionName",
@@ -157,13 +166,24 @@ func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) *mux.Router
 				fh.function = fn
 			}
 		}
-		var ht *mux.Route
 
+		var ht *mux.Route
 		if trigger.Spec.Prefix != nil && *trigger.Spec.Prefix != "" {
-			ht = muxRouter.PathPrefix(*trigger.Spec.Prefix).HandlerFunc(fh.handler)
+			if openTracingEnabled {
+				ht = muxRouter.PathPrefix(*trigger.Spec.Prefix).HandlerFunc(fh.handler)
+			} else {
+				handler := otel.GetHandlerWithOTEL(http.HandlerFunc(fh.handler), *trigger.Spec.Prefix)
+				ht = muxRouter.PathPrefix(*trigger.Spec.Prefix).Handler(handler)
+			}
 		} else {
-			ht = muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
+			if openTracingEnabled {
+				ht = muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
+			} else {
+				handler := otel.GetHandlerWithOTEL(http.HandlerFunc(fh.handler), trigger.Spec.RelativeURL)
+				ht = muxRouter.Handle(trigger.Spec.RelativeURL, handler)
+			}
 		}
+
 		methods := trigger.Spec.Methods
 		if len(trigger.Spec.Method) > 0 {
 			present := false
@@ -212,7 +232,14 @@ func (ts *HTTPTriggerSet) getRouter(fnTimeoutMap map[types.UID]int) *mux.Router
 			functionTimeoutMap:     fnTimeoutMap,
 			unTapServiceTimeout:    ts.unTapServiceTimeout,
 		}
-		muxRouter.PathPrefix(utils.UrlForFunction(fn.ObjectMeta.Name, fn.ObjectMeta.Namespace)).HandlerFunc(fh.handler)
+
+		route := utils.UrlForFunction(fn.ObjectMeta.Name, fn.ObjectMeta.Namespace)
+		if openTracingEnabled {
+			muxRouter.PathPrefix(route).HandlerFunc(fh.handler)
+		} else {
+			otelHandler := otel.GetHandlerWithOTEL(http.HandlerFunc(fh.handler), route)
+			muxRouter.PathPrefix(route).Handler(otelHandler)
+		}
 	}
 
 	// Healthz endpoint for the router.

pkg/router/router.go

@@ -53,11 +53,13 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/trace"
+	"go.opentelemetry.io/otel"
 	"go.uber.org/zap"
 
 	"github.com/fission/fission/pkg/crd"
 	executorClient "github.com/fission/fission/pkg/executor/client"
 	"github.com/fission/fission/pkg/throttler"
+	otelUtils "github.com/fission/fission/pkg/utils/otel"
 )
 
 // request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url
@@ -80,36 +82,38 @@ func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTrigger
 }
 
 func serve(ctx context.Context, logger *zap.Logger, port int, tracingSamplingRate float64,
-	httpTriggerSet *HTTPTriggerSet, displayAccessLog bool) {
+	httpTriggerSet *HTTPTriggerSet, displayAccessLog bool, openTracingEnabled bool) {
 	mr := router(ctx, logger, httpTriggerSet)
 	url := fmt.Sprintf(":%v", port)
 
-	err := http.ListenAndServe(url, &ochttp.Handler{
-		Handler: mr,
-		GetStartOptions: func(r *http.Request) trace.StartOptions {
-			// do not trace router healthz endpoint
-			if strings.Compare(r.URL.Path, "/router-healthz") == 0 {
+	var err error
+	if openTracingEnabled {
+		err = http.ListenAndServe(url, &ochttp.Handler{
+			Handler: mr,
+			GetStartOptions: func(r *http.Request) trace.StartOptions {
+				// do not trace router healthz endpoint
+				if strings.Compare(r.URL.Path, "/router-healthz") == 0 {
+					return trace.StartOptions{
+						Sampler: trace.NeverSample(),
+					}
+				}
+				if displayAccessLog {
+					reqMsg, err := httputil.DumpRequest(r, false)
+					if err != nil {
+						logger.Error("error dumping request", zap.Error(err))
+					}
+					logger.Info("request dump", zap.String("request", string(reqMsg)))
+				}
 				return trace.StartOptions{
-					Sampler: trace.NeverSample(),
+					Sampler: trace.ProbabilitySampler(tracingSamplingRate),
 				}
-			}
-			if displayAccessLog {
-				reqMsg, err := httputil.DumpRequest(r, false)
-				if err != nil {
-					logger.Error("error dumping request", zap.Error(err))
-				}
-				logger.Info("request dump", zap.String("request", string(reqMsg)))
-			}
-			return trace.StartOptions{
-				Sampler: trace.ProbabilitySampler(tracingSamplingRate),
-			}
-		},
-	})
+			},
+		})
+	} else {
+		err = http.ListenAndServe(url, otelUtils.GetHandlerWithOTEL(mr, "fission-router", otelUtils.UrlsToIgnore("/router-healthz")))
+	}
 	if err != nil {
-		logger.Error(
-			"HTTP server error",
-			zap.Error(err),
-		)
+		logger.Error("HTTP server error", zap.Error(err))
 	}
 }
 
@@ -122,7 +126,7 @@ func serveMetric(logger *zap.Logger) {
 }
 
 // Start starts a router
-func Start(logger *zap.Logger, port int, executorURL string) {
+func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
 	fmap := makeFunctionServiceMap(logger, time.Minute)
 
 	fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
@@ -252,7 +256,12 @@ func Start(logger *zap.Logger, port int, executorURL string) {
 	go serveMetric(logger)
 
 	logger.Info("starting router", zap.Int("port", port))
-	ctx, cancel := context.WithCancel(context.Background())
+
+	tracer := otel.Tracer("router")
+	ctx, span := tracer.Start(context.Background(), "router/Start")
+	defer span.End()
+
+	ctxWithCancel, cancel := context.WithCancel(ctx)
 	defer cancel()
-	serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog)
+	serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled)
 }

pkg/router/util/util.go

@@ -17,6 +17,8 @@ limitations under the License.
 package util
 
 import (
+	"net/http"
+
 	v1 "k8s.io/api/networking/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -100,3 +102,8 @@ func GetDeployLabels(trigger *fv1.HTTPTrigger) map[string]string {
 		"triggerNamespace": trigger.ObjectMeta.Namespace,
 	}
 }
+
+func IsWebsocketRequest(request *http.Request) bool {
+	return request.Header.Get("Upgrade") == "websocket" &&
+		request.Header.Get("Connection") == "Upgrade"
+}