Fission meets OpenTelemetry (#2157)

* add opentracing section and otelCollectorEndpoint
* initialize OTLP exporter
* pkg/controller: changes for context propagation
* pkg/executor: changes for context propagation
* pkg/fetcher: changes for context propagation
* pkg/router: changes for context propagation
* pkg/storagesvc: changes for context propagation
* set no default value for otel collector endpoint
* update readme and add notes to charts
* move common code to pkg/utils/otel
* adding fn and env as attributes
* don't use otelhttp transport for websocket
* URL ignore with common filter UrlsToIgnore

Note: The web socket example does not work when using OTEL HTTP. Here is an issue related to that on open-telemetry/opentelemetry-js-contrib.

Signed-off-by: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/storagesvc/client/client.go

@@ -27,10 +27,13 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
 	"go.opencensus.io/plugin/ochttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.uber.org/zap"
 	"golang.org/x/net/context/ctxhttp"
 
 	"github.com/fission/fission/pkg/storagesvc"
@@ -45,11 +48,20 @@ type (
 
 // Client creates a storage service client.
 func MakeClient(url string) *Client {
+	openTracingEnabled, err := strconv.ParseBool(os.Getenv("OPENTRACING_ENABLED"))
+	if err != nil {
+		fmt.Printf("error parsing OPENTRACING_ENABLED: %v\n", zap.Error(err))
+	}
+
+	var hc *http.Client
+	if openTracingEnabled {
+		hc = &http.Client{Transport: &ochttp.Transport{}}
+	} else {
+		hc = &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}
+	}
 	return &Client{
-		url: strings.TrimSuffix(url, "/") + "/v1",
-		httpClient: &http.Client{
-			Transport: &ochttp.Transport{},
-		},
+		url:        strings.TrimSuffix(url, "/") + "/v1",
+		httpClient: hc,
 	}
 }
 

pkg/storagesvc/client/storagesvc_test.go

@@ -95,8 +95,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) {
 	os.Setenv("STORAGE_S3_REGION", minioRegion)
 
 	storage := storagesvc.NewS3Storage()
-	_ = storagesvc.Start(logger, storage, port)
-
+	_ = storagesvc.Start(logger, storage, port, true)
 }
 
 func TestS3StorageService(t *testing.T) {
@@ -213,7 +212,7 @@ func TestLocalStorageService(t *testing.T) {
 	localPath := fmt.Sprintf("/tmp/%v", testID)
 	_ = os.Mkdir(localPath, os.ModePerm)
 	storage := storagesvc.NewLocalStorage(localPath)
-	_ = storagesvc.Start(logger, storage, port)
+	_ = storagesvc.Start(logger, storage, port, true)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))

pkg/storagesvc/storagesvc.go

@@ -29,6 +29,8 @@ import (
 	"github.com/pkg/errors"
 	"go.opencensus.io/plugin/ochttp"
 	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/utils/otel"
 )
 
 type (
@@ -132,7 +134,6 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
 			zap.String("filename", handler.Filename),
 		)
 	}
-
 }
 
 func (ss *StorageService) getIdFromRequest(r *http.Request) (string, error) {
@@ -199,7 +200,7 @@ func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int)
 	}
 }
 
-func (ss *StorageService) Start(port int) {
+func (ss *StorageService) Start(port int, openTracingEnabled bool) {
 	r := mux.NewRouter()
 	r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
 	r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
@@ -208,15 +209,19 @@ func (ss *StorageService) Start(port int) {
 
 	address := fmt.Sprintf(":%v", port)
 
-	err := http.ListenAndServe(address, &ochttp.Handler{
-		Handler: r,
-	})
-
+	var err error
+	if openTracingEnabled {
+		err = http.ListenAndServe(address, &ochttp.Handler{
+			Handler: r,
+		})
+	} else {
+		err = http.ListenAndServe(address, otel.GetHandlerWithOTEL(r, "fission-storagesvc", otel.UrlsToIgnore("/healthz")))
+	}
 	ss.logger.Fatal("done listening", zap.Error(err))
 }
 
 // Start runs storage service
-func Start(logger *zap.Logger, storage Storage, port int) error {
+func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error {
 	enablePruner := true
 	// create a storage client
 	storageClient, err := MakeStowClient(logger, storage)
@@ -226,7 +231,7 @@ func Start(logger *zap.Logger, storage Storage, port int) error {
 
 	// create http handlers
 	storageService := MakeStorageService(logger, storageClient, port)
-	go storageService.Start(port)
+	go storageService.Start(port, openTracingEnabled)
 
 	// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
 	if enablePruner {