Fission meets OpenTelemetry (#2157)

* add opentracing section and otelCollectorEndpoint
* initialize OTLP exporter
* pkg/controller: changes for context propagation
* pkg/executor: changes for context propagation
* pkg/fetcher: changes for context propagation
* pkg/router: changes for context propagation
* pkg/storagesvc: changes for context propagation
* set no default value for otel collector endpoint
* update readme and add notes to charts
* move common code to pkg/utils/otel
* adding fn and env as attributes
* don't use otelhttp transport for websocket
* URL ignore with common filter UrlsToIgnore

Note: The web socket example does not work when using OTEL HTTP. Here is an issue related to that on open-telemetry/opentelemetry-js-contrib.

Signed-off-by: Gaurav Gahlot <gauravgahlot0107@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/utils/otel/attributes.go

@@ -0,0 +1,24 @@
+package otel
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+
+	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+)
+
+/* GetAttributesForFunction returns a set of attributes for a function. Attributes returned:
+    function-name
+    function-namespace
+    environment-name
+    environment-namespace
+
+These attributes are tags that can be used to filter traces.
+*/
+func GetAttributesForFunction(fn *fv1.Function) []attribute.KeyValue {
+	return []attribute.KeyValue{
+		{Key: "function-name", Value: attribute.StringValue(fn.Name)},
+		{Key: "function-namespace", Value: attribute.StringValue(fn.Namespace)},
+		{Key: "environment-name", Value: attribute.StringValue(fn.Spec.Environment.Name)},
+		{Key: "environment-namespace", Value: attribute.StringValue(fn.Spec.Environment.Namespace)},
+	}
+}

pkg/utils/otel/handler.go

@@ -0,0 +1,31 @@
+package otel
+
+import (
+	"net/http"
+	"strings"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+)
+
+func UrlsToIgnore(ignoreEndpoints ...string) func(r *http.Request) bool {
+	return func(r *http.Request) bool {
+		for _, ignore := range ignoreEndpoints {
+			if strings.HasPrefix(r.URL.Path, ignore) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+func GetHandlerWithOTEL(h http.Handler, name string, filter ...otelhttp.Filter) http.Handler {
+	opts := []otelhttp.Option{
+		otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
+	}
+
+	for _, f := range filter {
+		opts = append(opts, otelhttp.WithFilter(f))
+	}
+
+	return otelhttp.NewHandler(h, name, opts...)
+}

pkg/utils/otel/provider.go

@@ -0,0 +1,61 @@
+package otel
+
+import (
+	"context"
+	"os"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+)
+
+// Initializes an OTLP exporter, and configures the corresponding trace and metric providers.
+func InitProvider(logger *zap.Logger, serviceName string) (func(), error) {
+	collectorEndpoint := os.Getenv("OTEL_COLLECTOR_ENDPOINT")
+	if collectorEndpoint == "" {
+		logger.Info("skipping trace exporter registration")
+		return nil, nil
+	}
+
+	ctx := context.Background()
+	res, err := resource.New(ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(serviceName),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	traceExporter, err := otlptracegrpc.New(ctx,
+		otlptracegrpc.WithInsecure(),
+		otlptracegrpc.WithEndpoint(collectorEndpoint),
+		otlptracegrpc.WithDialOption(grpc.WithBlock()),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+	tracerProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(bsp),
+	)
+
+	otel.SetTracerProvider(tracerProvider)
+	otel.SetTextMapPropagator(propagation.TraceContext{})
+
+	// Shutdown will flush any remaining spans and shut down the exporter.
+	return func() {
+		err := tracerProvider.Shutdown(ctx)
+		if err != nil {
+			logger.Fatal("error shutting down trace provider", zap.Error(err))
+		}
+	}, nil
+}

pkg/utils/tracing/tracing.go

@@ -0,0 +1,45 @@
+package tracing
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	"contrib.go.opencensus.io/exporter/jaeger"
+	"go.opencensus.io/trace"
+	"go.uber.org/zap"
+)
+
+func RegisterTraceExporter(logger *zap.Logger, collectorEndpoint, serviceName string) error {
+	if len(collectorEndpoint) == 0 {
+		logger.Info("skipping trace exporter registration")
+		return nil
+	}
+
+	exporter, err := jaeger.NewExporter(jaeger.Options{
+		CollectorEndpoint: collectorEndpoint,
+		Process: jaeger.Process{
+			ServiceName: serviceName,
+			Tags: []jaeger.Tag{
+				jaeger.BoolTag("fission", true),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	trace.RegisterExporter(exporter)
+
+	if strings.EqualFold(serviceName, "Fission-Fetcher") {
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+	} else {
+		samplingRate, err := strconv.ParseFloat(os.Getenv("TRACING_SAMPLING_RATE"), 32)
+		if err != nil {
+			return err
+		}
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.ProbabilitySampler(samplingRate)})
+	}
+
+	return nil
+}