enhancement: Add different samplers and propagators support with OpenTelemetry (#2201)

* Samplers: Check PR/helm values for supported types * Propagators: Check PR/helm values for supported types * Added tracing support via fission CLI * Use parentbased_traceidratio as default sampler with 0.1 ratio Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
2021-09-22 16:29:49 +05:30
parent 1347d73b04
commit 119d674207
9 changed files with 447 additions and 70 deletions
@@ -2,10 +2,15 @@ package otel

 import (
 	"context"
+	"fmt"
 	"os"
 	"strconv"
 	"strings"

+	"go.opentelemetry.io/contrib/propagators/aws/xray"
+	"go.opentelemetry.io/contrib/propagators/b3"
+	"go.opentelemetry.io/contrib/propagators/jaeger"
+	"go.opentelemetry.io/contrib/propagators/ot"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
@@ -20,9 +25,12 @@ import (
 )

 const (
-	OtelEnvPrefix      = "OTEL_"
-	OtelEndpointEnvVar = "OTEL_EXPORTER_OTLP_ENDPOINT"
-	OtelInsecureEnvVar = "OTEL_EXPORTER_OTLP_INSECURE"
+	OtelEnvPrefix        = "OTEL_"
+	OtelEndpointEnvVar   = "OTEL_EXPORTER_OTLP_ENDPOINT"
+	OtelInsecureEnvVar   = "OTEL_EXPORTER_OTLP_INSECURE"
+	OtelTracesSampler    = "OTEL_TRACES_SAMPLER"
+	OtelTracesSamplerArg = "OTEL_TRACES_SAMPLER_ARG"
+	OtelPropogaters      = "OTEL_PROPOGATORS"
 )

 type OtelConfig struct {
@@ -30,12 +38,118 @@ type OtelConfig struct {
 	insecure bool
 }

+/*
+Each Sampler type defines its own expected input, if any.
+Currently we get trace ratio for the case of,
+1. traceidratio
+2. parentbased_traceidratio
+*/
+func getSamplerArg() (float64, error) {
+	arg := os.Getenv(OtelTracesSamplerArg)
+	return strconv.ParseFloat(arg, 64)
+}
+
+/* GetPropogater returns a slice of propagators to be used by the OpenTelemetry
+provider.
+
+Supported providers:
+tracecontext - W3C Trace Context
+baggage - W3C Baggage
+b3 - B3 Single
+b3multi - B3 Multi
+jaeger - Jaeger uber-trace-id header
+xray - AWS X-Ray (third party)
+ottrace - OpenTracing Trace (third party)
+*/
+func GetPropogater(logger *zap.Logger) []propagation.TextMapPropagator {
+	propogatersEnv := os.Getenv(OtelPropogaters)
+	if propogatersEnv == "" {
+		return []propagation.TextMapPropagator{
+			propagation.TraceContext{}, propagation.Baggage{},
+		}
+	}
+	propogators := []propagation.TextMapPropagator{}
+	for _, prop := range strings.Split(propogatersEnv, ",") {
+		switch prop {
+		case "tracecontext":
+			propogators = append(propogators, propagation.TraceContext{})
+		case "baggage":
+			propogators = append(propogators, propagation.Baggage{})
+		case "b3multi":
+			propogators = append(propogators, b3.New(b3.WithInjectEncoding(b3.B3MultipleHeader)))
+		case "b3":
+			propogators = append(propogators, b3.New(b3.WithInjectEncoding(b3.B3SingleHeader)))
+		case "jaeger":
+			propogators = append(propogators, jaeger.Jaeger{})
+		case "xray":
+			propogators = append(propogators, xray.Propagator{})
+		case "ottrace":
+			propogators = append(propogators, ot.OT{})
+		default:
+			logger.Error("Unsupported propagation type", zap.String("propagation", prop))
+		}
+	}
+	if len(propogators) == 0 {
+		return []propagation.TextMapPropagator{
+			propagation.TraceContext{}, propagation.Baggage{},
+		}
+	}
+	return propogators
+}
+
+/*
+GetSampler returns a sampler that can be used to sample traces.
+This is based on https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#general-sdk-configuration
+We have to implement as open-telemetry Go sdk doesn't support configuration of different samplers.
+Once its added we may remove this code.
+
+Supported samplers:
+always_on - Sampler that always samples spans, regardless of the parent span's sampling decision.
+always_off - Sampler that never samples spans, regardless of the parent span's sampling decision.
+traceidratio - Sampler that samples probabalistically based on rate.
+parentbased_always_on - (default) Sampler that respects its parent span's sampling decision, but otherwise always samples.
+parentbased_always_off - Sampler that respects its parent span's sampling decision, but otherwise never samples.
+parentbased_traceidratio - Sampler that respects its parent span's sampling decision, but otherwise samples probabalistically based on rate.
+
+Environment variables:
+OTEL_TRACES_SAMPLER - Sampler to use(one of the above samplers)
+OTEL_TRACES_SAMPLER_ARG - Argument to pass to the sampler(float value)
+*/
+func GetSampler() (sdktrace.Sampler, error) {
+	samplerType := os.Getenv(OtelTracesSampler)
+	switch samplerType {
+	case "always_on":
+		return sdktrace.AlwaysSample(), nil
+	case "always_off":
+		return sdktrace.NeverSample(), nil
+	case "parentbased_always_on":
+		return sdktrace.ParentBased(sdktrace.AlwaysSample()), nil
+	case "parentbased_always_off":
+		return sdktrace.ParentBased(sdktrace.NeverSample()), nil
+	case "traceidratio":
+		arg, err := getSamplerArg()
+		if err != nil {
+			return nil, fmt.Errorf("invalid sampler arg: %w", err)
+		}
+		return sdktrace.TraceIDRatioBased(arg), nil
+	case "parentbased_traceidratio":
+		arg, err := getSamplerArg()
+		if err != nil {
+			return nil, fmt.Errorf("invalid sampler arg: %w", err)
+		}
+		return sdktrace.ParentBased(sdktrace.TraceIDRatioBased(arg)), nil
+	default:
+		return sdktrace.ParentBased(sdktrace.AlwaysSample()), nil
+	}
+}
+
+// parseOtelConfig parses the environment variables OTEL_EXPORTER_OTLP_ENDPOINT and
 func parseOtelConfig() OtelConfig {
 	config := OtelConfig{}
 	config.endpoint = os.Getenv(OtelEndpointEnvVar)
 	insecure, err := strconv.ParseBool(os.Getenv(OtelInsecureEnvVar))
 	if err != nil {
-		insecure = false
+		insecure = true
 	}
 	config.insecure = insecure
 	return config
@@ -44,7 +158,9 @@ func parseOtelConfig() OtelConfig {
 func getTraceExporter(ctx context.Context, logger *zap.Logger) (*otlptrace.Exporter, error) {
 	otelConfig := parseOtelConfig()
 	if otelConfig.endpoint == "" {
-		logger.Info("OTEL_EXPORTER_OTLP_ENDPOINT not set, skipping Opentelemtry tracing")
+		if logger != nil {
+			logger.Info("OTEL_EXPORTER_OTLP_ENDPOINT not set, skipping Opentelemtry tracing")
+		}
 		return nil, nil
 	}

@@ -75,10 +191,14 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
 	if err != nil {
 		return nil, err
 	}
+	sampler, err := GetSampler()
+	if err != nil {
+		return nil, err
+	}
 	tracerProvider := sdktrace.NewTracerProvider(
 		sdktrace.WithResource(res),
+		sdktrace.WithSampler(sampler),
 	)
-
 	traceExporter, err := getTraceExporter(ctx, logger)
 	if err != nil {
 		return nil, err
@@ -90,17 +210,16 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
 	}

 	otel.SetTracerProvider(tracerProvider)
-	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
-		propagation.TraceContext{}, propagation.Baggage{}))
-
+	propogaters := GetPropogater(logger)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propogaters...))
 	// Shutdown will flush any remaining spans and shut down the exporter.
 	return func(ctx context.Context) {
 		err := tracerProvider.Shutdown(ctx)
-		if err != nil {
+		if err != nil && logger != nil {
 			logger.Fatal("error shutting down trace provider", zap.Error(err))
 		}
 		if traceExporter != nil {
-			if err = traceExporter.Shutdown(ctx); err != nil {
+			if err = traceExporter.Shutdown(ctx); err != nil && logger != nil {
 				logger.Fatal("error shutting down trace exporter", zap.Error(err))
 			}
 		}