enhancement: Add different samplers and propagators support with OpenTelemetry (#2201)

* Samplers: Check PR/helm values for supported types * Propagators: Check PR/helm values for supported types * Added tracing support via fission CLI * Use parentbased_traceidratio as default sampler with 0.1 ratio Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
2021-09-22 16:29:49 +05:30
parent 1347d73b04
commit 119d674207
9 changed files with 447 additions and 70 deletions
@@ -2,10 +2,15 @@ package otel

 import (
 	"context"
+	"fmt"
 	"os"
 	"strconv"
 	"strings"

+	"go.opentelemetry.io/contrib/propagators/aws/xray"
+	"go.opentelemetry.io/contrib/propagators/b3"
+	"go.opentelemetry.io/contrib/propagators/jaeger"
+	"go.opentelemetry.io/contrib/propagators/ot"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
@@ -20,9 +25,12 @@ import (
 )

 const (
-	OtelEnvPrefix      = "OTEL_"
-	OtelEndpointEnvVar = "OTEL_EXPORTER_OTLP_ENDPOINT"
-	OtelInsecureEnvVar = "OTEL_EXPORTER_OTLP_INSECURE"
+	OtelEnvPrefix        = "OTEL_"
+	OtelEndpointEnvVar   = "OTEL_EXPORTER_OTLP_ENDPOINT"
+	OtelInsecureEnvVar   = "OTEL_EXPORTER_OTLP_INSECURE"
+	OtelTracesSampler    = "OTEL_TRACES_SAMPLER"
+	OtelTracesSamplerArg = "OTEL_TRACES_SAMPLER_ARG"
+	OtelPropogaters      = "OTEL_PROPOGATORS"
 )

 type OtelConfig struct {
@@ -30,12 +38,118 @@ type OtelConfig struct {
 	insecure bool
 }

+/*
+Each Sampler type defines its own expected input, if any.
+Currently we get trace ratio for the case of,
+1. traceidratio
+2. parentbased_traceidratio
+*/
+func getSamplerArg() (float64, error) {
+	arg := os.Getenv(OtelTracesSamplerArg)
+	return strconv.ParseFloat(arg, 64)
+}
+
+/* GetPropogater returns a slice of propagators to be used by the OpenTelemetry
+provider.
+
+Supported providers:
+tracecontext - W3C Trace Context
+baggage - W3C Baggage
+b3 - B3 Single
+b3multi - B3 Multi
+jaeger - Jaeger uber-trace-id header
+xray - AWS X-Ray (third party)
+ottrace - OpenTracing Trace (third party)
+*/
+func GetPropogater(logger *zap.Logger) []propagation.TextMapPropagator {
+	propogatersEnv := os.Getenv(OtelPropogaters)
+	if propogatersEnv == "" {
+		return []propagation.TextMapPropagator{
+			propagation.TraceContext{}, propagation.Baggage{},
+		}
+	}
+	propogators := []propagation.TextMapPropagator{}
+	for _, prop := range strings.Split(propogatersEnv, ",") {
+		switch prop {
+		case "tracecontext":
+			propogators = append(propogators, propagation.TraceContext{})
+		case "baggage":
+			propogators = append(propogators, propagation.Baggage{})
+		case "b3multi":
+			propogators = append(propogators, b3.New(b3.WithInjectEncoding(b3.B3MultipleHeader)))
+		case "b3":
+			propogators = append(propogators, b3.New(b3.WithInjectEncoding(b3.B3SingleHeader)))
+		case "jaeger":
+			propogators = append(propogators, jaeger.Jaeger{})
+		case "xray":
+			propogators = append(propogators, xray.Propagator{})
+		case "ottrace":
+			propogators = append(propogators, ot.OT{})
+		default:
+			logger.Error("Unsupported propagation type", zap.String("propagation", prop))
+		}
+	}
+	if len(propogators) == 0 {
+		return []propagation.TextMapPropagator{
+			propagation.TraceContext{}, propagation.Baggage{},
+		}
+	}
+	return propogators
+}
+
+/*
+GetSampler returns a sampler that can be used to sample traces.
+This is based on https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md#general-sdk-configuration
+We have to implement as open-telemetry Go sdk doesn't support configuration of different samplers.
+Once its added we may remove this code.
+
+Supported samplers:
+always_on - Sampler that always samples spans, regardless of the parent span's sampling decision.
+always_off - Sampler that never samples spans, regardless of the parent span's sampling decision.
+traceidratio - Sampler that samples probabalistically based on rate.
+parentbased_always_on - (default) Sampler that respects its parent span's sampling decision, but otherwise always samples.
+parentbased_always_off - Sampler that respects its parent span's sampling decision, but otherwise never samples.
+parentbased_traceidratio - Sampler that respects its parent span's sampling decision, but otherwise samples probabalistically based on rate.
+
+Environment variables:
+OTEL_TRACES_SAMPLER - Sampler to use(one of the above samplers)
+OTEL_TRACES_SAMPLER_ARG - Argument to pass to the sampler(float value)
+*/
+func GetSampler() (sdktrace.Sampler, error) {
+	samplerType := os.Getenv(OtelTracesSampler)
+	switch samplerType {
+	case "always_on":
+		return sdktrace.AlwaysSample(), nil
+	case "always_off":
+		return sdktrace.NeverSample(), nil
+	case "parentbased_always_on":
+		return sdktrace.ParentBased(sdktrace.AlwaysSample()), nil
+	case "parentbased_always_off":
+		return sdktrace.ParentBased(sdktrace.NeverSample()), nil
+	case "traceidratio":
+		arg, err := getSamplerArg()
+		if err != nil {
+			return nil, fmt.Errorf("invalid sampler arg: %w", err)
+		}
+		return sdktrace.TraceIDRatioBased(arg), nil
+	case "parentbased_traceidratio":
+		arg, err := getSamplerArg()
+		if err != nil {
+			return nil, fmt.Errorf("invalid sampler arg: %w", err)
+		}
+		return sdktrace.ParentBased(sdktrace.TraceIDRatioBased(arg)), nil
+	default:
+		return sdktrace.ParentBased(sdktrace.AlwaysSample()), nil
+	}
+}
+
+// parseOtelConfig parses the environment variables OTEL_EXPORTER_OTLP_ENDPOINT and
 func parseOtelConfig() OtelConfig {
 	config := OtelConfig{}
 	config.endpoint = os.Getenv(OtelEndpointEnvVar)
 	insecure, err := strconv.ParseBool(os.Getenv(OtelInsecureEnvVar))
 	if err != nil {
-		insecure = false
+		insecure = true
 	}
 	config.insecure = insecure
 	return config
@@ -44,7 +158,9 @@ func parseOtelConfig() OtelConfig {
 func getTraceExporter(ctx context.Context, logger *zap.Logger) (*otlptrace.Exporter, error) {
 	otelConfig := parseOtelConfig()
 	if otelConfig.endpoint == "" {
-		logger.Info("OTEL_EXPORTER_OTLP_ENDPOINT not set, skipping Opentelemtry tracing")
+		if logger != nil {
+			logger.Info("OTEL_EXPORTER_OTLP_ENDPOINT not set, skipping Opentelemtry tracing")
+		}
 		return nil, nil
 	}

@@ -75,10 +191,14 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
 	if err != nil {
 		return nil, err
 	}
+	sampler, err := GetSampler()
+	if err != nil {
+		return nil, err
+	}
 	tracerProvider := sdktrace.NewTracerProvider(
 		sdktrace.WithResource(res),
+		sdktrace.WithSampler(sampler),
 	)
-
 	traceExporter, err := getTraceExporter(ctx, logger)
 	if err != nil {
 		return nil, err
@@ -90,17 +210,16 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
 	}

 	otel.SetTracerProvider(tracerProvider)
-	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
-		propagation.TraceContext{}, propagation.Baggage{}))
-
+	propogaters := GetPropogater(logger)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propogaters...))
 	// Shutdown will flush any remaining spans and shut down the exporter.
 	return func(ctx context.Context) {
 		err := tracerProvider.Shutdown(ctx)
-		if err != nil {
+		if err != nil && logger != nil {
 			logger.Fatal("error shutting down trace provider", zap.Error(err))
 		}
 		if traceExporter != nil {
-			if err = traceExporter.Shutdown(ctx); err != nil {
+			if err = traceExporter.Shutdown(ctx); err != nil && logger != nil {
 				logger.Fatal("error shutting down trace exporter", zap.Error(err))
 			}
 		}
@@ -0,0 +1,181 @@
+package otel
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+
+	"go.opentelemetry.io/contrib/propagators/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/propagation"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+
+	"github.com/fission/fission/pkg/utils/loggerfactory"
+)
+
+func TestGetPropogater(t *testing.T) {
+	if OtelPropogaters != "OTEL_PROPOGATORS" {
+		t.Errorf("Expected OTEL_PROPOGATORS to be set, got %s", OtelPropogaters)
+	}
+	tests := []struct {
+		propogaterEnv string
+		propogaters   []propagation.TextMapPropagator
+	}{
+		{
+			"",
+			[]propagation.TextMapPropagator{propagation.TraceContext{}, propagation.Baggage{}},
+		},
+		{
+			"tracecontext,baggage",
+			[]propagation.TextMapPropagator{propagation.TraceContext{}, propagation.Baggage{}},
+		},
+		{
+			"jaeger",
+			[]propagation.TextMapPropagator{jaeger.Jaeger{}},
+		},
+		{
+			"baggage,tracecontext",
+			[]propagation.TextMapPropagator{propagation.Baggage{}, propagation.TraceContext{}},
+		},
+		{
+			"jaeger,baggage",
+			[]propagation.TextMapPropagator{jaeger.Jaeger{}, propagation.Baggage{}},
+		},
+	}
+	logger := loggerfactory.GetLogger()
+	for _, tt := range tests {
+		os.Setenv(OtelPropogaters, tt.propogaterEnv)
+		prop := GetPropogater(logger)
+		if prop == nil {
+			t.Errorf("GetPropogater() = %#v, want %#v", prop, tt.propogaters)
+		}
+		if len(prop) != len(tt.propogaters) {
+			t.Errorf("GetPropogater() = %#v, want %#v", prop, tt.propogaters)
+		}
+		if !reflect.DeepEqual(prop, tt.propogaters) {
+			t.Errorf("GetPropogater() = %#v, want %#v", prop, tt.propogaters)
+		}
+	}
+}
+
+func TestGetSampler(t *testing.T) {
+	if OtelTracesSampler != "OTEL_TRACES_SAMPLER" {
+		t.Errorf("Expected OTEL_TRACES_SAMPLER to be set, got %s", OtelTracesSampler)
+	}
+	if OtelTracesSamplerArg != "OTEL_TRACES_SAMPLER_ARG" {
+		t.Errorf("Expected OTEL_TRACES_SAMPLER_ARG to be set, got %s", OtelTracesSamplerArg)
+	}
+	if OtelPropogaters != "OTEL_PROPOGATORS" {
+		t.Errorf("Expected OTEL_PROPOGATORS to be set, got %s", OtelPropogaters)
+	}
+	tests := []struct {
+		sampler     string
+		samplerArg  string
+		wantSampler sdktrace.Sampler
+		wantError   error
+	}{
+		{
+			"",
+			"",
+			sdktrace.ParentBased(sdktrace.AlwaysSample()),
+			nil,
+		},
+		{
+			"always_on",
+			"",
+			sdktrace.AlwaysSample(),
+			nil,
+		},
+		{
+			"always_off",
+			"",
+			sdktrace.NeverSample(),
+			nil,
+		},
+		{
+			"parentbased_always_on",
+			"",
+			sdktrace.ParentBased(sdktrace.AlwaysSample()),
+			nil,
+		},
+		{
+			"parentbased_always_off",
+			"",
+			sdktrace.ParentBased(sdktrace.NeverSample()),
+			nil,
+		},
+		{
+			"traceidratio",
+			"0.5",
+			sdktrace.TraceIDRatioBased(0.5),
+			nil,
+		},
+		{
+			"traceidratio",
+			"",
+			nil,
+			errors.New("invalid sampler arg: strconv.ParseFloat: parsing \"\": invalid syntax"),
+		},
+		{
+			"parentbased_traceidratio",
+			"",
+			nil,
+			errors.New("invalid sampler arg: strconv.ParseFloat: parsing \"\": invalid syntax"),
+		},
+		{
+			"parentbased_traceidratio",
+			"0.01",
+			sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.01)),
+			nil,
+		},
+	}
+	for _, tt := range tests {
+		os.Setenv(OtelTracesSampler, tt.sampler)
+		os.Setenv(OtelTracesSamplerArg, tt.samplerArg)
+		gotSampler, gotError := GetSampler()
+		if !reflect.DeepEqual(gotSampler, tt.wantSampler) {
+			t.Errorf("GetSampler() gotSampler = %#v, want %#v", gotSampler, tt.wantSampler)
+		}
+		if fmt.Sprintf("%s", gotError) != fmt.Sprintf("%s", tt.wantError) {
+			t.Errorf("GetSampler() gotError = %#v, want %#v", gotError, tt.wantError)
+		}
+	}
+}
+
+func TestGetTraceExporter(t *testing.T) {
+	if OtelEndpointEnvVar != "OTEL_EXPORTER_OTLP_ENDPOINT" {
+		t.Errorf("Expected OTEL_EXPORTER_OTLP_ENDPOINT to be set, got %s", OtelEndpointEnvVar)
+	}
+	if OtelInsecureEnvVar != "OTEL_EXPORTER_OTLP_INSECURE" {
+		t.Errorf("Expected OTEL_EXPORTER_OTLP_INSECURE to be set, got %s", OtelInsecureEnvVar)
+	}
+	logger := loggerfactory.GetLogger()
+	ctx := context.Background()
+	tests := []struct {
+		oltpEndpoint string
+		oltpInsecure string
+		wantExporter *otlptrace.Exporter
+		wantError    error
+	}{
+		{
+			"",
+			"",
+			nil,
+			nil,
+		},
+	}
+	for _, tt := range tests {
+		os.Setenv(OtelEndpointEnvVar, tt.oltpEndpoint)
+		os.Setenv(OtelInsecureEnvVar, tt.oltpInsecure)
+		exporter, err := getTraceExporter(ctx, logger)
+		if !reflect.DeepEqual(exporter, tt.wantExporter) {
+			t.Errorf("getTraceExporter() exporter = %#v, want %#v", exporter, tt.wantExporter)
+		}
+		if fmt.Sprintf("%s", err) != fmt.Sprintf("%s", tt.wantError) {
+			t.Errorf("getTraceExporter() err = %#v, want %#v", err, tt.wantError)
+		}
+	}
+}