feature: Capture important events with span in fission and add trace id in logs (#2180)

* Capture important open telemetry events with span in fission * Add context to missing HTTP calls * Add Trace ID in logs * capture trace id in the proxy handler function * Always registry tracer to get traceID Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
2021-09-15 16:41:59 +05:30
parent 453debb6e9
commit 33473a4528
24 changed files with 468 additions and 209 deletions
@@ -1,9 +1,16 @@
 package otel

 import (
+	"context"
+
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+	appsv1 "k8s.io/api/apps/v1"
+	asv1 "k8s.io/api/autoscaling/v1"
+	apiv1 "k8s.io/api/core/v1"

 	fv1 "github.com/fission/fission/pkg/apis/core/v1"
+	"github.com/fission/fission/pkg/executor/fscache"
 )

 /* GetAttributesForFunction returns a set of attributes for a function. Attributes returned:
@@ -11,14 +18,112 @@ import (
    function-namespace
    environment-name
    environment-namespace
-
 These attributes are tags that can be used to filter traces.
 */
 func GetAttributesForFunction(fn *fv1.Function) []attribute.KeyValue {
-	return []attribute.KeyValue{
+	if fn == nil {
+		return []attribute.KeyValue{}
+	}
+	attrs := []attribute.KeyValue{
 		{Key: "function-name", Value: attribute.StringValue(fn.Name)},
 		{Key: "function-namespace", Value: attribute.StringValue(fn.Namespace)},
-		{Key: "environment-name", Value: attribute.StringValue(fn.Spec.Environment.Name)},
-		{Key: "environment-namespace", Value: attribute.StringValue(fn.Spec.Environment.Namespace)},
+	}
+	if fn.Spec.Environment.Name != "" {
+		attrs = append(attrs,
+			attribute.KeyValue{Key: "environment-name", Value: attribute.StringValue(fn.Spec.Environment.Name)},
+			attribute.KeyValue{Key: "environment-namespace", Value: attribute.StringValue(fn.Spec.Environment.Namespace)})
+	}
+	return attrs
+}
+
+func GetAttributesForEnv(env *fv1.Environment) []attribute.KeyValue {
+	if env == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "environment-name", Value: attribute.StringValue(env.Name)},
+		{Key: "environment-namespace", Value: attribute.StringValue(env.Namespace)}}
+}
+
+func GetAttributesForPackage(pkg *fv1.Package) []attribute.KeyValue {
+	if pkg == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "package-name", Value: attribute.StringValue(pkg.Name)},
+		{Key: "package-namespace", Value: attribute.StringValue(pkg.Namespace)}}
+}
+
+func GetAttributesForFuncSvc(fsvc *fscache.FuncSvc) []attribute.KeyValue {
+	if fsvc == nil {
+		return []attribute.KeyValue{}
+	}
+	var attrs []attribute.KeyValue
+	if fsvc.Function != nil {
+		attrs = append(attrs,
+			attribute.KeyValue{Key: "function-name", Value: attribute.StringValue(fsvc.Function.Name)},
+			attribute.KeyValue{Key: "function-namespace", Value: attribute.StringValue(fsvc.Function.Namespace)})
+	}
+	if fsvc.Environment != nil {
+		attrs = append(attrs,
+			attribute.KeyValue{Key: "environment-name", Value: attribute.StringValue(fsvc.Environment.Name)},
+			attribute.KeyValue{Key: "environment-namespace", Value: attribute.StringValue(fsvc.Environment.Namespace)})
+	}
+	if fsvc.Address != "" {
+		attrs = append(attrs, attribute.KeyValue{Key: "address", Value: attribute.StringValue(fsvc.Address)})
+	}
+	return attrs
+}
+
+func GetAttributesForPod(pod *apiv1.Pod) []attribute.KeyValue {
+	if pod == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "pod-name", Value: attribute.StringValue(pod.Name)},
+		{Key: "pod-namespace", Value: attribute.StringValue(pod.Namespace)},
+		{Key: "pod-ip", Value: attribute.StringValue(pod.Status.PodIP)},
 	}
 }
+
+func GetAttributesForDeployment(deployment *appsv1.Deployment) []attribute.KeyValue {
+	if deployment == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "deployment-name", Value: attribute.StringValue(deployment.Name)},
+		{Key: "deployment-namespace", Value: attribute.StringValue(deployment.Namespace)},
+	}
+}
+
+func GetAttributesForHPA(hpa *asv1.HorizontalPodAutoscaler) []attribute.KeyValue {
+	if hpa == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "hpa-name", Value: attribute.StringValue(hpa.Name)},
+		{Key: "hpa-namespace", Value: attribute.StringValue(hpa.Namespace)},
+	}
+}
+
+func GetAttributesForSvc(svc *apiv1.Service) []attribute.KeyValue {
+	if svc == nil {
+		return []attribute.KeyValue{}
+	}
+	return []attribute.KeyValue{
+		{Key: "svc-name", Value: attribute.StringValue(svc.Name)},
+		{Key: "svc-namespace", Value: attribute.StringValue(svc.Namespace)},
+	}
+}
+
+func MapToAttributes(m map[string]string) []attribute.KeyValue {
+	attrs := make([]attribute.KeyValue, 0, len(m))
+	for k, v := range m {
+		attrs = append(attrs, attribute.KeyValue{Key: attribute.Key(k), Value: attribute.StringValue(v)})
+	}
+	return attrs
+}
+
+func SpanTrackEvent(ctx context.Context, event string, attributes ...attribute.KeyValue) {
+	trace.SpanFromContext(ctx).AddEvent(event, trace.WithAttributes(attributes...))
+}
@@ -0,0 +1,30 @@
+/*
+Copyright 2021 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package otel
+
+import (
+	"context"
+
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+)
+
+func LoggerWithTraceID(context context.Context, logger *zap.Logger) *zap.Logger {
+	if span := trace.SpanContextFromContext(context); span.TraceID().IsValid() {
+		return logger.With(zap.String("trace_id", span.TraceID().String()))
+	}
+	return logger
+}
@@ -14,24 +14,13 @@ import (
 	"google.golang.org/grpc"
 )

-// Initializes an OTLP exporter, and configures the corresponding trace and metric providers.
-func InitProvider(logger *zap.Logger, serviceName string) (func(), error) {
+func getSpanProcessor(ctx context.Context, logger *zap.Logger) (*sdktrace.SpanProcessor, error) {
 	collectorEndpoint := os.Getenv("OTEL_COLLECTOR_ENDPOINT")
 	if collectorEndpoint == "" {
 		logger.Info("skipping trace exporter registration")
 		return nil, nil
 	}

-	ctx := context.Background()
-	res, err := resource.New(ctx,
-		resource.WithAttributes(
-			semconv.ServiceNameKey.String(serviceName),
-		),
-	)
-	if err != nil {
-		return nil, err
-	}
-
 	traceExporter, err := otlptracegrpc.New(ctx,
 		otlptracegrpc.WithInsecure(),
 		otlptracegrpc.WithEndpoint(collectorEndpoint),
@@ -42,12 +31,33 @@ func InitProvider(logger *zap.Logger, serviceName string) (func(), error) {
 	}

 	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+	return &bsp, nil
+}
+
+// Initializes an OTLP exporter, and configures the corresponding trace and metric providers.
+func InitProvider(logger *zap.Logger, serviceName string) (func(), error) {
+	ctx := context.Background()
+	res, err := resource.New(ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(serviceName),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
 	tracerProvider := sdktrace.NewTracerProvider(
 		sdktrace.WithSampler(sdktrace.AlwaysSample()),
 		sdktrace.WithResource(res),
-		sdktrace.WithSpanProcessor(bsp),
 	)

+	bsp, err := getSpanProcessor(ctx, logger)
+	if err != nil {
+		return nil, err
+	}
+	if bsp != nil {
+		tracerProvider.RegisterSpanProcessor(*bsp)
+	}
+
 	otel.SetTracerProvider(tracerProvider)
 	otel.SetTextMapPropagator(propagation.TraceContext{})

@@ -10,6 +10,17 @@ import (
 	"go.uber.org/zap"
 )

+func TracingEnabled(logger *zap.Logger) bool {
+	openTracingEnabled, err := strconv.ParseBool(os.Getenv("OPENTRACING_ENABLED"))
+	if err != nil {
+		if logger != nil {
+			logger.Error("Error parsing OpenTracing enabled flag", zap.Error(err))
+		}
+		return false
+	}
+	return openTracingEnabled
+}
+
 func RegisterTraceExporter(logger *zap.Logger, collectorEndpoint, serviceName string) error {
 	if len(collectorEndpoint) == 0 {
 		logger.Info("skipping trace exporter registration")