zitadel/backend/v3/instrumentation/tracing.go

package instrumentation

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"os"
	"slices"

	google_trace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
	"go.opentelemetry.io/contrib/exporters/autoexport"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
	"go.opentelemetry.io/otel/sdk/resource"
	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
	"go.opentelemetry.io/otel/trace"

	"github.com/zitadel/zitadel/internal/zerrors"
)

type TraceConfig struct {
	Fraction         float64
	Exporter         ExporterConfig
	TrustRemoteSpans bool
}

// TODO: remove for v5 release
type LegacyTraceConfig struct {
	Type      string
	Fraction  float64
	Endpoint  string
	ProjectID string
}

// SetLegacyConfig sets the fields of the TraceConfig based on the provided LegacyTraceConfig.
// If the Type field is already set to a value other than [ExporterTypeNone], the legacy config is ignored.
func (c *TraceConfig) SetLegacyConfig(lc *LegacyTraceConfig) {
	typ := c.Exporter.Type
	if lc == nil || !typ.isNone() {
		return
	}
	typeMap := map[string]ExporterType{
		"otel":   ExporterTypeGRPC,
		"google": ExporterTypeGoogle,
		"log":    ExporterTypeStdOut,
	}

	c.Fraction = lc.Fraction
	c.Exporter.Type = typeMap[lc.Type]
	c.Exporter.Endpoint = lc.Endpoint
	c.Exporter.GoogleProjectID = lc.ProjectID
}

type Tracer struct {
	trace.Tracer
}

func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindServer))
}

func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *Span) {
	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindServer))
}

func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindClient))
}

func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *Span) {
	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindClient))
}

func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *Span) {
	return t.newSpan(ctx, caller)
}

func (t *Tracer) newSpan(ctx context.Context, caller string, options ...trace.SpanStartOption) (context.Context, *Span) {
	return t.newSpanFromName(ctx, caller, options...)
}

func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...trace.SpanStartOption) (context.Context, *Span) {
	ctx, span := t.Start(ctx, name, options...)
	return ctx, CreateSpan(span)
}

func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span) {
	ctx, span := t.NewSpan(r.Context(), caller)
	r = r.WithContext(ctx)
	return r, span
}

type Span struct {
	span trace.Span
	opts []trace.SpanEndOption
}

func CreateSpan(span trace.Span) *Span {
	return &Span{span: span, opts: []trace.SpanEndOption{}}
}

func (s *Span) End() {
	if s.span == nil {
		return
	}

	s.span.End(s.opts...)
}

func (s *Span) EndWithError(err error) {
	s.SetStatusByError(err)
	s.End()
}

func (s *Span) SetStatusByError(err error) {
	if s.span == nil {
		return
	}
	if err != nil {
		s.span.RecordError(err)
		s.span.SetStatus(codes.Error, err.Error())
	}
	var zerr *zerrors.ZitadelError
	if errors.As(err, &zerr) {
		s.span.SetAttributes(
			attribute.Stringer("error_kind", zerr.Kind),
			attribute.String("error_msg", zerr.Message),
			attribute.String("error_id", zerr.ID),
		)
	}
}

func newTracerProvider(ctx context.Context, cfg TraceConfig, resource *resource.Resource) (_ *sdk_trace.TracerProvider, err error) {
	var exporter sdk_trace.SpanExporter
	switch cfg.Exporter.Type {
	case ExporterTypeAuto:
		// autoexport delegates exporter selection to the standard OTEL env vars.
		// We can't just call autoexport.NewSpanExporter unconditionally because
		// autoexport defaults to "otlp" when OTEL_TRACES_EXPORTER is unset, and
		// the OTLP exporter silently points at localhost:4318 even with no env
		// vars configured. That would cause every ZITADEL instance to start
		// attempting OTLP connections after upgrading, spamming logs with
		// connection errors.
		//
		// This guard is a heuristic: we check the env vars that realistically
		// indicate someone has intentionally configured OTEL export. It does not
		// cover every possible OTLP env var (e.g. OTEL_EXPORTER_OTLP_HEADERS or
		// OTEL_EXPORTER_OTLP_CERTIFICATE on their own), but those vars are
		// meaningless without an endpoint or exporter type being set too.
		if os.Getenv("OTEL_TRACES_EXPORTER") != "" ||
			os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" ||
			os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") != "" ||
			os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL") != "" ||
			os.Getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") != "" {
			exporter, err = autoexport.NewSpanExporter(ctx)
			if err == nil && autoexport.IsNoneSpanExporter(exporter) {
				exporter = nil
			}
		}
	case ExporterTypeUnspecified, ExporterTypeNone:
		// no exporter
	case ExporterTypeStdOut, ExporterTypeStdErr:
		exporter, err = traceStdOutExporter(cfg.Exporter)
	case ExporterTypeGRPC:
		exporter, err = traceGrpcExporter(ctx, cfg.Exporter)
	case ExporterTypeHTTP:
		exporter, err = traceHttpExporter(ctx, cfg.Exporter)
	case ExporterTypeGoogle:
		exporter, err = traceGoogleExporter(ctx, cfg.Exporter)
	case ExporterTypePrometheus:
		fallthrough // prometheus is not supported for traces
	default:
		err = errExporterType(cfg.Exporter.Type, "tracer")
	}
	if err != nil {
		return nil, fmt.Errorf("trace exporter: %w", err)
	}
	sampler := newSampler(cfg)
	opts := []sdk_trace.TracerProviderOption{
		sdk_trace.WithResource(resource),
		sdk_trace.WithSampler(sampler),
	}
	if exporter != nil {
		opts = append(opts, sdk_trace.WithBatcher(exporter, sdk_trace.WithBatchTimeout(cfg.Exporter.BatchDuration)))
	}
	tracerProvider := sdk_trace.NewTracerProvider(opts...)
	return tracerProvider, nil
}

func traceStdOutExporter(cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
	options := []stdouttrace.Option{
		stdouttrace.WithPrettyPrint(),
	}
	if cfg.Type == ExporterTypeStdErr {
		options = append(options, stdouttrace.WithWriter(os.Stderr))
	}
	exporter, err := stdouttrace.New(options...)
	if err != nil {
		return nil, err
	}
	return exporter, nil
}

func traceGrpcExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
	var grpcOpts []otlptracegrpc.Option
	if cfg.Endpoint != "" {
		grpcOpts = append(grpcOpts, otlptracegrpc.WithEndpoint(cfg.Endpoint))
	}
	if cfg.Insecure {
		grpcOpts = append(grpcOpts, otlptracegrpc.WithInsecure())
	}

	exporter, err := otlptracegrpc.New(ctx, grpcOpts...)
	if err != nil {
		return nil, err
	}
	return exporter, nil
}

func traceHttpExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
	var httpOpts []otlptracehttp.Option
	if cfg.Endpoint != "" {
		httpOpts = append(httpOpts, otlptracehttp.WithEndpoint(cfg.Endpoint))
	}
	if cfg.Insecure {
		httpOpts = append(httpOpts, otlptracehttp.WithInsecure())
	}

	exporter, err := otlptracehttp.New(ctx, httpOpts...)
	if err != nil {
		return nil, err
	}
	return exporter, nil
}

func traceGoogleExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
	exporter, err := google_trace.New(
		google_trace.WithContext(ctx),
		google_trace.WithProjectID(cfg.GoogleProjectID),
	)
	if err != nil {
		return nil, err
	}
	return exporter, nil
}

// newSampler returns a sampler decorator which behaves differently,
// based on the parent of the span. If the span has no parent and is of kind server,
// the decorated sampler is used to make sampling decision.
// If the span has a parent, depending on whether the parent is remote and whether it
// is sampled, one of the following samplers will apply:
//   - remote parent sampled -> always sample
//   - remote parent not sampled -> sample based on the decorated sampler (fraction based)
//   - local parent sampled -> always sample
//   - local parent not sampled -> never sample
func newSampler(cfg TraceConfig) sdk_trace.Sampler {
	fraction := sdk_trace.TraceIDRatioBased(cfg.Fraction)
	return sdk_trace.ParentBased(
		spanKindBased(fraction, trace.SpanKindServer),
		sdk_trace.WithRemoteParentNotSampled(fraction),
	)
}

type spanKindSampler struct {
	sampler sdk_trace.Sampler
	kinds   []trace.SpanKind
}

// ShouldSample implements the [sdk_trace.Sampler] interface.
// It will not sample any spans which do not match the configured span kinds.
// For spans which do match, the decorated sampler is used to make the sampling decision.
func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
	psc := trace.SpanContextFromContext(p.ParentContext)
	if !slices.Contains(sk.kinds, p.Kind) {
		return sdk_trace.SamplingResult{
			Decision:   sdk_trace.Drop,
			Tracestate: psc.TraceState(),
		}
	}
	s := sk.sampler.ShouldSample(p)
	return s
}

func (sk spanKindSampler) Description() string {
	return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
		sk.sampler.Description(),
		sk.kinds,
	)
}

// spanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
// If the span kind does not match one of the configured kinds, it will not be sampled.
// If the span kind matches, the decorated sampler is used to make sampling decision.
func spanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
	return spanKindSampler{
		sampler: sampler,
		kinds:   kinds,
	}
}