andwati/zitadel
1
0
Fork 0
mirror of https://github.com/andwati/zitadel.git synced 2026-10-18 05:52:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): improved instrumentation for observability (#11159)

Browse Source 
# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4
This commit is contained in:
Tim Möhlmann
2026-01-12 05:51:39 +00:00
committed by GitHub
parent d7a9e9d48b
commit eb22b58756
160 changed files with 4320 additions and 2135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/telemetry/tracing/caller.go
-22
View File
@@ -1,22 +0,0 @@
package tracing
import (
"runtime"
"github.com/zitadel/logging"
)
func GetCaller() string {
fpcs := make([]uintptr, 1)
n := runtime.Callers(3, fpcs)
if n == 0 {
logging.WithFields("logID", "TRACE-rWjfC").Debug("no caller")
return ""
}
caller := runtime.FuncForPC(fpcs[0] - 1)
if caller == nil {
logging.WithFields("logID", "TRACE-25POw").Debug("caller was nil")
return ""
}
return caller.Name()
}
internal/telemetry/tracing/config/config.go
-34
View File
@@ -1,34 +0,0 @@
package config
import (
"github.com/zitadel/zitadel/internal/telemetry/tracing/google"
"github.com/zitadel/zitadel/internal/telemetry/tracing/log"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Type string
Config map[string]interface{} `mapstructure:",remain"`
}
func (c *Config) NewTracer() error {
t, ok := tracer[c.Type]
if !ok {
return zerrors.ThrowInternalf(nil, "TRACE-dsbjh", "config type %s not supported", c.Type)
}
return t(c.Config)
}
var tracer = map[string]func(map[string]interface{}) error{
"otel": otel.NewTracerFromConfig,
"google": google.NewTracer,
"log": log.NewTracer,
"none": NoTracer,
"": NoTracer,
}
func NoTracer(_ map[string]interface{}) error {
return nil
}
internal/telemetry/tracing/google/google_tracer.go
-41
View File
@@ -1,41 +0,0 @@
package google
import (
texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
)
type Config struct {
ProjectID string
Fraction float64
ServiceName string
}
func NewTracer(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.ProjectID, _ = rawConfig["projectid"].(string)
c.ServiceName, _ = rawConfig["servicename"].(string)
c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
if err != nil {
return err
}
return c.NewTracer()
}
type Tracer struct {
otel.Tracer
}
func (c *Config) NewTracer() error {
sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := texporter.New(texporter.WithProjectID(c.ProjectID))
if err != nil {
return err
}
tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
return err
}
internal/telemetry/tracing/log/config.go
-39
View File
@@ -1,39 +0,0 @@
package log
import (
stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
)
type Config struct {
Fraction float64
ServiceName string
}
func NewTracer(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
c.ServiceName, _ = rawConfig["servicename"].(string)
if err != nil {
return err
}
return c.NewTracer()
}
type Tracer struct {
otel.Tracer
}
func (c *Config) NewTracer() error {
sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := stdout.New(stdout.WithPrettyPrint())
if err != nil {
return err
}
tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
return err
}
internal/telemetry/tracing/otel/config.go
-77
View File
@@ -1,77 +0,0 @@
package otel
import (
"context"
"strconv"
otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Fraction float64
Endpoint string
ServiceName string
}
func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.Endpoint, _ = rawConfig["endpoint"].(string)
c.ServiceName, _ = rawConfig["servicename"].(string)
c.Fraction, err = FractionFromConfig(rawConfig["fraction"])
if err != nil {
return err
}
return c.NewTracer()
}
func FractionFromConfig(i interface{}) (float64, error) {
if i == nil {
return 0, nil
}
switch fraction := i.(type) {
case float64:
return fraction, nil
case int:
return float64(fraction), nil
case string:
f, err := strconv.ParseFloat(fraction, 64)
if err != nil {
return 0, zerrors.ThrowInternal(err, "OTEL-SAfe1", "could not map fraction")
}
return f, nil
default:
return 0, zerrors.ThrowInternal(nil, "OTEL-Dd2s", "could not map fraction, unknown type")
}
}
func (c *Config) NewTracer() error {
sampler := NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := otlpgrpc.New(context.Background(), otlpgrpc.WithEndpoint(c.Endpoint), otlpgrpc.WithInsecure())
if err != nil {
return err
}
tracing.T, err = NewTracer(sampler, exporter, c.ServiceName)
return err
}
// NewSampler returns a sampler decorator which behaves differently,
// based on the parent of the span. If the span has no parent and is of kind server,
// the decorated sampler is used to make sampling decision.
// If the span has a parent, depending on whether the parent is remote and whether it
// is sampled, one of the following samplers will apply:
// - remote parent sampled -> always sample
// - remote parent not sampled -> sample based on the decorated sampler (fraction based)
// - local parent sampled -> always sample
// - local parent not sampled -> never sample
func NewSampler(sampler sdk_trace.Sampler) sdk_trace.Sampler {
return sdk_trace.ParentBased(
tracing.SpanKindBased(sampler, api_trace.SpanKindServer),
sdk_trace.WithRemoteParentNotSampled(sampler),
)
}
internal/telemetry/tracing/otel/open_telemetry.go
-77
View File
@@ -1,77 +0,0 @@
package otel
import (
"context"
"net/http"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
type Tracer struct {
Exporter api_trace.Tracer
sampler sdk_trace.Sampler
}
func NewTracer(sampler sdk_trace.Sampler, exporter sdk_trace.SpanExporter, serviceName string) (*Tracer, error) {
resource, err := otel_resource.ResourceWithService(serviceName)
if err != nil {
return nil, err
}
tp := sdk_trace.NewTracerProvider(
sdk_trace.WithSampler(sampler),
sdk_trace.WithBatcher(exporter),
sdk_trace.WithResource(resource),
)
otel.SetTracerProvider(tp)
tc := propagation.TraceContext{}
otel.SetTextMapPropagator(tc)
return &Tracer{Exporter: tp.Tracer(""), sampler: sampler}, nil
}
func (t *Tracer) Sampler() sdk_trace.Sampler {
return t.sampler
}
func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
}
func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
}
func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
}
func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
}
func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller)
}
func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, caller, options...)
}
func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
ctx, span := t.Exporter.Start(ctx, name, options...)
return ctx, tracing.CreateSpan(span)
}
func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
ctx, span := t.NewSpan(r.Context(), caller)
r = r.WithContext(ctx)
return r, span
}
internal/telemetry/tracing/sampler.go
-46
View File
@@ -1,46 +0,0 @@
package tracing
import (
"fmt"
"slices"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
type spanKindSampler struct {
sampler sdk_trace.Sampler
kinds []trace.SpanKind
}
// ShouldSample implements the [sdk_trace.Sampler] interface.
// It will not sample any spans which do not match the configured span kinds.
// For spans which do match, the decorated sampler is used to make the sampling decision.
func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
psc := trace.SpanContextFromContext(p.ParentContext)
if !slices.Contains(sk.kinds, p.Kind) {
return sdk_trace.SamplingResult{
Decision: sdk_trace.Drop,
Tracestate: psc.TraceState(),
}
}
s := sk.sampler.ShouldSample(p)
return s
}
func (sk spanKindSampler) Description() string {
return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
sk.sampler.Description(),
sk.kinds,
)
}
// SpanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
// If the span kind does not match one of the configured kinds, it will not be sampled.
// If the span kind matches, the decorated sampler is used to make sampling decision.
func SpanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
return spanKindSampler{
sampler: sampler,
kinds: kinds,
}
}
internal/telemetry/tracing/sampler_test.go
-80
View File
@@ -1,80 +0,0 @@
package tracing
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
func TestSpanKindBased(t *testing.T) {
type args struct {
sampler sdk_trace.Sampler
kinds []trace.SpanKind
}
type want struct {
description string
sampled int
}
tests := []struct {
name string
args args
want want
}{
{
"never sample, no sample",
args{
sampler: sdk_trace.NeverSample(),
kinds: []trace.SpanKind{trace.SpanKindServer},
},
want{
description: "SpanKindBased{sampler:AlwaysOffSampler,kinds:[server]}",
sampled: 0,
},
},
{
"always sample, no kind, no sample",
args{
sampler: sdk_trace.AlwaysSample(),
kinds: nil,
},
want{
description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[]}",
sampled: 0,
},
},
{
"always sample, 2 kinds, 2 samples",
args{
sampler: sdk_trace.AlwaysSample(),
kinds: []trace.SpanKind{trace.SpanKindServer, trace.SpanKindClient},
},
want{
description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[server client]}",
sampled: 2,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sampler := SpanKindBased(tt.args.sampler, tt.args.kinds...)
assert.Equal(t, tt.want.description, sampler.Description())
p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))
tr := p.Tracer("test")
var sampled int
for i := trace.SpanKindUnspecified; i <= trace.SpanKindConsumer; i++ {
ctx := context.Background()
_, span := tr.Start(ctx, "test", trace.WithSpanKind(i))
if span.SpanContext().IsSampled() {
sampled++
}
}
assert.Equal(t, tt.want.sampled, sampled)
})
}
}
internal/telemetry/tracing/span.go
-46
View File
@@ -1,46 +0,0 @@
package tracing
import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
)
type Span struct {
span trace.Span
opts []trace.SpanEndOption
}
func CreateSpan(span trace.Span) *Span {
return &Span{span: span, opts: []trace.SpanEndOption{}}
}
func (s *Span) End() {
if s.span == nil {
return
}
s.span.End(s.opts...)
}
func (s *Span) EndWithError(err error) {
s.SetStatusByError(err)
s.End()
}
func (s *Span) SetStatusByError(err error) {
if s.span == nil {
return
}
if err != nil {
// trace.WithErrorStatus(codes.Error)
s.span.RecordError(err)
s.span.SetAttributes(
attribute.KeyValue{},
)
}
code, msg, id, _ := gerrors.ExtractZITADELError(err)
s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
}
internal/telemetry/tracing/tracing.go
+19 -57
View File
@@ -1,81 +1,43 @@
// Package tracing acts as a shim to [tracing],
// so we can migrate the tracing package without changing all imports at once.
package tracing
import (
"context"
"net/http"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
)
type Tracer interface {
NewSpan(ctx context.Context, caller string) (context.Context, *Span)
NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
Sampler() sdk_trace.Sampler
func NewSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewSpan(ctx, instrumentation.GetCallingFunc(1))
}
var T Tracer
func Sampler() sdk_trace.Sampler {
if T == nil {
return sdk_trace.NeverSample()
}
return T.Sampler()
func NewNamedSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span) {
return tracing.NewNamedSpan(ctx, name)
}
func NewSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewSpan(ctx, GetCaller())
func NewClientSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewClientSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewSpan(ctx, name)
func NewServerSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewServerSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewClientSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewClientSpan(ctx, GetCaller())
func NewClientInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewClientInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewServerSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewServerSpan(ctx, GetCaller())
func NewServerInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewServerInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewClientInterceptorSpan(ctx, GetCaller())
}
func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewServerInterceptorSpan(ctx, GetCaller())
}
func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
if T == nil {
return r, CreateSpan(nil)
}
return T.NewSpanHTTP(r, GetCaller())
func NewSpanHTTP(r *http.Request) (*http.Request, *instrumentation.Span) {
return tracing.GlobalTracer().NewSpanHTTP(r, instrumentation.GetCallingFunc(1))
}
func TraceIDFromCtx(ctx context.Context) string {
return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
return tracing.TraceIDFromCtx(ctx)
}