feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

backend/v3/domain/defaults.go

@@ -1,18 +1,12 @@
 package domain
 
 import (
-	"log/slog"
-
 	"github.com/zitadel/zitadel/backend/v3/storage/database"
 	"github.com/zitadel/zitadel/backend/v3/storage/eventstore"
-	"github.com/zitadel/zitadel/backend/v3/telemetry/logging"
-	"github.com/zitadel/zitadel/backend/v3/telemetry/tracing"
 )
 
 var (
 	pool             database.Pool
-	tracer           tracing.Tracer
-	logger           logging.Logger = *logging.NewLogger(slog.Default())
 	legacyEventstore eventstore.LegacyEventstore
 )
 
@@ -20,14 +14,6 @@ func SetPool(p database.Pool) {
 	pool = p
 }
 
-func SetTracer(t tracing.Tracer) {
-	tracer = t
-}
-
-func SetLogger(l logging.Logger) {
-	logger = l
-}
-
 func SetLegacyEventstore(es eventstore.LegacyEventstore) {
 	legacyEventstore = es
 }

backend/v3/domain/invoker_logging.go

@@ -3,6 +3,8 @@ package domain
 import (
 	"context"
 	"time"
+
+	slogctx "github.com/veqryn/slog-context"
 )
 
 // loggingInvoker decorates each command with logging.
@@ -19,6 +21,7 @@ func NewLoggingInvoker(next Invoker) *loggingInvoker {
 
 func (i *loggingInvoker) Invoke(ctx context.Context, executor Executor, opts *InvokeOpts) (err error) {
 	start := time.Now()
+	logger := slogctx.FromCtx(ctx)
 	logger.InfoContext(ctx, "invoking", "name", executor.String())
 
 	err = i.execute(ctx, executor, opts)

backend/v3/domain/invoker_tracing.go

@@ -3,6 +3,8 @@ package domain
 import (
 	"context"
 	"fmt"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
 )
 
 // traceInvoker decorates each command with tracing.
@@ -17,12 +19,9 @@ func NewTraceInvoker(next Invoker) *traceInvoker {
 }
 
 func (i *traceInvoker) Invoke(ctx context.Context, executor Executor, opts *InvokeOpts) (err error) {
-	ctx, span := tracer.Start(ctx, fmt.Sprintf("%T", executor))
+	ctx, span := tracing.NewNamedSpan(ctx, fmt.Sprintf("%T", executor))
 	defer func() {
-		if err != nil {
-			span.RecordError(err)
-		}
-		span.End()
+		span.EndWithError(err)
 	}()
 
 	return i.execute(ctx, executor, opts)

backend/v3/instrumentation/caller.go

@@ -0,0 +1,46 @@
+package instrumentation
+
+import (
+	"log/slog"
+	"runtime"
+)
+
+type Caller struct {
+	Function string
+	File     string
+	Line     int
+}
+
+// GetCaller returns the caller information
+// skipping the given number of stack frames relative to the caller of this function.
+func GetCaller(skip int) (Caller, bool) {
+	pc, file, line, ok := runtime.Caller(skip + 1)
+	if !ok {
+		slog.Default().Debug("no caller info", "id", "TRACE-3r8bE")
+		return Caller{}, false
+	}
+
+	f := runtime.FuncForPC(pc)
+	if f == nil {
+		slog.Default().Debug("caller was nil", "id", "TRACE-25POw")
+		return Caller{}, false
+	}
+	return Caller{
+		Function: f.Name(),
+		File:     file,
+		Line:     line,
+	}, true
+}
+
+const CallerUnknown = "unknown"
+
+// GetCallingFunc returns the caller function name,
+// skipping the given number of stack frames relative to the caller of this function.
+// If the caller cannot be determined, [CallerUnknown] is returned.
+func GetCallingFunc(skip int) string {
+	caller, ok := GetCaller(skip + 1)
+	if !ok {
+		return CallerUnknown
+	}
+	return caller.Function
+}

backend/v3/instrumentation/caller_test.go

@@ -0,0 +1,48 @@
+package instrumentation
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetCaller(t *testing.T) {
+	_, _, prevLine, _ := runtime.Caller(0)
+	caller, ok := GetCaller(0)
+	require.True(t, ok)
+	want := Caller{
+		Function: "github.com/zitadel/zitadel/backend/v3/instrumentation.TestGetCaller",
+		File:     "backend/v3/instrumentation/caller_test.go",
+		Line:     prevLine + 1,
+	}
+	assert.Equal(t, want.Function, caller.Function, "function")
+	assert.Contains(t, caller.File, want.File, "file")
+	assert.Equal(t, want.Line, caller.Line, "line")
+}
+
+func TestGetCallingFunc(t *testing.T) {
+	tests := []struct {
+		name string
+		skip int
+		want string
+	}{
+		{
+			name: "test function",
+			skip: 0,
+			want: "github.com/zitadel/zitadel/backend/v3/instrumentation.TestGetCallingFunc.func1",
+		},
+		{
+			name: "unknown caller",
+			skip: 100,
+			want: CallerUnknown,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			callerFunc := GetCallingFunc(tt.skip)
+			assert.Equal(t, tt.want, callerFunc, "function")
+		})
+	}
+}

backend/v3/instrumentation/exportertype_enumer.go

@@ -0,0 +1,114 @@
+// Code generated by "enumer -type=ExporterType -trimprefix=ExporterType -text -linecomment"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _ExporterTypeName = "NoneStdOutStdErrGRPCHTTPGooglePrometheus"
+
+var _ExporterTypeIndex = [...]uint8{0, 0, 4, 10, 16, 20, 24, 30, 40}
+
+const _ExporterTypeLowerName = "nonestdoutstderrgrpchttpgoogleprometheus"
+
+func (i ExporterType) String() string {
+	if i < 0 || i >= ExporterType(len(_ExporterTypeIndex)-1) {
+		return fmt.Sprintf("ExporterType(%d)", i)
+	}
+	return _ExporterTypeName[_ExporterTypeIndex[i]:_ExporterTypeIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _ExporterTypeNoOp() {
+	var x [1]struct{}
+	_ = x[ExporterTypeUnspecified-(0)]
+	_ = x[ExporterTypeNone-(1)]
+	_ = x[ExporterTypeStdOut-(2)]
+	_ = x[ExporterTypeStdErr-(3)]
+	_ = x[ExporterTypeGRPC-(4)]
+	_ = x[ExporterTypeHTTP-(5)]
+	_ = x[ExporterTypeGoogle-(6)]
+	_ = x[ExporterTypePrometheus-(7)]
+}
+
+var _ExporterTypeValues = []ExporterType{ExporterTypeUnspecified, ExporterTypeNone, ExporterTypeStdOut, ExporterTypeStdErr, ExporterTypeGRPC, ExporterTypeHTTP, ExporterTypeGoogle, ExporterTypePrometheus}
+
+var _ExporterTypeNameToValueMap = map[string]ExporterType{
+	_ExporterTypeName[0:0]:        ExporterTypeUnspecified,
+	_ExporterTypeLowerName[0:0]:   ExporterTypeUnspecified,
+	_ExporterTypeName[0:4]:        ExporterTypeNone,
+	_ExporterTypeLowerName[0:4]:   ExporterTypeNone,
+	_ExporterTypeName[4:10]:       ExporterTypeStdOut,
+	_ExporterTypeLowerName[4:10]:  ExporterTypeStdOut,
+	_ExporterTypeName[10:16]:      ExporterTypeStdErr,
+	_ExporterTypeLowerName[10:16]: ExporterTypeStdErr,
+	_ExporterTypeName[16:20]:      ExporterTypeGRPC,
+	_ExporterTypeLowerName[16:20]: ExporterTypeGRPC,
+	_ExporterTypeName[20:24]:      ExporterTypeHTTP,
+	_ExporterTypeLowerName[20:24]: ExporterTypeHTTP,
+	_ExporterTypeName[24:30]:      ExporterTypeGoogle,
+	_ExporterTypeLowerName[24:30]: ExporterTypeGoogle,
+	_ExporterTypeName[30:40]:      ExporterTypePrometheus,
+	_ExporterTypeLowerName[30:40]: ExporterTypePrometheus,
+}
+
+var _ExporterTypeNames = []string{
+	_ExporterTypeName[0:0],
+	_ExporterTypeName[0:4],
+	_ExporterTypeName[4:10],
+	_ExporterTypeName[10:16],
+	_ExporterTypeName[16:20],
+	_ExporterTypeName[20:24],
+	_ExporterTypeName[24:30],
+	_ExporterTypeName[30:40],
+}
+
+// ExporterTypeString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func ExporterTypeString(s string) (ExporterType, error) {
+	if val, ok := _ExporterTypeNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _ExporterTypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to ExporterType values", s)
+}
+
+// ExporterTypeValues returns all values of the enum
+func ExporterTypeValues() []ExporterType {
+	return _ExporterTypeValues
+}
+
+// ExporterTypeStrings returns a slice of all String values of the enum
+func ExporterTypeStrings() []string {
+	strs := make([]string, len(_ExporterTypeNames))
+	copy(strs, _ExporterTypeNames)
+	return strs
+}
+
+// IsAExporterType returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i ExporterType) IsAExporterType() bool {
+	for _, v := range _ExporterTypeValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for ExporterType
+func (i ExporterType) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for ExporterType
+func (i *ExporterType) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = ExporterTypeString(string(text))
+	return err
+}

backend/v3/instrumentation/instrumentation.go

@@ -0,0 +1,86 @@
+// Package instrumentation initializes and configures telemetry and logging instrumentation.
+// After [Start] is called, the instrumentation providers can be accessed using global getters.
+package instrumentation
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"sync"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/trace"
+)
+
+const Name = "zitadel/backend/v3/instrumentation/tracing"
+
+type Config struct {
+	ServiceName string
+	Trace       TraceConfig
+	Metric      MetricConfig
+	Log         LogConfig
+	Profile     ProfileConfig
+}
+
+type ShutdownFunc func(context.Context) error
+
+var (
+	startOnce sync.Once
+	shutdown  ShutdownFunc
+	errStart  error
+)
+
+// Start initializes global instrumentation once, based on the provided configuration.
+// It is safe to call multiple times; subsequent calls will return the same shutdown function and error as the first call.
+func Start(ctx context.Context, cfg Config) (ShutdownFunc, error) {
+	startOnce.Do(func() {
+		errStart = startProfiler(cfg.Profile, cfg.ServiceName)
+		if errStart != nil {
+			return
+		}
+		var logProvider *log.LoggerProvider
+		logProvider, shutdown, errStart = setupOTelSDK(ctx, cfg)
+		if errStart != nil {
+			return
+		}
+		setLogger(logProvider, cfg.Log)
+	})
+	return shutdown, errStart
+}
+
+// NewTracer creates a new tracer from the global provider.
+// See [trace.TracerProvider.Tracer] for details.
+func NewTracer(name string, options ...trace.TracerOption) *Tracer {
+	provider := otel.GetTracerProvider()
+	return &Tracer{provider.Tracer(name, options...)}
+}
+
+// NewMeter creates a new meter from the global provider.
+// See [metric.MeterProvider.Meter] for details.
+func NewMeter(name string, options ...metric.MeterOption) *Meter {
+	provider := otel.GetMeterProvider()
+	return &Meter{
+		Meter: provider.Meter(name, options...),
+	}
+}
+
+// Logger returns the globally configured logger.
+func Logger() *slog.Logger {
+	return slog.Default()
+}
+
+func RequestFilter(ignoredPrefix ...string) otelhttp.Filter {
+	return func(r *http.Request) bool {
+		return !slices.ContainsFunc(
+			ignoredPrefix,
+			func(endpoint string) bool {
+				return strings.HasPrefix(r.URL.RequestURI(), endpoint)
+			},
+		)
+	}
+}

backend/v3/instrumentation/instrumentation_test.go

@@ -0,0 +1,40 @@
+package instrumentation
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRequestFilter(t *testing.T) {
+	filter := RequestFilter("/foo", "/bar")
+	var tests = []struct {
+		name string
+		r    *http.Request
+		want bool
+	}{
+		{
+			name: "foo subpath false",
+			r:    httptest.NewRequest("POST", "/foo/some", nil),
+			want: false,
+		},
+		{
+			name: "foo exact false",
+			r:    httptest.NewRequest("POST", "/foo", nil),
+			want: false,
+		},
+		{
+			name: "other path true",
+			r:    httptest.NewRequest("POST", "/other", nil),
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := filter(tt.r)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

backend/v3/instrumentation/logformat_enumer.go

@@ -0,0 +1,102 @@
+// Code generated by "enumer -type=LogFormat -trimprefix=LogFormat -text -linecomment"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _LogFormatName = "DisabledTextJSONGCP"
+
+var _LogFormatIndex = [...]uint8{0, 0, 8, 12, 16, 19}
+
+const _LogFormatLowerName = "disabledtextjsongcp"
+
+func (i LogFormat) String() string {
+	if i < 0 || i >= LogFormat(len(_LogFormatIndex)-1) {
+		return fmt.Sprintf("LogFormat(%d)", i)
+	}
+	return _LogFormatName[_LogFormatIndex[i]:_LogFormatIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _LogFormatNoOp() {
+	var x [1]struct{}
+	_ = x[LogFormatUndefined-(0)]
+	_ = x[LogFormatDisabled-(1)]
+	_ = x[LogFormatText-(2)]
+	_ = x[LogFormatJSON-(3)]
+	_ = x[LogFormatGCP-(4)]
+}
+
+var _LogFormatValues = []LogFormat{LogFormatUndefined, LogFormatDisabled, LogFormatText, LogFormatJSON, LogFormatGCP}
+
+var _LogFormatNameToValueMap = map[string]LogFormat{
+	_LogFormatName[0:0]:        LogFormatUndefined,
+	_LogFormatLowerName[0:0]:   LogFormatUndefined,
+	_LogFormatName[0:8]:        LogFormatDisabled,
+	_LogFormatLowerName[0:8]:   LogFormatDisabled,
+	_LogFormatName[8:12]:       LogFormatText,
+	_LogFormatLowerName[8:12]:  LogFormatText,
+	_LogFormatName[12:16]:      LogFormatJSON,
+	_LogFormatLowerName[12:16]: LogFormatJSON,
+	_LogFormatName[16:19]:      LogFormatGCP,
+	_LogFormatLowerName[16:19]: LogFormatGCP,
+}
+
+var _LogFormatNames = []string{
+	_LogFormatName[0:0],
+	_LogFormatName[0:8],
+	_LogFormatName[8:12],
+	_LogFormatName[12:16],
+	_LogFormatName[16:19],
+}
+
+// LogFormatString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func LogFormatString(s string) (LogFormat, error) {
+	if val, ok := _LogFormatNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _LogFormatNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to LogFormat values", s)
+}
+
+// LogFormatValues returns all values of the enum
+func LogFormatValues() []LogFormat {
+	return _LogFormatValues
+}
+
+// LogFormatStrings returns a slice of all String values of the enum
+func LogFormatStrings() []string {
+	strs := make([]string, len(_LogFormatNames))
+	copy(strs, _LogFormatNames)
+	return strs
+}
+
+// IsALogFormat returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i LogFormat) IsALogFormat() bool {
+	for _, v := range _LogFormatValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for LogFormat
+func (i LogFormat) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for LogFormat
+func (i *LogFormat) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = LogFormatString(string(text))
+	return err
+}

backend/v3/instrumentation/logger.go

@@ -0,0 +1,93 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdoutlog"
+	sdk_log "go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func newLoggerProvider(ctx context.Context, cfg ExporterConfig, resource *resource.Resource) (_ *sdk_log.LoggerProvider, err error) {
+	var exporter sdk_log.Exporter
+	switch cfg.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no exporter
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		exporter, err = logStdOutExporter(cfg)
+	case ExporterTypeGRPC:
+		exporter, err = logGrpcExporter(ctx, cfg)
+	case ExporterTypeHTTP:
+		exporter, err = logHttpExporter(ctx, cfg)
+	case ExporterTypeGoogle, ExporterTypePrometheus:
+		fallthrough // google, prometheus are not supported for logs
+	default:
+		err = errExporterType(cfg.Type, "logger")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("log exporter: %w", err)
+	}
+	opts := []sdk_log.LoggerProviderOption{
+		sdk_log.WithResource(resource),
+	}
+	if exporter != nil {
+		opts = append(opts, sdk_log.WithProcessor(
+			sdk_log.NewBatchProcessor(
+				exporter,
+				sdk_log.WithExportInterval(cfg.BatchDuration),
+			),
+		))
+	}
+	loggerProvider := sdk_log.NewLoggerProvider(opts...)
+	return loggerProvider, nil
+}
+
+func logStdOutExporter(cfg ExporterConfig) (sdk_log.Exporter, error) {
+	options := []stdoutlog.Option{
+		stdoutlog.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdoutlog.WithWriter(os.Stderr))
+	}
+	exporter, err := stdoutlog.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func logGrpcExporter(ctx context.Context, cfg ExporterConfig) (sdk_log.Exporter, error) {
+	var grpcOpts []otlploggrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlploggrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlploggrpc.WithInsecure())
+	}
+
+	exporter, err := otlploggrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func logHttpExporter(ctx context.Context, cfg ExporterConfig) (sdk_log.Exporter, error) {
+	var httpOpts []otlploghttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlploghttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlploghttp.WithInsecure())
+	}
+
+	exporter, err := otlploghttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}

backend/v3/instrumentation/logger_test.go

@@ -0,0 +1,81 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func Test_newLoggerProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     ExporterConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeNone,
+			},
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeStdOut,
+			},
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeStdErr,
+			},
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: ExporterConfig{
+				Type:     ExporterTypeGRPC,
+				Endpoint: "localhost:4317",
+			},
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: ExporterConfig{
+				Type:     ExporterTypeHTTP,
+				Endpoint: "localhost:4318",
+			},
+		},
+		{
+			name: "google exporter returns error",
+			cfg: ExporterConfig{
+				Type:            ExporterTypeGoogle,
+				GoogleProjectID: "test-project",
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns error",
+			cfg: ExporterConfig{
+				Type: ExporterTypePrometheus,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newLoggerProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/instrumentation/logging/connect_handler.go

@@ -0,0 +1,71 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+
+	"connectrpc.com/connect"
+	slogctx "github.com/veqryn/slog-context"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewConnectInterceptor(next connect.UnaryFunc, ignoredMethodSuffixes ...string) connect.UnaryFunc {
+	return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
+		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+			return strings.HasSuffix(req.Spec().Procedure, s)
+		}) {
+			return next(ctx, req)
+		}
+
+		logger := instrumentation.Logger()
+		ctx = instrumentation.SetConnectRequestDetails(ctx, req)
+		ctx = slogctx.NewCtx(ctx, logger)
+
+		resp, err := next(ctx, req)
+		var code connect.Code
+		if err != nil {
+			code = connect.CodeOf(err)
+		}
+		logger.Log(ctx,
+			assertConnectLevel(code),
+			"connect RPC served",
+			"code", code,
+		)
+
+		return resp, err
+	}
+}
+
+func assertConnectLevel(code connect.Code) slog.Level {
+	switch code {
+	case connect.CodeUnimplemented:
+		return slog.LevelDebug
+
+	// client errors
+	case connect.CodeCanceled,
+		connect.CodeInvalidArgument,
+		connect.CodeNotFound,
+		connect.CodeAlreadyExists,
+		connect.CodePermissionDenied,
+		connect.CodeResourceExhausted,
+		connect.CodeFailedPrecondition,
+		connect.CodeOutOfRange,
+		connect.CodeUnauthenticated:
+		return slog.LevelWarn
+
+	// server errors
+	case connect.CodeUnknown,
+		connect.CodeAborted,
+		connect.CodeDeadlineExceeded,
+		connect.CodeInternal,
+		connect.CodeUnavailable,
+		connect.CodeDataLoss:
+		return slog.LevelError
+
+	default: // includes 0 code when no error
+		return slog.LevelInfo
+	}
+}

backend/v3/instrumentation/logging/grpc_handler.go

@@ -0,0 +1,74 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+
+	slogctx "github.com/veqryn/slog-context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewGrpcInterceptor(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) {
+		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+			return strings.HasSuffix(info.FullMethod, s)
+		}) {
+			return next(ctx, req)
+		}
+
+		logger := instrumentation.Logger()
+		ctx = instrumentation.SetGrpcRequestDetails(ctx, info)
+		ctx = slogctx.NewCtx(ctx, logger)
+
+		resp, err := next(ctx, req)
+		var code codes.Code
+		if err != nil {
+			code = status.Code(err)
+		}
+		logger.Log(ctx,
+			assertGrpcLevel(code),
+			"gRPC served",
+			"code", code,
+		)
+
+		return resp, err
+	}
+}
+
+func assertGrpcLevel(code codes.Code) slog.Level {
+	switch code {
+	case codes.Unimplemented:
+		return slog.LevelDebug
+
+	// client errors
+	case codes.Canceled,
+		codes.InvalidArgument,
+		codes.NotFound,
+		codes.AlreadyExists,
+		codes.PermissionDenied,
+		codes.ResourceExhausted,
+		codes.FailedPrecondition,
+		codes.OutOfRange,
+		codes.Unauthenticated:
+		return slog.LevelWarn
+
+	// server errors
+	case codes.Unknown,
+		codes.Aborted,
+		codes.DeadlineExceeded,
+		codes.Internal,
+		codes.Unavailable,
+		codes.DataLoss:
+		return slog.LevelError
+	case codes.OK:
+		fallthrough
+	default: // includes 0 code when no error
+		return slog.LevelInfo
+	}
+}

backend/v3/instrumentation/logging/http_handler.go

@@ -0,0 +1,63 @@
+package logging
+
+import (
+	"log/slog"
+	"net/http"
+
+	slogctx "github.com/veqryn/slog-context"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewHandler(next http.Handler, service string, ignoredPrefix ...string) http.Handler {
+	filter := instrumentation.RequestFilter(ignoredPrefix...)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !filter(r) {
+			next.ServeHTTP(w, r)
+			return
+		}
+		logger := instrumentation.Logger()
+		ctx := instrumentation.SetHttpRequestDetails(r.Context(), service, r)
+		ctx = slogctx.NewCtx(ctx, logger)
+		sw := &statusWriter{ResponseWriter: w}
+
+		next.ServeHTTP(sw, r.WithContext(ctx))
+
+		logger.Log(ctx,
+			LogLevelFromStatus(sw.status),
+			"http request served",
+			"status", sw.status,
+		)
+	})
+}
+
+func LogLevelFromStatus(status int) slog.Level {
+	if status < 200 {
+		return slog.LevelDebug
+	}
+	if status < 400 {
+		return slog.LevelInfo
+	}
+	if status < 500 {
+		return slog.LevelWarn
+	}
+	return slog.LevelError
+}
+
+// statusWriter is a [http.ResponseWriter] that captures the status code for logging.
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func (w *statusWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.status = http.StatusOK
+	}
+	return w.ResponseWriter.Write(p)
+}
+
+func (w *statusWriter) WriteHeader(statusCode int) {
+	w.status = statusCode
+	w.ResponseWriter.WriteHeader(statusCode)
+}

backend/v3/instrumentation/metric.go

@@ -0,0 +1,286 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	google_metric "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
+	sdk_metric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type MetricConfig struct {
+	Exporter ExporterConfig
+}
+
+type Meter struct {
+	Meter             metric.Meter
+	Counters          sync.Map
+	UpDownSumObserver sync.Map
+	ValueObservers    sync.Map
+	Histograms        sync.Map
+}
+
+// TODO: remove for v5 release
+type LegacyMetricConfig struct {
+	Type string
+}
+
+func (c *MetricConfig) SetLegacyConfig(lc *LegacyMetricConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	if lc.Type == "otel" {
+		c.Exporter.Type = ExporterTypePrometheus
+	}
+}
+
+var hasPrometheusExporter bool
+
+func (m *Meter) GetExporter() http.Handler {
+	if hasPrometheusExporter {
+		return promhttp.Handler()
+	}
+	return http.NotFoundHandler()
+}
+
+func (m *Meter) RegisterCounter(name, description string) error {
+	if _, exists := m.Counters.Load(name); exists {
+		return nil
+	}
+	counter, err := m.Meter.Int64Counter(name, metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+	m.Counters.Store(name, counter)
+	return nil
+}
+
+func (m *Meter) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	counter, exists := m.Counters.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
+	}
+	counter.(metric.Int64Counter).Add(ctx, value, MapToAddOption(labels)...)
+	return nil
+}
+
+func (m *Meter) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	histogram, exists := m.Histograms.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-5wwb1", "Errors.Metrics.Histogram.NotFound")
+	}
+	histogram.(metric.Float64Histogram).Record(ctx, value, MapToRecordOption(labels)...)
+	return nil
+}
+
+func (m *Meter) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	if _, exists := m.Histograms.Load(name); exists {
+		return nil
+	}
+
+	histogram, err := m.Meter.Float64Histogram(name,
+		metric.WithDescription(description),
+		metric.WithUnit(unit),
+		metric.WithExplicitBucketBoundaries(buckets...),
+	)
+	if err != nil {
+		return err
+	}
+
+	m.Histograms.Store(name, histogram)
+	return nil
+}
+
+func (m *Meter) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	counter, err := m.Meter.Int64ObservableUpDownCounter(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, counter)
+	return nil
+}
+
+func (m *Meter) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	gauge, err := m.Meter.Int64ObservableGauge(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, gauge)
+	return nil
+}
+
+func MapToAddOption(labels map[string]attribute.Value) []metric.AddOption {
+	return []metric.AddOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func MapToRecordOption(labels map[string]attribute.Value) []metric.RecordOption {
+	return []metric.RecordOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func labelsToAttributes(labels map[string]attribute.Value) []attribute.KeyValue {
+	if labels == nil {
+		return nil
+	}
+	attributes := make([]attribute.KeyValue, 0, len(labels))
+	for key, value := range labels {
+		attributes = append(attributes, attribute.KeyValue{
+			Key:   attribute.Key(key),
+			Value: value,
+		})
+	}
+	return attributes
+}
+
+func newMeterProvider(ctx context.Context, cfg MetricConfig, resource *resource.Resource) (_ *sdk_metric.MeterProvider, err error) {
+	var readerOption sdk_metric.Option
+	switch cfg.Exporter.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no reader option
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		readerOption, err = metricStdOutOption(cfg.Exporter)
+	case ExporterTypeGRPC:
+		readerOption, err = metricGrpcOption(ctx, cfg.Exporter)
+	case ExporterTypeHTTP:
+		readerOption, err = metricHttpOption(ctx, cfg.Exporter)
+	case ExporterTypeGoogle:
+		readerOption, err = metricGoogleOption(cfg.Exporter)
+	case ExporterTypePrometheus:
+		readerOption, err = metricPrometheusOption()
+	default:
+		err = errExporterType(cfg.Exporter.Type, "metrics")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("meter reader: %w", err)
+	}
+
+	// create a view to filter out unwanted attributes
+	view := sdk_metric.NewView(
+		sdk_metric.Instrument{
+			Scope: instrumentation.Scope{Name: otelhttp.ScopeName},
+		},
+		sdk_metric.Stream{
+			AttributeFilter: attribute.NewAllowKeysFilter("http.method", "http.status_code", "http.target"),
+		},
+	)
+
+	opts := []sdk_metric.Option{
+		sdk_metric.WithResource(resource),
+		sdk_metric.WithView(view),
+	}
+	if readerOption != nil {
+		opts = append(opts, readerOption)
+	}
+	meterProvider := sdk_metric.NewMeterProvider(opts...)
+	return meterProvider, nil
+}
+
+func metricStdOutOption(cfg ExporterConfig) (sdk_metric.Option, error) {
+	options := []stdoutmetric.Option{
+		stdoutmetric.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdoutmetric.WithWriter(os.Stderr))
+	}
+	exporter, err := stdoutmetric.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricGrpcOption(ctx context.Context, cfg ExporterConfig) (sdk_metric.Option, error) {
+	var grpcOpts []otlpmetricgrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlpmetricgrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlpmetricgrpc.WithInsecure())
+	}
+
+	exporter, err := otlpmetricgrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricHttpOption(ctx context.Context, cfg ExporterConfig) (sdk_metric.Option, error) {
+	var httpOpts []otlpmetrichttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlpmetrichttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlpmetrichttp.WithInsecure())
+	}
+
+	exporter, err := otlpmetrichttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricGoogleOption(cfg ExporterConfig) (sdk_metric.Option, error) {
+	exporter, err := google_metric.New(
+		google_metric.WithProjectID(cfg.GoogleProjectID),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricPrometheusOption() (sdk_metric.Option, error) {
+	prom, err := prometheus.New(prometheus.WithoutScopeInfo())
+	if err != nil {
+		return nil, err
+	}
+	hasPrometheusExporter = true
+	return sdk_metric.WithReader(prom), nil
+}

backend/v3/instrumentation/metric_test.go

@@ -0,0 +1,186 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func TestMetricConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string // description of this test case
+		target MetricConfig
+		lc     *LegacyMetricConfig
+		want   MetricConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_newMeterProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     MetricConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "unsupported exporter returns error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterType(-1),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdErr,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeHTTP,
+					Endpoint: "localhost:4318",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "google exporter return error (no credentials)",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeGoogle,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGRPC,
+					Endpoint: "localhost:4317",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/instrumentation/metrics/http_handler.go

@@ -0,0 +1,135 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"strings"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+const (
+	RequestCounter               = "http.server.request_count"
+	RequestCountDescription      = "Request counter"
+	TotalRequestCounter          = "http.server.total_request_count"
+	TotalRequestDescription      = "Total return code counter"
+	ReturnCodeCounter            = "http.server.return_code_counter"
+	ReturnCodeCounterDescription = "Return code counter"
+	Method                       = "method"
+	URI                          = "uri"
+	ReturnCode                   = "return_code"
+)
+
+type Handler struct {
+	handler http.Handler
+	methods []MetricType
+	filter  otelhttp.Filter
+}
+
+type MetricType int32
+
+const (
+	MetricTypeTotalCount MetricType = iota
+	MetricTypeStatusCode
+	MetricTypeRequestCount
+)
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	RequestURI *string
+	Status     int
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+type Filter func(*http.Request) bool
+
+func NewHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
+	h := Handler{
+		handler: handler,
+		methods: metricMethods,
+		filter:  instrumentation.RequestFilter(ignoredEndpoints...),
+	}
+	return &h
+}
+
+type key int
+
+const requestURI key = iota
+
+func SetRequestURIPattern(ctx context.Context, pattern string) {
+	uri, ok := ctx.Value(requestURI).(*string)
+	if !ok {
+		return
+	}
+	*uri = pattern
+}
+
+// ServeHTTP serves HTTP requests (http.Handler)
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(h.methods) == 0 {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	if !h.filter(r) {
+		// Simply pass through to the handler if a filter rejects the request
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	uri := strings.Split(r.RequestURI, "?")[0]
+	recorder := &StatusRecorder{
+		ResponseWriter: w,
+		RequestURI:     &uri,
+		Status:         200,
+	}
+	r = r.WithContext(context.WithValue(r.Context(), requestURI, &uri))
+	h.handler.ServeHTTP(recorder, r)
+	if h.containsMetricsMethod(MetricTypeRequestCount) {
+		RegisterRequestCounter(recorder, r)
+	}
+	if h.containsMetricsMethod(MetricTypeTotalCount) {
+		RegisterTotalRequestCounter(r)
+	}
+	if h.containsMetricsMethod(MetricTypeStatusCode) {
+		RegisterRequestCodeCounter(recorder, r)
+	}
+}
+
+func (h *Handler) containsMetricsMethod(method MetricType) bool {
+	for _, m := range h.methods {
+		if m == method {
+			return true
+		}
+	}
+	return false
+}
+
+func RegisterRequestCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]attribute.Value{
+		URI:    attribute.StringValue(*recorder.RequestURI),
+		Method: attribute.StringValue(r.Method),
+	}
+	RegisterCounter(RequestCounter, RequestCountDescription)
+	AddCount(r.Context(), RequestCounter, 1, labels)
+}
+
+func RegisterTotalRequestCounter(r *http.Request) {
+	RegisterCounter(TotalRequestCounter, TotalRequestDescription)
+	AddCount(r.Context(), TotalRequestCounter, 1, nil)
+}
+
+func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
+	var labels = map[string]attribute.Value{
+		URI:        attribute.StringValue(*recorder.RequestURI),
+		Method:     attribute.StringValue(r.Method),
+		ReturnCode: attribute.IntValue(recorder.Status),
+	}
+	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
+	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
+}

backend/v3/instrumentation/metrics/metric.go

@@ -0,0 +1,71 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+const (
+	ActiveSessionCounter            = "zitadel.active_session_counter"
+	ActiveSessionCounterDescription = "Active session counter"
+	SpoolerDivCounter               = "zitadel.spooler_div_milliseconds"
+	SpoolerDivCounterDescription    = "Spooler div from last successful run to now in milliseconds"
+	Database                        = "database"
+	ViewName                        = "view_name"
+)
+
+type Metrics interface {
+	GetExporter() http.Handler
+	RegisterCounter(name, description string) error
+	AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error
+	AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error
+	RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error
+	RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error
+	RegisterHistogram(name, description, unit string, buckets []float64) error
+}
+
+var globalMeter = sync.OnceValue(func() Metrics {
+	return instrumentation.NewMeter(
+		instrumentation.Name,
+		metric.WithInstrumentationVersion(build.Version()),
+	)
+})
+
+func GlobalMeter() Metrics {
+	return globalMeter()
+}
+
+func GetExporter() http.Handler {
+	return globalMeter().GetExporter()
+}
+
+func RegisterCounter(name, description string) error {
+	return globalMeter().RegisterCounter(name, description)
+}
+
+func AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	return globalMeter().AddCount(ctx, name, value, labels)
+}
+
+func AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	return globalMeter().AddHistogramMeasurement(ctx, name, value, labels)
+}
+
+func RegisterHistogram(name, description, unit string, buckets []float64) error {
+	return globalMeter().RegisterHistogram(name, description, unit, buckets)
+}
+
+func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return globalMeter().RegisterUpDownSumObserver(name, description, callbackFunc)
+}
+
+func RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return globalMeter().RegisterValueObserver(name, description, callbackFunc)
+}

backend/v3/instrumentation/metrics/mock.go

@@ -0,0 +1,95 @@
+package metrics
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// MockMetrics implements the metrics.Metrics interface for testing
+type MockMetrics struct {
+	mu              sync.RWMutex
+	histogramValues map[string][]float64
+	counterValues   map[string]int64
+	histogramLabels map[string][]map[string]attribute.Value
+	counterLabels   map[string][]map[string]attribute.Value
+}
+
+var _ Metrics = new(MockMetrics)
+
+// NewMockMetrics creates a new Metrics instance for testing
+func NewMockMetrics() *MockMetrics {
+	return &MockMetrics{
+		histogramValues: make(map[string][]float64),
+		counterValues:   make(map[string]int64),
+		histogramLabels: make(map[string][]map[string]attribute.Value),
+		counterLabels:   make(map[string][]map[string]attribute.Value),
+	}
+}
+
+func (m *MockMetrics) GetExporter() http.Handler {
+	return nil
+}
+
+func (m *MockMetrics) GetMetricsProvider() metric.MeterProvider {
+	return nil
+}
+
+func (m *MockMetrics) RegisterCounter(name, description string) error {
+	return nil
+}
+
+func (m *MockMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.counterValues[name] += value
+	m.counterLabels[name] = append(m.counterLabels[name], labels)
+	return nil
+}
+
+func (m *MockMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.histogramValues[name] = append(m.histogramValues[name], value)
+	m.histogramLabels[name] = append(m.histogramLabels[name], labels)
+	return nil
+}
+
+func (m *MockMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (m *MockMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	return nil
+}
+
+func (m *MockMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	return nil
+}
+
+func (m *MockMetrics) GetHistogramValues(name string) []float64 {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.histogramValues[name]
+}
+
+func (m *MockMetrics) GetHistogramLabels(name string) []map[string]attribute.Value {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.histogramLabels[name]
+}
+
+func (m *MockMetrics) GetCounterValue(name string) int64 {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.counterValues[name]
+}
+
+func (m *MockMetrics) GetCounterLabels(name string) []map[string]attribute.Value {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.counterLabels[name]
+}

backend/v3/instrumentation/otel.go

@@ -0,0 +1,138 @@
+package instrumentation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/log/global"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+//go:generate enumer -type=ExporterType -trimprefix=ExporterType -text -linecomment
+type ExporterType int
+
+// ExporterType defines the type of exporter to use.
+// - metrics supports all types
+// - tracing supports all types except Prometheus
+// - logging supports all types except Google and Prometheus
+// - profiling supports only Google and None
+const (
+	// Empty line comment sets empty string of unspecified value
+	ExporterTypeUnspecified ExporterType = iota //
+	ExporterTypeNone
+	ExporterTypeStdOut
+	ExporterTypeStdErr
+	ExporterTypeGRPC
+	ExporterTypeHTTP
+	ExporterTypeGoogle
+	ExporterTypePrometheus
+)
+
+// isNone returns true if the ExporterType is either Unspecified or None.
+func (e ExporterType) isNone() bool {
+	return e == ExporterTypeUnspecified || e == ExporterTypeNone
+}
+
+type ExporterConfig struct {
+	Type            ExporterType
+	Endpoint        string
+	Insecure        bool
+	BatchDuration   time.Duration
+	GoogleProjectID string // only for tracing, metrics and profiler
+}
+
+// setupOTelSDK bootstraps the OpenTelemetry pipeline.
+// If it does not return an error, make sure to call shutdown for proper cleanup.
+func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, ShutdownFunc, error) {
+	var (
+		shutdownFuncs []ShutdownFunc
+		err           error
+	)
+
+	// shutdown calls cleanup functions registered via shutdownFuncs.
+	// The errors from the calls are joined.
+	// Each registered cleanup will be invoked once.
+	shutdown := func(ctx context.Context) error {
+		var err error
+		for _, fn := range shutdownFuncs {
+			err = errors.Join(err, fn(ctx))
+		}
+		shutdownFuncs = nil
+		return err
+	}
+
+	// handleErr calls shutdown for cleanup and makes sure that all errors are returned.
+	handleErr := func(inErr error) {
+		err = errors.Join(inErr, shutdown(ctx))
+	}
+
+	// Set up propagator.
+	prop := newPropagator()
+	otel.SetTextMapPropagator(prop)
+
+	resource, err := resourceWithService("ZITADEL")
+	if err != nil {
+		return nil, shutdown, err
+	}
+
+	// Set up trace provider.
+	tracerProvider, err := newTracerProvider(ctx, cfg.Trace, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown)
+	otel.SetTracerProvider(tracerProvider)
+
+	// Set up meter provider.
+	meterProvider, err := newMeterProvider(ctx, cfg.Metric, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown)
+	otel.SetMeterProvider(meterProvider)
+	// Set up logger provider.
+	loggerProvider, err := newLoggerProvider(ctx, cfg.Log.Exporter, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown)
+	global.SetLoggerProvider(loggerProvider)
+
+	return loggerProvider, shutdown, err
+}
+
+func newPropagator() propagation.TextMapPropagator {
+	return propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+}
+
+func resourceWithService(serviceName string) (*resource.Resource, error) {
+	attributes := []attribute.KeyValue{
+		semconv.ServiceNameKey.String(serviceName),
+	}
+	if build.Version() != "" {
+		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
+	}
+	return resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes("", attributes...),
+	)
+}
+
+func errExporterType(typ ExporterType, instrument string) error {
+	return fmt.Errorf("exporter type \"%v\" unsupported for %s", typ, instrument)
+}

backend/v3/instrumentation/otel_test.go

@@ -0,0 +1,37 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExporterType_isNone(t *testing.T) {
+	tests := []struct {
+		name string
+		e    ExporterType
+		want bool
+	}{
+		{
+			name: "unspecified is none",
+			e:    ExporterTypeUnspecified,
+			want: true,
+		},
+		{
+			name: "none is none",
+			e:    ExporterTypeNone,
+			want: true,
+		},
+		{
+			name: "stdout is not none",
+			e:    ExporterTypeStdOut,
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.e.isNone()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

backend/v3/instrumentation/profiler.go

@@ -0,0 +1,44 @@
+package instrumentation
+
+import (
+	"cloud.google.com/go/profiler"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+type ProfileConfig struct {
+	Exporter ExporterConfig
+}
+
+// TODO: remove for v5 release
+type LegacyProfileConfig struct {
+	Type      string
+	ProjectID string
+}
+
+func (c *ProfileConfig) SetLegacyConfig(lc *LegacyProfileConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	if lc.Type == "google" {
+		c.Exporter.Type = ExporterTypeGoogle
+		c.Exporter.GoogleProjectID = lc.ProjectID
+	}
+}
+
+func startProfiler(cfg ProfileConfig, serviceName string) error {
+	typ := cfg.Exporter.Type
+	if typ.isNone() {
+		return nil
+	}
+	if typ != ExporterTypeGoogle {
+		return errExporterType(typ, "profiler")
+	}
+	pc := profiler.Config{
+		Service:        serviceName,
+		ServiceVersion: build.Version(),
+		ProjectID:      cfg.Exporter.GoogleProjectID,
+	}
+	return profiler.Start(pc)
+}

backend/v3/instrumentation/profiler_test.go

@@ -0,0 +1,137 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProfileConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string
+		target ProfileConfig
+		lc     *LegacyProfileConfig
+		want   ProfileConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "project-id",
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "project-id",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_startProfiler(t *testing.T) {
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     ProfileConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "google exporter returns error (no credentials)",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "test-project",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "stdout exporter returns error",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotErr := startProfiler(tt.cfg, "service-name")
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+			} else {
+				assert.NoError(t, gotErr)
+			}
+		})
+	}
+}

backend/v3/instrumentation/sampler_test.go

@@ -0,0 +1,80 @@
+package instrumentation
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func TestSpanKindBased(t *testing.T) {
+	type args struct {
+		sampler sdk_trace.Sampler
+		kinds   []trace.SpanKind
+	}
+	type want struct {
+		description string
+		sampled     int
+	}
+	tests := []struct {
+		name string
+		args args
+		want want
+	}{
+		{
+			"never sample, no sample",
+			args{
+				sampler: sdk_trace.NeverSample(),
+				kinds:   []trace.SpanKind{trace.SpanKindServer},
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOffSampler,kinds:[server]}",
+				sampled:     0,
+			},
+		},
+		{
+			"always sample, no kind, no sample",
+			args{
+				sampler: sdk_trace.AlwaysSample(),
+				kinds:   nil,
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[]}",
+				sampled:     0,
+			},
+		},
+		{
+			"always sample, 2 kinds, 2 samples",
+			args{
+				sampler: sdk_trace.AlwaysSample(),
+				kinds:   []trace.SpanKind{trace.SpanKindServer, trace.SpanKindClient},
+			},
+			want{
+				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[server client]}",
+				sampled:     2,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sampler := spanKindBased(tt.args.sampler, tt.args.kinds...)
+			assert.Equal(t, tt.want.description, sampler.Description())
+
+			p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))
+			tr := p.Tracer("test")
+
+			var sampled int
+			for i := trace.SpanKindUnspecified; i <= trace.SpanKindConsumer; i++ {
+				ctx := context.Background()
+				_, span := tr.Start(ctx, "test", trace.WithSpanKind(i))
+				if span.SpanContext().IsSampled() {
+					sampled++
+				}
+			}
+
+			assert.Equal(t, tt.want.sampled, sampled)
+		})
+	}
+}

backend/v3/instrumentation/slog.go

@@ -0,0 +1,197 @@
+package instrumentation
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"connectrpc.com/connect"
+	"github.com/muhlemmer/sloggcp"
+	"github.com/rs/xid"
+	slogmulti "github.com/samber/slog-multi"
+	slogctx "github.com/veqryn/slog-context"
+	slogotel "github.com/veqryn/slog-context/otel"
+	"go.opentelemetry.io/contrib/bridges/otelslog"
+	"go.opentelemetry.io/otel/sdk/log"
+	"google.golang.org/grpc"
+
+	http_util "github.com/zitadel/zitadel/internal/api/http"
+)
+
+type LogFormat int
+
+//go:generate enumer -type=LogFormat -trimprefix=LogFormat -text -linecomment
+const (
+	// Empty line comment sets empty string of unspecified value
+	LogFormatUndefined LogFormat = iota //
+	LogFormatDisabled
+	LogFormatText
+	LogFormatJSON
+	// JSON formatted logs compatible with Google Cloud Platform
+	LogFormatGCP
+)
+
+type LogConfig struct {
+	Level           slog.Level
+	Format          LogFormat
+	AddSource       bool
+	GoogleProjectID string
+	GoogleLogName   string
+	Exporter        ExporterConfig
+}
+
+// setLogger configures the global slog logger.
+// Logs are sent to StdErr and/or the [log.LoggerProvider].
+//
+// When present in the context, each line emitted contains:
+//   - Trace ID
+//   - Span ID
+//   - Service name
+//   - Sanitized request hosts from [http_util.DomainCtx]
+//   - Request details, such as method and path.
+func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
+	options := &slog.HandlerOptions{
+		AddSource: cfg.AddSource,
+		Level:     cfg.Level,
+	}
+	var stdErrHandler slog.Handler
+	switch cfg.Format {
+	case LogFormatUndefined:
+		return // uses default slog logger
+	case LogFormatDisabled:
+		stdErrHandler = slog.DiscardHandler
+	case LogFormatText:
+		stdErrHandler = slog.NewTextHandler(os.Stderr, options)
+	case LogFormatJSON:
+		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
+	case LogFormatGCP:
+		options.ReplaceAttr = sloggcp.ReplaceAttr
+		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
+	}
+
+	stdErrHandler = slogctx.NewHandler(
+		stdErrHandler,
+		&slogctx.HandlerOptions{
+			Prependers: []slogctx.AttrExtractor{
+				domainExtractor,
+				requestExtractor,
+				slogotel.ExtractTraceSpanID,
+				slogctx.ExtractPrepended,
+			},
+		},
+	)
+
+	otelHandler := otelslog.NewHandler(
+		Name,
+		otelslog.WithLoggerProvider(provider),
+	)
+
+	logger := slog.New(slogmulti.Fanout(stdErrHandler, otelHandler))
+	logger.Info("structured logger configured", "config_level", cfg.Level, "format", cfg.Format)
+	slog.SetDefault(logger)
+}
+
+const (
+	ProtocolHttp    = "http"
+	ProtocolConnect = "connect"
+	ProtocolGrpc    = "grpc"
+)
+
+// SetHttpRequestDetails adds static details to each context aware log entry.
+func SetHttpRequestDetails(ctx context.Context, service string, request *http.Request) context.Context {
+	now := time.Now()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolHttp,
+		service:     service,
+		http_method: request.Method,
+		path:        request.URL.Path,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+// SetConnectRequestDetails adds static details to each context aware log entry.
+func SetConnectRequestDetails(ctx context.Context, request connect.AnyRequest) context.Context {
+	now := time.Now()
+	spec := request.Spec()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolConnect,
+		service:     serviceFromRPCMethod(spec.Procedure),
+		http_method: request.HTTPMethod(),
+		path:        spec.Procedure,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+func SetGrpcRequestDetails(ctx context.Context, info *grpc.UnaryServerInfo) context.Context {
+	now := time.Now()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolGrpc,
+		service:     serviceFromRPCMethod(info.FullMethod),
+		http_method: http.MethodPost, // gRPC always uses POST
+		path:        info.FullMethod,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+type ctxKey struct{}
+
+type requestDetails struct {
+	protocol    string
+	service     string
+	http_method string
+	path        string
+	requestID   xid.ID
+	start       time.Time
+}
+
+func (r *requestDetails) attrs() []slog.Attr {
+	attrs := make([]slog.Attr, 0, 6)
+	if r.protocol != "" {
+		attrs = append(attrs, slog.String("protocol", r.protocol))
+	}
+	if r.service != "" {
+		attrs = append(attrs, slog.String("service", r.service))
+	}
+	if r.http_method != "" {
+		attrs = append(attrs, slog.String("http_method", r.http_method))
+	}
+	if r.path != "" {
+		attrs = append(attrs, slog.String("path", r.path))
+	}
+	if !r.requestID.IsZero() {
+		attrs = append(attrs, slog.String("request_id", r.requestID.String()))
+	}
+	if !r.start.IsZero() {
+		attrs = append(attrs, slog.Duration("duration", time.Since(r.start)))
+	}
+	return attrs
+}
+
+func serviceFromRPCMethod(fullMethod string) string {
+	parts := strings.Split(fullMethod, "/")
+	if len(parts) >= 2 {
+		return parts[1]
+	}
+	return "unknown"
+}
+
+// domainExtractor sets the sanitized request hosts from [http_util.DomainCtx] to a log entry.
+func domainExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	return []slog.Attr{
+		slog.Any("domain", http_util.DomainContext(ctx)),
+	}
+}
+
+// requestExtractor sets the request details from [requestDetails] to a log entry.
+func requestExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	if r, ok := ctx.Value(ctxKey{}).(*requestDetails); ok {
+		return r.attrs()
+	}
+	return nil
+}

backend/v3/instrumentation/tracing.go

@@ -0,0 +1,269 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"slices"
+
+	google_trace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
+)
+
+type TraceConfig struct {
+	Fraction float64
+	Exporter ExporterConfig
+}
+
+// TODO: remove for v5 release
+type LegacyTraceConfig struct {
+	Type      string
+	Fraction  float64
+	Endpoint  string
+	ProjectID string
+}
+
+// SetLegacyConfig sets the fields of the TraceConfig based on the provided LegacyTraceConfig.
+// If the Type field is already set to a value other than [ExporterTypeNone], the legacy config is ignored.
+func (c *TraceConfig) SetLegacyConfig(lc *LegacyTraceConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	typeMap := map[string]ExporterType{
+		"otel":   ExporterTypeGRPC,
+		"google": ExporterTypeGoogle,
+		"log":    ExporterTypeStdOut,
+	}
+
+	c.Fraction = lc.Fraction
+	c.Exporter.Type = typeMap[lc.Type]
+	c.Exporter.Endpoint = lc.Endpoint
+	c.Exporter.GoogleProjectID = lc.ProjectID
+}
+
+type Tracer struct {
+	trace.Tracer
+}
+
+func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindServer))
+}
+
+func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindServer))
+}
+
+func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindClient))
+}
+
+func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindClient))
+}
+
+func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller)
+}
+
+func (t *Tracer) newSpan(ctx context.Context, caller string, options ...trace.SpanStartOption) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, caller, options...)
+}
+
+func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...trace.SpanStartOption) (context.Context, *Span) {
+	ctx, span := t.Start(ctx, name, options...)
+	return ctx, CreateSpan(span)
+}
+
+func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span) {
+	ctx, span := t.NewSpan(r.Context(), caller)
+	r = r.WithContext(ctx)
+	return r, span
+}
+
+type Span struct {
+	span trace.Span
+	opts []trace.SpanEndOption
+}
+
+func CreateSpan(span trace.Span) *Span {
+	return &Span{span: span, opts: []trace.SpanEndOption{}}
+}
+
+func (s *Span) End() {
+	if s.span == nil {
+		return
+	}
+
+	s.span.End(s.opts...)
+}
+
+func (s *Span) EndWithError(err error) {
+	s.SetStatusByError(err)
+	s.End()
+}
+
+func (s *Span) SetStatusByError(err error) {
+	if s.span == nil {
+		return
+	}
+	if err != nil {
+		s.span.RecordError(err)
+	}
+
+	code, msg, id := gerrors.ExtractZITADELError(err)
+	s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
+}
+
+func newTracerProvider(ctx context.Context, cfg TraceConfig, resource *resource.Resource) (_ *sdk_trace.TracerProvider, err error) {
+	var exporter sdk_trace.SpanExporter
+	switch cfg.Exporter.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no exporter
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		exporter, err = traceStdOutExporter(cfg.Exporter)
+	case ExporterTypeGRPC:
+		exporter, err = traceGrpcExporter(ctx, cfg.Exporter)
+	case ExporterTypeHTTP:
+		exporter, err = traceHttpExporter(ctx, cfg.Exporter)
+	case ExporterTypeGoogle:
+		exporter, err = traceGoogleExporter(ctx, cfg.Exporter)
+	case ExporterTypePrometheus:
+		fallthrough // prometheus is not supported for logs
+	default:
+		err = errExporterType(cfg.Exporter.Type, "tracer")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("trace exporter: %w", err)
+	}
+	sampler := newSampler(cfg)
+	opts := []sdk_trace.TracerProviderOption{
+		sdk_trace.WithResource(resource),
+		sdk_trace.WithSampler(sampler),
+	}
+	if exporter != nil {
+		opts = append(opts, sdk_trace.WithBatcher(exporter))
+	}
+	tracerProvider := sdk_trace.NewTracerProvider(opts...)
+	return tracerProvider, nil
+}
+
+func traceStdOutExporter(cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	options := []stdouttrace.Option{
+		stdouttrace.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdouttrace.WithWriter(os.Stderr))
+	}
+	exporter, err := stdouttrace.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceGrpcExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	var grpcOpts []otlptracegrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlptracegrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlptracegrpc.WithInsecure())
+	}
+
+	exporter, err := otlptracegrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceHttpExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	var httpOpts []otlptracehttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlptracehttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlptracehttp.WithInsecure())
+	}
+
+	exporter, err := otlptracehttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceGoogleExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	exporter, err := google_trace.New(
+		google_trace.WithContext(ctx),
+		google_trace.WithProjectID(cfg.GoogleProjectID),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+// newSampler returns a sampler decorator which behaves differently,
+// based on the parent of the span. If the span has no parent and is of kind server,
+// the decorated sampler is used to make sampling decision.
+// If the span has a parent, depending on whether the parent is remote and whether it
+// is sampled, one of the following samplers will apply:
+//   - remote parent sampled -> always sample
+//   - remote parent not sampled -> sample based on the decorated sampler (fraction based)
+//   - local parent sampled -> always sample
+//   - local parent not sampled -> never sample
+func newSampler(cfg TraceConfig) sdk_trace.Sampler {
+	fraction := sdk_trace.TraceIDRatioBased(cfg.Fraction)
+	return sdk_trace.ParentBased(
+		spanKindBased(fraction, trace.SpanKindServer),
+		sdk_trace.WithRemoteParentNotSampled(fraction),
+	)
+}
+
+type spanKindSampler struct {
+	sampler sdk_trace.Sampler
+	kinds   []trace.SpanKind
+}
+
+// ShouldSample implements the [sdk_trace.Sampler] interface.
+// It will not sample any spans which do not match the configured span kinds.
+// For spans which do match, the decorated sampler is used to make the sampling decision.
+func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
+	psc := trace.SpanContextFromContext(p.ParentContext)
+	if !slices.Contains(sk.kinds, p.Kind) {
+		return sdk_trace.SamplingResult{
+			Decision:   sdk_trace.Drop,
+			Tracestate: psc.TraceState(),
+		}
+	}
+	s := sk.sampler.ShouldSample(p)
+	return s
+}
+
+func (sk spanKindSampler) Description() string {
+	return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
+		sk.sampler.Description(),
+		sk.kinds,
+	)
+}
+
+// spanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
+// If the span kind does not match one of the configured kinds, it will not be sampled.
+// If the span kind matches, the decorated sampler is used to make sampling decision.
+func spanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
+	return spanKindSampler{
+		sampler: sampler,
+		kinds:   kinds,
+	}
+}

backend/v3/instrumentation/tracing/http_handler.go

@@ -0,0 +1,23 @@
+package tracing
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewHandler(handler http.Handler, ignoredPrefix ...string) http.Handler {
+	return otelhttp.NewHandler(handler,
+		"zitadel",
+		otelhttp.WithFilter(instrumentation.RequestFilter(ignoredPrefix...)),
+		otelhttp.WithPublicEndpoint(),
+		otelhttp.WithSpanNameFormatter(spanNameFormatter),
+		otelhttp.WithMeterProvider(otel.GetMeterProvider()))
+}
+
+func spanNameFormatter(_ string, r *http.Request) string {
+	return r.URL.Path
+}

backend/v3/instrumentation/tracing/tracing.go

@@ -0,0 +1,65 @@
+// Package tracing provides helper functions to create spans for telemetry tracing.
+package tracing
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	api_trace "go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+type Tracer interface {
+	NewSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewClientSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewServerSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span)
+	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span)
+	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *instrumentation.Span)
+}
+
+var globalTracer = sync.OnceValue(func() Tracer {
+	return instrumentation.NewTracer(
+		instrumentation.Name,
+		api_trace.WithInstrumentationVersion(build.Version()),
+	)
+})
+
+func GlobalTracer() Tracer {
+	return globalTracer()
+}
+
+func NewSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewSpan(ctx, name)
+}
+
+func NewClientSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewClientSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewServerSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewServerSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewClientInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewServerInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewSpanHTTP(r *http.Request) (*http.Request, *instrumentation.Span) {
+	return globalTracer().NewSpanHTTP(r, instrumentation.GetCallingFunc(1))
+}
+
+func TraceIDFromCtx(ctx context.Context) string {
+	return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
+}

backend/v3/instrumentation/tracing_test.go

@@ -0,0 +1,191 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func TestTraceConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string
+		target TraceConfig
+		lc     *LegacyTraceConfig
+		want   TraceConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Fraction: 0.5,
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGoogle,
+					Endpoint: "endpoint",
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Fraction: 0.5,
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGoogle,
+					Endpoint: "endpoint",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_newTracerProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     TraceConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdErr,
+				},
+			},
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGRPC,
+					Endpoint: "localhost:4317",
+				},
+			},
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeHTTP,
+					Endpoint: "http://localhost:4318/v1/traces",
+				},
+			},
+		},
+		{
+			name: "google exporter return error (no credentials)",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeGoogle,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid exporter type returns error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterType(-1),
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newTracerProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/telemetry/logging/logger.go

@@ -1,12 +0,0 @@
-package logging
-
-import "log/slog"
-
-// Logger abstracts [slog.Logger] not sure if thats needed
-type Logger struct {
-	*slog.Logger
-}
-
-func NewLogger(logger *slog.Logger) *Logger {
-	return &Logger{Logger: logger}
-}

backend/v3/telemetry/metric/doc.go

@@ -1,2 +0,0 @@
-// implementation of otel metrics
-package metric

backend/v3/telemetry/tracing/tracer.go

@@ -1,24 +0,0 @@
-package tracing
-
-import (
-	"context"
-
-	"go.opentelemetry.io/otel/trace"
-	"go.opentelemetry.io/otel/trace/noop"
-)
-
-// Tracer is a wrapper around the OpenTelemetry Tracer interface.
-type Tracer struct {
-	trace.Tracer
-}
-
-var noopTracer = Tracer{
-	Tracer: noop.NewTracerProvider().Tracer(""),
-}
-
-func (t *Tracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
-	if t.Tracer == nil {
-		return noopTracer.Start(ctx, spanName, opts...)
-	}
-	return t.Tracer.Start(ctx, spanName, opts...)
-}