feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

.devcontainer/devcontainer.json

@@ -43,6 +43,10 @@
 	"containerEnv": {
 		"DEVCONTAINER_LOCAL_WORKSPACE_ROOT": "${localWorkspaceFolder}"
 	},
+	"remoteEnv": {
+		"PATH": "${containerEnv:PATH}:/workspaces/zitadel/.artifacts/bin/linux/amd64"
+	},
+	"postStartCommand": "echo 'export PATH=\"${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\"' >> ~/.bashrc",
 	"customizations": {
 		"vscode": {
 			"extensions": [

apps/api/project.json

@@ -118,7 +118,7 @@
             "dependsOn": [
                 "generate"
             ],
-            "command": "go test -race -coverprofile=profile.api.test-unit.cov -coverpkg=./internal/...  ./...",
+            "command": "go test -race -coverprofile=profile.api.test-unit.cov -coverpkg=./internal/...,./backend/...  ./...",
             "inputs": [
                 "sources",
                 "{workspaceRoot}/go*"
@@ -224,6 +224,7 @@
             "executor": "nx:run-commands",
             "options": {
                 "commands": [
+                    "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/daixiang0/[email protected]",
                     "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/dmarkham/[email protected]",
                     "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install go.uber.org/mock/[email protected]",
                     "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install golang.org/x/tools/cmd/[email protected]",

backend/v3/domain/defaults.go

@@ -1,18 +1,12 @@
 package domain
 
 import (
-	"log/slog"
-
 	"github.com/zitadel/zitadel/backend/v3/storage/database"
 	"github.com/zitadel/zitadel/backend/v3/storage/eventstore"
-	"github.com/zitadel/zitadel/backend/v3/telemetry/logging"
-	"github.com/zitadel/zitadel/backend/v3/telemetry/tracing"
 )
 
 var (
 	pool             database.Pool
-	tracer           tracing.Tracer
-	logger           logging.Logger = *logging.NewLogger(slog.Default())
 	legacyEventstore eventstore.LegacyEventstore
 )
 
@@ -20,14 +14,6 @@ func SetPool(p database.Pool) {
 	pool = p
 }
 
-func SetTracer(t tracing.Tracer) {
-	tracer = t
-}
-
-func SetLogger(l logging.Logger) {
-	logger = l
-}
-
 func SetLegacyEventstore(es eventstore.LegacyEventstore) {
 	legacyEventstore = es
 }

backend/v3/domain/invoker_logging.go

@@ -3,6 +3,8 @@ package domain
 import (
 	"context"
 	"time"
+
+	slogctx "github.com/veqryn/slog-context"
 )
 
 // loggingInvoker decorates each command with logging.
@@ -19,6 +21,7 @@ func NewLoggingInvoker(next Invoker) *loggingInvoker {
 
 func (i *loggingInvoker) Invoke(ctx context.Context, executor Executor, opts *InvokeOpts) (err error) {
 	start := time.Now()
+	logger := slogctx.FromCtx(ctx)
 	logger.InfoContext(ctx, "invoking", "name", executor.String())
 
 	err = i.execute(ctx, executor, opts)

backend/v3/domain/invoker_tracing.go

@@ -3,6 +3,8 @@ package domain
 import (
 	"context"
 	"fmt"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
 )
 
 // traceInvoker decorates each command with tracing.
@@ -17,12 +19,9 @@ func NewTraceInvoker(next Invoker) *traceInvoker {
 }
 
 func (i *traceInvoker) Invoke(ctx context.Context, executor Executor, opts *InvokeOpts) (err error) {
-	ctx, span := tracer.Start(ctx, fmt.Sprintf("%T", executor))
+	ctx, span := tracing.NewNamedSpan(ctx, fmt.Sprintf("%T", executor))
 	defer func() {
-		if err != nil {
-			span.RecordError(err)
-		}
-		span.End()
+		span.EndWithError(err)
 	}()
 
 	return i.execute(ctx, executor, opts)

backend/v3/instrumentation/caller.go

@@ -0,0 +1,46 @@
+package instrumentation
+
+import (
+	"log/slog"
+	"runtime"
+)
+
+type Caller struct {
+	Function string
+	File     string
+	Line     int
+}
+
+// GetCaller returns the caller information
+// skipping the given number of stack frames relative to the caller of this function.
+func GetCaller(skip int) (Caller, bool) {
+	pc, file, line, ok := runtime.Caller(skip + 1)
+	if !ok {
+		slog.Default().Debug("no caller info", "id", "TRACE-3r8bE")
+		return Caller{}, false
+	}
+
+	f := runtime.FuncForPC(pc)
+	if f == nil {
+		slog.Default().Debug("caller was nil", "id", "TRACE-25POw")
+		return Caller{}, false
+	}
+	return Caller{
+		Function: f.Name(),
+		File:     file,
+		Line:     line,
+	}, true
+}
+
+const CallerUnknown = "unknown"
+
+// GetCallingFunc returns the caller function name,
+// skipping the given number of stack frames relative to the caller of this function.
+// If the caller cannot be determined, [CallerUnknown] is returned.
+func GetCallingFunc(skip int) string {
+	caller, ok := GetCaller(skip + 1)
+	if !ok {
+		return CallerUnknown
+	}
+	return caller.Function
+}

backend/v3/instrumentation/caller_test.go

@@ -0,0 +1,48 @@
+package instrumentation
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetCaller(t *testing.T) {
+	_, _, prevLine, _ := runtime.Caller(0)
+	caller, ok := GetCaller(0)
+	require.True(t, ok)
+	want := Caller{
+		Function: "github.com/zitadel/zitadel/backend/v3/instrumentation.TestGetCaller",
+		File:     "backend/v3/instrumentation/caller_test.go",
+		Line:     prevLine + 1,
+	}
+	assert.Equal(t, want.Function, caller.Function, "function")
+	assert.Contains(t, caller.File, want.File, "file")
+	assert.Equal(t, want.Line, caller.Line, "line")
+}
+
+func TestGetCallingFunc(t *testing.T) {
+	tests := []struct {
+		name string
+		skip int
+		want string
+	}{
+		{
+			name: "test function",
+			skip: 0,
+			want: "github.com/zitadel/zitadel/backend/v3/instrumentation.TestGetCallingFunc.func1",
+		},
+		{
+			name: "unknown caller",
+			skip: 100,
+			want: CallerUnknown,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			callerFunc := GetCallingFunc(tt.skip)
+			assert.Equal(t, tt.want, callerFunc, "function")
+		})
+	}
+}

backend/v3/instrumentation/exportertype_enumer.go

@@ -0,0 +1,114 @@
+// Code generated by "enumer -type=ExporterType -trimprefix=ExporterType -text -linecomment"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _ExporterTypeName = "NoneStdOutStdErrGRPCHTTPGooglePrometheus"
+
+var _ExporterTypeIndex = [...]uint8{0, 0, 4, 10, 16, 20, 24, 30, 40}
+
+const _ExporterTypeLowerName = "nonestdoutstderrgrpchttpgoogleprometheus"
+
+func (i ExporterType) String() string {
+	if i < 0 || i >= ExporterType(len(_ExporterTypeIndex)-1) {
+		return fmt.Sprintf("ExporterType(%d)", i)
+	}
+	return _ExporterTypeName[_ExporterTypeIndex[i]:_ExporterTypeIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _ExporterTypeNoOp() {
+	var x [1]struct{}
+	_ = x[ExporterTypeUnspecified-(0)]
+	_ = x[ExporterTypeNone-(1)]
+	_ = x[ExporterTypeStdOut-(2)]
+	_ = x[ExporterTypeStdErr-(3)]
+	_ = x[ExporterTypeGRPC-(4)]
+	_ = x[ExporterTypeHTTP-(5)]
+	_ = x[ExporterTypeGoogle-(6)]
+	_ = x[ExporterTypePrometheus-(7)]
+}
+
+var _ExporterTypeValues = []ExporterType{ExporterTypeUnspecified, ExporterTypeNone, ExporterTypeStdOut, ExporterTypeStdErr, ExporterTypeGRPC, ExporterTypeHTTP, ExporterTypeGoogle, ExporterTypePrometheus}
+
+var _ExporterTypeNameToValueMap = map[string]ExporterType{
+	_ExporterTypeName[0:0]:        ExporterTypeUnspecified,
+	_ExporterTypeLowerName[0:0]:   ExporterTypeUnspecified,
+	_ExporterTypeName[0:4]:        ExporterTypeNone,
+	_ExporterTypeLowerName[0:4]:   ExporterTypeNone,
+	_ExporterTypeName[4:10]:       ExporterTypeStdOut,
+	_ExporterTypeLowerName[4:10]:  ExporterTypeStdOut,
+	_ExporterTypeName[10:16]:      ExporterTypeStdErr,
+	_ExporterTypeLowerName[10:16]: ExporterTypeStdErr,
+	_ExporterTypeName[16:20]:      ExporterTypeGRPC,
+	_ExporterTypeLowerName[16:20]: ExporterTypeGRPC,
+	_ExporterTypeName[20:24]:      ExporterTypeHTTP,
+	_ExporterTypeLowerName[20:24]: ExporterTypeHTTP,
+	_ExporterTypeName[24:30]:      ExporterTypeGoogle,
+	_ExporterTypeLowerName[24:30]: ExporterTypeGoogle,
+	_ExporterTypeName[30:40]:      ExporterTypePrometheus,
+	_ExporterTypeLowerName[30:40]: ExporterTypePrometheus,
+}
+
+var _ExporterTypeNames = []string{
+	_ExporterTypeName[0:0],
+	_ExporterTypeName[0:4],
+	_ExporterTypeName[4:10],
+	_ExporterTypeName[10:16],
+	_ExporterTypeName[16:20],
+	_ExporterTypeName[20:24],
+	_ExporterTypeName[24:30],
+	_ExporterTypeName[30:40],
+}
+
+// ExporterTypeString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func ExporterTypeString(s string) (ExporterType, error) {
+	if val, ok := _ExporterTypeNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _ExporterTypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to ExporterType values", s)
+}
+
+// ExporterTypeValues returns all values of the enum
+func ExporterTypeValues() []ExporterType {
+	return _ExporterTypeValues
+}
+
+// ExporterTypeStrings returns a slice of all String values of the enum
+func ExporterTypeStrings() []string {
+	strs := make([]string, len(_ExporterTypeNames))
+	copy(strs, _ExporterTypeNames)
+	return strs
+}
+
+// IsAExporterType returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i ExporterType) IsAExporterType() bool {
+	for _, v := range _ExporterTypeValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for ExporterType
+func (i ExporterType) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for ExporterType
+func (i *ExporterType) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = ExporterTypeString(string(text))
+	return err
+}

backend/v3/instrumentation/instrumentation.go

@@ -0,0 +1,86 @@
+// Package instrumentation initializes and configures telemetry and logging instrumentation.
+// After [Start] is called, the instrumentation providers can be accessed using global getters.
+package instrumentation
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"sync"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/trace"
+)
+
+const Name = "zitadel/backend/v3/instrumentation/tracing"
+
+type Config struct {
+	ServiceName string
+	Trace       TraceConfig
+	Metric      MetricConfig
+	Log         LogConfig
+	Profile     ProfileConfig
+}
+
+type ShutdownFunc func(context.Context) error
+
+var (
+	startOnce sync.Once
+	shutdown  ShutdownFunc
+	errStart  error
+)
+
+// Start initializes global instrumentation once, based on the provided configuration.
+// It is safe to call multiple times; subsequent calls will return the same shutdown function and error as the first call.
+func Start(ctx context.Context, cfg Config) (ShutdownFunc, error) {
+	startOnce.Do(func() {
+		errStart = startProfiler(cfg.Profile, cfg.ServiceName)
+		if errStart != nil {
+			return
+		}
+		var logProvider *log.LoggerProvider
+		logProvider, shutdown, errStart = setupOTelSDK(ctx, cfg)
+		if errStart != nil {
+			return
+		}
+		setLogger(logProvider, cfg.Log)
+	})
+	return shutdown, errStart
+}
+
+// NewTracer creates a new tracer from the global provider.
+// See [trace.TracerProvider.Tracer] for details.
+func NewTracer(name string, options ...trace.TracerOption) *Tracer {
+	provider := otel.GetTracerProvider()
+	return &Tracer{provider.Tracer(name, options...)}
+}
+
+// NewMeter creates a new meter from the global provider.
+// See [metric.MeterProvider.Meter] for details.
+func NewMeter(name string, options ...metric.MeterOption) *Meter {
+	provider := otel.GetMeterProvider()
+	return &Meter{
+		Meter: provider.Meter(name, options...),
+	}
+}
+
+// Logger returns the globally configured logger.
+func Logger() *slog.Logger {
+	return slog.Default()
+}
+
+func RequestFilter(ignoredPrefix ...string) otelhttp.Filter {
+	return func(r *http.Request) bool {
+		return !slices.ContainsFunc(
+			ignoredPrefix,
+			func(endpoint string) bool {
+				return strings.HasPrefix(r.URL.RequestURI(), endpoint)
+			},
+		)
+	}
+}

backend/v3/instrumentation/instrumentation_test.go

@@ -0,0 +1,40 @@
+package instrumentation
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRequestFilter(t *testing.T) {
+	filter := RequestFilter("/foo", "/bar")
+	var tests = []struct {
+		name string
+		r    *http.Request
+		want bool
+	}{
+		{
+			name: "foo subpath false",
+			r:    httptest.NewRequest("POST", "/foo/some", nil),
+			want: false,
+		},
+		{
+			name: "foo exact false",
+			r:    httptest.NewRequest("POST", "/foo", nil),
+			want: false,
+		},
+		{
+			name: "other path true",
+			r:    httptest.NewRequest("POST", "/other", nil),
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := filter(tt.r)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

backend/v3/instrumentation/logformat_enumer.go

@@ -0,0 +1,102 @@
+// Code generated by "enumer -type=LogFormat -trimprefix=LogFormat -text -linecomment"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _LogFormatName = "DisabledTextJSONGCP"
+
+var _LogFormatIndex = [...]uint8{0, 0, 8, 12, 16, 19}
+
+const _LogFormatLowerName = "disabledtextjsongcp"
+
+func (i LogFormat) String() string {
+	if i < 0 || i >= LogFormat(len(_LogFormatIndex)-1) {
+		return fmt.Sprintf("LogFormat(%d)", i)
+	}
+	return _LogFormatName[_LogFormatIndex[i]:_LogFormatIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _LogFormatNoOp() {
+	var x [1]struct{}
+	_ = x[LogFormatUndefined-(0)]
+	_ = x[LogFormatDisabled-(1)]
+	_ = x[LogFormatText-(2)]
+	_ = x[LogFormatJSON-(3)]
+	_ = x[LogFormatGCP-(4)]
+}
+
+var _LogFormatValues = []LogFormat{LogFormatUndefined, LogFormatDisabled, LogFormatText, LogFormatJSON, LogFormatGCP}
+
+var _LogFormatNameToValueMap = map[string]LogFormat{
+	_LogFormatName[0:0]:        LogFormatUndefined,
+	_LogFormatLowerName[0:0]:   LogFormatUndefined,
+	_LogFormatName[0:8]:        LogFormatDisabled,
+	_LogFormatLowerName[0:8]:   LogFormatDisabled,
+	_LogFormatName[8:12]:       LogFormatText,
+	_LogFormatLowerName[8:12]:  LogFormatText,
+	_LogFormatName[12:16]:      LogFormatJSON,
+	_LogFormatLowerName[12:16]: LogFormatJSON,
+	_LogFormatName[16:19]:      LogFormatGCP,
+	_LogFormatLowerName[16:19]: LogFormatGCP,
+}
+
+var _LogFormatNames = []string{
+	_LogFormatName[0:0],
+	_LogFormatName[0:8],
+	_LogFormatName[8:12],
+	_LogFormatName[12:16],
+	_LogFormatName[16:19],
+}
+
+// LogFormatString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func LogFormatString(s string) (LogFormat, error) {
+	if val, ok := _LogFormatNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _LogFormatNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to LogFormat values", s)
+}
+
+// LogFormatValues returns all values of the enum
+func LogFormatValues() []LogFormat {
+	return _LogFormatValues
+}
+
+// LogFormatStrings returns a slice of all String values of the enum
+func LogFormatStrings() []string {
+	strs := make([]string, len(_LogFormatNames))
+	copy(strs, _LogFormatNames)
+	return strs
+}
+
+// IsALogFormat returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i LogFormat) IsALogFormat() bool {
+	for _, v := range _LogFormatValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for LogFormat
+func (i LogFormat) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for LogFormat
+func (i *LogFormat) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = LogFormatString(string(text))
+	return err
+}

backend/v3/instrumentation/logger.go

@@ -0,0 +1,93 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdoutlog"
+	sdk_log "go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func newLoggerProvider(ctx context.Context, cfg ExporterConfig, resource *resource.Resource) (_ *sdk_log.LoggerProvider, err error) {
+	var exporter sdk_log.Exporter
+	switch cfg.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no exporter
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		exporter, err = logStdOutExporter(cfg)
+	case ExporterTypeGRPC:
+		exporter, err = logGrpcExporter(ctx, cfg)
+	case ExporterTypeHTTP:
+		exporter, err = logHttpExporter(ctx, cfg)
+	case ExporterTypeGoogle, ExporterTypePrometheus:
+		fallthrough // google, prometheus are not supported for logs
+	default:
+		err = errExporterType(cfg.Type, "logger")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("log exporter: %w", err)
+	}
+	opts := []sdk_log.LoggerProviderOption{
+		sdk_log.WithResource(resource),
+	}
+	if exporter != nil {
+		opts = append(opts, sdk_log.WithProcessor(
+			sdk_log.NewBatchProcessor(
+				exporter,
+				sdk_log.WithExportInterval(cfg.BatchDuration),
+			),
+		))
+	}
+	loggerProvider := sdk_log.NewLoggerProvider(opts...)
+	return loggerProvider, nil
+}
+
+func logStdOutExporter(cfg ExporterConfig) (sdk_log.Exporter, error) {
+	options := []stdoutlog.Option{
+		stdoutlog.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdoutlog.WithWriter(os.Stderr))
+	}
+	exporter, err := stdoutlog.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func logGrpcExporter(ctx context.Context, cfg ExporterConfig) (sdk_log.Exporter, error) {
+	var grpcOpts []otlploggrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlploggrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlploggrpc.WithInsecure())
+	}
+
+	exporter, err := otlploggrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func logHttpExporter(ctx context.Context, cfg ExporterConfig) (sdk_log.Exporter, error) {
+	var httpOpts []otlploghttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlploghttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlploghttp.WithInsecure())
+	}
+
+	exporter, err := otlploghttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}

backend/v3/instrumentation/logger_test.go

@@ -0,0 +1,81 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func Test_newLoggerProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     ExporterConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeNone,
+			},
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeStdOut,
+			},
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: ExporterConfig{
+				Type: ExporterTypeStdErr,
+			},
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: ExporterConfig{
+				Type:     ExporterTypeGRPC,
+				Endpoint: "localhost:4317",
+			},
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: ExporterConfig{
+				Type:     ExporterTypeHTTP,
+				Endpoint: "localhost:4318",
+			},
+		},
+		{
+			name: "google exporter returns error",
+			cfg: ExporterConfig{
+				Type:            ExporterTypeGoogle,
+				GoogleProjectID: "test-project",
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns error",
+			cfg: ExporterConfig{
+				Type: ExporterTypePrometheus,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newLoggerProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/instrumentation/logging/connect_handler.go

@@ -0,0 +1,71 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+
+	"connectrpc.com/connect"
+	slogctx "github.com/veqryn/slog-context"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewConnectInterceptor(next connect.UnaryFunc, ignoredMethodSuffixes ...string) connect.UnaryFunc {
+	return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
+		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+			return strings.HasSuffix(req.Spec().Procedure, s)
+		}) {
+			return next(ctx, req)
+		}
+
+		logger := instrumentation.Logger()
+		ctx = instrumentation.SetConnectRequestDetails(ctx, req)
+		ctx = slogctx.NewCtx(ctx, logger)
+
+		resp, err := next(ctx, req)
+		var code connect.Code
+		if err != nil {
+			code = connect.CodeOf(err)
+		}
+		logger.Log(ctx,
+			assertConnectLevel(code),
+			"connect RPC served",
+			"code", code,
+		)
+
+		return resp, err
+	}
+}
+
+func assertConnectLevel(code connect.Code) slog.Level {
+	switch code {
+	case connect.CodeUnimplemented:
+		return slog.LevelDebug
+
+	// client errors
+	case connect.CodeCanceled,
+		connect.CodeInvalidArgument,
+		connect.CodeNotFound,
+		connect.CodeAlreadyExists,
+		connect.CodePermissionDenied,
+		connect.CodeResourceExhausted,
+		connect.CodeFailedPrecondition,
+		connect.CodeOutOfRange,
+		connect.CodeUnauthenticated:
+		return slog.LevelWarn
+
+	// server errors
+	case connect.CodeUnknown,
+		connect.CodeAborted,
+		connect.CodeDeadlineExceeded,
+		connect.CodeInternal,
+		connect.CodeUnavailable,
+		connect.CodeDataLoss:
+		return slog.LevelError
+
+	default: // includes 0 code when no error
+		return slog.LevelInfo
+	}
+}

backend/v3/instrumentation/logging/grpc_handler.go

@@ -0,0 +1,74 @@
+package logging
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+
+	slogctx "github.com/veqryn/slog-context"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewGrpcInterceptor(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) {
+		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+			return strings.HasSuffix(info.FullMethod, s)
+		}) {
+			return next(ctx, req)
+		}
+
+		logger := instrumentation.Logger()
+		ctx = instrumentation.SetGrpcRequestDetails(ctx, info)
+		ctx = slogctx.NewCtx(ctx, logger)
+
+		resp, err := next(ctx, req)
+		var code codes.Code
+		if err != nil {
+			code = status.Code(err)
+		}
+		logger.Log(ctx,
+			assertGrpcLevel(code),
+			"gRPC served",
+			"code", code,
+		)
+
+		return resp, err
+	}
+}
+
+func assertGrpcLevel(code codes.Code) slog.Level {
+	switch code {
+	case codes.Unimplemented:
+		return slog.LevelDebug
+
+	// client errors
+	case codes.Canceled,
+		codes.InvalidArgument,
+		codes.NotFound,
+		codes.AlreadyExists,
+		codes.PermissionDenied,
+		codes.ResourceExhausted,
+		codes.FailedPrecondition,
+		codes.OutOfRange,
+		codes.Unauthenticated:
+		return slog.LevelWarn
+
+	// server errors
+	case codes.Unknown,
+		codes.Aborted,
+		codes.DeadlineExceeded,
+		codes.Internal,
+		codes.Unavailable,
+		codes.DataLoss:
+		return slog.LevelError
+	case codes.OK:
+		fallthrough
+	default: // includes 0 code when no error
+		return slog.LevelInfo
+	}
+}

backend/v3/instrumentation/logging/http_handler.go

@@ -0,0 +1,63 @@
+package logging
+
+import (
+	"log/slog"
+	"net/http"
+
+	slogctx "github.com/veqryn/slog-context"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewHandler(next http.Handler, service string, ignoredPrefix ...string) http.Handler {
+	filter := instrumentation.RequestFilter(ignoredPrefix...)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !filter(r) {
+			next.ServeHTTP(w, r)
+			return
+		}
+		logger := instrumentation.Logger()
+		ctx := instrumentation.SetHttpRequestDetails(r.Context(), service, r)
+		ctx = slogctx.NewCtx(ctx, logger)
+		sw := &statusWriter{ResponseWriter: w}
+
+		next.ServeHTTP(sw, r.WithContext(ctx))
+
+		logger.Log(ctx,
+			LogLevelFromStatus(sw.status),
+			"http request served",
+			"status", sw.status,
+		)
+	})
+}
+
+func LogLevelFromStatus(status int) slog.Level {
+	if status < 200 {
+		return slog.LevelDebug
+	}
+	if status < 400 {
+		return slog.LevelInfo
+	}
+	if status < 500 {
+		return slog.LevelWarn
+	}
+	return slog.LevelError
+}
+
+// statusWriter is a [http.ResponseWriter] that captures the status code for logging.
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func (w *statusWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.status = http.StatusOK
+	}
+	return w.ResponseWriter.Write(p)
+}
+
+func (w *statusWriter) WriteHeader(statusCode int) {
+	w.status = statusCode
+	w.ResponseWriter.WriteHeader(statusCode)
+}

backend/v3/instrumentation/metric.go

@@ -0,0 +1,286 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	google_metric "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
+	sdk_metric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+type MetricConfig struct {
+	Exporter ExporterConfig
+}
+
+type Meter struct {
+	Meter             metric.Meter
+	Counters          sync.Map
+	UpDownSumObserver sync.Map
+	ValueObservers    sync.Map
+	Histograms        sync.Map
+}
+
+// TODO: remove for v5 release
+type LegacyMetricConfig struct {
+	Type string
+}
+
+func (c *MetricConfig) SetLegacyConfig(lc *LegacyMetricConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	if lc.Type == "otel" {
+		c.Exporter.Type = ExporterTypePrometheus
+	}
+}
+
+var hasPrometheusExporter bool
+
+func (m *Meter) GetExporter() http.Handler {
+	if hasPrometheusExporter {
+		return promhttp.Handler()
+	}
+	return http.NotFoundHandler()
+}
+
+func (m *Meter) RegisterCounter(name, description string) error {
+	if _, exists := m.Counters.Load(name); exists {
+		return nil
+	}
+	counter, err := m.Meter.Int64Counter(name, metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+	m.Counters.Store(name, counter)
+	return nil
+}
+
+func (m *Meter) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
+	counter, exists := m.Counters.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
+	}
+	counter.(metric.Int64Counter).Add(ctx, value, MapToAddOption(labels)...)
+	return nil
+}
+
+func (m *Meter) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
+	histogram, exists := m.Histograms.Load(name)
+	if !exists {
+		return zerrors.ThrowNotFound(nil, "METER-5wwb1", "Errors.Metrics.Histogram.NotFound")
+	}
+	histogram.(metric.Float64Histogram).Record(ctx, value, MapToRecordOption(labels)...)
+	return nil
+}
+
+func (m *Meter) RegisterHistogram(name, description, unit string, buckets []float64) error {
+	if _, exists := m.Histograms.Load(name); exists {
+		return nil
+	}
+
+	histogram, err := m.Meter.Float64Histogram(name,
+		metric.WithDescription(description),
+		metric.WithUnit(unit),
+		metric.WithExplicitBucketBoundaries(buckets...),
+	)
+	if err != nil {
+		return err
+	}
+
+	m.Histograms.Store(name, histogram)
+	return nil
+}
+
+func (m *Meter) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	counter, err := m.Meter.Int64ObservableUpDownCounter(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, counter)
+	return nil
+}
+
+func (m *Meter) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
+	if _, exists := m.UpDownSumObserver.Load(name); exists {
+		return nil
+	}
+
+	gauge, err := m.Meter.Int64ObservableGauge(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
+	if err != nil {
+		return err
+	}
+
+	m.UpDownSumObserver.Store(name, gauge)
+	return nil
+}
+
+func MapToAddOption(labels map[string]attribute.Value) []metric.AddOption {
+	return []metric.AddOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func MapToRecordOption(labels map[string]attribute.Value) []metric.RecordOption {
+	return []metric.RecordOption{metric.WithAttributes(labelsToAttributes(labels)...)}
+}
+
+func labelsToAttributes(labels map[string]attribute.Value) []attribute.KeyValue {
+	if labels == nil {
+		return nil
+	}
+	attributes := make([]attribute.KeyValue, 0, len(labels))
+	for key, value := range labels {
+		attributes = append(attributes, attribute.KeyValue{
+			Key:   attribute.Key(key),
+			Value: value,
+		})
+	}
+	return attributes
+}
+
+func newMeterProvider(ctx context.Context, cfg MetricConfig, resource *resource.Resource) (_ *sdk_metric.MeterProvider, err error) {
+	var readerOption sdk_metric.Option
+	switch cfg.Exporter.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no reader option
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		readerOption, err = metricStdOutOption(cfg.Exporter)
+	case ExporterTypeGRPC:
+		readerOption, err = metricGrpcOption(ctx, cfg.Exporter)
+	case ExporterTypeHTTP:
+		readerOption, err = metricHttpOption(ctx, cfg.Exporter)
+	case ExporterTypeGoogle:
+		readerOption, err = metricGoogleOption(cfg.Exporter)
+	case ExporterTypePrometheus:
+		readerOption, err = metricPrometheusOption()
+	default:
+		err = errExporterType(cfg.Exporter.Type, "metrics")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("meter reader: %w", err)
+	}
+
+	// create a view to filter out unwanted attributes
+	view := sdk_metric.NewView(
+		sdk_metric.Instrument{
+			Scope: instrumentation.Scope{Name: otelhttp.ScopeName},
+		},
+		sdk_metric.Stream{
+			AttributeFilter: attribute.NewAllowKeysFilter("http.method", "http.status_code", "http.target"),
+		},
+	)
+
+	opts := []sdk_metric.Option{
+		sdk_metric.WithResource(resource),
+		sdk_metric.WithView(view),
+	}
+	if readerOption != nil {
+		opts = append(opts, readerOption)
+	}
+	meterProvider := sdk_metric.NewMeterProvider(opts...)
+	return meterProvider, nil
+}
+
+func metricStdOutOption(cfg ExporterConfig) (sdk_metric.Option, error) {
+	options := []stdoutmetric.Option{
+		stdoutmetric.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdoutmetric.WithWriter(os.Stderr))
+	}
+	exporter, err := stdoutmetric.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricGrpcOption(ctx context.Context, cfg ExporterConfig) (sdk_metric.Option, error) {
+	var grpcOpts []otlpmetricgrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlpmetricgrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlpmetricgrpc.WithInsecure())
+	}
+
+	exporter, err := otlpmetricgrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricHttpOption(ctx context.Context, cfg ExporterConfig) (sdk_metric.Option, error) {
+	var httpOpts []otlpmetrichttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlpmetrichttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlpmetrichttp.WithInsecure())
+	}
+
+	exporter, err := otlpmetrichttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricGoogleOption(cfg ExporterConfig) (sdk_metric.Option, error) {
+	exporter, err := google_metric.New(
+		google_metric.WithProjectID(cfg.GoogleProjectID),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return sdk_metric.WithReader(
+		sdk_metric.NewPeriodicReader(
+			exporter,
+			sdk_metric.WithInterval(cfg.BatchDuration),
+		),
+	), nil
+}
+
+func metricPrometheusOption() (sdk_metric.Option, error) {
+	prom, err := prometheus.New(prometheus.WithoutScopeInfo())
+	if err != nil {
+		return nil, err
+	}
+	hasPrometheusExporter = true
+	return sdk_metric.WithReader(prom), nil
+}

backend/v3/instrumentation/metric_test.go

@@ -0,0 +1,186 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func TestMetricConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string // description of this test case
+		target MetricConfig
+		lc     *LegacyMetricConfig
+		want   MetricConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyMetricConfig{
+				Type: "otel",
+			},
+			want: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_newMeterProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     MetricConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "unsupported exporter returns error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterType(-1),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdErr,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeHTTP,
+					Endpoint: "localhost:4318",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "google exporter return error (no credentials)",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeGoogle,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: MetricConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGRPC,
+					Endpoint: "localhost:4317",
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/instrumentation/metrics/http_handler.go

@@ -5,7 +5,10 @@ import (
 	"net/http"
 	"strings"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 )
 
 const (
@@ -23,7 +26,7 @@ const (
 type Handler struct {
 	handler http.Handler
 	methods []MetricType
-	filters []Filter
+	filter  otelhttp.Filter
 }
 
 type MetricType int32
@@ -47,13 +50,11 @@ func (r *StatusRecorder) WriteHeader(status int) {
 
 type Filter func(*http.Request) bool
 
-func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
+func NewHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
 	h := Handler{
 		handler: handler,
 		methods: metricMethods,
-	}
-	if len(ignoredEndpoints) > 0 {
-		h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
+		filter:  instrumentation.RequestFilter(ignoredEndpoints...),
 	}
 	return &h
 }
@@ -76,12 +77,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		h.handler.ServeHTTP(w, r)
 		return
 	}
-	for _, f := range h.filters {
-		if !f(r) {
-			// Simply pass through to the handler if a filter rejects the request
-			h.handler.ServeHTTP(w, r)
-			return
-		}
+	if !h.filter(r) {
+		// Simply pass through to the handler if a filter rejects the request
+		h.handler.ServeHTTP(w, r)
+		return
 	}
 	uri := strings.Split(r.RequestURI, "?")[0]
 	recorder := &StatusRecorder{
@@ -134,14 +133,3 @@ func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
 	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
 	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
 }
-
-func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
-	return func(r *http.Request) bool {
-		for _, endpoint := range endpoints {
-			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
-				return false
-			}
-		}
-		return true
-	}
-}

backend/v3/instrumentation/metrics/metric.go

@@ -3,9 +3,13 @@ package metrics
 import (
 	"context"
 	"net/http"
+	"sync"
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/cmd/build"
 )
 
 const (
@@ -19,7 +23,6 @@ const (
 
 type Metrics interface {
 	GetExporter() http.Handler
-	GetMetricsProvider() metric.MeterProvider
 	RegisterCounter(name, description string) error
 	AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error
 	AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error
@@ -28,60 +31,41 @@ type Metrics interface {
 	RegisterHistogram(name, description, unit string, buckets []float64) error
 }
 
-var M Metrics
+var globalMeter = sync.OnceValue(func() Metrics {
+	return instrumentation.NewMeter(
+		instrumentation.Name,
+		metric.WithInstrumentationVersion(build.Version()),
+	)
+})
 
-func GetExporter() http.Handler {
-	if M == nil {
-		return nil
-	}
-	return M.GetExporter()
+func GlobalMeter() Metrics {
+	return globalMeter()
 }
 
-func GetMetricsProvider() metric.MeterProvider {
-	if M == nil {
-		return nil
-	}
-	return M.GetMetricsProvider()
+func GetExporter() http.Handler {
+	return globalMeter().GetExporter()
 }
 
 func RegisterCounter(name, description string) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterCounter(name, description)
+	return globalMeter().RegisterCounter(name, description)
 }
 
 func AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
-	if M == nil {
-		return nil
-	}
-	return M.AddCount(ctx, name, value, labels)
+	return globalMeter().AddCount(ctx, name, value, labels)
 }
 
 func AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
-	if M == nil {
-		return nil
-	}
-	return M.AddHistogramMeasurement(ctx, name, value, labels)
+	return globalMeter().AddHistogramMeasurement(ctx, name, value, labels)
 }
 
 func RegisterHistogram(name, description, unit string, buckets []float64) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterHistogram(name, description, unit, buckets)
+	return globalMeter().RegisterHistogram(name, description, unit, buckets)
 }
 
 func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterUpDownSumObserver(name, description, callbackFunc)
+	return globalMeter().RegisterUpDownSumObserver(name, description, callbackFunc)
 }
 
 func RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterValueObserver(name, description, callbackFunc)
+	return globalMeter().RegisterValueObserver(name, description, callbackFunc)
 }

backend/v3/instrumentation/otel.go

@@ -0,0 +1,138 @@
+package instrumentation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/log/global"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/log"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+//go:generate enumer -type=ExporterType -trimprefix=ExporterType -text -linecomment
+type ExporterType int
+
+// ExporterType defines the type of exporter to use.
+// - metrics supports all types
+// - tracing supports all types except Prometheus
+// - logging supports all types except Google and Prometheus
+// - profiling supports only Google and None
+const (
+	// Empty line comment sets empty string of unspecified value
+	ExporterTypeUnspecified ExporterType = iota //
+	ExporterTypeNone
+	ExporterTypeStdOut
+	ExporterTypeStdErr
+	ExporterTypeGRPC
+	ExporterTypeHTTP
+	ExporterTypeGoogle
+	ExporterTypePrometheus
+)
+
+// isNone returns true if the ExporterType is either Unspecified or None.
+func (e ExporterType) isNone() bool {
+	return e == ExporterTypeUnspecified || e == ExporterTypeNone
+}
+
+type ExporterConfig struct {
+	Type            ExporterType
+	Endpoint        string
+	Insecure        bool
+	BatchDuration   time.Duration
+	GoogleProjectID string // only for tracing, metrics and profiler
+}
+
+// setupOTelSDK bootstraps the OpenTelemetry pipeline.
+// If it does not return an error, make sure to call shutdown for proper cleanup.
+func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, ShutdownFunc, error) {
+	var (
+		shutdownFuncs []ShutdownFunc
+		err           error
+	)
+
+	// shutdown calls cleanup functions registered via shutdownFuncs.
+	// The errors from the calls are joined.
+	// Each registered cleanup will be invoked once.
+	shutdown := func(ctx context.Context) error {
+		var err error
+		for _, fn := range shutdownFuncs {
+			err = errors.Join(err, fn(ctx))
+		}
+		shutdownFuncs = nil
+		return err
+	}
+
+	// handleErr calls shutdown for cleanup and makes sure that all errors are returned.
+	handleErr := func(inErr error) {
+		err = errors.Join(inErr, shutdown(ctx))
+	}
+
+	// Set up propagator.
+	prop := newPropagator()
+	otel.SetTextMapPropagator(prop)
+
+	resource, err := resourceWithService("ZITADEL")
+	if err != nil {
+		return nil, shutdown, err
+	}
+
+	// Set up trace provider.
+	tracerProvider, err := newTracerProvider(ctx, cfg.Trace, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, tracerProvider.Shutdown)
+	otel.SetTracerProvider(tracerProvider)
+
+	// Set up meter provider.
+	meterProvider, err := newMeterProvider(ctx, cfg.Metric, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, meterProvider.Shutdown)
+	otel.SetMeterProvider(meterProvider)
+	// Set up logger provider.
+	loggerProvider, err := newLoggerProvider(ctx, cfg.Log.Exporter, resource)
+	if err != nil {
+		handleErr(err)
+		return nil, shutdown, err
+	}
+	shutdownFuncs = append(shutdownFuncs, loggerProvider.Shutdown)
+	global.SetLoggerProvider(loggerProvider)
+
+	return loggerProvider, shutdown, err
+}
+
+func newPropagator() propagation.TextMapPropagator {
+	return propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+}
+
+func resourceWithService(serviceName string) (*resource.Resource, error) {
+	attributes := []attribute.KeyValue{
+		semconv.ServiceNameKey.String(serviceName),
+	}
+	if build.Version() != "" {
+		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
+	}
+	return resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes("", attributes...),
+	)
+}
+
+func errExporterType(typ ExporterType, instrument string) error {
+	return fmt.Errorf("exporter type \"%v\" unsupported for %s", typ, instrument)
+}

backend/v3/instrumentation/otel_test.go

@@ -0,0 +1,37 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExporterType_isNone(t *testing.T) {
+	tests := []struct {
+		name string
+		e    ExporterType
+		want bool
+	}{
+		{
+			name: "unspecified is none",
+			e:    ExporterTypeUnspecified,
+			want: true,
+		},
+		{
+			name: "none is none",
+			e:    ExporterTypeNone,
+			want: true,
+		},
+		{
+			name: "stdout is not none",
+			e:    ExporterTypeStdOut,
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.e.isNone()
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

backend/v3/instrumentation/profiler.go

@@ -0,0 +1,44 @@
+package instrumentation
+
+import (
+	"cloud.google.com/go/profiler"
+
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+type ProfileConfig struct {
+	Exporter ExporterConfig
+}
+
+// TODO: remove for v5 release
+type LegacyProfileConfig struct {
+	Type      string
+	ProjectID string
+}
+
+func (c *ProfileConfig) SetLegacyConfig(lc *LegacyProfileConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	if lc.Type == "google" {
+		c.Exporter.Type = ExporterTypeGoogle
+		c.Exporter.GoogleProjectID = lc.ProjectID
+	}
+}
+
+func startProfiler(cfg ProfileConfig, serviceName string) error {
+	typ := cfg.Exporter.Type
+	if typ.isNone() {
+		return nil
+	}
+	if typ != ExporterTypeGoogle {
+		return errExporterType(typ, "profiler")
+	}
+	pc := profiler.Config{
+		Service:        serviceName,
+		ServiceVersion: build.Version(),
+		ProjectID:      cfg.Exporter.GoogleProjectID,
+	}
+	return profiler.Start(pc)
+}

backend/v3/instrumentation/profiler_test.go

@@ -0,0 +1,137 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProfileConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string
+		target ProfileConfig
+		lc     *LegacyProfileConfig
+		want   ProfileConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "project-id",
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyProfileConfig{
+				Type:      "google",
+				ProjectID: "project-id",
+			},
+			want: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "project-id",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_startProfiler(t *testing.T) {
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     ProfileConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "google exporter returns error (no credentials)",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type:            ExporterTypeGoogle,
+					GoogleProjectID: "test-project",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "stdout exporter returns error",
+			cfg: ProfileConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotErr := startProfiler(tt.cfg, "service-name")
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+			} else {
+				assert.NoError(t, gotErr)
+			}
+		})
+	}
+}

backend/v3/instrumentation/sampler_test.go

@@ -1,4 +1,4 @@
-package tracing
+package instrumentation
 
 import (
 	"context"
@@ -59,7 +59,7 @@ func TestSpanKindBased(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			sampler := SpanKindBased(tt.args.sampler, tt.args.kinds...)
+			sampler := spanKindBased(tt.args.sampler, tt.args.kinds...)
 			assert.Equal(t, tt.want.description, sampler.Description())
 
 			p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))

backend/v3/instrumentation/slog.go

@@ -0,0 +1,197 @@
+package instrumentation
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"connectrpc.com/connect"
+	"github.com/muhlemmer/sloggcp"
+	"github.com/rs/xid"
+	slogmulti "github.com/samber/slog-multi"
+	slogctx "github.com/veqryn/slog-context"
+	slogotel "github.com/veqryn/slog-context/otel"
+	"go.opentelemetry.io/contrib/bridges/otelslog"
+	"go.opentelemetry.io/otel/sdk/log"
+	"google.golang.org/grpc"
+
+	http_util "github.com/zitadel/zitadel/internal/api/http"
+)
+
+type LogFormat int
+
+//go:generate enumer -type=LogFormat -trimprefix=LogFormat -text -linecomment
+const (
+	// Empty line comment sets empty string of unspecified value
+	LogFormatUndefined LogFormat = iota //
+	LogFormatDisabled
+	LogFormatText
+	LogFormatJSON
+	// JSON formatted logs compatible with Google Cloud Platform
+	LogFormatGCP
+)
+
+type LogConfig struct {
+	Level           slog.Level
+	Format          LogFormat
+	AddSource       bool
+	GoogleProjectID string
+	GoogleLogName   string
+	Exporter        ExporterConfig
+}
+
+// setLogger configures the global slog logger.
+// Logs are sent to StdErr and/or the [log.LoggerProvider].
+//
+// When present in the context, each line emitted contains:
+//   - Trace ID
+//   - Span ID
+//   - Service name
+//   - Sanitized request hosts from [http_util.DomainCtx]
+//   - Request details, such as method and path.
+func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
+	options := &slog.HandlerOptions{
+		AddSource: cfg.AddSource,
+		Level:     cfg.Level,
+	}
+	var stdErrHandler slog.Handler
+	switch cfg.Format {
+	case LogFormatUndefined:
+		return // uses default slog logger
+	case LogFormatDisabled:
+		stdErrHandler = slog.DiscardHandler
+	case LogFormatText:
+		stdErrHandler = slog.NewTextHandler(os.Stderr, options)
+	case LogFormatJSON:
+		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
+	case LogFormatGCP:
+		options.ReplaceAttr = sloggcp.ReplaceAttr
+		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
+	}
+
+	stdErrHandler = slogctx.NewHandler(
+		stdErrHandler,
+		&slogctx.HandlerOptions{
+			Prependers: []slogctx.AttrExtractor{
+				domainExtractor,
+				requestExtractor,
+				slogotel.ExtractTraceSpanID,
+				slogctx.ExtractPrepended,
+			},
+		},
+	)
+
+	otelHandler := otelslog.NewHandler(
+		Name,
+		otelslog.WithLoggerProvider(provider),
+	)
+
+	logger := slog.New(slogmulti.Fanout(stdErrHandler, otelHandler))
+	logger.Info("structured logger configured", "config_level", cfg.Level, "format", cfg.Format)
+	slog.SetDefault(logger)
+}
+
+const (
+	ProtocolHttp    = "http"
+	ProtocolConnect = "connect"
+	ProtocolGrpc    = "grpc"
+)
+
+// SetHttpRequestDetails adds static details to each context aware log entry.
+func SetHttpRequestDetails(ctx context.Context, service string, request *http.Request) context.Context {
+	now := time.Now()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolHttp,
+		service:     service,
+		http_method: request.Method,
+		path:        request.URL.Path,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+// SetConnectRequestDetails adds static details to each context aware log entry.
+func SetConnectRequestDetails(ctx context.Context, request connect.AnyRequest) context.Context {
+	now := time.Now()
+	spec := request.Spec()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolConnect,
+		service:     serviceFromRPCMethod(spec.Procedure),
+		http_method: request.HTTPMethod(),
+		path:        spec.Procedure,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+func SetGrpcRequestDetails(ctx context.Context, info *grpc.UnaryServerInfo) context.Context {
+	now := time.Now()
+	return context.WithValue(ctx, ctxKey{}, &requestDetails{
+		protocol:    ProtocolGrpc,
+		service:     serviceFromRPCMethod(info.FullMethod),
+		http_method: http.MethodPost, // gRPC always uses POST
+		path:        info.FullMethod,
+		requestID:   xid.NewWithTime(now),
+		start:       now,
+	})
+}
+
+type ctxKey struct{}
+
+type requestDetails struct {
+	protocol    string
+	service     string
+	http_method string
+	path        string
+	requestID   xid.ID
+	start       time.Time
+}
+
+func (r *requestDetails) attrs() []slog.Attr {
+	attrs := make([]slog.Attr, 0, 6)
+	if r.protocol != "" {
+		attrs = append(attrs, slog.String("protocol", r.protocol))
+	}
+	if r.service != "" {
+		attrs = append(attrs, slog.String("service", r.service))
+	}
+	if r.http_method != "" {
+		attrs = append(attrs, slog.String("http_method", r.http_method))
+	}
+	if r.path != "" {
+		attrs = append(attrs, slog.String("path", r.path))
+	}
+	if !r.requestID.IsZero() {
+		attrs = append(attrs, slog.String("request_id", r.requestID.String()))
+	}
+	if !r.start.IsZero() {
+		attrs = append(attrs, slog.Duration("duration", time.Since(r.start)))
+	}
+	return attrs
+}
+
+func serviceFromRPCMethod(fullMethod string) string {
+	parts := strings.Split(fullMethod, "/")
+	if len(parts) >= 2 {
+		return parts[1]
+	}
+	return "unknown"
+}
+
+// domainExtractor sets the sanitized request hosts from [http_util.DomainCtx] to a log entry.
+func domainExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	return []slog.Attr{
+		slog.Any("domain", http_util.DomainContext(ctx)),
+	}
+}
+
+// requestExtractor sets the request details from [requestDetails] to a log entry.
+func requestExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	if r, ok := ctx.Value(ctxKey{}).(*requestDetails); ok {
+		return r.attrs()
+	}
+	return nil
+}

backend/v3/instrumentation/tracing.go

@@ -0,0 +1,269 @@
+package instrumentation
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"slices"
+
+	google_trace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
+)
+
+type TraceConfig struct {
+	Fraction float64
+	Exporter ExporterConfig
+}
+
+// TODO: remove for v5 release
+type LegacyTraceConfig struct {
+	Type      string
+	Fraction  float64
+	Endpoint  string
+	ProjectID string
+}
+
+// SetLegacyConfig sets the fields of the TraceConfig based on the provided LegacyTraceConfig.
+// If the Type field is already set to a value other than [ExporterTypeNone], the legacy config is ignored.
+func (c *TraceConfig) SetLegacyConfig(lc *LegacyTraceConfig) {
+	typ := c.Exporter.Type
+	if lc == nil || !typ.isNone() {
+		return
+	}
+	typeMap := map[string]ExporterType{
+		"otel":   ExporterTypeGRPC,
+		"google": ExporterTypeGoogle,
+		"log":    ExporterTypeStdOut,
+	}
+
+	c.Fraction = lc.Fraction
+	c.Exporter.Type = typeMap[lc.Type]
+	c.Exporter.Endpoint = lc.Endpoint
+	c.Exporter.GoogleProjectID = lc.ProjectID
+}
+
+type Tracer struct {
+	trace.Tracer
+}
+
+func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindServer))
+}
+
+func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindServer))
+}
+
+func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, name, trace.WithSpanKind(trace.SpanKindClient))
+}
+
+func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller, trace.WithSpanKind(trace.SpanKindClient))
+}
+
+func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *Span) {
+	return t.newSpan(ctx, caller)
+}
+
+func (t *Tracer) newSpan(ctx context.Context, caller string, options ...trace.SpanStartOption) (context.Context, *Span) {
+	return t.newSpanFromName(ctx, caller, options...)
+}
+
+func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...trace.SpanStartOption) (context.Context, *Span) {
+	ctx, span := t.Start(ctx, name, options...)
+	return ctx, CreateSpan(span)
+}
+
+func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span) {
+	ctx, span := t.NewSpan(r.Context(), caller)
+	r = r.WithContext(ctx)
+	return r, span
+}
+
+type Span struct {
+	span trace.Span
+	opts []trace.SpanEndOption
+}
+
+func CreateSpan(span trace.Span) *Span {
+	return &Span{span: span, opts: []trace.SpanEndOption{}}
+}
+
+func (s *Span) End() {
+	if s.span == nil {
+		return
+	}
+
+	s.span.End(s.opts...)
+}
+
+func (s *Span) EndWithError(err error) {
+	s.SetStatusByError(err)
+	s.End()
+}
+
+func (s *Span) SetStatusByError(err error) {
+	if s.span == nil {
+		return
+	}
+	if err != nil {
+		s.span.RecordError(err)
+	}
+
+	code, msg, id := gerrors.ExtractZITADELError(err)
+	s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
+}
+
+func newTracerProvider(ctx context.Context, cfg TraceConfig, resource *resource.Resource) (_ *sdk_trace.TracerProvider, err error) {
+	var exporter sdk_trace.SpanExporter
+	switch cfg.Exporter.Type {
+	case ExporterTypeUnspecified, ExporterTypeNone:
+		// no exporter
+	case ExporterTypeStdOut, ExporterTypeStdErr:
+		exporter, err = traceStdOutExporter(cfg.Exporter)
+	case ExporterTypeGRPC:
+		exporter, err = traceGrpcExporter(ctx, cfg.Exporter)
+	case ExporterTypeHTTP:
+		exporter, err = traceHttpExporter(ctx, cfg.Exporter)
+	case ExporterTypeGoogle:
+		exporter, err = traceGoogleExporter(ctx, cfg.Exporter)
+	case ExporterTypePrometheus:
+		fallthrough // prometheus is not supported for logs
+	default:
+		err = errExporterType(cfg.Exporter.Type, "tracer")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("trace exporter: %w", err)
+	}
+	sampler := newSampler(cfg)
+	opts := []sdk_trace.TracerProviderOption{
+		sdk_trace.WithResource(resource),
+		sdk_trace.WithSampler(sampler),
+	}
+	if exporter != nil {
+		opts = append(opts, sdk_trace.WithBatcher(exporter))
+	}
+	tracerProvider := sdk_trace.NewTracerProvider(opts...)
+	return tracerProvider, nil
+}
+
+func traceStdOutExporter(cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	options := []stdouttrace.Option{
+		stdouttrace.WithPrettyPrint(),
+	}
+	if cfg.Type == ExporterTypeStdErr {
+		options = append(options, stdouttrace.WithWriter(os.Stderr))
+	}
+	exporter, err := stdouttrace.New(options...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceGrpcExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	var grpcOpts []otlptracegrpc.Option
+	if cfg.Endpoint != "" {
+		grpcOpts = append(grpcOpts, otlptracegrpc.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		grpcOpts = append(grpcOpts, otlptracegrpc.WithInsecure())
+	}
+
+	exporter, err := otlptracegrpc.New(ctx, grpcOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceHttpExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	var httpOpts []otlptracehttp.Option
+	if cfg.Endpoint != "" {
+		httpOpts = append(httpOpts, otlptracehttp.WithEndpoint(cfg.Endpoint))
+	}
+	if cfg.Insecure {
+		httpOpts = append(httpOpts, otlptracehttp.WithInsecure())
+	}
+
+	exporter, err := otlptracehttp.New(ctx, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+func traceGoogleExporter(ctx context.Context, cfg ExporterConfig) (sdk_trace.SpanExporter, error) {
+	exporter, err := google_trace.New(
+		google_trace.WithContext(ctx),
+		google_trace.WithProjectID(cfg.GoogleProjectID),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return exporter, nil
+}
+
+// newSampler returns a sampler decorator which behaves differently,
+// based on the parent of the span. If the span has no parent and is of kind server,
+// the decorated sampler is used to make sampling decision.
+// If the span has a parent, depending on whether the parent is remote and whether it
+// is sampled, one of the following samplers will apply:
+//   - remote parent sampled -> always sample
+//   - remote parent not sampled -> sample based on the decorated sampler (fraction based)
+//   - local parent sampled -> always sample
+//   - local parent not sampled -> never sample
+func newSampler(cfg TraceConfig) sdk_trace.Sampler {
+	fraction := sdk_trace.TraceIDRatioBased(cfg.Fraction)
+	return sdk_trace.ParentBased(
+		spanKindBased(fraction, trace.SpanKindServer),
+		sdk_trace.WithRemoteParentNotSampled(fraction),
+	)
+}
+
+type spanKindSampler struct {
+	sampler sdk_trace.Sampler
+	kinds   []trace.SpanKind
+}
+
+// ShouldSample implements the [sdk_trace.Sampler] interface.
+// It will not sample any spans which do not match the configured span kinds.
+// For spans which do match, the decorated sampler is used to make the sampling decision.
+func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
+	psc := trace.SpanContextFromContext(p.ParentContext)
+	if !slices.Contains(sk.kinds, p.Kind) {
+		return sdk_trace.SamplingResult{
+			Decision:   sdk_trace.Drop,
+			Tracestate: psc.TraceState(),
+		}
+	}
+	s := sk.sampler.ShouldSample(p)
+	return s
+}
+
+func (sk spanKindSampler) Description() string {
+	return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
+		sk.sampler.Description(),
+		sk.kinds,
+	)
+}
+
+// spanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
+// If the span kind does not match one of the configured kinds, it will not be sampled.
+// If the span kind matches, the decorated sampler is used to make sampling decision.
+func spanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
+	return spanKindSampler{
+		sampler: sampler,
+		kinds:   kinds,
+	}
+}

backend/v3/instrumentation/tracing/http_handler.go

@@ -0,0 +1,23 @@
+package tracing
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+)
+
+func NewHandler(handler http.Handler, ignoredPrefix ...string) http.Handler {
+	return otelhttp.NewHandler(handler,
+		"zitadel",
+		otelhttp.WithFilter(instrumentation.RequestFilter(ignoredPrefix...)),
+		otelhttp.WithPublicEndpoint(),
+		otelhttp.WithSpanNameFormatter(spanNameFormatter),
+		otelhttp.WithMeterProvider(otel.GetMeterProvider()))
+}
+
+func spanNameFormatter(_ string, r *http.Request) string {
+	return r.URL.Path
+}

backend/v3/instrumentation/tracing/tracing.go

@@ -0,0 +1,65 @@
+// Package tracing provides helper functions to create spans for telemetry tracing.
+package tracing
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	api_trace "go.opentelemetry.io/otel/trace"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/cmd/build"
+)
+
+type Tracer interface {
+	NewSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewClientSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewServerSpan(ctx context.Context, caller string) (context.Context, *instrumentation.Span)
+	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span)
+	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span)
+	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *instrumentation.Span)
+}
+
+var globalTracer = sync.OnceValue(func() Tracer {
+	return instrumentation.NewTracer(
+		instrumentation.Name,
+		api_trace.WithInstrumentationVersion(build.Version()),
+	)
+})
+
+func GlobalTracer() Tracer {
+	return globalTracer()
+}
+
+func NewSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewSpan(ctx, name)
+}
+
+func NewClientSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewClientSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewServerSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewServerSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewClientInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return globalTracer().NewServerInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
+}
+
+func NewSpanHTTP(r *http.Request) (*http.Request, *instrumentation.Span) {
+	return globalTracer().NewSpanHTTP(r, instrumentation.GetCallingFunc(1))
+}
+
+func TraceIDFromCtx(ctx context.Context) string {
+	return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
+}

backend/v3/instrumentation/tracing_test.go

@@ -0,0 +1,191 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+func TestTraceConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name   string
+		target TraceConfig
+		lc     *LegacyTraceConfig
+		want   TraceConfig
+	}{
+		{
+			name: "nil legacy config does not change target",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: nil,
+			want: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "non-none target type does not change target",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type none)",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Fraction: 0.5,
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGoogle,
+					Endpoint: "endpoint",
+				},
+			},
+		},
+		{
+			name: "sets fields from legacy config (type unspecified)",
+			target: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeUnspecified,
+				},
+			},
+			lc: &LegacyTraceConfig{
+				Type:     "google",
+				Fraction: 0.5,
+				Endpoint: "endpoint",
+			},
+			want: TraceConfig{
+				Fraction: 0.5,
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGoogle,
+					Endpoint: "endpoint",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.target.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.target)
+		})
+	}
+}
+
+func Test_newTracerProvider(t *testing.T) {
+	resource, err := resource.New(t.Context())
+	require.NoError(t, err)
+
+	tests := []struct {
+		name string // description of this test case
+		// Named input parameters for target function.
+		cfg     TraceConfig
+		wantErr bool
+	}{
+		{
+			name: "none exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeNone,
+				},
+			},
+		},
+		{
+			name: "stdout exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdOut,
+				},
+			},
+		},
+		{
+			name: "stderr exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeStdErr,
+				},
+			},
+		},
+		{
+			name: "grpc exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeGRPC,
+					Endpoint: "localhost:4317",
+				},
+			},
+		},
+		{
+			name: "http exporter returns no error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type:     ExporterTypeHTTP,
+					Endpoint: "http://localhost:4318/v1/traces",
+				},
+			},
+		},
+		{
+			name: "google exporter return error (no credentials)",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypeGoogle,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "prometheus exporter returns error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterTypePrometheus,
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid exporter type returns error",
+			cfg: TraceConfig{
+				Exporter: ExporterConfig{
+					Type: ExporterType(-1),
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, gotErr := newTracerProvider(t.Context(), tt.cfg, resource)
+			if tt.wantErr {
+				assert.Error(t, gotErr)
+				assert.Nil(t, got)
+			} else {
+				assert.NoError(t, gotErr)
+				assert.NotNil(t, got)
+			}
+		})
+	}
+}

backend/v3/telemetry/logging/logger.go

@@ -1,12 +0,0 @@
-package logging
-
-import "log/slog"
-
-// Logger abstracts [slog.Logger] not sure if thats needed
-type Logger struct {
-	*slog.Logger
-}
-
-func NewLogger(logger *slog.Logger) *Logger {
-	return &Logger{Logger: logger}
-}

backend/v3/telemetry/metric/doc.go

@@ -1,2 +0,0 @@
-// implementation of otel metrics
-package metric

backend/v3/telemetry/tracing/tracer.go

@@ -1,24 +0,0 @@
-package tracing
-
-import (
-	"context"
-
-	"go.opentelemetry.io/otel/trace"
-	"go.opentelemetry.io/otel/trace/noop"
-)
-
-// Tracer is a wrapper around the OpenTelemetry Tracer interface.
-type Tracer struct {
-	trace.Tracer
-}
-
-var noopTracer = Tracer{
-	Tracer: noop.NewTracerProvider().Tracer(""),
-}
-
-func (t *Tracer) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
-	if t.Tracer == nil {
-		return noopTracer.Start(ctx, spanName, opts...)
-	}
-	return t.Tracer.Start(ctx, spanName, opts...)
-}

cmd/admin/admin.go

@@ -3,6 +3,7 @@ package admin
 import (
 	_ "embed"
 	"errors"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 
@@ -18,7 +19,12 @@ func New() *cobra.Command {
 		Short:      "The ZITADEL admin CLI lets you interact with your instance",
 		Long:       `The ZITADEL admin CLI lets you interact with your instance`,
 		Deprecated: "please use subcommands directly, e.g. `zitadel start`",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel admin command failed", "err", err)
+				}
+			}()
 			return errors.New("no additional command provided")
 		},
 	}

cmd/defaults.yaml

@@ -1,13 +1,94 @@
+Instrumentation:
+  ServiceName: "zitadel" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud. Need to configure GoogleProjectID below!
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud. Need to configure GoogleProjectID below!
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1m # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging
+    #   - "text": Logs are printed on StdErr as human-readable text
+    #   - "json": Logs are printed on StdErr as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
+    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables OTEL log exporter
+      #   - "stdOut": Exports OTEL logs to standard output
+      #   - "stdErr": Exports OTEL logs to standard error
+      #   - "grpc": Exports OTEL logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports OTEL logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+  Profile:
+    Exporter:
+      # The following profiler types are supported:
+      # "none": Disables profiling
+      # "google": Exports profiling data to Google Cloud Profiler
+      Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+      # Project ID for Google Cloud Profiler
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
+# Legacy logging (Deprecated, see Instrumentation.Slog)
 Log:
   Level: info # ZITADEL_LOG_LEVEL
   Formatter:
     Format: text # ZITADEL_LOG_FORMATTER_FORMAT
 
-# Exposes metrics on /debug/metrics
+# Exposes metrics on /debug/metrics (Deprecated, see Instrumentation.Metric.Prometheus)
 Metrics:
   # Select type otel (OpenTelemetry) or none (disables collection and endpoint)
   Type: otel # ZITADEL_METRICS_TYPE
 
+
+# Collects and exports tracing information (Deprecated, see Instrumentation.Trace)
 Tracing:
   # Choose one in "otel", "google", "log" and "none"
   # Depending on the type there are different configuration options

cmd/initialise/init.go

@@ -3,6 +3,8 @@ package initialise
 import (
 	"context"
 	"embed"
+	"fmt"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -43,10 +45,15 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel init command failed", "err", err)
+				}
+			}()
 			config := MustNewConfig(viper.GetViper())
 
-			InitAll(cmd.Context(), config)
+			return InitAll(cmd.Context(), config)
 		},
 	}
 
@@ -54,16 +61,21 @@ The user provided by flags needs privileges to
 	return cmd
 }
 
-func InitAll(ctx context.Context, config *Config) {
+func InitAll(ctx context.Context, config *Config) error {
 	err := initialise(ctx, config.Database,
 		VerifyUser(config.Database.Username(), config.Database.Password()),
 		VerifyDatabase(config.Database.DatabaseName()),
 		VerifyGrant(config.Database.DatabaseName(), config.Database.Username()),
 	)
-	logging.OnError(err).Fatal("unable to initialize the database")
+	if err != nil {
+		return fmt.Errorf("initialize database failed: %w", err)
+	}
 
 	err = verifyZitadel(ctx, config.Database)
-	logging.OnError(err).Fatal("unable to initialize ZITADEL")
+	if err != nil {
+		return fmt.Errorf("initialize ZITADEL failed: %w", err)
+	}
+	return nil
 }
 
 func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {

cmd/key/key.go

@@ -2,6 +2,7 @@ package key
 
 import (
 	"io"
+	"log/slog"
 	"os"
 	"strings"
 
@@ -44,7 +45,13 @@ Requirements:
 		Example: `new -f keys.yaml
 new key1=somekey key2=anotherkey
 new -f keys.yaml key2=anotherkey`,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel keys new command failed", "err", err)
+				}
+			}()
+
 			keys, err := keysFromArgs(args)
 			if err != nil {
 				return err

cmd/mirror/auth.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	_ "embed"
+	"errors"
 	"io"
 	"strconv"
 	"time"
@@ -22,9 +23,16 @@ func authCmd() *cobra.Command {
 		Long: `mirrors the auth requests table from one database to another
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Only auth requests are mirrored`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copyAuth(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/config.go

@@ -1,13 +1,16 @@
 package mirror
 
 import (
+	"context"
 	_ "embed"
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -16,7 +19,6 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/id"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
 )
 
 type Migration struct {
@@ -26,9 +28,10 @@ type Migration struct {
 	EventBulkSize     uint32
 	MaxAuthRequestAge time.Duration
 
-	Log     *logging.Config
-	Machine *id.Config
-	Metrics metrics.Config
+	Log             *logging.Config
+	Machine         *id.Config
+	Instrumentation instrumentation.Config
+	Metrics         instrumentation.LegacyMetricConfig
 }
 
 var (
@@ -36,19 +39,24 @@ var (
 	defaultConfig []byte
 )
 
-func mustNewMigrationConfig(v *viper.Viper) *Migration {
+func mustNewMigrationConfig(ctx context.Context, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
 	config := new(Migration)
 	mustNewConfig(v, config)
 
-	err := config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
 
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
+	// Legacy logger
+	err = config.Log.SetLogger()
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
-	return config
+	return config, shutdown, nil
 }
 
 func mustNewProjectionsConfig(v *viper.Viper) *ProjectionsConfig {

cmd/mirror/defaults.yaml

@@ -1,3 +1,81 @@
+Instrumentation:
+  ServiceName: "zitadel-mirror" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud. Need to configure GoogleProjectID below!
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud. Need to configure GoogleProjectID below!
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1m # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging
+    #   - "text": Logs are printed on StdErr as human-readable text
+    #   - "json": Logs are printed on StdErr as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
+    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables OTEL log exporter
+      #   - "stdOut": Exports OTEL logs to standard output
+      #   - "stdErr": Exports OTEL logs to standard error
+      #   - "grpc": Exports OTEL logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports OTEL logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+  Profile:
+    Exporter:
+      # The following profiler types are supported:
+      # "none": Disables profiling
+      # "google": Exports profiling data to Google Cloud Profiler
+      Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+      # Project ID for Google Cloud Profiler
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
 Source:
   cockroach:
     Host: localhost # ZITADEL_SOURCE_COCKROACH_HOST

cmd/mirror/event_store.go

@@ -33,9 +33,16 @@ func eventstoreCmd() *cobra.Command {
 		Long: `mirrors the eventstore of an instance from one database to another
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Migrate only copies events2 and unique constraints`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copyEventstore(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/mirror.go

@@ -3,6 +3,9 @@ package mirror
 import (
 	"bytes"
 	_ "embed"
+	"errors"
+	"fmt"
+	"log/slog"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -44,18 +47,34 @@ Order of execution:
 				logging.WithFields("file", file).OnError(err).Warn("unable to read config file")
 			}
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel mirror command failed", "err", err)
+				}
+			}()
+
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return fmt.Errorf("unable to create migration config: %w", err)
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+
 			projectionConfig := mustNewProjectionsConfig(viper.GetViper())
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Fatal("unable to read master key")
+			if err != nil {
+				return fmt.Errorf("unable to read master key: %w", err)
+			}
 
 			copySystem(cmd.Context(), config)
 			copyAuth(cmd.Context(), config)
 			copyEventstore(cmd.Context(), config)
 
 			projections(cmd.Context(), projectionConfig, masterKey)
+			return nil
 		},
 	}
 

cmd/mirror/system.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	_ "embed"
+	"errors"
 	"io"
 	"time"
 
@@ -21,9 +22,16 @@ func systemCmd() *cobra.Command {
 		Long: `mirrors the system tables of ZITADEL from one database to another
 ZITADEL needs to be initialized
 Only keys and assets are mirrored`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copySystem(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/verify.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 	"slices"
 
@@ -20,9 +21,16 @@ func verifyCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "verify",
 		Short: "counts if source and dest have the same amount of entries",
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			verifyMigration(cmd.Context(), config)
+			return nil
 		},
 	}
 }

cmd/setup/cleanup.go

@@ -2,6 +2,7 @@ package setup
 
 import (
 	"context"
+	"errors"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -19,9 +20,16 @@ func NewCleanup() *cobra.Command {
 		Use:   "cleanup",
 		Short: "cleans up migration if they got stuck",
 		Long:  `cleans up migration if they got stuck`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			Cleanup(cmd.Context(), config)
+			return nil
 		},
 	}
 }

cmd/setup/config.go

@@ -2,6 +2,9 @@ package setup
 
 import (
 	"bytes"
+	"context"
+	"errors"
+	"fmt"
 	"strings"
 	"time"
 
@@ -9,6 +12,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -27,7 +31,6 @@ import (
 	"github.com/zitadel/zitadel/internal/notification/handlers"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
 )
 
 type Config struct {
@@ -40,8 +43,9 @@ type Config struct {
 	ExternalDomain  string
 	ExternalPort    uint16
 	ExternalSecure  bool
+	Instrumentation instrumentation.Config
 	Log             *logging.Config
-	Metrics         metrics.Config
+	Metrics         *instrumentation.LegacyMetricConfig
 	EncryptionKeys  *encryption.EncryptionKeyConfig
 	DefaultInstance command.InstanceSetup
 	Machine         *id.Config
@@ -66,7 +70,7 @@ type InitProjections struct {
 	BulkLimit        uint64
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
@@ -85,20 +89,28 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read default config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read default config: %w", err)
+	}
+
+	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
 
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.
 	config.DefaultInstance.RolePermissionMappings = config.InternalAuthZ.RolePermissionMappings
 
-	return config
+	return config, shutdown, nil
 }
 
 type Steps struct {

cmd/setup/config_test.go

@@ -241,7 +241,8 @@ Actions:
 			v := viper.New()
 			v.SetConfigType("yaml")
 			require.NoError(t, v.ReadConfig(strings.NewReader(tt.args.yaml)))
-			got := MustNewConfig(v)
+			got, _, err := NewConfig(t.Context(), v)
+			require.NoError(t, err)
 			tt.want(t, got)
 		})
 	}

cmd/setup/setup.go

@@ -6,6 +6,7 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
@@ -61,23 +62,45 @@ func New() *cobra.Command {
 		Long: `sets up data to start ZITADEL.
 Requirements:
 - postgreSQL`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel setup command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return fmt.Errorf("invalid tlsMode: %w", err)
+			}
 
 			err = BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
 
 			err = bindForMirror(cmd)
-			logging.OnError(err).Fatal("unable to bind \"for-mirror\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"for-mirror\" flag: %w", err)
+			}
+
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			config := MustNewConfig(viper.GetViper())
 			steps := MustNewSteps(viper.New())
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return fmt.Errorf("no master key provided: %w", err)
+			}
 
 			Setup(cmd.Context(), config, steps, masterKey)
+			return nil
 		},
 	}
 

cmd/setup/steps.yaml

@@ -1,3 +1,82 @@
+Instrumentation:
+  ServiceName: "zitadel-setup" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud Trace
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud Trace
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging to standard error
+    #   - "text": Logs are printed as human-readable text
+    #   - "json": Logs are printed as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging
+    StdErr: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables log exporting
+      #   - "stdOut": Exports logs to standard output
+      #   - "stdErr": Exports logs to standard error
+      #   - "grpc": Exports logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_GOOGLEPROJECTID
+  Profile:
+    # The following profiler types are supported:
+    # "none": Disables profiling
+    # "google": Exports profiling data to Google Cloud Profiler
+    Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+    # Project ID for Google Cloud Profiler
+    GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
 # By using the FirstInstance section, you can overwrite the DefaultInstance configuration for the first instance created by zitadel setup.
 FirstInstance:
   # If set to true zitadel is setup without initial data

cmd/start/config.go

@@ -1,12 +1,16 @@
 package start
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -34,12 +38,10 @@ import (
 	"github.com/zitadel/zitadel/internal/query/projection"
 	"github.com/zitadel/zitadel/internal/serviceping"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
-	profiler "github.com/zitadel/zitadel/internal/telemetry/profiler/config"
-	tracing "github.com/zitadel/zitadel/internal/telemetry/tracing/config"
 )
 
 type Config struct {
+	Instrumentation     instrumentation.Config
 	Log                 *logging.Config
 	Port                uint16
 	ExternalPort        uint16
@@ -53,9 +55,9 @@ type Config struct {
 	WebAuthNName        string
 	Database            database.Config
 	Caches              *connector.CachesConfig
-	Tracing             tracing.Config
-	Metrics             metrics.Config
-	Profiler            profiler.Config
+	Tracing             *instrumentation.LegacyTraceConfig
+	Metrics             *instrumentation.LegacyMetricConfig
+	Profiler            *instrumentation.LegacyProfileConfig
 	Projections         projection.Config
 	Notifications       handlers.WorkerConfig
 	Executions          execution.WorkerConfig
@@ -93,21 +95,26 @@ type QuotasConfig struct {
 	Execution *logstore.EmitterConfig
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config, err := readConfig(v)
-	logging.OnError(err).Fatal("unable to read config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read config: %w", err)
+	}
 
+	config.Instrumentation.Trace.SetLegacyConfig(config.Tracing)
+	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	config.Instrumentation.Profile.SetLegacyConfig(config.Profiler)
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+
+	// Legacy logger
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Tracing.NewTracer()
-	logging.OnError(err).Fatal("unable to set tracer")
-
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
-
-	err = config.Profiler.NewProfiler()
-	logging.OnError(err).Fatal("unable to set profiler")
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 	if config.Actions != nil {
@@ -115,12 +122,14 @@ func MustNewConfig(v *viper.Viper) *Config {
 	}
 
 	err = config.SystemDefaults.Validate()
-	logging.OnError(err).Fatal("system defaults config invalid")
-
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("system defaults config invalid: %w", err)
+	}
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.
 	config.DefaultInstance.RolePermissionMappings = config.InternalAuthZ.RolePermissionMappings
 
-	return config
+	return config, shutdown, nil
 }
 
 func readConfig(v *viper.Viper) (*Config, error) {

cmd/start/start.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/tls"
 	_ "embed"
+	"errors"
 	"fmt"
+	"log/slog"
 	"math"
 	"net/http"
 	"os"
@@ -127,12 +129,25 @@ func New(server chan<- *Server) *cobra.Command {
 		Long: `starts ZITADEL.
 Requirements:
 - postgreSQL`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			err := cmd_tls.ModeFromFlag(cmd)
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start command failed", "err", err)
+				}
+			}()
+
+			err = cmd_tls.ModeFromFlag(cmd)
 			if err != nil {
 				return err
 			}
-			config := MustNewConfig(viper.GetViper())
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
 				return err

cmd/start/start_from_init.go

@@ -2,10 +2,12 @@ package start
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
@@ -24,31 +26,52 @@ Last ZITADEL starts.
 
 Requirements:
 - postgreSQL`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-init command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return fmt.Errorf("invalid tlsMode: %w", err)
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return fmt.Errorf("no master key provided: %w", err)
+			}
 
 			initCtx, cancel := context.WithCancel(cmd.Context())
 			initialise.InitAll(initCtx, initialise.MustNewConfig(viper.GetViper()))
 			cancel()
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}
 

cmd/start/start_from_setup.go

@@ -2,10 +2,11 @@ package start
 
 import (
 	"context"
+	"errors"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
@@ -24,27 +25,48 @@ Requirements:
 - database
 - database is initialized
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-setup command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return err
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return err
+			}
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return err
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}
 

cmd/zitadel.go

@@ -36,7 +36,8 @@ func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return errors.New("no additional command provided")
 		},
-		Version: build.Version(),
+		Version:      build.Version(),
+		SilenceUsage: true,
 	}
 
 	viper.AutomaticEnv()

go.mod

@@ -13,6 +13,7 @@ require (
 	dario.cat/mergo v1.0.2
 	github.com/BurntSushi/toml v1.5.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0
 	github.com/Masterminds/squirrel v1.5.4
 	github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b
@@ -66,6 +67,7 @@ require (
 	github.com/muesli/gamut v0.3.1
 	github.com/muhlemmer/gu v0.3.1
 	github.com/muhlemmer/httpforwarded v0.1.0
+	github.com/muhlemmer/sloggcp v0.1.0
 	github.com/nicksnyder/go-i18n/v2 v2.4.0
 	github.com/pashagolub/pgxmock/v4 v4.9.0
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
@@ -80,7 +82,9 @@ require (
 	github.com/riverqueue/rivercontrib/otelriver v0.5.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/cors v1.11.1
+	github.com/rs/xid v1.6.0
 	github.com/russellhaering/goxmldsig v1.5.0
+	github.com/samber/slog-multi v1.6.0
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/shopspring/decimal v1.4.0
 	github.com/sirupsen/logrus v1.9.3
@@ -92,33 +96,45 @@ require (
 	github.com/stretchr/testify v1.11.1
 	github.com/ttacon/libphonenumber v1.2.1
 	github.com/twilio/twilio-go v1.28.6
+	github.com/veqryn/slog-context v0.8.0
+	github.com/veqryn/slog-context/otel v0.8.0
 	github.com/zitadel/exifremove v0.1.0
 	github.com/zitadel/logging v0.6.2
 	github.com/zitadel/oidc/v3 v3.45.0
 	github.com/zitadel/passwap v0.10.0
 	github.com/zitadel/saml v0.4.1
 	github.com/zitadel/schema v1.3.1
+	go.opentelemetry.io/contrib/bridges/otelslog v0.13.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
-	go.opentelemetry.io/otel v1.38.0
+	go.opentelemetry.io/otel v1.39.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.60.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0
-	go.opentelemetry.io/otel/metric v1.38.0
-	go.opentelemetry.io/otel/sdk v1.38.0
-	go.opentelemetry.io/otel/sdk/metric v1.38.0
-	go.opentelemetry.io/otel/trace v1.38.0
+	go.opentelemetry.io/otel/log v0.14.0
+	go.opentelemetry.io/otel/metric v1.39.0
+	go.opentelemetry.io/otel/sdk v1.39.0
+	go.opentelemetry.io/otel/sdk/log v0.14.0
+	go.opentelemetry.io/otel/sdk/metric v1.39.0
+	go.opentelemetry.io/otel/trace v1.39.0
 	go.uber.org/mock v0.6.0
 	golang.org/x/crypto v0.44.0
 	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
 	golang.org/x/net v0.47.0
 	golang.org/x/oauth2 v0.33.0
 	golang.org/x/sync v0.18.0
-	golang.org/x/sys v0.38.0
+	golang.org/x/sys v0.39.0
 	golang.org/x/text v0.31.0
 	google.golang.org/api v0.256.0
-	google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba
-	google.golang.org/grpc v1.76.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217
+	google.golang.org/grpc v1.77.0
 	google.golang.org/protobuf v1.36.10
 	sigs.k8s.io/yaml v1.6.0
 )
@@ -133,8 +149,7 @@ require (
 	cloud.google.com/go/monitoring v1.24.2 // indirect
 	cloud.google.com/go/trace v1.11.6 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
-	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.0 // indirect
@@ -145,7 +160,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
+	github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -159,7 +174,7 @@ require (
 	github.com/dsoprea/go-png-image-structure v0.0.0-20210512210324-29b889a6093d // indirect
 	github.com/dsoprea/go-utility v0.0.0-20221003172846-a3e1774ef349 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
@@ -218,12 +233,13 @@ require (
 	github.com/prometheus/otlptranslator v0.0.2 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
 	github.com/riverqueue/river/rivershared v0.26.0 // indirect
-	github.com/rs/xid v1.6.0 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
+	github.com/samber/lo v1.52.0 // indirect
+	github.com/samber/slog-common v0.19.0 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
-	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
@@ -235,17 +251,16 @@ require (
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	github.com/zeebo/errs v1.4.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	nhooyr.io/websocket v1.8.11 // indirect

go.sum

@@ -41,8 +41,8 @@ github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4=
-github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.30.0 h1:5eCqTd9rTwMlE62z0xFdzPJ+3pji75hJrwq1jrCjo5w=
@@ -133,8 +133,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls=
-github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
+github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
+github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
 github.com/cockroachdb/cockroach-go/v2 v2.4.0 h1:7K5vpE3m7LylIbmpbr4eEhApDTPMgFgR+eDPy1sdJjM=
 github.com/cockroachdb/cockroach-go/v2 v2.4.0/go.mod h1:9U179XbCx4qFWtNhc7BiWLPfuyMVQ7qdAhfrwLz1vH0=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
@@ -208,10 +208,10 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
-github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
-github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
+github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
+github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
+github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
+github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
 github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -589,6 +589,8 @@ github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM=
 github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM=
 github.com/muhlemmer/httpforwarded v0.1.0 h1:x4DLrzXdliq8mprgUMR0olDvHGkou5BJsK/vWUetyzY=
 github.com/muhlemmer/httpforwarded v0.1.0/go.mod h1:yo9czKedo2pdZhoXe+yDkGVbU0TJ0q9oQ90BVoDEtw0=
+github.com/muhlemmer/sloggcp v0.1.0 h1:iWaksY9nzkqlEecQPkfd7hbASqaIQPSoM7kldC2kAmc=
+github.com/muhlemmer/sloggcp v0.1.0/go.mod h1:2JKYrbP9Xc/azEDCIhvMrKP4/bxm6CqjdnMPS3OfpT0=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -717,6 +719,12 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
+github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
+github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
+github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI=
+github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M=
+github.com/samber/slog-multi v1.6.0 h1:i1uBY+aaln6ljwdf7Nrt4Sys8Kk6htuYuXDHWJsHtZg=
+github.com/samber/slog-multi v1.6.0/go.mod h1:qTqzmKdPpT0h4PFsTN5rYRgLwom1v+fNGuIrl1Xnnts=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
@@ -756,8 +764,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
 github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
-github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE=
-github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g=
+github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
+github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
@@ -797,6 +805,10 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/veqryn/slog-context v0.8.0 h1:lDhwAgjwx52K5StqqQzi5d0Y/F4SNyGZbsXGd8MtucM=
+github.com/veqryn/slog-context v0.8.0/go.mod h1:8rsT72p0kzzN9lmkwtabIhxg7ZkpnKblt9x3Eix8Tc0=
+github.com/veqryn/slog-context/otel v0.8.0 h1:tuMAAb/aXImUBTXz+RmNkTu6eBdzZAde8cZM/UL0HdI=
+github.com/veqryn/slog-context/otel v0.8.0/go.mod h1:CHln1I9F42pK+GvLKMiH/QzIdzTf2n3R0pkHa9+vYzQ=
 github.com/wcharczuk/go-chart/v2 v2.1.0/go.mod h1:yx7MvAVNcP/kN9lKXM/NTce4au4DFN99j6i1OwDclNA=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
@@ -812,8 +824,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
-github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 github.com/zitadel/exifremove v0.1.0 h1:qD50ezWsfeeqfcvs79QyyjVfK+snN12v0U0deaU8aKg=
 github.com/zitadel/exifremove v0.1.0/go.mod h1:rzKJ3woL/Rz2KthVBiSBKIBptNTvgmk9PLaeUKTm+ek=
 github.com/zitadel/logging v0.6.2 h1:MW2kDDR0ieQynPZ0KIZPrh9ote2WkxfBif5QoARDQcU=
@@ -833,34 +843,54 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw=
-go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k=
+go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s=
+go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4=
+go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=
+go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
+go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
+go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0 h1:EKpiGphOYq3CYnIe2eX9ftUkyU+Y8Dtte8OaWyHJ4+I=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.15.0/go.mod h1:nWFP7C+T8TygkTjJ7mAyEaFaE7wNfms3nV/vexZ6qt0=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0 h1:nKP4Z2ejtHn3yShBb+2KawiXgpn8In5cT7aO2wXuOTE=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0/go.mod h1:NwjeBbNigsO4Aj9WgM0C+cKIrxsZUaRmZUO7A8I7u8o=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU=
 go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo=
 go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 h1:B/g+qde6Mkzxbry5ZZag0l7QrQBCtVm7lVjaLgmpje8=
+go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0/go.mod h1:mOJK8eMmgW6ocDJn6Bn11CcZ05gi3P8GylBXEkZtbgA=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
-go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
+go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
+go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
+go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
+go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
+go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
+go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
+go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
+go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
+go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
+go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
+go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
+go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
+go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
+go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -998,8 +1028,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -1056,10 +1086,10 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
 google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
-google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4=
-google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -1074,8 +1104,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
-google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
+google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
+google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

internal/actions/http_module_test.go

@@ -3,7 +3,6 @@ package actions
 import (
 	"bytes"
 	"context"
-	"errors"
 	"io"
 	"net"
 	"net/http"
@@ -92,13 +91,13 @@ func Test_isHostBlocked(t *testing.T) {
 			name: "looked up failure",
 			fields: fields{
 				lookup: func(host string) ([]net.IP, error) {
-					return nil, errors.New("some error")
+					return nil, io.EOF
 				},
 			},
 			args: args{
 				address: mustNewURL(t, "https://test2.com/hodor"),
 			},
-			want: zerrors.ThrowInternal(nil, "ACTIO-4m9s2", "lookup failed"),
+			want: zerrors.ThrowInternal(io.EOF, "ACTIO-4m9s2", "lookup failed"),
 		},
 	}
 	for _, tt := range tests {

internal/api/api.go

@@ -12,10 +12,12 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
 	"github.com/zitadel/logging"
+	"go.opentelemetry.io/otel"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server"
@@ -26,7 +28,6 @@ import (
 	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 	instance_pb "github.com/zitadel/zitadel/pkg/grpc/instance/v2"
@@ -131,7 +132,12 @@ func New(
 	if err != nil {
 		return nil, err
 	}
-	api.connectOTELInterceptor, err = otelconnect.NewInterceptor()
+	api.connectOTELInterceptor, err = otelconnect.NewInterceptor(
+		otelconnect.WithTracerProvider(otel.GetTracerProvider()),
+		otelconnect.WithMeterProvider(otel.GetMeterProvider()),
+		otelconnect.WithPropagator(otel.GetTextMapPropagator()),
+		otelconnect.WithoutServerPeerAttributes(),
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -199,6 +205,7 @@ func (a *API) registerConnectServer(service server.ConnectServer) {
 		connect_middleware.CallDurationHandler(),
 		a.connectOTELInterceptor,
 		connect_middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+		connect_middleware.LogHandler(grpc_api.Probes...),
 		connect_middleware.NoCacheInterceptor(),
 		connect_middleware.InstanceInterceptor(a.queries, a.externalDomain, a.translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName, instance_pb.InstanceService_ServiceDesc.ServiceName),
 		connect_middleware.AccessStorageInterceptor(a.accessInterceptor.AccessService()),

internal/api/assets/asset.go

@@ -79,7 +79,7 @@ type ErrorHandler func(w http.ResponseWriter, r *http.Request, err error, defaul
 func DefaultErrorHandler(translator *i18n.Translator) func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 	return func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 		logging.WithFields("uri", r.RequestURI).WithError(err).Warn("error occurred on asset api")
-		code, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+		code, ok := http_util.ZitadelErrorToHTTPStatusCode(r.Context(), err)
 		if !ok {
 			code = defaultCode
 		}

internal/api/grpc/gerrors/zitadel_errors.go

@@ -1,10 +1,13 @@
 package gerrors
 
 import (
+	"context"
 	"errors"
+	"log/slog"
 
 	"connectrpc.com/connect"
 	"github.com/jackc/pgx/v5/pgconn"
+	slogctx "github.com/veqryn/slog-context"
 	"github.com/zitadel/logging"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -16,16 +19,15 @@ import (
 	"github.com/zitadel/zitadel/pkg/grpc/message"
 )
 
-func ZITADELToGRPCError(err error) error {
+func ZITADELToGRPCError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err, "code", code)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -38,20 +40,21 @@ func ZITADELToGRPCError(err error) error {
 	return s.Err()
 }
 
-func ZITADELToConnectError(err error) error {
+func ZITADELToConnectError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
 	connectError := new(connect.Error)
 	if errors.As(err, &connectError) {
+		// Connect error may be returned by other middlewares,
+		// so we assume it's a client error and log as warning.
+		slogctx.FromCtx(ctx).WarnContext(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
 		return err
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -62,44 +65,62 @@ func ZITADELToConnectError(err error) error {
 	return cErr
 }
 
-func ExtractZITADELError(err error) (c codes.Code, msg, id string, ok bool) {
+func ExtractZITADELError(err error) (code codes.Code, msg, id string) {
 	if err == nil {
-		return codes.OK, "", "", false
+		return codes.OK, "", ""
 	}
+	code, msg, id, _ = extractError(err)
+	return code, msg, id
+}
+
+func extractError(err error) (c codes.Code, msg, id string, lvl slog.Level) {
 	connErr := new(pgconn.ConnectError)
 	if ok := errors.As(err, &connErr); ok {
-		return codes.Internal, "db connection error", "", true
+		return codes.Internal, "db connection error", "", slog.LevelError
 	}
-	zitadelErr := new(zerrors.ZitadelError)
-	if ok := errors.As(err, &zitadelErr); !ok {
-		return codes.Unknown, err.Error(), "", false
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return codes.Unknown, err.Error(), "", slog.LevelError
 	}
-	switch {
-	case zerrors.IsErrorAlreadyExists(err):
-		return codes.AlreadyExists, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsDeadlineExceeded(err):
-		return codes.DeadlineExceeded, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsInternal(err):
-		return codes.Internal, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsErrorInvalidArgument(err):
-		return codes.InvalidArgument, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsNotFound(err):
-		return codes.NotFound, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPermissionDenied(err):
-		return codes.PermissionDenied, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPreconditionFailed(err):
-		return codes.FailedPrecondition, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnauthenticated(err):
-		return codes.Unauthenticated, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnavailable(err):
-		return codes.Unavailable, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnimplemented(err):
-		return codes.Unimplemented, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsResourceExhausted(err):
-		return codes.ResourceExhausted, zitadelErr.GetMessage(), zitadelErr.GetID(), true
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		c, lvl = codes.AlreadyExists, slog.LevelWarn
+	case zerrors.KindDeadlineExceeded:
+		c, lvl = codes.DeadlineExceeded, slog.LevelError
+	case zerrors.KindInternal:
+		c, lvl = codes.Internal, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		c, lvl = codes.InvalidArgument, slog.LevelWarn
+	case zerrors.KindNotFound:
+		c, lvl = codes.NotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		c, lvl = codes.PermissionDenied, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		c, lvl = codes.FailedPrecondition, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		c, lvl = codes.Unauthenticated, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		c, lvl = codes.Unavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		c, lvl = codes.Unimplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		c, lvl = codes.ResourceExhausted, slog.LevelError
+	case zerrors.KindDataLoss:
+		c, lvl = codes.DataLoss, slog.LevelError
+	case zerrors.KindCanceled:
+		c, lvl = codes.Canceled, slog.LevelWarn
+	case zerrors.KindAborted:
+		c, lvl = codes.Aborted, slog.LevelError
+	case zerrors.KindOutOfRange:
+		c, lvl = codes.OutOfRange, slog.LevelWarn
+	case zerrors.KindUnknown:
+		c, lvl = codes.Unknown, slog.LevelError
 	default:
-		return codes.Unknown, err.Error(), "", false
+		c, lvl = codes.Unknown, slog.LevelError
 	}
+	return c, msg, id, lvl
 }
 
 func getErrorInfo(id, key string, err error) protoadapt.MessageV1 {

internal/api/grpc/gerrors/zitadel_errors_test.go

@@ -3,6 +3,7 @@ package gerrors
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -41,7 +42,7 @@ func TestCaosToGRPCError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := ZITADELToGRPCError(tt.args.err); (err != nil) != tt.wantErr {
+			if err := ZITADELToGRPCError(t.Context(), tt.args.err); (err != nil) != tt.wantErr {
 				t.Errorf("ZITADELToGRPCError() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
@@ -96,7 +97,7 @@ func Test_getErrorInfo(t *testing.T) {
 	}
 }
 
-func Test_Extract(t *testing.T) {
+func Test_extractError(t *testing.T) {
 	type args struct {
 		err error
 	}
@@ -106,7 +107,7 @@ func Test_Extract(t *testing.T) {
 		wantC   codes.Code
 		wantMsg string
 		wantID  string
-		wantOk  bool
+		wantLvl slog.Level
 	}{
 		{
 			"already exists",
@@ -114,7 +115,7 @@ func Test_Extract(t *testing.T) {
 			codes.AlreadyExists,
 			"already exists",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"deadline exceeded",
@@ -122,7 +123,7 @@ func Test_Extract(t *testing.T) {
 			codes.DeadlineExceeded,
 			"deadline exceeded",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"internal error",
@@ -130,7 +131,7 @@ func Test_Extract(t *testing.T) {
 			codes.Internal,
 			"internal error",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"invalid argument",
@@ -138,7 +139,7 @@ func Test_Extract(t *testing.T) {
 			codes.InvalidArgument,
 			"invalid argument",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"not found",
@@ -146,7 +147,7 @@ func Test_Extract(t *testing.T) {
 			codes.NotFound,
 			"not found",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"permission denied",
@@ -154,7 +155,7 @@ func Test_Extract(t *testing.T) {
 			codes.PermissionDenied,
 			"permission denied",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"precondition failed",
@@ -162,7 +163,7 @@ func Test_Extract(t *testing.T) {
 			codes.FailedPrecondition,
 			"precondition failed",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unauthenticated",
@@ -170,7 +171,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unauthenticated,
 			"unauthenticated",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unavailable",
@@ -178,7 +179,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unavailable,
 			"unavailable",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unimplemented",
@@ -186,7 +187,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unimplemented,
 			"unimplemented",
 			"id",
-			true,
+			slog.LevelInfo,
 		},
 		{
 			"exhausted",
@@ -194,7 +195,7 @@ func Test_Extract(t *testing.T) {
 			codes.ResourceExhausted,
 			"exhausted",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unknown",
@@ -202,12 +203,12 @@ func Test_Extract(t *testing.T) {
 			codes.Unknown,
 			"unknown",
 			"",
-			false,
+			slog.LevelError,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotC, gotMsg, gotID, gotOk := ExtractZITADELError(tt.args.err)
+			gotC, gotMsg, gotID := ExtractZITADELError(tt.args.err)
 			if gotC != tt.wantC {
 				t.Errorf("extract() gotC = %v, want %v", gotC, tt.wantC)
 			}
@@ -217,9 +218,6 @@ func Test_Extract(t *testing.T) {
 			if gotID != tt.wantID {
 				t.Errorf("extract() gotID = %v, want %v", gotID, tt.wantID)
 			}
-			if gotOk != tt.wantOk {
-				t.Errorf("extract() gotOk = %v, want %v", gotOk, tt.wantOk)
-			}
 		})
 	}
 }

internal/api/grpc/server/connect_middleware/activity_interceptor.go

@@ -19,7 +19,7 @@ func ActivityInterceptor() connect.UnaryInterceptorFunc {
 			ctx = activityInfoFromGateway(ctx, req.Header()).SetMethod(req.Spec().Procedure).IntoContext(ctx)
 			resp, err := handler(ctx, req)
 			if isResourceAPI(req.Spec().Procedure) {
-				code, _, _, _ := gerrors.ExtractZITADELError(err)
+				code, _, _ := gerrors.ExtractZITADELError(err)
 				ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 				activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 			}

internal/api/grpc/server/connect_middleware/error_interceptor.go

@@ -19,5 +19,5 @@ func ErrorHandler() connect.UnaryInterceptorFunc {
 
 func toConnectError(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc) (connect.AnyResponse, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToConnectError(err) // TODO !
+	return resp, gerrors.ZITADELToConnectError(ctx, err)
 }

internal/api/grpc/server/connect_middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req connect.AnyRequest, handler connec
 func addInstanceByDomain(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (connect.AnyResponse, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr))

internal/api/grpc/server/connect_middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package connect_middleware
+
+import (
+	"connectrpc.com/connect"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredPrefix ...string) connect.UnaryInterceptorFunc {
+	return func(next connect.UnaryFunc) connect.UnaryFunc {
+		return logging.NewConnectInterceptor(next, ignoredPrefix...)
+	}
+}

internal/api/grpc/server/connect_middleware/metrics_interceptor.go

@@ -10,8 +10,8 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"google.golang.org/grpc/codes"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/gateway.go

@@ -20,10 +20,10 @@ import (
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	client_middleware "github.com/zitadel/zitadel/internal/api/grpc/client/middleware"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -209,7 +209,7 @@ func addInterceptors(
 	handler = http_mw.CallDurationHandler(handler)
 	handler = http_mw.CORSInterceptor(handler)
 	handler = http_mw.RobotsTagHandler(handler)
-	handler = http_mw.DefaultTelemetryHandler(handler)
+	handler = http_mw.DefaultTraceHandler(handler)
 	handler = http_mw.ActivityHandler(handler)
 	// For some non-obvious reason, the exhaustedCookieInterceptor sends the SetCookie header
 	// only if it follows the http_mw.DefaultTelemetryHandler
@@ -218,6 +218,7 @@ func addInterceptors(
 		metrics.MetricTypeTotalCount,
 		metrics.MetricTypeStatusCode,
 	}, http_utils.Probes...)(handler)
+	handler = http_mw.LogHandler("grpc_gateway", http_utils.Probes...)(handler)
 	return handler
 }
 

internal/api/grpc/server/middleware/activity_interceptor.go

@@ -18,7 +18,7 @@ func ActivityInterceptor() grpc.UnaryServerInterceptor {
 		ctx = activityInfoFromGateway(ctx).SetMethod(info.FullMethod).IntoContext(ctx)
 		resp, err := handler(ctx, req)
 		if isResourceAPI(info.FullMethod) {
-			code, _, _, _ := gerrors.ExtractZITADELError(err)
+			code, _, _ := gerrors.ExtractZITADELError(err)
 			ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 			activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 		}

internal/api/grpc/server/middleware/error_interceptor.go

@@ -17,5 +17,5 @@ func ErrorHandler() grpc.UnaryServerInterceptor {
 
 func toGRPCError(ctx context.Context, req interface{}, handler grpc.UnaryHandler) (interface{}, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToGRPCError(err)
+	return resp, gerrors.ZITADELToGRPCError(ctx, err)
 }

internal/api/grpc/server/middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req interface{}, handler grpc.UnaryHan
 func addInstanceByDomain(ctx context.Context, req interface{}, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (interface{}, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, status.Error(codes.NotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr).Error())

internal/api/grpc/server/middleware/log_interceptor.go

@@ -0,0 +1,11 @@
+package middleware
+
+import (
+	"google.golang.org/grpc"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return logging.NewGrpcInterceptor(ignoredMethodSuffixes...)
+}

internal/api/grpc/server/middleware/metrics_interceptor.go

@@ -9,8 +9,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/status"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/server.go

@@ -11,6 +11,7 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
@@ -19,7 +20,6 @@ import (
 	"github.com/zitadel/zitadel/internal/logstore"
 	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
 )
 
@@ -71,6 +71,7 @@ func CreateServer(
 			grpc_middleware.ChainUnaryServer(
 				middleware.CallDurationHandler(),
 				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+				middleware.LogHandler(grpc_api.Probes...),
 				middleware.NoCacheInterceptor(),
 				middleware.InstanceInterceptor(queries, externalDomain, translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
 				middleware.AccessStorageInterceptor(accessSvc),

internal/api/grpc/user/v2/integration_test/intent_test.go

@@ -12,13 +12,13 @@ import (
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/sink"
 	"github.com/zitadel/zitadel/pkg/grpc/object/v2"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"google.golang.org/protobuf/types/known/structpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestServer_StartIdentityProviderIntent(t *testing.T) {
@@ -631,7 +631,7 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 						DisplayName:       gu.Ptr("displayname"),
 					},
 					Email: &user.SetHumanEmail{
-						Email: "[email protected]",
+						Email:        "[email protected]",
 						Verification: &user.SetHumanEmail_SendCode{SendCode: &user.SendEmailVerificationCode{}},
 					},
 				},
@@ -1037,4 +1037,4 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 			assert.EqualExportedValues(t, tt.want, got)
 		})
 	}
-}
+}

internal/api/http/error.go

@@ -1,49 +1,77 @@
 package http
 
 import (
-	"errors"
+	"context"
+	"log/slog"
 	"net/http"
 
+	slogctx "github.com/veqryn/slog-context"
+
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func ZitadelErrorToHTTPStatusCode(err error) (statusCode int, ok bool) {
+func ZitadelErrorToHTTPStatusCode(ctx context.Context, err error) (statusCode int, ok bool) {
 	if err == nil {
 		return http.StatusOK, true
 	}
-	//nolint:errorlint
-	switch err.(type) {
-	case *zerrors.AlreadyExistsError:
-		return http.StatusConflict, true
-	case *zerrors.DeadlineExceededError:
-		return http.StatusGatewayTimeout, true
-	case *zerrors.InternalError:
-		return http.StatusInternalServerError, true
-	case *zerrors.InvalidArgumentError:
-		return http.StatusBadRequest, true
-	case *zerrors.NotFoundError:
-		return http.StatusNotFound, true
-	case *zerrors.PermissionDeniedError:
-		return http.StatusForbidden, true
-	case *zerrors.PreconditionFailedError:
-		// use the same code as grpc-gateway:
-		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
-		return http.StatusBadRequest, true
-	case *zerrors.UnauthenticatedError:
-		return http.StatusUnauthorized, true
-	case *zerrors.UnavailableError:
-		return http.StatusServiceUnavailable, true
-	case *zerrors.UnimplementedError:
-		return http.StatusNotImplemented, true
-	case *zerrors.ResourceExhaustedError:
-		return http.StatusTooManyRequests, true
-	default:
-		c := new(zerrors.ZitadelError)
-		if errors.As(err, &c) {
-			return ZitadelErrorToHTTPStatusCode(errors.Unwrap(err))
-		}
+	statusCode, key, id, lvl := extractError(err)
+	msg := key
+	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
+	if statusCode == statusUnknown {
 		return http.StatusInternalServerError, false
 	}
+	return statusCode, true
+}
+
+const statusUnknown = 0
+
+func extractError(err error) (statusCode int, msg, id string, lvl slog.Level) {
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return statusUnknown, err.Error(), "", slog.LevelError
+	}
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		statusCode, lvl = http.StatusConflict, slog.LevelError
+	case zerrors.KindDeadlineExceeded:
+		statusCode, lvl = http.StatusGatewayTimeout, slog.LevelError
+	case zerrors.KindInternal:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindNotFound:
+		statusCode, lvl = http.StatusNotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		statusCode, lvl = http.StatusForbidden, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		// use the same code as grpc-gateway:
+		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		statusCode, lvl = http.StatusUnauthorized, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		statusCode, lvl = http.StatusServiceUnavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		statusCode, lvl = http.StatusNotImplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		statusCode, lvl = http.StatusTooManyRequests, slog.LevelError
+	case zerrors.KindCanceled:
+		statusCode, lvl = 499, slog.LevelWarn
+	case zerrors.KindDataLoss:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindOutOfRange:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindAborted:
+		statusCode, lvl = http.StatusConflict, slog.LevelWarn
+	case zerrors.KindUnknown:
+		fallthrough
+	default:
+		statusCode, lvl = statusUnknown, slog.LevelError
+	}
+	return statusCode, msg, id, lvl
 }
 
 func HTTPStatusCodeToZitadelError(parent error, statusCode int, id string, message string) error {

internal/api/http/error_test.go

@@ -3,6 +3,7 @@ package http
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 
@@ -118,7 +119,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 			wantOk:         true,
 		},
 		{
-			name: "no caos/zitadel error",
+			name: "no zitadel error",
 			args: args{
 				err: errors.New("error"),
 			},
@@ -128,7 +129,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(tt.args.err)
+			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(t.Context(), tt.args.err)
 			if gotStatusCode != tt.wantStatusCode {
 				t.Errorf("ZitadelErrorToHTTPStatusCode() gotStatusCode = %v, want %v", gotStatusCode, tt.wantStatusCode)
 			}
@@ -272,9 +273,9 @@ func TestHTTPStatusCodeToZitadelError(t *testing.T) {
 				statusCode: 1000,
 				id:         "id",
 				message:    "message",
-				parent:     errors.New("parent error"),
+				parent:     io.EOF,
 			},
-			wantErr: zerrors.ThrowUnknown(nil, "id", "message"),
+			wantErr: zerrors.ThrowUnknown(io.EOF, "id", "message"),
 		},
 	}
 	for _, tt := range tests {

internal/api/http/middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		return logging.NewHandler(h, service, ignoredPrefix...)
+	}
+}

internal/api/http/middleware/metrics_interceptor.go

@@ -3,11 +3,11 @@ package middleware
 import (
 	"net/http"
 
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethods ...string) func(http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
-		return metrics.NewMetricsHandler(handler, metricTypes, ignoredMethods...)
+		return metrics.NewHandler(handler, metricTypes, ignoredMethods...)
 	}
 }

internal/api/http/middleware/telemetry_interceptor.go

@@ -1,18 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/telemetry"
-)
-
-func DefaultTelemetryHandler(handler http.Handler) http.Handler {
-	return TelemetryHandler(http_utils.Probes...)(handler)
-}
-
-func TelemetryHandler(ignoredMethods ...string) func(http.Handler) http.Handler {
-	return func(handler http.Handler) http.Handler {
-		return telemetry.TelemetryHandler(handler, ignoredMethods...)
-	}
-}

internal/api/http/middleware/trace_interceptor.go

@@ -0,0 +1,18 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
+	http_utils "github.com/zitadel/zitadel/internal/api/http"
+)
+
+func DefaultTraceHandler(handler http.Handler) http.Handler {
+	return TraceHandler(http_utils.Probes...)(handler)
+}
+
+func TraceHandler(ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(handler http.Handler) http.Handler {
+		return tracing.NewHandler(handler, ignoredPrefix...)
+	}
+}

internal/api/http/request_context.go

@@ -3,6 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"net"
 	"net/url"
 )
@@ -67,6 +68,22 @@ func (r *DomainCtx) Origin() string {
 	return fmt.Sprintf("%s://%s", r.Protocol, host)
 }
 
+var _ slog.LogValuer = (*DomainCtx)(nil)
+
+func (r *DomainCtx) LogValue() slog.Value {
+	values := make([]slog.Attr, 0, 3)
+	if r.InstanceHost != "" {
+		values = append(values, slog.String("instance_host", r.InstanceHost))
+	}
+	if r.PublicHost != "" {
+		values = append(values, slog.String("public_host", r.PublicHost))
+	}
+	if r.Protocol != "" {
+		values = append(values, slog.String("protocol", r.Protocol))
+	}
+	return slog.GroupValue(values...)
+}
+
 func DomainContext(ctx context.Context) *DomainCtx {
 	o, ok := ctx.Value(domainCtx).(*DomainCtx)
 	if !ok {

internal/api/oidc/auth_request.go

@@ -44,7 +44,7 @@ const (
 func (o *OPStorage) CreateAuthRequest(ctx context.Context, req *oidc.AuthRequest, userID string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -167,7 +167,7 @@ func (o *OPStorage) audienceFromProjectID(ctx context.Context, projectID string)
 func (o *OPStorage) AuthRequestByID(ctx context.Context, id string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -206,7 +206,7 @@ func (o *OPStorage) decryptGrant(grant string) (string, error) {
 func (o *OPStorage) SaveAuthCode(ctx context.Context, id, code string) (err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -249,7 +249,7 @@ func (o *OPStorage) TerminateSession(ctx context.Context, userID, clientID strin
 func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessions []command.HumanSignOutSession, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	userAgentID, ok := middleware.UserAgentIDFromCtx(ctx)
@@ -276,7 +276,7 @@ func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessio
 func (o *OPStorage) TerminateSessionFromRequest(ctx context.Context, endSessionRequest *op.EndSessionRequest) (redirectURI string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -505,7 +505,7 @@ func (o *OPStorage) revokeTokenV1(ctx context.Context, token, userID, clientID s
 func (o *OPStorage) GetRefreshTokenInfo(ctx context.Context, clientID string, token string) (userID string, tokenID string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/client.go

@@ -36,8 +36,6 @@ const (
 	ClaimUserGroups                 = ScopeUserGroups
 	ScopeCustomUserGroups           = "urn:zitadel:iam:user:groups"
 	ClaimCustomUserGroups           = ScopeCustomUserGroups
-
-	oidcCtx = "oidc"
 )
 
 // GetClientByClientID implements the op.Storage interface to retrieve an OIDC client by its ID.
@@ -46,7 +44,7 @@ const (
 func (o *OPStorage) GetClientByClientID(ctx context.Context, id string) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	client, err := o.query.ActiveOIDCClientByID(ctx, id, false)
@@ -188,7 +186,7 @@ func userinfoClaims(userInfo *oidc.UserInfo) func(c *actions.FieldConfig) interf
 func (s *Server) VerifyClient(ctx context.Context, r *op.Request[op.ClientCredentials]) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/error.go

@@ -1,6 +1,7 @@
 package oidc
 
 import (
+	"context"
 	"errors"
 
 	"github.com/zitadel/oidc/v3/pkg/oidc"
@@ -15,7 +16,7 @@ import (
 // When err is already of the correct type is passed as-is.
 // If the err is a Zitadel error, it is transformed with a proper HTTP status code.
 // Unknown errors are treated as internal server errors.
-func oidcError(err error) error {
+func oidcError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
@@ -37,7 +38,7 @@ func oidcError(err error) error {
 		errors.As(err, &zError)
 	}
 
-	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	newOidcErr := oidc.ErrServerError
 	if statusCode < 500 {
 		newOidcErr = oidc.ErrInvalidRequest

internal/api/oidc/error_test.go

@@ -56,7 +56,7 @@ func Test_oidcError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := oidcError(tt.err)
+			err := oidcError(t.Context(), tt.err)
 			require.ErrorIs(t, err, tt.wantErr)
 		})
 	}

internal/api/oidc/introspect.go

@@ -20,7 +20,7 @@ import (
 func (s *Server) Introspect(ctx context.Context, r *op.Request[op.IntrospectionRequest]) (resp *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	ctx, cancel := context.WithCancel(ctx)

internal/api/oidc/key.go

@@ -150,7 +150,7 @@ func (k *publicKeyCache) getKey(ctx context.Context, keyID string) (_ *cachedPub
 func (k *publicKeyCache) verifySignature(ctx context.Context, jws *jose.JSONWebSignature, checkKeyExpiry bool) (_ []byte, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/op.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zitadel/oidc/v3/pkg/oidc"
 	"github.com/zitadel/oidc/v3/pkg/op"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/assets"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -21,7 +22,6 @@ import (
 	"github.com/zitadel/zitadel/internal/domain/federatedlogout"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -173,7 +173,8 @@ func NewServer(
 		op.WithFallbackLogger(fallbackLogger),
 		op.WithHTTPMiddleware(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("oidc"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/oidc/server.go

@@ -117,7 +117,7 @@ func (s *Server) Ready(ctx context.Context, r *op.Request[struct{}]) (_ *op.Resp
 func (s *Server) Discovery(ctx context.Context, r *op.Request[struct{}]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	restrictions, err := s.query.GetInstanceRestrictions(ctx)

internal/api/oidc/token_client_credentials.go

@@ -16,7 +16,7 @@ func (s *Server) ClientCredentialsExchange(ctx context.Context, r *op.ClientRequ
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 	client, ok := r.Client.(*clientCredentialsClient)
 	if !ok {

internal/api/oidc/token_code.go

@@ -19,7 +19,7 @@ func (s *Server) CodeExchange(ctx context.Context, r *op.ClientRequest[oidc.Acce
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_device.go

@@ -18,7 +18,7 @@ func (s *Server) DeviceToken(ctx context.Context, r *op.ClientRequest[oidc.Devic
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_exchange.go

@@ -32,7 +32,7 @@ func init() {
 func (s *Server) TokenExchange(ctx context.Context, r *op.ClientRequest[oidc.TokenExchangeRequest]) (_ *op.Response, err error) {
 	resp, err := s.tokenExchange(ctx, r)
 	if err != nil {
-		return nil, oidcError(err)
+		return nil, oidcError(ctx, err)
 	}
 	return resp, nil
 }

internal/api/oidc/token_jwt_profile.go

@@ -18,7 +18,7 @@ func (s *Server) JWTProfile(ctx context.Context, r *op.Request[oidc.JWTProfileGr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	user, err := s.verifyJWTProfile(ctx, r.Data)

internal/api/oidc/token_refresh.go

@@ -18,7 +18,7 @@ func (s *Server) RefreshToken(ctx context.Context, r *op.ClientRequest[oidc.Refr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/userinfo.go

@@ -30,7 +30,7 @@ import (
 func (s *Server) UserInfo(ctx context.Context, r *op.Request[oidc.UserInfoRequest]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	token, err := s.verifyAccessToken(ctx, r.Data.AccessToken)

internal/api/saml/provider.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/zitadel/saml/pkg/provider"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
@@ -17,7 +18,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -71,7 +71,8 @@ func NewProvider(
 	options := []provider.Option{
 		provider.WithHttpInterceptors(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("saml"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/scim/serrors/errors.go

@@ -127,7 +127,8 @@ func ThrowPayloadTooLarge(parent error) error {
 }
 
 func IsScimOrZitadelError(err error) bool {
-	return IsScimError(err) || zerrors.IsZitadelError(err)
+	_, zok := zerrors.AsZitadelError(err)
+	return IsScimError(err) || zok
 }
 
 func IsScimError(err error) bool {
@@ -174,7 +175,7 @@ func MapToScimError(ctx context.Context, translator *i18n.Translator, err error)
 		}
 	}
 
-	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	if !ok {
 		statusCode = http.StatusInternalServerError
 	}

internal/api/ui/console/console.go

@@ -114,7 +114,12 @@ func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, call
 	handler.Use(security, limitingAccessInterceptor.WithoutLimiting().Handle)
 
 	env := handler.NewRoute().Path(envRequestPath).Subrouter()
-	env.Use(callDurationInterceptor, middleware.TelemetryHandler(), instanceHandler)
+	env.Use(
+		callDurationInterceptor,
+		middleware.TraceHandler(),
+		middleware.LogHandler("console"),
+		instanceHandler,
+	)
 	env.HandleFunc("", func(w http.ResponseWriter, r *http.Request) {
 		url := http_util.BuildOrigin(r.Host, externalSecure)
 		ctx := r.Context()