feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

backend/v3/domain/invoker_logging.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"time"
 
-	slogctx "github.com/veqryn/slog-context"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 )
 
 // loggingInvoker decorates each command with logging.
@@ -21,7 +21,7 @@ func NewLoggingInvoker(next Invoker) *loggingInvoker {
 
 func (i *loggingInvoker) Invoke(ctx context.Context, executor Executor, opts *InvokeOpts) (err error) {
 	start := time.Now()
-	logger := slogctx.FromCtx(ctx)
+	logger := logging.FromCtx(ctx)
 	logger.InfoContext(ctx, "invoking", "name", executor.String())
 
 	err = i.execute(ctx, executor, opts)

backend/v3/instrumentation/instrumentation.go

@@ -4,7 +4,6 @@ package instrumentation
 
 import (
 	"context"
-	"log/slog"
 	"net/http"
 	"slices"
 	"strings"
@@ -69,11 +68,6 @@ func NewMeter(name string, options ...metric.MeterOption) *Meter {
 	}
 }
 
-// Logger returns the globally configured logger.
-func Logger() *slog.Logger {
-	return slog.Default()
-}
-
 func RequestFilter(ignoredPrefix ...string) otelhttp.Filter {
 	return func(r *http.Request) bool {
 		return !slices.ContainsFunc(

backend/v3/instrumentation/logformat_enumer.go

@@ -24,7 +24,7 @@ func (i LogFormat) String() string {
 // Re-run the stringer command to generate them again.
 func _LogFormatNoOp() {
 	var x [1]struct{}
-	_ = x[LogFormatUndefined-(0)]
+	_ = x[LogFormatUnspecified-(0)]
 	_ = x[LogFormatDisabled-(1)]
 	_ = x[LogFormatText-(2)]
 	_ = x[LogFormatJSON-(3)]
@@ -32,11 +32,11 @@ func _LogFormatNoOp() {
 	_ = x[LogFormatGCPErrorReporting-(5)]
 }
 
-var _LogFormatValues = []LogFormat{LogFormatUndefined, LogFormatDisabled, LogFormatText, LogFormatJSON, LogFormatGCP, LogFormatGCPErrorReporting}
+var _LogFormatValues = []LogFormat{LogFormatUnspecified, LogFormatDisabled, LogFormatText, LogFormatJSON, LogFormatGCP, LogFormatGCPErrorReporting}
 
 var _LogFormatNameToValueMap = map[string]LogFormat{
-	_LogFormatName[0:0]:        LogFormatUndefined,
-	_LogFormatLowerName[0:0]:   LogFormatUndefined,
+	_LogFormatName[0:0]:        LogFormatUnspecified,
+	_LogFormatLowerName[0:0]:   LogFormatUnspecified,
 	_LogFormatName[0:8]:        LogFormatDisabled,
 	_LogFormatLowerName[0:8]:   LogFormatDisabled,
 	_LogFormatName[8:12]:       LogFormatText,

backend/v3/instrumentation/logging/connect_handler.go

@@ -1,34 +0,0 @@
-package logging
-
-import (
-	"context"
-	"slices"
-	"strings"
-
-	"connectrpc.com/connect"
-	slogctx "github.com/veqryn/slog-context"
-
-	"github.com/zitadel/zitadel/backend/v3/instrumentation"
-)
-
-func NewConnectInterceptor(next connect.UnaryFunc, ignoredMethodSuffixes ...string) connect.UnaryFunc {
-	return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
-		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
-			return strings.HasSuffix(req.Spec().Procedure, s)
-		}) {
-			return next(ctx, req)
-		}
-
-		logger := instrumentation.Logger()
-		ctx = instrumentation.SetConnectRequestDetails(ctx, req)
-		ctx = slogctx.NewCtx(ctx, logger)
-
-		resp, err := next(ctx, req)
-		var code connect.Code
-		if err != nil {
-			code = connect.CodeOf(err)
-		}
-		logger.InfoContext(ctx, "connect RPC request", "code", code)
-		return resp, err
-	}
-}

backend/v3/instrumentation/logging/grpc_handler.go

@@ -1,36 +0,0 @@
-package logging
-
-import (
-	"context"
-	"slices"
-	"strings"
-
-	slogctx "github.com/veqryn/slog-context"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	"github.com/zitadel/zitadel/backend/v3/instrumentation"
-)
-
-func NewGrpcInterceptor(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
-	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) {
-		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
-			return strings.HasSuffix(info.FullMethod, s)
-		}) {
-			return next(ctx, req)
-		}
-
-		logger := instrumentation.Logger()
-		ctx = instrumentation.SetGrpcRequestDetails(ctx, info)
-		ctx = slogctx.NewCtx(ctx, logger)
-
-		resp, err := next(ctx, req)
-		var code codes.Code
-		if err != nil {
-			code = status.Code(err)
-		}
-		logger.InfoContext(ctx, "gRPC request", "code", code)
-		return resp, err
-	}
-}

backend/v3/instrumentation/logging/http_handler.go

@@ -1,45 +0,0 @@
-package logging
-
-import (
-	"net/http"
-
-	slogctx "github.com/veqryn/slog-context"
-
-	"github.com/zitadel/zitadel/backend/v3/instrumentation"
-)
-
-func NewHandler(next http.Handler, service string, ignoredPrefix ...string) http.Handler {
-	filter := instrumentation.RequestFilter(ignoredPrefix...)
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if !filter(r) {
-			next.ServeHTTP(w, r)
-			return
-		}
-		logger := instrumentation.Logger()
-		ctx := instrumentation.SetHttpRequestDetails(r.Context(), service, r)
-		ctx = slogctx.NewCtx(ctx, logger)
-		sw := &statusWriter{ResponseWriter: w}
-
-		next.ServeHTTP(sw, r.WithContext(ctx))
-
-		logger.InfoContext(ctx, "http request", "status", sw.status)
-	})
-}
-
-// statusWriter is a [http.ResponseWriter] that captures the status code for logging.
-type statusWriter struct {
-	http.ResponseWriter
-	status int
-}
-
-func (w *statusWriter) Write(p []byte) (int, error) {
-	if w.status == 0 {
-		w.status = http.StatusOK
-	}
-	return w.ResponseWriter.Write(p)
-}
-
-func (w *statusWriter) WriteHeader(statusCode int) {
-	w.status = statusCode
-	w.ResponseWriter.WriteHeader(statusCode)
-}

backend/v3/instrumentation/logging/logging.go

@@ -0,0 +1,221 @@
+// Package logging provides utilities for structured logging with context support.
+// It builds on top of slog and slog-context to offer a consistent logging experience
+// across different parts of the application by categorizing logs into different streams.
+//
+// The package uses the global [slog.Default] logger as the base logger,
+// which is configured at application startup to set the desired logging level and output format.
+// It provides functions to create new loggers for specific streams and to
+// add logging capabilities to contexts.
+// Streams are defined using the [Stream] enumeration.
+// Log context can be created using [NewCtx], and loggers can be retrieved from contexts using [FromCtx].
+// Streams are typically initialized at the start of different application components
+// (e.g., request handling, event processing) to ensure that all logs generated within those components
+// are tagged appropriately.
+//
+// Example usage:
+//
+//	// Initialize a context for request handling, typically done in middleware
+//	ctx := logging.NewCtx(context.Background(), logging.StreamRequest, slog.String("request_id", "12345"))
+//	// Somewhere deeper in the call stack
+//	logging.Info(ctx, "Something to log")
+//
+// This will produce a log entry with the stream set to "request" and include the request ID.
+package logging
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"os"
+	"runtime"
+	"time"
+
+	slogctx "github.com/veqryn/slog-context"
+	"github.com/zitadel/sloggcp"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/cmd/build"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+// Stream represents a logging stream for categorizing log entries.
+// This is a type alias for [instrumentation.Stream] to expose it in this package.
+type Stream = instrumentation.Stream
+
+const (
+	StreamRuntime      = instrumentation.StreamRuntime      // Application runtime logs.
+	StreamReady        = instrumentation.StreamReady        // Readiness and liveness checks.
+	StreamRequest      = instrumentation.StreamRequest      // API request handling.
+	StreamEventPusher  = instrumentation.StreamEventPusher  // Event pushing to the database.
+	StreamEventHandler = instrumentation.StreamEventHandler // Event handling and processing.
+	StreamQueue        = instrumentation.StreamQueue        // Queue operations and job processing.
+)
+
+var noop = slog.New(slog.DiscardHandler)
+
+// New creates a new logger with the given stream and additional arguments.
+func New(stream Stream, args ...any) *slog.Logger {
+	if !instrumentation.IsStreamEnabled(stream) {
+		return noop
+	}
+	args = append(args,
+		slog.String("stream", stream.String()),
+		slog.String("version", build.Version()),
+	)
+	return slog.Default().With(args...)
+}
+
+// NewCtx creates a new context with a logger for the given stream and additional arguments.
+// Use the [FromCtx] or other helpers to retrieve the logger from the context.
+// An existing logger in the context will be replaced.
+func NewCtx(ctx context.Context, stream Stream, args ...any) context.Context {
+	logger := New(stream, args...)
+	return ToCtx(ctx, logger)
+}
+
+// ToCtx adds the given logger to the context.
+// See [slogctx.NewCtx].
+func ToCtx(ctx context.Context, logger *slog.Logger) context.Context {
+	return slogctx.NewCtx(ctx, logger)
+}
+
+// FromCtx retrieves the logger from the context.
+// See [slogctx.FromCtx].
+func FromCtx(ctx context.Context) *slog.Logger {
+	return slogctx.FromCtx(ctx)
+}
+
+// With adds the given arguments to the logger in the context.
+// See [slogctx.With].
+func With(ctx context.Context, args ...any) context.Context {
+	return slogctx.With(ctx, args...)
+}
+
+// WithGroup adds a group to the logger in the context.
+// See [slogctx.WithGroup].
+func WithGroup(ctx context.Context, name string) context.Context {
+	return slogctx.WithGroup(ctx, name)
+}
+
+// Log logs a message with the given level and arguments using the logger from the context.
+// See [slogctx.Log].
+func Log(ctx context.Context, level slog.Level, msg string, args ...any) {
+	log(ctx, FromCtx(ctx), level, msg, 1, args...)
+}
+
+// Debug logs a debug message using the logger from the context.
+// See [slogctx.Debug].
+func Debug(ctx context.Context, msg string, args ...any) {
+	log(ctx, FromCtx(ctx), slog.LevelDebug, msg, 1, args...)
+}
+
+// Info logs an info message using the logger from the context.
+// See [slogctx.Info].
+func Info(ctx context.Context, msg string, args ...any) {
+	log(ctx, FromCtx(ctx), slog.LevelInfo, msg, 1, args...)
+}
+
+// Warn logs a warning message using the logger from the context.
+// See [slogctx.Warn].
+func Warn(ctx context.Context, msg string, args ...any) {
+	log(ctx, FromCtx(ctx), slog.LevelWarn, msg, 1, args...)
+}
+
+// Error logs an error message using the logger from the context.
+// See [slogctx.Error].
+func Error(ctx context.Context, msg string, args ...any) {
+	log(ctx, FromCtx(ctx), slog.LevelError, msg, 1, args...)
+}
+
+// WithError adds an error attribute to the logger from the context and returns the new logger.
+// If the error is not a [zerrors.ZitadelError], it is wrapped in a generic ZitadelError with kind [zerrors.KindUnknown].
+func WithError(ctx context.Context, err error) *ErrorContextLogger {
+	var target *zerrors.ZitadelError
+	if !errors.As(err, &target) {
+		target = zerrors.CreateZitadelError(zerrors.KindUnknown, err, "LOG-Ao5ch", "an unknown error occurred", 1)
+	}
+	return &ErrorContextLogger{
+		ctx:          ctx,
+		logger:       slogctx.FromCtx(ctx).With(slogctx.Err(target)),
+		canTerminate: true,
+	}
+}
+
+// OnError returns a logger that includes the error as an attribute when err is non-nil.
+// If err is nil, it returns a no-op logger.
+// If err is not a [zerrors.ZitadelError], it is wrapped as a generic ZitadelError of kind [zerrors.KindUnknown].
+func OnError(ctx context.Context, err error) *ErrorContextLogger {
+	if err == nil {
+		return &ErrorContextLogger{ctx, noop, false}
+	}
+	var target *zerrors.ZitadelError
+	if !errors.As(err, &target) {
+		target = zerrors.CreateZitadelError(zerrors.KindUnknown, err, "LOG-ii6Pi", "an unknown error occurred", 1)
+	}
+	return &ErrorContextLogger{
+		ctx:          ctx,
+		logger:       slogctx.FromCtx(ctx).With(slogctx.Err(target)),
+		canTerminate: true,
+	}
+}
+
+type ErrorContextLogger struct {
+	ctx    context.Context
+	logger *slog.Logger
+	// canTerminate sets whether Panic/Fatal should actually call panic or os.Exit.
+	// False when OnError returned a no-op logger, true in all other cases.
+	canTerminate bool
+}
+
+func (l *ErrorContextLogger) Debug(msg string, args ...any) {
+	log(l.ctx, l.logger, slog.LevelDebug, msg, 1, args...)
+}
+
+func (l *ErrorContextLogger) Info(msg string, args ...any) {
+	log(l.ctx, l.logger, slog.LevelInfo, msg, 1, args...)
+}
+
+func (l *ErrorContextLogger) Warn(msg string, args ...any) {
+	log(l.ctx, l.logger, slog.LevelWarn, msg, 1, args...)
+}
+
+func (l *ErrorContextLogger) Error(msg string, args ...any) {
+	log(l.ctx, l.logger, slog.LevelError, msg, 1, args...)
+}
+
+// Panic logs a [sloggcp.LevelAlert] leveled message and panics.
+// If the logger was created via [OnError] with a nil error, this method does nothing.
+func (l *ErrorContextLogger) Panic(msg string, args ...any) {
+	log(l.ctx, l.logger, sloggcp.LevelAlert, msg, 1, args...)
+	if l.canTerminate {
+		panic(msg)
+	}
+}
+
+// Fatal logs a [sloggcp.LevelEmergency] leveled message and exits the application with code 1.
+// If the logger was created via [OnError] with a nil error, this method does nothing.
+func (l *ErrorContextLogger) Fatal(msg string, args ...any) {
+	log(l.ctx, l.logger, sloggcp.LevelEmergency, msg, 1, args...)
+	if l.canTerminate {
+		exit(1)
+	}
+}
+
+// exit is a variable to allow testing of Fatal without exiting the test process.
+var exit = os.Exit
+
+// log is a helper function that logs a message with the given level and arguments using the provided logger.
+func log(ctx context.Context, logger *slog.Logger, level slog.Level, msg string, skip int, args ...any) {
+	handler := logger.Handler()
+	if !handler.Enabled(ctx, level) {
+		return
+	}
+	var pcs [1]uintptr
+	if instrumentation.IsAddSourceEnabled() {
+		runtime.Callers(skip+2, pcs[:])
+
+	}
+	r := slog.NewRecord(time.Now(), level, msg, pcs[0])
+	r.Add(args...)
+	_ = logger.Handler().Handle(ctx, r)
+}

backend/v3/instrumentation/logging/logging_test.go

@@ -0,0 +1,583 @@
+package logging
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/internal/zerrors"
+)
+
+var globalLock sync.Mutex
+
+type testLogEntry struct {
+	Level string
+	Msg   string
+	Err   *testLogError `json:",omitempty"`
+	Foo   string
+	Group struct {
+		V int
+	}
+}
+
+type testLogError struct {
+	Kind    string
+	Parent  string
+	Message string
+	ID      string
+}
+
+func init() {
+	// Only enable StreamRuntime for tests
+	instrumentation.EnableStreams(StreamRuntime)
+}
+
+// prepareDefaultLogger sets the global default logger to a JSON logger writing to a buffer.
+// It returns a function that MUST be called to retrieve exactly one logged entry and release the global lock.
+func prepareDefaultLogger() (done func() (*testLogEntry, error)) {
+	globalLock.Lock()
+	var buf bytes.Buffer
+	logger := slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{
+		Level:     slog.LevelDebug,
+		AddSource: false,
+	}))
+	slog.SetDefault(logger)
+
+	return func() (*testLogEntry, error) {
+		defer globalLock.Unlock()
+		if buf.Len() == 0 {
+			return nil, nil
+		}
+
+		entry := new(testLogEntry)
+		decoder := json.NewDecoder(&buf)
+		if err := decoder.Decode(entry); err != nil {
+			return nil, err
+		}
+		return entry, nil
+	}
+}
+
+func TestNew(t *testing.T) {
+	tests := []struct {
+		name   string
+		stream Stream
+		want   *testLogEntry
+	}{
+		{
+			name:   "enabled stream",
+			stream: StreamRuntime,
+			want: &testLogEntry{
+				Level: "INFO",
+				Msg:   "test message",
+				Foo:   "bar",
+			},
+		},
+		{
+			name:   "disabled stream",
+			stream: StreamEventPusher,
+			want:   nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			done := prepareDefaultLogger()
+			logger := New(tt.stream, slog.String("foo", "bar"))
+			logger.Info("test message")
+			got, err := done()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestFromCtx(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+		slog.String("foo", "bar"),
+	)
+	FromCtx(ctx).Info("test message")
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestWith(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	ctx = With(ctx, slog.String("foo", "bar"))
+	FromCtx(ctx).Info("test message")
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestWithGroup(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	ctx = WithGroup(ctx, "group")
+	ctx = With(ctx, slog.Int("v", 42))
+	FromCtx(ctx).Info("test message")
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Group: struct{ V int }{V: 42},
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestLog(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	Log(ctx, slog.LevelInfo, "test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestDebug(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	Debug(ctx, "test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "DEBUG",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestInfo(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	Info(ctx, "test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestWarn(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	Warn(ctx, "test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "WARN",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestError(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	Error(ctx, "test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "ERROR",
+		Msg:   "test message",
+		Foo:   "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestWithError(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	err := errors.New("some error")
+	logger := WithError(ctx, err)
+	logger.Info("test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Err: &testLogError{
+			Kind:    "Unknown",
+			Parent:  "some error",
+			Message: "an unknown error occurred",
+			ID:      "LOG-Ao5ch",
+		},
+		Foo: "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestOnError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want *testLogEntry
+	}{
+		{
+			name: "nil error",
+			err:  nil,
+		},
+		{
+			name: "non-zitadel error",
+			err:  errors.New("some error"),
+			want: &testLogEntry{
+				Level: "INFO",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "Unknown",
+					Parent:  "some error",
+					Message: "an unknown error occurred",
+					ID:      "LOG-ii6Pi",
+				},
+				Foo: "bar",
+			},
+		},
+		{
+			name: "zitadel error",
+			err: zerrors.CreateZitadelError(
+				zerrors.KindNotFound,
+				errors.New("parent error"),
+				"ZIT-404",
+				"resource not found",
+				0,
+			),
+			want: &testLogEntry{
+				Level: "INFO",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "NotFound",
+					Parent:  "parent error",
+					Message: "resource not found",
+					ID:      "ZIT-404",
+				},
+				Foo: "bar",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			done := prepareDefaultLogger()
+			ctx := NewCtx(
+				t.Context(),
+				StreamRuntime,
+			)
+			logger := OnError(ctx, tt.err)
+			logger.Info("test message", slog.String("foo", "bar"))
+			got, err := done()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestErrorContextLogger_Debug(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	WithError(ctx, nil).Debug("test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "DEBUG",
+		Msg:   "test message",
+		Err: &testLogError{
+			Kind:    "Unknown",
+			Parent:  "",
+			Message: "an unknown error occurred",
+			ID:      "LOG-Ao5ch",
+		},
+		Foo: "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestErrorContextLogger_Info(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	WithError(ctx, nil).Info("test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "INFO",
+		Msg:   "test message",
+		Err: &testLogError{
+			Kind:    "Unknown",
+			Parent:  "",
+			Message: "an unknown error occurred",
+			ID:      "LOG-Ao5ch",
+		},
+		Foo: "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestErrorContextLogger_Warn(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	WithError(ctx, nil).Warn("test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "WARN",
+		Msg:   "test message",
+		Err: &testLogError{
+			Kind:    "Unknown",
+			Parent:  "",
+			Message: "an unknown error occurred",
+			ID:      "LOG-Ao5ch",
+		},
+		Foo: "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestErrorContextLogger_Error(t *testing.T) {
+	done := prepareDefaultLogger()
+	ctx := NewCtx(
+		t.Context(),
+		StreamRuntime,
+	)
+	WithError(ctx, nil).Error("test message", slog.String("foo", "bar"))
+	want := &testLogEntry{
+		Level: "ERROR",
+		Msg:   "test message",
+		Err: &testLogError{
+			Kind:    "Unknown",
+			Parent:  "",
+			Message: "an unknown error occurred",
+			ID:      "LOG-Ao5ch",
+		},
+		Foo: "bar",
+	}
+	got, err := done()
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestErrorContextLogger_Panic(t *testing.T) {
+	tests := []struct {
+		name      string
+		construct func(context.Context, error) *ErrorContextLogger
+		err       error
+		want      *testLogEntry
+		wantPanic any
+	}{
+		{
+			name:      "with zitadel error",
+			construct: WithError,
+			err: zerrors.CreateZitadelError(
+				zerrors.KindNotFound,
+				errors.New("parent error"),
+				"ZIT-404",
+				"resource not found",
+				0,
+			),
+			want: &testLogEntry{
+				Level: "ERROR+4",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "NotFound",
+					Parent:  "parent error",
+					Message: "resource not found",
+					ID:      "ZIT-404",
+				},
+				Foo: "bar",
+			},
+			wantPanic: "test message",
+		},
+		{
+			name:      "on nil error",
+			construct: OnError,
+			err:       nil,
+			want:      nil,
+			wantPanic: nil,
+		},
+		{
+			name:      "on non-zitadel error",
+			construct: OnError,
+			err:       errors.New("some error"),
+			want: &testLogEntry{
+				Level: "ERROR+4",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "Unknown",
+					Parent:  "some error",
+					Message: "an unknown error occurred",
+					ID:      "LOG-ii6Pi",
+				},
+				Foo: "bar",
+			},
+			wantPanic: "test message",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			done := prepareDefaultLogger()
+			ctx := NewCtx(
+				t.Context(),
+				StreamRuntime,
+			)
+			var (
+				gotPanic any
+				gotEntry *testLogEntry
+				err      error
+			)
+
+			// need to check in defer because of panic
+			defer func() {
+				gotPanic = recover()
+				gotEntry, err = done()
+				require.NoError(t, err)
+				assert.Equal(t, tt.want, gotEntry)
+				assert.Equal(t, tt.wantPanic, gotPanic)
+			}()
+			tt.construct(ctx, tt.err).Panic("test message", slog.String("foo", "bar"))
+		})
+	}
+}
+
+func TestErrorContextLogger_Fatal(t *testing.T) {
+	tests := []struct {
+		name      string
+		construct func(context.Context, error) *ErrorContextLogger
+		err       error
+		want      *testLogEntry
+		wantExit  int
+	}{
+		{
+			name:      "with zitadel error",
+			construct: WithError,
+			err: zerrors.CreateZitadelError(
+				zerrors.KindNotFound,
+				errors.New("parent error"),
+				"ZIT-404",
+				"resource not found",
+				0,
+			),
+			want: &testLogEntry{
+				Level: "ERROR+6",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "NotFound",
+					Parent:  "parent error",
+					Message: "resource not found",
+					ID:      "ZIT-404",
+				},
+				Foo: "bar",
+			},
+			wantExit: 1,
+		},
+		{
+			name:      "on nil error",
+			construct: OnError,
+			err:       nil,
+			want:      nil,
+			wantExit:  0,
+		},
+		{
+			name:      "on non-zitadel error",
+			construct: OnError,
+			err:       errors.New("some error"),
+			want: &testLogEntry{
+				Level: "ERROR+6",
+				Msg:   "test message",
+				Err: &testLogError{
+					Kind:    "Unknown",
+					Parent:  "some error",
+					Message: "an unknown error occurred",
+					ID:      "LOG-ii6Pi",
+				},
+				Foo: "bar",
+			},
+			wantExit: 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			done := prepareDefaultLogger()
+			ctx := NewCtx(
+				t.Context(),
+				StreamRuntime,
+			)
+			var gotExit int
+			exit = func(code int) {
+				gotExit = code
+			}
+			defer func() {
+				exit = os.Exit
+			}()
+
+			tt.construct(ctx, tt.err).Fatal("test message", slog.String("foo", "bar"))
+			gotEntry, err := done()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, gotEntry)
+			assert.Equal(t, tt.wantExit, gotExit)
+		})
+	}
+}

backend/v3/instrumentation/metrics/http_handler.go

@@ -1,14 +1,10 @@
 package metrics
 
 import (
-	"context"
 	"net/http"
 	"strings"
 
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"go.opentelemetry.io/otel/attribute"
-
-	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 )
 
 const (
@@ -23,12 +19,6 @@ const (
 	ReturnCode                   = "return_code"
 )
 
-type Handler struct {
-	handler http.Handler
-	methods []MetricType
-	filter  otelhttp.Filter
-}
-
 type MetricType int32
 
 const (
@@ -37,82 +27,13 @@ const (
 	MetricTypeRequestCount
 )
 
-type StatusRecorder struct {
-	http.ResponseWriter
-	RequestURI *string
-	Status     int
+type StatusRecorder interface {
+	Status() int
 }
 
-func (r *StatusRecorder) WriteHeader(status int) {
-	r.Status = status
-	r.ResponseWriter.WriteHeader(status)
-}
-
-type Filter func(*http.Request) bool
-
-func NewHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
-	h := Handler{
-		handler: handler,
-		methods: metricMethods,
-		filter:  instrumentation.RequestFilter(ignoredEndpoints...),
-	}
-	return &h
-}
-
-type key int
-
-const requestURI key = iota
-
-func SetRequestURIPattern(ctx context.Context, pattern string) {
-	uri, ok := ctx.Value(requestURI).(*string)
-	if !ok {
-		return
-	}
-	*uri = pattern
-}
-
-// ServeHTTP serves HTTP requests (http.Handler)
-func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if len(h.methods) == 0 {
-		h.handler.ServeHTTP(w, r)
-		return
-	}
-	if !h.filter(r) {
-		// Simply pass through to the handler if a filter rejects the request
-		h.handler.ServeHTTP(w, r)
-		return
-	}
-	uri := strings.Split(r.RequestURI, "?")[0]
-	recorder := &StatusRecorder{
-		ResponseWriter: w,
-		RequestURI:     &uri,
-		Status:         200,
-	}
-	r = r.WithContext(context.WithValue(r.Context(), requestURI, &uri))
-	h.handler.ServeHTTP(recorder, r)
-	if h.containsMetricsMethod(MetricTypeRequestCount) {
-		RegisterRequestCounter(recorder, r)
-	}
-	if h.containsMetricsMethod(MetricTypeTotalCount) {
-		RegisterTotalRequestCounter(r)
-	}
-	if h.containsMetricsMethod(MetricTypeStatusCode) {
-		RegisterRequestCodeCounter(recorder, r)
-	}
-}
-
-func (h *Handler) containsMetricsMethod(method MetricType) bool {
-	for _, m := range h.methods {
-		if m == method {
-			return true
-		}
-	}
-	return false
-}
-
-func RegisterRequestCounter(recorder *StatusRecorder, r *http.Request) {
+func RegisterRequestCounter(recorder StatusRecorder, r *http.Request) {
 	var labels = map[string]attribute.Value{
-		URI:    attribute.StringValue(*recorder.RequestURI),
+		URI:    attribute.StringValue(baseURI(r)),
 		Method: attribute.StringValue(r.Method),
 	}
 	RegisterCounter(RequestCounter, RequestCountDescription)
@@ -124,12 +45,16 @@ func RegisterTotalRequestCounter(r *http.Request) {
 	AddCount(r.Context(), TotalRequestCounter, 1, nil)
 }
 
-func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
+func RegisterRequestCodeCounter(recorder StatusRecorder, r *http.Request) {
 	var labels = map[string]attribute.Value{
-		URI:        attribute.StringValue(*recorder.RequestURI),
+		URI:        attribute.StringValue(baseURI(r)),
 		Method:     attribute.StringValue(r.Method),
-		ReturnCode: attribute.IntValue(recorder.Status),
+		ReturnCode: attribute.IntValue(recorder.Status()),
 	}
 	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
 	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
 }
+
+func baseURI(r *http.Request) string {
+	return strings.Split(r.RequestURI, "?")[0]
+}

backend/v3/instrumentation/slog.go

@@ -3,22 +3,20 @@ package instrumentation
 import (
 	"context"
 	"log/slog"
-	"net/http"
 	"os"
-	"strings"
+	"slices"
+	"sync/atomic"
 	"time"
 
-	"connectrpc.com/connect"
 	"github.com/rs/xid"
 	slogmulti "github.com/samber/slog-multi"
 	slogctx "github.com/veqryn/slog-context"
 	slogotel "github.com/veqryn/slog-context/otel"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 	"github.com/zitadel/sloggcp"
 	"go.opentelemetry.io/contrib/bridges/otelslog"
 	"go.opentelemetry.io/otel/sdk/log"
-	"google.golang.org/grpc"
 
-	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -27,7 +25,7 @@ type LogFormat int
 //go:generate enumer -type=LogFormat -trimprefix=LogFormat -text -linecomment -transform=snake
 const (
 	// Empty line comment sets empty string of unspecified value
-	LogFormatUndefined LogFormat = iota //
+	LogFormatUnspecified LogFormat = iota //
 	LogFormatDisabled
 	LogFormatText
 	LogFormatJSON
@@ -36,19 +34,74 @@ const (
 	LogFormatGCPErrorReporting
 )
 
+func (f LogFormat) isDisabled() bool {
+	return f == LogFormatUnspecified || f == LogFormatDisabled
+}
+
 type LogConfig struct {
 	Level     slog.Level
+	Streams   []Stream
+	Mask      MaskConfig
 	Format    LogFormat
 	AddSource bool
 	Errors    ErrorConfig
 	Exporter  ExporterConfig
 }
 
+func (c *LogConfig) SetLegacyConfig(lc *old_logging.Config) {
+	if lc == nil || !c.Format.isDisabled() {
+		return
+	}
+	err := c.Level.UnmarshalText([]byte(lc.Level))
+	if err != nil {
+		c.Level = slog.LevelInfo
+	}
+	c.Format, err = LogFormatString(lc.Formatter.Format)
+	if err != nil {
+		c.Format = LogFormatText
+	}
+	c.AddSource = lc.AddSource
+}
+
+func (c LogConfig) replacer() replacer {
+	var replacers []replacer
+	if len(c.Mask.Keys) > 0 {
+		replacers = append(replacers, c.Mask.replacer())
+	}
+	if c.Format == LogFormatGCP {
+		replacers = append(replacers, sloggcp.ReplaceAttr)
+	}
+	if c.Format == LogFormatGCPErrorReporting {
+		replacers = append(replacers, errReplacer())
+	}
+	return chainReplacers(replacers...)
+}
+
+type MaskConfig struct {
+	Keys  []string
+	Value string
+}
+
+func (c MaskConfig) replacer() replacer {
+	return func(_ []string, a slog.Attr) slog.Attr {
+		if slices.Contains(c.Keys, a.Key) {
+			a.Value = slog.StringValue(c.Value)
+		}
+		return a
+	}
+}
+
 type ErrorConfig struct {
 	ReportLocation bool
 	StackTrace     bool
 }
 
+var addSourceEnabled atomic.Bool
+
+func IsAddSourceEnabled() bool {
+	return addSourceEnabled.Load()
+}
+
 // setLogger configures the global slog logger.
 // Logs are sent to [os.Stderr] and/or the [log.LoggerProvider].
 //
@@ -60,12 +113,15 @@ type ErrorConfig struct {
 //   - Request details, such as method and path.
 func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
 	options := &slog.HandlerOptions{
-		AddSource: cfg.AddSource,
-		Level:     cfg.Level,
+		AddSource:   cfg.AddSource,
+		Level:       cfg.Level,
+		ReplaceAttr: cfg.replacer(),
 	}
+	addSourceEnabled.Store(cfg.AddSource)
+
 	var stdErrHandler slog.Handler
 	switch cfg.Format {
-	case LogFormatUndefined:
+	case LogFormatUnspecified:
 		return // uses default slog logger
 	case LogFormatDisabled:
 		stdErrHandler = slog.DiscardHandler
@@ -74,14 +130,13 @@ func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
 	case LogFormatJSON:
 		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
 	case LogFormatGCP:
-		options.ReplaceAttr = sloggcp.ReplaceAttr
 		stdErrHandler = slog.NewJSONHandler(os.Stderr, options)
 	case LogFormatGCPErrorReporting:
 		zerrors.GCPErrorReportingEnabled(true)
-		options.ReplaceAttr = replaceErrAttr
 		stdErrHandler = sloggcp.NewErrorReportingHandler(os.Stderr, options)
 	}
 
+	EnableStreams(cfg.Streams...)
 	zerrors.EnableReportLocation(cfg.Errors.ReportLocation)
 	zerrors.EnableStackTrace(cfg.Errors.StackTrace)
 
@@ -89,8 +144,8 @@ func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
 		stdErrHandler,
 		&slogctx.HandlerOptions{
 			Prependers: []slogctx.AttrExtractor{
-				domainExtractor,
-				requestExtractor,
+				instanceExtractor,
+				requestIDExtractor,
 				slogotel.ExtractTraceSpanID,
 				slogctx.ExtractPrepended,
 			},
@@ -113,106 +168,65 @@ const (
 	ProtocolGrpc    = "grpc"
 )
 
-// SetHttpRequestDetails adds static details to each context aware log entry.
-func SetHttpRequestDetails(ctx context.Context, service string, request *http.Request) context.Context {
-	now := time.Now()
-	return context.WithValue(ctx, ctxKey{}, &requestDetails{
-		protocol:    ProtocolHttp,
-		service:     service,
-		http_method: request.Method,
-		path:        request.URL.Path,
-		requestID:   xid.NewWithTime(now),
-		start:       now,
-	})
+// Instance is a minimal interface for logging the instance ID.
+type Instance interface {
+	InstanceID() string
 }
 
-// SetConnectRequestDetails adds static details to each context aware log entry.
-func SetConnectRequestDetails(ctx context.Context, request connect.AnyRequest) context.Context {
-	now := time.Now()
-	spec := request.Spec()
-	return context.WithValue(ctx, ctxKey{}, &requestDetails{
-		protocol:    ProtocolConnect,
-		service:     serviceFromRPCMethod(spec.Procedure),
-		http_method: request.HTTPMethod(),
-		path:        spec.Procedure,
-		requestID:   xid.NewWithTime(now),
-		start:       now,
-	})
+// SetInstance adds the instance to the context for logging.
+func SetInstance(ctx context.Context, instance Instance) context.Context {
+	return context.WithValue(ctx, ctxKeyInstance, instance)
 }
 
-func SetGrpcRequestDetails(ctx context.Context, info *grpc.UnaryServerInfo) context.Context {
-	now := time.Now()
-	return context.WithValue(ctx, ctxKey{}, &requestDetails{
-		protocol:    ProtocolGrpc,
-		service:     serviceFromRPCMethod(info.FullMethod),
-		http_method: http.MethodPost, // gRPC always uses POST
-		path:        info.FullMethod,
-		requestID:   xid.NewWithTime(now),
-		start:       now,
-	})
+// SetRequest generates a new [xid.ID] based on the passed request timestamp
+// and adds it to the context.
+func SetRequestID(ctx context.Context, ts time.Time) context.Context {
+	return context.WithValue(ctx, ctxKeyRequestID, xid.NewWithTime(ts))
 }
 
-type ctxKey struct{}
+type ctxKeyType int
 
-type requestDetails struct {
-	protocol    string
-	service     string
-	http_method string
-	path        string
-	requestID   xid.ID
-	start       time.Time
-}
+const (
+	ctxKeyRequestID ctxKeyType = iota
+	ctxKeyInstance
+)
 
-func (r *requestDetails) attrs() []slog.Attr {
-	attrs := make([]slog.Attr, 0, 6)
-	if r.protocol != "" {
-		attrs = append(attrs, slog.String("protocol", r.protocol))
-	}
-	if r.service != "" {
-		attrs = append(attrs, slog.String("service", r.service))
-	}
-	if r.http_method != "" {
-		attrs = append(attrs, slog.String("http_method", r.http_method))
-	}
-	if r.path != "" {
-		attrs = append(attrs, slog.String("path", r.path))
-	}
-	if !r.requestID.IsZero() {
-		attrs = append(attrs, slog.String("request_id", r.requestID.String()))
-	}
-	if !r.start.IsZero() {
-		attrs = append(attrs, slog.Duration("duration", time.Since(r.start)))
-	}
-	return attrs
-}
-
-func serviceFromRPCMethod(fullMethod string) string {
-	parts := strings.Split(fullMethod, "/")
-	if len(parts) >= 2 {
-		return parts[1]
-	}
-	return "unknown"
-}
-
-// domainExtractor sets the sanitized request hosts from [http_util.DomainCtx] to a log entry.
-func domainExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
-	return []slog.Attr{
-		slog.Any("domain", http_util.DomainContext(ctx)),
-	}
-}
-
-// requestExtractor sets the request details from [requestDetails] to a log entry.
-func requestExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
-	if r, ok := ctx.Value(ctxKey{}).(*requestDetails); ok {
-		return r.attrs()
+// instanceExtractor sets the instance ID from [Instance] to a log entry.
+func instanceExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	if instance, ok := ctx.Value(ctxKeyInstance).(Instance); ok {
+		return []slog.Attr{
+			slog.String("instance", instance.InstanceID()),
+		}
 	}
 	return nil
 }
 
-// replaceErrAttr renames the "err" attribute to the Google Cloud Platform compatible error key.
-func replaceErrAttr(groups []string, a slog.Attr) slog.Attr {
-	if len(groups) == 0 && a.Key == "err" {
-		a.Key = sloggcp.ErrorKey
+// requestIDExtractor sets the request XID to a log entry.
+func requestIDExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	if r, ok := ctx.Value(ctxKeyRequestID).(xid.ID); ok {
+		return []slog.Attr{
+			slog.String("request_id", r.String()),
+		}
+	}
+	return nil
+}
+
+type replacer func(groups []string, a slog.Attr) slog.Attr
+
+func chainReplacers(replacers ...replacer) replacer {
+	return func(groups []string, a slog.Attr) slog.Attr {
+		for _, r := range replacers {
+			a = r(groups, a)
+		}
+		return a
+	}
+}
+
+func errReplacer() replacer {
+	return func(groups []string, a slog.Attr) slog.Attr {
+		if len(groups) == 0 && a.Key == "err" {
+			a.Key = sloggcp.ErrorKey
+		}
+		return a
 	}
-	return a
 }

backend/v3/instrumentation/slog_test.go

@@ -0,0 +1,138 @@
+package instrumentation
+
+import (
+	"log/slog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
+	"github.com/zitadel/sloggcp"
+)
+
+func TestLogConfig_SetLegacyConfig(t *testing.T) {
+	tests := []struct {
+		name string // description of this test case
+		lc   *old_logging.Config
+		c    *LogConfig
+		want *LogConfig
+	}{
+		{
+			name: "nil legacy config does not change log config",
+			lc:   nil,
+			c:    &LogConfig{Level: slog.LevelInfo},
+			want: &LogConfig{Level: slog.LevelInfo},
+		},
+		{
+			name: "legacy config sets log level when format is disabled",
+			lc:   &old_logging.Config{Level: "debug"},
+			c:    &LogConfig{Level: slog.LevelInfo},
+			want: &LogConfig{Level: slog.LevelDebug},
+		},
+		{
+			name: "legacy config does not change log config when format is not disabled",
+			lc:   &old_logging.Config{Level: "debug"},
+			c:    &LogConfig{Level: slog.LevelInfo, Format: LogFormatJSON},
+			want: &LogConfig{Level: slog.LevelInfo, Format: LogFormatJSON},
+		},
+		{
+			name: "invalid legacy log level defaults to info",
+			lc:   &old_logging.Config{Level: "invalid"},
+			c:    &LogConfig{Level: slog.LevelDebug},
+			want: &LogConfig{Level: slog.LevelInfo},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.c.SetLegacyConfig(tt.lc)
+			assert.Equal(t, tt.want, tt.c)
+		})
+	}
+}
+
+func TestLogConfig_replacer(t *testing.T) {
+	type args struct {
+		groups []string
+		a      slog.Attr
+	}
+	tests := []struct {
+		name string // description of this test case
+		c    LogConfig
+		args args
+		want slog.Attr
+	}{
+		{
+			name: "empty config does not change attribute",
+			c:    LogConfig{},
+			args: args{
+				a: slog.String("key", "value"),
+			},
+			want: slog.String("key", "value"),
+		},
+		{
+			name: "masking configured key",
+			c: LogConfig{
+				Mask: MaskConfig{
+					Keys:  []string{"sensitive"},
+					Value: "masked",
+				},
+			},
+			args: args{
+				a: slog.String("sensitive", "value"),
+			},
+			want: slog.String("sensitive", "masked"),
+		},
+		{
+			name: "masking configured key in any group",
+			c: LogConfig{
+				Mask: MaskConfig{
+					Keys:  []string{"sensitive"},
+					Value: "masked",
+				},
+			},
+			args: args{
+				groups: []string{"a", "b"},
+				a:      slog.String("sensitive", "value"),
+			},
+			want: slog.String("sensitive", "masked"),
+		},
+		{
+			name: "not masking unmatched key",
+			c: LogConfig{
+				Mask: MaskConfig{
+					Keys:  []string{"sensitive"},
+					Value: "masked",
+				},
+			},
+			args: args{
+				a: slog.String("unmatched", "value"),
+			},
+			want: slog.String("unmatched", "value"),
+		},
+		{
+			name: "sloggcp replacer",
+			c: LogConfig{
+				Format: LogFormatGCP,
+			},
+			args: args{
+				a: slog.Any("level", slog.LevelInfo),
+			},
+			want: slog.String(sloggcp.SeverityKey, sloggcp.InfoSeverity),
+		},
+		{
+			name: "errReplacer",
+			c: LogConfig{
+				Format: LogFormatGCPErrorReporting,
+			},
+			args: args{
+				a: slog.String("err", "some error"),
+			},
+			want: slog.String(sloggcp.ErrorKey, "some error"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tt.c.replacer()(tt.args.groups, tt.args.a)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

backend/v3/instrumentation/stream.go

@@ -0,0 +1,29 @@
+package instrumentation
+
+import "sync"
+
+type Stream int
+
+//go:generate enumer -type=Stream -trimprefix=Stream -transform=snake -text
+const (
+	StreamRuntime      Stream = iota // Top-level commands, such as starting the application or running migrations.
+	StreamReady                      // Readiness and liveness checks.
+	StreamRequest                    // API request handling.
+	StreamEventPusher                // Event pushing to the database.
+	StreamEventHandler               // Event handling and processing.
+	StreamQueue                      // Queue operations and job processing.
+)
+
+var enabledStreams sync.Map
+
+func EnableStreams(streams ...Stream) {
+	enabledStreams.Clear()
+	for _, stream := range streams {
+		enabledStreams.Store(stream, struct{}{})
+	}
+}
+
+func IsStreamEnabled(stream Stream) bool {
+	_, ok := enabledStreams.Load(stream)
+	return ok
+}

backend/v3/instrumentation/stream_enumer.go

@@ -0,0 +1,106 @@
+// Code generated by "enumer -type=Stream -trimprefix=Stream -transform=snake -text"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _StreamName = "runtimereadyrequestevent_pusherevent_handlerqueue"
+
+var _StreamIndex = [...]uint8{0, 7, 12, 19, 31, 44, 49}
+
+const _StreamLowerName = "runtimereadyrequestevent_pusherevent_handlerqueue"
+
+func (i Stream) String() string {
+	if i < 0 || i >= Stream(len(_StreamIndex)-1) {
+		return fmt.Sprintf("Stream(%d)", i)
+	}
+	return _StreamName[_StreamIndex[i]:_StreamIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _StreamNoOp() {
+	var x [1]struct{}
+	_ = x[StreamRuntime-(0)]
+	_ = x[StreamReady-(1)]
+	_ = x[StreamRequest-(2)]
+	_ = x[StreamEventPusher-(3)]
+	_ = x[StreamEventHandler-(4)]
+	_ = x[StreamQueue-(5)]
+}
+
+var _StreamValues = []Stream{StreamRuntime, StreamReady, StreamRequest, StreamEventPusher, StreamEventHandler, StreamQueue}
+
+var _StreamNameToValueMap = map[string]Stream{
+	_StreamName[0:7]:        StreamRuntime,
+	_StreamLowerName[0:7]:   StreamRuntime,
+	_StreamName[7:12]:       StreamReady,
+	_StreamLowerName[7:12]:  StreamReady,
+	_StreamName[12:19]:      StreamRequest,
+	_StreamLowerName[12:19]: StreamRequest,
+	_StreamName[19:31]:      StreamEventPusher,
+	_StreamLowerName[19:31]: StreamEventPusher,
+	_StreamName[31:44]:      StreamEventHandler,
+	_StreamLowerName[31:44]: StreamEventHandler,
+	_StreamName[44:49]:      StreamQueue,
+	_StreamLowerName[44:49]: StreamQueue,
+}
+
+var _StreamNames = []string{
+	_StreamName[0:7],
+	_StreamName[7:12],
+	_StreamName[12:19],
+	_StreamName[19:31],
+	_StreamName[31:44],
+	_StreamName[44:49],
+}
+
+// StreamString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StreamString(s string) (Stream, error) {
+	if val, ok := _StreamNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _StreamNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to Stream values", s)
+}
+
+// StreamValues returns all values of the enum
+func StreamValues() []Stream {
+	return _StreamValues
+}
+
+// StreamStrings returns a slice of all String values of the enum
+func StreamStrings() []string {
+	strs := make([]string, len(_StreamNames))
+	copy(strs, _StreamNames)
+	return strs
+}
+
+// IsAStream returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i Stream) IsAStream() bool {
+	for _, v := range _StreamValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for Stream
+func (i Stream) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for Stream
+func (i *Stream) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = StreamString(string(text))
+	return err
+}

backend/v3/instrumentation/stream_test.go

@@ -0,0 +1,13 @@
+package instrumentation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStream(t *testing.T) {
+	EnableStreams(StreamRuntime)
+	assert.True(t, IsStreamEnabled(StreamRuntime))
+	assert.False(t, IsStreamEnabled(StreamQueue))
+}

backend/v3/instrumentation/tracing.go

@@ -2,6 +2,7 @@ package instrumentation
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -9,6 +10,7 @@ import (
 
 	google_trace "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
@@ -16,7 +18,7 @@ import (
 	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/trace"
 
-	"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
+	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
 type TraceConfig struct {
@@ -119,10 +121,16 @@ func (s *Span) SetStatusByError(err error) {
 	}
 	if err != nil {
 		s.span.RecordError(err)
+		s.span.SetStatus(codes.Error, err.Error())
+	}
+	var zerr *zerrors.ZitadelError
+	if errors.As(err, &zerr) {
+		s.span.SetAttributes(
+			attribute.Stringer("error_kind", zerr.Kind),
+			attribute.String("error_msg", zerr.Message),
+			attribute.String("error_id", zerr.ID),
+		)
 	}
-
-	code, msg, id := gerrors.ExtractZITADELError(err)
-	s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
 }
 
 func newTracerProvider(ctx context.Context, cfg TraceConfig, resource *resource.Resource) (_ *sdk_trace.TracerProvider, err error) {

backend/v3/instrumentation/tracing/http_handler.go

@@ -1,23 +0,0 @@
-package tracing
-
-import (
-	"net/http"
-
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
-	"go.opentelemetry.io/otel"
-
-	"github.com/zitadel/zitadel/backend/v3/instrumentation"
-)
-
-func NewHandler(handler http.Handler, ignoredPrefix ...string) http.Handler {
-	return otelhttp.NewHandler(handler,
-		"zitadel",
-		otelhttp.WithFilter(instrumentation.RequestFilter(ignoredPrefix...)),
-		otelhttp.WithPublicEndpoint(),
-		otelhttp.WithSpanNameFormatter(spanNameFormatter),
-		otelhttp.WithMeterProvider(otel.GetMeterProvider()))
-}
-
-func spanNameFormatter(_ string, r *http.Request) string {
-	return r.URL.Path
-}

backend/v3/storage/database/events_testing/events_test.go

@@ -52,8 +52,8 @@ var (
 	SystemClient        system.SystemServiceClient
 	OrgClient           v2beta_org.OrganizationServiceClient
 	ProjectClient       v2beta_project.ProjectServiceClient
-	SessionClient session.SessionServiceClient
-	UserClient    user.UserServiceClient
+	SessionClient       session.SessionServiceClient
+	UserClient          user.UserServiceClient
 	AdminClient         admin.AdminServiceClient
 	MgmtClient          mgmt.ManagementServiceClient
 	AuthorizationClient authorization.AuthorizationServiceClient