feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

internal/eventstore/handler/v2/failed_event.go

@@ -1,10 +1,13 @@
 package handler
 
 import (
+	"context"
 	"database/sql"
 	_ "embed"
+	"log/slog"
 	"time"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
@@ -25,6 +28,14 @@ type failure struct {
 	err           error
 }
 
+func (f *failure) LogValue() slog.Value {
+	return slog.GroupValue(
+		slog.Uint64("sequence", f.sequence),
+		slog.String("instance", f.instance),
+		slog.String("aggregate_id", f.aggregateID),
+	)
+}
+
 func failureFromEvent(event eventstore.Event, err error) *failure {
 	return &failure{
 		sequence:      event.Sequence(),
@@ -47,15 +58,16 @@ func failureFromStatement(statement *Statement, err error) *failure {
 	}
 }
 
-func (h *Handler) handleFailedStmt(tx *sql.Tx, f *failure) (shouldContinue bool) {
+func (h *Handler) handleFailedStmt(ctx context.Context, tx *sql.Tx, f *failure) (shouldContinue bool) {
+	ctx = logging.With(ctx, "failure", f)
 	failureCount, err := h.failureCount(tx, f)
 	if err != nil {
-		h.logFailure(f).WithError(err).Warn("unable to get failure count")
+		logging.Warn(ctx, "unable to get failure count", "err", err)
 		return false
 	}
 	failureCount += 1
 	err = h.setFailureCount(tx, failureCount, f)
-	h.logFailure(f).OnError(err).Warn("unable to update failure count")
+	logging.OnError(ctx, err).Warn("unable to update failure count")
 
 	return failureCount >= h.maxFailureCount
 }

internal/eventstore/handler/v2/field_handler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/shopspring/decimal"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -79,8 +80,7 @@ func (h *FieldHandler) Trigger(ctx context.Context, opts ...TriggerOpt) (err err
 			wg.Done()
 		}
 		wg.Wait()
-		h.log().OnError(err).Info("process events failed")
-		h.log().WithField("iteration", i).Debug("trigger iteration")
+		logging.Debug(ctx, "trigger iteration", "iteration", i)
 		if !additionalIteration || err != nil {
 			return err
 		}
@@ -93,7 +93,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
 		if errors.As(err, &pgErr) {
 			// error returned if the row is currently locked by another connection
 			if pgErr.Code == "55P03" {
-				h.log().Debug("state already locked")
+				logging.WithError(ctx, err).Info("another handler is already updating this projection")
 				err = nil
 				additionalIteration = false
 			}
@@ -117,7 +117,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
 	defer func() {
 		if err != nil && !errors.Is(err, &executionError{}) {
 			rollbackErr := tx.Rollback()
-			h.log().OnError(rollbackErr).Debug("unable to rollback tx")
+			logging.OnError(ctx, rollbackErr).Error("unable to rollback tx")
 			return
 		}
 		commitErr := tx.Commit()
@@ -156,7 +156,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
 		return additionalIteration, err
 	}
 	if len(events) == 0 {
-		err = h.setState(tx, currentState)
+		err = h.setState(ctx, tx, currentState)
 		return additionalIteration, err
 	}
 
@@ -165,7 +165,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
 		return false, err
 	}
 
-	err = h.setState(tx, currentState)
+	err = h.setState(ctx, tx, currentState)
 
 	return additionalIteration, err
 }
@@ -173,7 +173,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
 func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState *state) (_ []eventstore.FillFieldsEvent, additionalIteration bool, err error) {
 	events, err := h.es.Filter(ctx, h.eventQuery(currentState).SetTx(tx))
 	if err != nil || len(events) == 0 {
-		h.log().OnError(err).Debug("filter eventstore failed")
+		logging.OnError(ctx, err).Debug("filter eventstore failed")
 		return nil, false, err
 	}
 	eventAmount := len(events)

internal/eventstore/handler/v2/handler.go

@@ -4,15 +4,17 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"log/slog"
 	"math/rand"
 	"slices"
 	"sync"
 	"time"
 
 	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/rs/xid"
 	"github.com/shopspring/decimal"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/api/call"
 	"github.com/zitadel/zitadel/internal/database"
@@ -80,11 +82,12 @@ var _ migration.Migration = (*Handler)(nil)
 // Execute implements migration.Migration.
 func (h *Handler) Execute(ctx context.Context, startedEvent eventstore.Event) error {
 	start := time.Now()
-	logging.WithFields("projection", h.ProjectionName()).Info("projection starts prefilling")
+	ctx = logging.With(ctx, "projection", h.ProjectionName())
+	logging.Info(ctx, "projection starts prefilling")
 	logTicker := time.NewTicker(30 * time.Second)
 	go func() {
 		for range logTicker.C {
-			logging.WithFields("projection", h.ProjectionName()).Info("projection is prefilling")
+			logging.Info(ctx, "projection is prefilling")
 		}
 	}()
 
@@ -123,7 +126,7 @@ func (h *Handler) Execute(ctx context.Context, startedEvent eventstore.Event) er
 	wg.Wait()
 
 	logTicker.Stop()
-	logging.WithFields("projection", h.ProjectionName(), "took", time.Since(start)).Info("projections ended prefilling")
+	logging.Info(ctx, "projections ended prefilling", "took", time.Since(start))
 	return nil
 }
 
@@ -170,7 +173,7 @@ func NewHandler(
 		aggregates[reducer.Aggregate] = eventTypes
 	}
 
-	metrics := NewProjectionMetrics()
+	metrics := NewProjectionMetrics(ctx)
 
 	handler := &Handler{
 		projection:             projection,
@@ -203,6 +206,7 @@ func NewHandler(
 }
 
 func (h *Handler) Start(ctx context.Context) {
+	ctx = logging.NewCtx(ctx, logging.StreamEventHandler, slog.String("projection", h.ProjectionName()))
 	go h.schedule(ctx)
 	if h.triggerWithoutEvents != nil {
 		return
@@ -267,10 +271,12 @@ func (h *Handler) schedule(ctx context.Context) {
 			t.Stop()
 			return
 		case <-t.C:
+			jobCtx := logging.With(ctx, "job_id", xid.New(), "invoker", "schedule")
 			instances, err := h.queryInstances()
-			h.log().OnError(err).Debug("unable to query instances")
-
-			h.triggerInstances(call.WithTimestamp(ctx), instances)
+			if err != nil {
+				logging.Debug(jobCtx, "unable to query instances", "err", err)
+			}
+			h.triggerInstances(call.WithTimestamp(jobCtx), instances)
 			t.Reset(h.requeueEvery)
 		}
 	}
@@ -286,12 +292,12 @@ func (h *Handler) triggerInstances(ctx context.Context, instances []string, trig
 		if err == nil {
 			continue
 		}
-		h.log().WithField("instance", instance).WithError(err).Debug("trigger failed")
+		logging.Debug(instanceCtx, "trigger failed", "err", err)
 		time.Sleep(h.retryFailedAfter)
 		// retry if trigger failed
 		for ; err != nil; _, err = h.Trigger(instanceCtx, triggerOpts...) {
 			time.Sleep(h.retryFailedAfter)
-			h.log().WithField("instance", instance).WithError(err).Debug("trigger failed")
+			logging.Debug(instanceCtx, "trigger failed", "err", err)
 		}
 	}
 }
@@ -302,28 +308,29 @@ func randomizeStart(min, maxSeconds float64) time.Duration {
 }
 
 func (h *Handler) subscribe(ctx context.Context) {
+	defer logging.Info(ctx, "handler has shutdown")
 	queue := make(chan eventstore.Event, 100)
 	subscription := eventstore.SubscribeEventTypes(queue, h.eventTypes)
 	for {
 		select {
 		case <-ctx.Done():
 			subscription.Unsubscribe()
-			h.log().Debug("shutdown")
 			return
 		case event := <-queue:
 			events := checkAdditionalEvents(queue, event)
 			solvedInstances := make([]string, 0, len(events))
 			queueCtx := call.WithTimestamp(ctx)
+			queueCtx = logging.With(queueCtx, "job_id", xid.New(), "invoker", "subscribe")
 			for _, e := range events {
 				if slices.Contains(solvedInstances, e.Aggregate().InstanceID) {
 					continue
 				}
 				queueCtx = authz.WithInstanceID(queueCtx, e.Aggregate().InstanceID)
-				_, err := h.Trigger(queueCtx)
-				h.log().OnError(err).Debug("trigger of queued event failed")
-				if err == nil {
-					solvedInstances = append(solvedInstances, e.Aggregate().InstanceID)
+				if _, err := h.Trigger(queueCtx); err != nil {
+					logging.Warn(queueCtx, "trigger of queued event failed", "err", err)
+					continue
 				}
+				solvedInstances = append(solvedInstances, e.Aggregate().InstanceID)
 			}
 		}
 	}
@@ -468,8 +475,7 @@ func (h *Handler) Trigger(ctx context.Context, opts ...TriggerOpt) (_ context.Co
 			wg.Done()
 		}
 		wg.Wait()
-		h.log().OnError(err).Info("process events failed")
-		h.log().WithField("iteration", i).Debug("trigger iteration")
+		logging.Debug(ctx, "trigger iteration", "iteration", i)
 		if !additionalIteration || err != nil {
 			return call.ResetTimestamp(ctx), err
 		}
@@ -523,7 +529,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 		if errors.As(err, &pgErr) {
 			// error returned if the row is currently locked by another connection
 			if pgErr.Code == "55P03" {
-				h.log().Debug("state already locked")
+				logging.WithError(ctx, err).Info("another handler is already updating this projection")
 				err = nil
 				additionalIteration = false
 			}
@@ -549,7 +555,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 	defer func() {
 		if err != nil && !errors.Is(err, &executionError{}) {
 			rollbackErr := tx.Rollback()
-			h.log().OnError(rollbackErr).Debug("unable to rollback tx")
+			logging.OnError(ctx, rollbackErr).Error("unable to rollback tx")
 			return
 		}
 		commitErr := tx.Commit()
@@ -569,7 +575,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 		return false, err
 	}
 	if !hasLocked {
-		h.log().Debug("skip execution, projection already locked")
+		logging.Info(ctx, "another handler is already updating this projection")
 		return false, nil
 	}
 
@@ -611,12 +617,12 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 	}()
 
 	if len(statements) == 0 {
-		err = h.setState(tx, currentState)
+		err = h.setState(ctx, tx, currentState)
 		return additionalIteration, err
 	}
 
 	lastProcessedIndex, err := h.executeStatements(ctx, tx, statements)
-	h.log().OnError(err).WithField("lastProcessedIndex", lastProcessedIndex).Debug("execution of statements failed")
+	logging.OnError(ctx, err).Debug("execution of statements failed", "lastProcessedIndex", lastProcessedIndex)
 	if lastProcessedIndex < 0 {
 		return false, err
 	}
@@ -628,7 +634,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
 	currentState.sequence = statements[lastProcessedIndex].Sequence
 	currentState.eventTimestamp = statements[lastProcessedIndex].CreationDate
 
-	setStateErr := h.setState(tx, currentState)
+	setStateErr := h.setState(ctx, tx, currentState)
 	if setStateErr != nil {
 		err = setStateErr
 	}
@@ -647,12 +653,12 @@ func (h *Handler) generateStatements(ctx context.Context, tx *sql.Tx, currentSta
 
 	events, err := h.es.Filter(ctx, h.eventQuery(currentState).SetTx(tx))
 	if err != nil {
-		h.log().WithError(err).Debug("filter eventstore failed")
+		logging.WithError(ctx, err).Debug("filter eventstore failed")
 		return nil, false, err
 	}
 	eventAmount := len(events)
 
-	statements, err := h.eventsToStatements(tx, events, currentState)
+	statements, err := h.eventsToStatements(ctx, tx, events, currentState)
 	if err != nil || len(statements) == 0 {
 		return nil, false, err
 	}
@@ -716,17 +722,17 @@ func (h *Handler) executeStatement(ctx context.Context, tx *sql.Tx, statement *S
 
 	_, err = tx.ExecContext(ctx, "SAVEPOINT exec_stmt")
 	if err != nil {
-		h.log().WithError(err).Debug("create savepoint failed")
+		logging.WithError(ctx, err).Debug("create savepoint failed")
 		return err
 	}
 
 	if err = statement.Execute(ctx, tx, h.projection.Name()); err != nil {
-		h.log().WithError(err).Error("statement execution failed")
+		logging.WithError(ctx, err).Error("statement execution failed")
 
 		_, rollbackErr := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT exec_stmt")
-		h.log().OnError(rollbackErr).Error("rollback to savepoint failed")
+		logging.OnError(ctx, rollbackErr).Debug("rollback to savepoint failed")
 
-		shouldContinue := h.handleFailedStmt(tx, failureFromStatement(statement, err))
+		shouldContinue := h.handleFailedStmt(ctx, tx, failureFromStatement(statement, err))
 		if shouldContinue {
 			return nil
 		}

internal/eventstore/handler/v2/init.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 
 	"github.com/jackc/pgx/v5/pgconn"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/eventstore/handler"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
@@ -198,15 +198,18 @@ func (h *Handler) Init(ctx context.Context) error {
 	if err != nil {
 		return zerrors.ThrowInternal(err, "CRDB-SAdf2", "begin failed")
 	}
+	ctx = logging.With(ctx, "projection", h.ProjectionName())
 	for i, execute := range check.Init().Executes {
-		logging.WithFields("projection", h.projection.Name(), "execute", i).Debug("executing check")
+		logging.Debug(ctx, "executing check", "execute", i)
 		next, err := execute(ctx, tx, h.projection.Name())
 		if err != nil {
-			logging.OnError(tx.Rollback()).Debug("unable to rollback")
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logging.Error(ctx, "unable to rollback", "err", rollbackErr)
+			}
 			return err
 		}
 		if !next {
-			logging.WithFields("projection", h.projection.Name(), "execute", i).Debug("projection set up")
+			logging.Debug(ctx, "projection set up", "execute", i)
 			break
 		}
 	}

internal/eventstore/handler/v2/log.go

@@ -1,23 +0,0 @@
-package handler
-
-import (
-	"github.com/zitadel/logging"
-
-	"github.com/zitadel/zitadel/internal/eventstore"
-)
-
-func (h *Handler) log() *logging.Entry {
-	return logging.WithFields("projection", h.projection.Name())
-}
-
-func (h *Handler) logFailure(fail *failure) *logging.Entry {
-	return h.log().WithField("sequence", fail.sequence).
-		WithField("instance", fail.instance).
-		WithField("aggregate", fail.aggregateID)
-}
-
-func (h *Handler) logEvent(event eventstore.Event) *logging.Entry {
-	return h.log().WithField("sequence", event.Sequence()).
-		WithField("instance", event.Aggregate().InstanceID).
-		WithField("aggregate", event.Aggregate().Type)
-}

internal/eventstore/handler/v2/metrics.go

@@ -3,9 +3,9 @@ package handler
 import (
 	"context"
 
-	"github.com/zitadel/logging"
 	"go.opentelemetry.io/otel/attribute"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
@@ -22,32 +22,32 @@ type ProjectionMetrics struct {
 	provider metrics.Metrics
 }
 
-func NewProjectionMetrics() *ProjectionMetrics {
-	return newProjectionMetrics(metrics.GlobalMeter())
+func NewProjectionMetrics(ctx context.Context) *ProjectionMetrics {
+	return newProjectionMetrics(ctx, metrics.GlobalMeter())
 }
 
-func newProjectionMetrics(m metrics.Metrics) *ProjectionMetrics {
+func newProjectionMetrics(ctx context.Context, m metrics.Metrics) *ProjectionMetrics {
 	projectionMetrics := &ProjectionMetrics{provider: m}
 
 	err := projectionMetrics.provider.RegisterCounter(
 		ProjectionEventsProcessed,
 		"Number of events reduced to process projection updates",
 	)
-	logging.OnError(err).Error("failed to register projection events processed counter")
+	logging.OnError(ctx, err).Error("failed to register projection events processed counter")
 	err = projectionMetrics.provider.RegisterHistogram(
 		ProjectionHandleTimerMetric,
 		"Time taken to process a projection update",
 		"s",
 		[]float64{0.005, 0.01, 0.05, 0.1, 1, 5, 10, 30, 60, 120},
 	)
-	logging.OnError(err).Error("failed to register projection handle timer metric")
+	logging.OnError(ctx, err).Error("failed to register projection handle timer metric")
 	err = projectionMetrics.provider.RegisterHistogram(
 		ProjectionStateLatencyMetric,
 		"When finishing processing a batch of events, this track the age of the last events seen from current time",
 		"s",
 		[]float64{0.1, 0.5, 1, 5, 10, 30, 60, 300, 600, 1800},
 	)
-	logging.OnError(err).Error("failed to register projection state latency metric")
+	logging.OnError(ctx, err).Error("failed to register projection state latency metric")
 	return projectionMetrics
 }
 
@@ -55,7 +55,7 @@ func (m *ProjectionMetrics) ProjectionUpdateTiming(ctx context.Context, projecti
 	err := m.provider.AddHistogramMeasurement(ctx, ProjectionHandleTimerMetric, duration, map[string]attribute.Value{
 		ProjectionLabel: attribute.StringValue(projection),
 	})
-	logging.OnError(err).Error("failed to add projection trigger timing")
+	logging.OnError(ctx, err).Error("failed to add projection trigger timing")
 }
 
 func (m *ProjectionMetrics) ProjectionEventsProcessed(ctx context.Context, projection string, count int64, success bool) {
@@ -63,12 +63,12 @@ func (m *ProjectionMetrics) ProjectionEventsProcessed(ctx context.Context, proje
 		ProjectionLabel: attribute.StringValue(projection),
 		SuccessLabel:    attribute.BoolValue(success),
 	})
-	logging.OnError(err).Error("failed to add projection events processed metric")
+	logging.OnError(ctx, err).Error("failed to add projection events processed metric")
 }
 
 func (m *ProjectionMetrics) ProjectionStateLatency(ctx context.Context, projection string, latency float64) {
 	err := m.provider.AddHistogramMeasurement(ctx, ProjectionStateLatencyMetric, latency, map[string]attribute.Value{
 		ProjectionLabel: attribute.StringValue(projection),
 	})
-	logging.OnError(err).Error("failed to add projection state latency metric")
+	logging.OnError(ctx, err).Error("failed to add projection state latency metric")
 }

internal/eventstore/handler/v2/metrics_test.go

@@ -13,14 +13,14 @@ import (
 
 func TestNewProjectionMetrics(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	metrics := newProjectionMetrics(mockMetrics)
+	metrics := newProjectionMetrics(t.Context(), mockMetrics)
 	require.NotNil(t, metrics)
 	assert.NotNil(t, metrics.provider)
 }
 
 func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	projectionMetrics := newProjectionMetrics(mockMetrics)
+	projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -39,7 +39,7 @@ func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
 
 func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	projectionMetrics := newProjectionMetrics(mockMetrics)
+	projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -59,7 +59,7 @@ func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
 
 func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	projectionMetrics := newProjectionMetrics(mockMetrics)
+	projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -78,7 +78,7 @@ func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
 
 func TestProjectionMetrics_Integration(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	projectionMetrics := newProjectionMetrics(mockMetrics)
+	projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"

internal/eventstore/handler/v2/state.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/shopspring/decimal"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -55,7 +56,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx) (currentState *s
 		offset,
 	)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
-		h.log().WithError(err).Debug("unable to query current state")
+		logging.WithError(ctx, err).Debug("unable to query current state")
 		return nil, err
 	}
 
@@ -69,7 +70,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx) (currentState *s
 	return currentState, nil
 }
 
-func (h *Handler) setState(tx *sql.Tx, updatedState *state) error {
+func (h *Handler) setState(ctx context.Context, tx *sql.Tx, updatedState *state) error {
 	res, err := tx.Exec(updateStateStmt,
 		h.projection.Name(),
 		updatedState.instanceID,
@@ -81,12 +82,14 @@ func (h *Handler) setState(tx *sql.Tx, updatedState *state) error {
 		updatedState.offset,
 	)
 	if err != nil {
-		h.log().WithError(err).Warn("unable to update state")
-		return zerrors.ThrowInternal(err, "V2-WF23g2", "unable to update state")
+		err = zerrors.ThrowInternal(err, "V2-WF23g2", "unable to update state")
+		logging.Warn(ctx, "unable to update state", "err", err)
+		return err
 	}
 	if affected, err := res.RowsAffected(); affected == 0 {
-		h.log().OnError(err).Error("unable to check if states are updated")
-		return zerrors.ThrowInternal(err, "V2-FGEKi", "unable to update state")
+		err = zerrors.ThrowInternal(err, "V2-FGEKi", "unable to update state")
+		logging.Error(ctx, "unable to check if states are updated", "err", err)
+		return err
 	}
 	return nil
 }

internal/eventstore/handler/v2/state_test.go

@@ -137,7 +137,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
 			h := &Handler{
 				projection: tt.fields.projection,
 			}
-			err = h.setState(tx, tt.args.updatedState)
+			err = h.setState(t.Context(), tx, tt.args.updatedState)
 
 			tt.isErr(t, err)
 			tt.fields.mock.Assert(t)

internal/eventstore/handler/v2/statement.go

@@ -11,9 +11,9 @@ import (
 	"time"
 
 	"github.com/shopspring/decimal"
-	"github.com/zitadel/logging"
 	"golang.org/x/exp/constraints"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/zerrors"
@@ -39,7 +39,7 @@ func (s *executionError) Unwrap() error {
 	return s.parent
 }
 
-func (h *Handler) eventsToStatements(tx *sql.Tx, events []eventstore.Event, currentState *state) (statements []*Statement, err error) {
+func (h *Handler) eventsToStatements(ctx context.Context, tx *sql.Tx, events []eventstore.Event, currentState *state) (statements []*Statement, err error) {
 	statements = make([]*Statement, 0, len(events))
 
 	previousPosition := currentState.position
@@ -47,8 +47,8 @@ func (h *Handler) eventsToStatements(tx *sql.Tx, events []eventstore.Event, curr
 	for _, event := range events {
 		statement, err := h.reduce(event)
 		if err != nil {
-			h.logEvent(event).WithError(err).Error("reduce failed")
-			if shouldContinue := h.handleFailedStmt(tx, failureFromEvent(event, err)); shouldContinue {
+			logging.Error(ctx, "reduce failed", "err", err, "event", failureFromEvent(event, err))
+			if shouldContinue := h.handleFailedStmt(ctx, tx, failureFromEvent(event, err)); shouldContinue {
 				continue
 			}
 			return statements, &executionError{err}
@@ -577,7 +577,9 @@ func NewCol(name string, value interface{}) Column {
 func NewJSONCol(name string, value interface{}) Column {
 	marshalled, err := json.Marshal(value)
 	if err != nil {
-		logging.WithFields("column", name).WithError(err).Panic("unable to marshal column")
+		err = zerrors.ThrowInternal(err, "V2-aez5X", "unable to marshal column")
+		ctx := logging.NewCtx(context.Background(), logging.StreamEventHandler)
+		logging.OnError(ctx, err).Panic("unable to marshal column", "column", name, "err", err)
 	}
 
 	return NewCol(name, marshalled)