feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

internal/api/authz/instance.go

@@ -6,6 +6,7 @@ import (
 
 	"golang.org/x/text/language"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/internal/execution/target"
 	"github.com/zitadel/zitadel/internal/feature"
 )
@@ -107,11 +108,12 @@ func GetFeatures(ctx context.Context) feature.Features {
 }
 
 func WithInstance(ctx context.Context, instance Instance) context.Context {
+	ctx = instrumentation.SetInstance(ctx, instance)
 	return context.WithValue(ctx, instanceKey, instance)
 }
 
 func WithInstanceID(ctx context.Context, id string) context.Context {
-	return context.WithValue(ctx, instanceKey, &instance{id: id})
+	return WithInstance(ctx, &instance{id: id})
 }
 
 func WithDefaultLanguage(ctx context.Context, defaultLanguage language.Tag) context.Context {

internal/api/grpc/gerrors/zitadel_errors.go

@@ -7,13 +7,12 @@ import (
 
 	"connectrpc.com/connect"
 	"github.com/jackc/pgx/v5/pgconn"
-	slogctx "github.com/veqryn/slog-context"
-	"github.com/zitadel/logging"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/protoadapt"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	commandErrors "github.com/zitadel/zitadel/internal/command/errors"
 	"github.com/zitadel/zitadel/internal/zerrors"
 	"github.com/zitadel/zitadel/pkg/grpc/message"
@@ -27,13 +26,13 @@ func ZITADELToGRPCError(ctx context.Context, err error) error {
 	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
-	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err, "code", code)
+	logging.Log(ctx, lvl, msg, "err", err, "code", code)
 
 	errorInfo := getErrorInfo(id, key, err)
 
 	s, err := status.New(code, msg).WithDetails(errorInfo)
 	if err != nil {
-		logging.WithError(err).WithField("logID", "GRPC-gIeRw").Debug("unable to add detail")
+		logging.WithError(ctx, err).Debug("unable to add error detail")
 		return status.New(code, msg).Err()
 	}
 
@@ -48,13 +47,13 @@ func ZITADELToConnectError(ctx context.Context, err error) error {
 	if errors.As(err, &connectError) {
 		// Connect error may be returned by other middlewares,
 		// so we assume it's a client error and log as warning.
-		slogctx.FromCtx(ctx).WarnContext(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
+		logging.Warn(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
 		return err
 	}
 	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
-	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
+	logging.Log(ctx, lvl, msg, "err", err)
 
 	errorInfo := getErrorInfo(id, key, err)
 

internal/api/grpc/server/connect_middleware/log_interceptor.go

@@ -1,13 +1,55 @@
 package connect_middleware
 
 import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+	"time"
+
 	"connectrpc.com/connect"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
 )
 
-func LogHandler(ignoredPrefix ...string) connect.UnaryInterceptorFunc {
+func LogHandler(ignoredMethodSuffixes ...string) connect.UnaryInterceptorFunc {
 	return func(next connect.UnaryFunc) connect.UnaryFunc {
-		return logging.NewConnectInterceptor(next, ignoredPrefix...)
+		return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
+			if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+				return strings.HasSuffix(req.Spec().Procedure, s)
+			}) {
+				return next(ctx, req)
+			}
+			start := time.Now()
+			ctx = logging.NewCtx(ctx, logging.StreamRequest)
+			ctx = instrumentation.SetRequestID(ctx, start)
+
+			resp, err := next(ctx, req)
+			var code connect.Code
+			if err != nil {
+				code = connect.CodeOf(err)
+			}
+			spec := req.Spec()
+			logging.Info(ctx, "request served",
+				slog.String("protocol", "connect"),
+				slog.Any("domain", http_util.DomainContext(ctx)),
+				slog.String("service", serviceFromRPCMethod(spec.Procedure)),
+				slog.String("http_method", req.HTTPMethod()),
+				slog.String("path", spec.Procedure),
+				slog.Any("code", code),
+				slog.Duration("duration", time.Since(start)),
+			)
+			return resp, err
+		}
 	}
 }
+
+func serviceFromRPCMethod(fullMethod string) string {
+	parts := strings.Split(fullMethod, "/")
+	if len(parts) >= 2 {
+		return parts[1]
+	}
+	return "unknown"
+}

internal/api/grpc/server/gateway.go

@@ -284,5 +284,4 @@ func setRequestURIPattern(ctx context.Context) {
 	}
 	span := trace.SpanFromContext(ctx)
 	span.SetName(pattern)
-	metrics.SetRequestURIPattern(ctx, pattern)
 }

internal/api/grpc/server/middleware/log_interceptor.go

@@ -1,11 +1,55 @@
 package middleware
 
 import (
-	"google.golang.org/grpc"
+	"context"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
 
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
 )
 
 func LogHandler(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
-	return logging.NewGrpcInterceptor(ignoredMethodSuffixes...)
+	return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) {
+		if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
+			return strings.HasSuffix(info.FullMethod, s)
+		}) {
+			return next(ctx, req)
+		}
+		start := time.Now()
+		ctx = logging.NewCtx(ctx, logging.StreamRequest)
+		ctx = instrumentation.SetRequestID(ctx, start)
+
+		resp, err := next(ctx, req)
+		var code codes.Code
+		if err != nil {
+			code = status.Code(err)
+		}
+		logging.Info(ctx, "request served",
+			slog.String("protocol", "grpc"),
+			slog.Any("domain", http_util.DomainContext(ctx)),
+			slog.String("service", serviceFromRPCMethod(info.FullMethod)),
+			slog.String("http_method", http.MethodPost), // gRPC always uses POST
+			slog.String("path", info.FullMethod),
+			slog.Any("code", code),
+			slog.Duration("duration", time.Since(start)),
+		)
+		return resp, err
+	}
+}
+
+func serviceFromRPCMethod(fullMethod string) string {
+	parts := strings.Split(fullMethod, "/")
+	if len(parts) >= 2 {
+		return parts[1]
+	}
+	return "unknown"
 }

internal/api/http/error.go

@@ -5,8 +5,7 @@ import (
 	"log/slog"
 	"net/http"
 
-	slogctx "github.com/veqryn/slog-context"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -17,7 +16,7 @@ func ZitadelErrorToHTTPStatusCode(ctx context.Context, err error) (statusCode in
 	statusCode, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
-	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
+	logging.Log(ctx, lvl, msg, "err", err)
 	if statusCode == statusUnknown {
 		return http.StatusInternalServerError, false
 	}

internal/api/http/middleware/access_interceptor.go

@@ -133,7 +133,7 @@ func (a *AccessInterceptor) handle(publicAuthPathPrefixes ...string) func(http.H
 		return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 			ctx := request.Context()
 			tracingCtx, checkSpan := tracing.NewNamedSpan(ctx, "checkAccessQuota")
-			wrappedWriter := &statusRecorder{ResponseWriter: writer, status: 0}
+			wrappedWriter := newStatusWriter(writer)
 			limited := a.Limit(wrappedWriter, request.WithContext(tracingCtx), publicAuthPathPrefixes...)
 			checkSpan.End()
 			if limited {
@@ -151,7 +151,7 @@ func (a *AccessInterceptor) handle(publicAuthPathPrefixes ...string) func(http.H
 	}
 }
 
-func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusRecorder, writer http.ResponseWriter, request *http.Request, notCountable bool) {
+func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusWriter, writer http.ResponseWriter, request *http.Request, notCountable bool) {
 	if !a.logstoreSvc.Enabled() {
 		return
 	}
@@ -178,17 +178,3 @@ func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusR
 		NotCountable:    notCountable,
 	})
 }
-
-type statusRecorder struct {
-	http.ResponseWriter
-	status       int
-	ignoreWrites bool
-}
-
-func (r *statusRecorder) WriteHeader(status int) {
-	if r.ignoreWrites {
-		return
-	}
-	r.status = status
-	r.ResponseWriter.WriteHeader(status)
-}

internal/api/http/middleware/log_interceptor.go

@@ -1,13 +1,39 @@
 package middleware
 
 import (
+	"log/slog"
 	"net/http"
+	"time"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
 )
 
 func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
-	return func(h http.Handler) http.Handler {
-		return logging.NewHandler(h, service, ignoredPrefix...)
+	return func(next http.Handler) http.Handler {
+		filter := instrumentation.RequestFilter(ignoredPrefix...)
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if !filter(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+			start := time.Now()
+			ctx := logging.NewCtx(r.Context(), logging.StreamRequest)
+			ctx = instrumentation.SetRequestID(ctx, start)
+			sw := newStatusWriter(w)
+
+			next.ServeHTTP(sw, r.WithContext(ctx))
+
+			logging.Info(ctx, "request served",
+				slog.String("protocol", "http"),
+				slog.Any("domain", http_util.DomainContext(ctx)),
+				slog.String("service", service),
+				slog.String("http_method", r.Method), // gRPC always uses POST
+				slog.String("path", r.URL.Path),
+				slog.Int("status", sw.status),
+				slog.Duration("duration", time.Since(start)),
+			)
+		})
 	}
 }

internal/api/http/middleware/metrics_interceptor.go

@@ -3,11 +3,57 @@ package middleware
 import (
 	"net/http"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethods ...string) func(http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
-		return metrics.NewHandler(handler, metricTypes, ignoredMethods...)
+		return &Handler{
+			handler: handler,
+			methods: metricTypes,
+			filter:  instrumentation.RequestFilter(ignoredMethods...),
+		}
 	}
 }
+
+type Handler struct {
+	handler http.Handler
+	methods []metrics.MetricType
+	filter  otelhttp.Filter
+}
+
+// ServeHTTP implements [http.Handler]
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(h.methods) == 0 {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	if !h.filter(r) {
+		// Simply pass through to the handler if a filter rejects the request
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	recorder := newStatusWriter(w)
+	h.handler.ServeHTTP(recorder, r)
+	if h.containsMetricsMethod(metrics.MetricTypeRequestCount) {
+		metrics.RegisterRequestCounter(recorder, r)
+	}
+	if h.containsMetricsMethod(metrics.MetricTypeTotalCount) {
+		metrics.RegisterTotalRequestCounter(r)
+	}
+	if h.containsMetricsMethod(metrics.MetricTypeStatusCode) {
+		metrics.RegisterRequestCodeCounter(recorder, r)
+	}
+}
+
+func (h *Handler) containsMetricsMethod(method metrics.MetricType) bool {
+	for _, m := range h.methods {
+		if m == method {
+			return true
+		}
+	}
+	return false
+}

internal/api/http/middleware/middleware.go

@@ -0,0 +1,29 @@
+package middleware
+
+import "net/http"
+
+// statusWriter is a [http.ResponseWriter] that captures the status code for logging.
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func newStatusWriter(w http.ResponseWriter) *statusWriter {
+	return &statusWriter{ResponseWriter: w}
+}
+
+func (w *statusWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.status = http.StatusOK
+	}
+	return w.ResponseWriter.Write(p)
+}
+
+func (w *statusWriter) WriteHeader(statusCode int) {
+	w.status = statusCode
+	w.ResponseWriter.WriteHeader(statusCode)
+}
+
+func (w *statusWriter) Status() int {
+	return w.status
+}

internal/api/http/middleware/middleware_test.go

@@ -1,9 +1,12 @@
 package middleware
 
 import (
+	"net/http/httptest"
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/text/language"
 
 	"github.com/zitadel/zitadel/internal/i18n"
@@ -17,3 +20,38 @@ func TestMain(m *testing.M) {
 	i18n.SupportLanguages(SupportedLanguages...)
 	os.Exit(m.Run())
 }
+
+func Test_statusWriter(t *testing.T) {
+	tests := []struct {
+		name        string
+		writeStatus int
+		wantStatus  int
+	}{
+		{
+			name:       "default status is 200",
+			wantStatus: 200,
+		},
+		{
+			name:        "writes status code and body correctly",
+			writeStatus: 201,
+			wantStatus:  201,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			const body = "Hello, World!"
+			rec := httptest.NewRecorder()
+			sw := newStatusWriter(rec)
+			if tt.writeStatus != 0 {
+				sw.WriteHeader(tt.writeStatus)
+			}
+			_, err := sw.Write([]byte(body))
+			require.NoError(t, err)
+			assert.Equal(t, tt.wantStatus, sw.Status())
+
+			resp := rec.Result()
+			assert.Equal(t, tt.wantStatus, resp.StatusCode)
+			assert.Equal(t, body, rec.Body.String())
+		})
+	}
+}

internal/api/http/middleware/trace_interceptor.go

@@ -3,7 +3,10 @@ package middleware
 import (
 	"net/http"
 
-	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 )
 
@@ -13,6 +16,16 @@ func DefaultTraceHandler(handler http.Handler) http.Handler {
 
 func TraceHandler(ignoredPrefix ...string) func(http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
-		return tracing.NewHandler(handler, ignoredPrefix...)
+		return otelhttp.NewHandler(handler,
+			"zitadel",
+			otelhttp.WithFilter(instrumentation.RequestFilter(ignoredPrefix...)),
+			otelhttp.WithPublicEndpoint(),
+			otelhttp.WithSpanNameFormatter(spanNameFormatter),
+			otelhttp.WithMeterProvider(otel.GetMeterProvider()),
+		)
 	}
 }
+
+func spanNameFormatter(_ string, r *http.Request) string {
+	return r.URL.Path
+}

internal/api/ui/login/external_provider_handler.go

@@ -1611,5 +1611,5 @@ func WrapIdPError(err error) *IdPError {
 	if errors.As(err, &zErr) {
 		id = zErr.ID
 	}
-	return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
+	return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal", 1)}
 }