feat(telemetry): unify request details (#11509)

# Which Problems Are Solved

In the "new" structured logging, request details were added in different
middlewares, such as request, instance and user IDs. This meant the the
upstream request logging middleware did not have access to metadata that
got added later, resulting in incomplete logs. Furthermore it was not
possible to correlate an API error response to log output.

# How the Problems Are Solved

A mutable request details object is added to the context early on. When
the api authz function run, the instance and user IDs are added to this
object as they become available. Every logline emitted after this
(time-wise) will then all contain these details under the `request` log
group.

<details>

<summary>example output in JSON</summary>

```json
{
  "time": "2026-03-13T19:21:20.890428806Z",
  "level": "INFO",
  "source": {
    "function": "github.com/zitadel/zitadel/internal/api/grpc/server/connect_middleware.LogHandler.func1.1",
    "file": "/workspaces/zitadel/internal/api/grpc/server/connect_middleware/log_interceptor.go",
    "line": 34
  },
  "msg": "request served",
  "request": {
    "id": "d6q67c04vtjmi77cbbpg",
    "instance_host": "localhost:8080",
    "instance_id": "362349751439458307",
    "user_id": "362349751440048131"
  },
  "TraceID": "a9e0fee3522224f3583bbdcda737f4b4",
  "SpanID": "a563056eee36920a",
  "stream": "request",
  "version": "2026-03-13T19:20:59Z",
  "protocol": "connect",
  "service": "zitadel.user.v2.UserService",
  "http_method": "POST",
  "path": "/zitadel.user.v2.UserService/ListUsers",
  "code": "code_0",
  "duration": 12254350
}
```

</details>

Request IDs are now also returned with a response header or metadata.
Depending on the protocol:

- HTTP calls always return the request ID as header, regardless of
status
- gRPC calls always return the request ID as header, even if there was
an error
- connect RPC calls returns the request ID as header on success, trailer
in case of error. This is because header must be set on the response
object, which is nil in case of error. When there is an error, metadata
can be added which are then sent as trailers.

# Additional Changes

- Use the existing call duration middleware for both request ID and
logging for a consistent request start timestamp in all layers.
- Upgrade sloggcp for some fixes (notably TraceID)
- Modify the NoCache middleware so it uses `SetHeaders` instead of
`SendHeaders`. The latter prevented any other handler from setting
headers, including the new request ID middleware.

# Additional Context

Follow up on demo of:
  - https://github.com/zitadel/zitadel/pull/11159
  - https://github.com/zitadel/zitadel/pull/11435
  - backport to v4

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: muhlemmer <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: Marco A. <[email protected]>

internal/api/http/middleware/log_interceptor.go

@@ -3,13 +3,15 @@ package middleware
 import (
 	"log/slog"
 	"net/http"
-	"time"
 
 	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
-	http_util "github.com/zitadel/zitadel/internal/api/http"
+	"github.com/zitadel/zitadel/internal/api/call"
 )
 
+// LogHandler is an HTTP middleware that logs the request details
+// including protocol, domain, service, HTTP method, path, response code, and duration.
+// It depends on [CallDurationHandler] and [RequestDetailsHandler] to set the request start time and ID in the context.
 func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		filter := instrumentation.RequestFilter(ignoredPrefix...)
@@ -18,21 +20,18 @@ func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http
 				next.ServeHTTP(w, r)
 				return
 			}
-			start := time.Now()
 			ctx := logging.NewCtx(r.Context(), logging.StreamRequest)
-			ctx = instrumentation.SetRequestID(ctx, start)
 			sw := newStatusWriter(w)
 
 			next.ServeHTTP(sw, r.WithContext(ctx))
 
 			logging.Info(ctx, "request served",
 				slog.String("protocol", "http"),
-				slog.Any("domain", http_util.DomainContext(ctx)),
 				slog.String("service", service),
-				slog.String("http_method", r.Method), // gRPC always uses POST
+				slog.String("http_method", r.Method),
 				slog.String("path", r.URL.Path),
 				slog.Int("status", sw.status),
-				slog.Duration("duration", time.Since(start)),
+				slog.Duration("duration", call.Took(ctx)),
 			)
 		})
 	}

internal/api/http/middleware/request_details_interceptor.go

@@ -0,0 +1,23 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
+)
+
+// RequestDetailsHandler is a HTTP middleware that sets a request ID in the context
+// and adds the ID to the response headers.
+// It depends on [CallDurationHandler] to set the request start time in the context.
+func RequestDetailsHandler() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			domainCtx := http_util.DomainContext(r.Context())
+			ctx := instrumentation.WithRequestDetails(r.Context(), domainCtx.InstanceHost, domainCtx.PublicHost)
+			id := instrumentation.GetRequestID(ctx)
+			w.Header().Set(http_util.XRequestID, id.String())
+			next.ServeHTTP(w, r.WithContext(ctx))
+		})
+	}
+}

internal/api/http/middleware/request_details_interceptor_test.go

@@ -0,0 +1,97 @@
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/rs/xid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/internal/api/call"
+	http_util "github.com/zitadel/zitadel/internal/api/http"
+)
+
+func TestRequestDetailsHandler(t *testing.T) {
+	tests := []struct {
+		name         string
+		setupContext bool
+		status       int
+	}{
+		{
+			name:         "sets request ID header on success",
+			setupContext: true,
+			status:       http.StatusOK,
+		},
+		{
+			name:         "sets request ID header on error",
+			setupContext: true,
+			status:       http.StatusInternalServerError,
+		},
+		{
+			name:         "sets request ID header without call context",
+			setupContext: false,
+			status:       http.StatusOK,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create test request
+			req := httptest.NewRequest(http.MethodGet, "/test", nil)
+
+			// Setup context with call duration if needed
+			if tt.setupContext {
+				req = req.WithContext(call.WithTimestamp(req.Context()))
+			}
+
+			rec := httptest.NewRecorder()
+
+			// Create test handler that returns the specified status
+			handler := RequestDetailsHandler()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// Verify request ID is in context
+				requestID := instrumentation.GetRequestID(r.Context())
+				assert.False(t, requestID.IsNil(), "Request ID should be present in context")
+
+				w.WriteHeader(tt.status)
+			}))
+
+			handler.ServeHTTP(rec, req)
+
+			// Verify response
+			resp := rec.Result()
+			assert.Equal(t, tt.status, resp.StatusCode)
+
+			// Verify x-request-id header is set
+			requestID := resp.Header.Get(http_util.XRequestID)
+			require.NotEmpty(t, requestID, "x-request-id header should be set")
+
+			// Verify the x-request-id header is a valid xid format
+			_, err := xid.FromString(requestID)
+			require.NoError(t, err, "x-request-id header should be a valid xid")
+		})
+	}
+}
+
+func TestRequestDetailsHandler_Stability(t *testing.T) {
+	// Test that the same request gets consistent request ID
+	req := httptest.NewRequest(http.MethodGet, "/test", nil)
+	req = req.WithContext(call.WithTimestamp(req.Context()))
+	rec := httptest.NewRecorder()
+
+	var capturedID string
+	handler := RequestDetailsHandler()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		id := instrumentation.GetRequestID(r.Context())
+		require.False(t, id.IsNil(), "Request ID should be present in context")
+		capturedID = id.String()
+	}))
+
+	handler.ServeHTTP(rec, req)
+
+	resp := rec.Result()
+	headerID := resp.Header.Get(http_util.XRequestID)
+
+	// Verify the ID in context matches the ID in header
+	assert.Equal(t, capturedID, headerID, "Request ID in context should match header")
+}