andwati/zitadel
1
0
Fork 0
mirror of https://github.com/andwati/zitadel.git synced 2026-11-04 21:32:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): unify request details (#11509)

Browse Source 
# Which Problems Are Solved

In the "new" structured logging, request details were added in different
middlewares, such as request, instance and user IDs. This meant the the
upstream request logging middleware did not have access to metadata that
got added later, resulting in incomplete logs. Furthermore it was not
possible to correlate an API error response to log output.

# How the Problems Are Solved

A mutable request details object is added to the context early on. When
the api authz function run, the instance and user IDs are added to this
object as they become available. Every logline emitted after this
(time-wise) will then all contain these details under the `request` log
group.

<details>

<summary>example output in JSON</summary>

```json
{
  "time": "2026-03-13T19:21:20.890428806Z",
  "level": "INFO",
  "source": {
    "function": "github.com/zitadel/zitadel/internal/api/grpc/server/connect_middleware.LogHandler.func1.1",
    "file": "/workspaces/zitadel/internal/api/grpc/server/connect_middleware/log_interceptor.go",
    "line": 34
  },
  "msg": "request served",
  "request": {
    "id": "d6q67c04vtjmi77cbbpg",
    "instance_host": "localhost:8080",
    "instance_id": "362349751439458307",
    "user_id": "362349751440048131"
  },
  "TraceID": "a9e0fee3522224f3583bbdcda737f4b4",
  "SpanID": "a563056eee36920a",
  "stream": "request",
  "version": "2026-03-13T19:20:59Z",
  "protocol": "connect",
  "service": "zitadel.user.v2.UserService",
  "http_method": "POST",
  "path": "/zitadel.user.v2.UserService/ListUsers",
  "code": "code_0",
  "duration": 12254350
}
```

</details>

Request IDs are now also returned with a response header or metadata.
Depending on the protocol:

- HTTP calls always return the request ID as header, regardless of
status
- gRPC calls always return the request ID as header, even if there was
an error
- connect RPC calls returns the request ID as header on success, trailer
in case of error. This is because header must be set on the response
object, which is nil in case of error. When there is an error, metadata
can be added which are then sent as trailers.

# Additional Changes

- Use the existing call duration middleware for both request ID and
logging for a consistent request start timestamp in all layers.
- Upgrade sloggcp for some fixes (notably TraceID)
- Modify the NoCache middleware so it uses `SetHeaders` instead of
`SendHeaders`. The latter prevented any other handler from setting
headers, including the new request ID middleware.

# Additional Context

Follow up on demo of:
  - https://github.com/zitadel/zitadel/pull/11159
  - https://github.com/zitadel/zitadel/pull/11435
  - backport to v4

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: muhlemmer <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: Marco A. <[email protected]>
This commit is contained in:
Tim Möhlmann
2026-03-18 14:07:33 +01:00
committed by GitHub
co-authored by Copilot muhlemmer Copilot Autofix powered by AI Marco A.
parent f138082b64
commit 997aa607c4
28 changed files with 765 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/v3/instrumentation/instrumentation.go
+2
View File
@@ -68,6 +68,8 @@ func NewMeter(name string, options ...metric.MeterOption) *Meter {
}
}
// RequestFilter returns an [otelhttp.Filter] that excludes requests
// with paths starting with any of the specified prefixes.
func RequestFilter(ignoredPrefix ...string) otelhttp.Filter {
return func(r *http.Request) bool {
return !slices.ContainsFunc(
backend/v3/instrumentation/request.go
+110
View File
@@ -0,0 +1,110 @@
package instrumentation
import (
"context"
"log/slog"
"sync"
"time"
"github.com/rs/xid"
"github.com/zitadel/zitadel/internal/api/call"
)
// allow injection for testing
var xidWithTime = xid.NewWithTime
// WithRequestDetails creates a new context with request details, including a unique request ID.
// The request ID is generated using the call timestamp as returned by [call.FromContext].
//
// Note: The returned context carries a mutable pointer to requestDetails.
// This allows subsequent calls to SetInstanceID/SetUserID to update the details
// visible to the logger, even if the logger holds the original context created here.
func WithRequestDetails(ctx context.Context, instanceHost, publicHost string) context.Context {
details := &requestDetails{
id: xidWithTime(call.FromContext(ctx)),
instanceHost: instanceHost,
publicHost: publicHost,
}
return context.WithValue(ctx, ctxKey{}, details)
}
func getRequestDetails(ctx context.Context) (*requestDetails, bool) {
details, ok := ctx.Value(ctxKey{}).(*requestDetails)
return details, ok
}
// SetInstanceID sets the instance ID in the request details stored in the context.
// It uses a mutex to update the shared state safely, ensuring upstream loggers see the change.
func SetInstanceID(ctx context.Context, instanceID string) {
if details, ok := getRequestDetails(ctx); ok {
details.mtx.Lock()
details.instanceID = instanceID
details.mtx.Unlock()
}
}
// SetUserID sets the user ID in the request details stored in the context.
// It uses a mutex to update the shared state safely, ensuring upstream loggers see the change.
func SetUserID(ctx context.Context, userID string) {
if details, ok := getRequestDetails(ctx); ok {
details.mtx.Lock()
details.userID = userID
details.mtx.Unlock()
}
}
// GetRequestID retrieves the request ID from the context.
// [xid.NilID] is returned if no request details are found in the context.
func GetRequestID(ctx context.Context) xid.ID {
d, ok := getRequestDetails(ctx)
if !ok {
return xid.NilID()
}
d.mtx.Lock()
defer d.mtx.Unlock()
return d.id
}
type ctxKey struct{}
type requestDetails struct {
mtx sync.Mutex
id xid.ID
instanceHost string
publicHost string // may optionally be set through header (login v2)
instanceID string
userID string
}
func (d *requestDetails) slogAttributes() []any {
attributes := make([]any, 0, 5)
d.mtx.Lock()
defer d.mtx.Unlock()
attributes = append(attributes,
slog.String("id", d.id.String()),
slog.String("instance_host", d.instanceHost),
)
if d.publicHost != "" {
attributes = append(attributes, slog.String("public_host", d.publicHost))
}
if d.instanceID != "" {
attributes = append(attributes, slog.String("instance_id", d.instanceID))
}
if d.userID != "" {
attributes = append(attributes, slog.String("user_id", d.userID))
}
return attributes
}
func requestDetailsExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
if d, ok := getRequestDetails(ctx); ok {
return []slog.Attr{
slog.Group("request", d.slogAttributes()...),
}
}
return nil
}
backend/v3/instrumentation/request_test.go
+230
View File
@@ -0,0 +1,230 @@
package instrumentation
import (
"context"
"log/slog"
"testing"
"time"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var testID = xid.New()
func TestSetInstanceID(t *testing.T) {
originalXidWithTime := xidWithTime
xidWithTime = func(time.Time) xid.ID {
return testID
}
t.Cleanup(func() {
xidWithTime = originalXidWithTime
})
tests := []struct {
name string
ctx context.Context
instanceID string
wantDetails *requestDetails
}{
{
name: "no request details in context",
ctx: context.Background(),
wantDetails: nil,
},
{
name: "request details in context",
ctx: WithRequestDetails(context.Background(), "instanceHost", "publicHost"),
instanceID: "instanceID",
wantDetails: &requestDetails{
id: testID,
instanceID: "instanceID",
instanceHost: "instanceHost",
publicHost: "publicHost",
userID: "",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
SetInstanceID(tt.ctx, tt.instanceID)
got, ok := getRequestDetails(tt.ctx)
require.Equal(t, tt.wantDetails != nil, ok)
assert.Equal(t, tt.wantDetails, got)
})
}
}
func TestSetUserID(t *testing.T) {
originalXidWithTime := xidWithTime
xidWithTime = func(time.Time) xid.ID {
return testID
}
t.Cleanup(func() {
xidWithTime = originalXidWithTime
})
tests := []struct {
name string
ctx context.Context
userID string
wantDetails *requestDetails
}{
{
name: "no request details in context",
ctx: context.Background(),
wantDetails: nil,
},
{
name: "request details in context",
ctx: WithRequestDetails(context.Background(), "instanceHost", "publicHost"),
userID: "userID",
wantDetails: &requestDetails{
id: testID,
instanceHost: "instanceHost",
publicHost: "publicHost",
userID: "userID",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
SetUserID(tt.ctx, tt.userID)
got, ok := getRequestDetails(tt.ctx)
require.Equal(t, tt.wantDetails != nil, ok)
assert.Equal(t, tt.wantDetails, got)
})
}
}
func TestGetRequestID(t *testing.T) {
originalXidWithTime := xidWithTime
xidWithTime = func(time.Time) xid.ID {
return testID
}
t.Cleanup(func() {
xidWithTime = originalXidWithTime
})
reqCtx := WithRequestDetails(context.Background(), "instanceHost", "publicHost")
tests := []struct {
name string
ctx context.Context
wantID xid.ID
}{
{
name: "no request ID in context",
ctx: context.Background(),
wantID: xid.NilID(),
},
{
name: "request ID in context",
ctx: reqCtx,
wantID: testID,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GetRequestID(tt.ctx)
assert.Equal(t, tt.wantID, got)
})
}
}
func Test_requestDetails_slogAttributes(t *testing.T) {
type fields struct {
id xid.ID
instanceHost string
publicHost string
instanceID string
userID string
}
tests := []struct {
name string
fields fields
want []any
}{
{
name: "all fields set",
fields: fields{
id: testID,
instanceHost: "instanceHost",
publicHost: "publicHost",
instanceID: "instanceID",
userID: "userID",
},
want: []any{
slog.String("id", testID.String()),
slog.String("instance_host", "instanceHost"),
slog.String("public_host", "publicHost"),
slog.String("instance_id", "instanceID"),
slog.String("user_id", "userID"),
},
},
{
name: "only required fields set",
fields: fields{
id: testID,
instanceHost: "instanceHost",
publicHost: "publicHost",
},
want: []any{
slog.String("id", testID.String()),
slog.String("instance_host", "instanceHost"),
slog.String("public_host", "publicHost"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d := &requestDetails{
id: tt.fields.id,
instanceHost: tt.fields.instanceHost,
publicHost: tt.fields.publicHost,
instanceID: tt.fields.instanceID,
userID: tt.fields.userID,
}
got := d.slogAttributes()
assert.Equal(t, tt.want, got)
})
}
}
func Test_requestDetailsExtractor(t *testing.T) {
originalXidWithTime := xidWithTime
xidWithTime = func(time.Time) xid.ID {
return testID
}
t.Cleanup(func() {
xidWithTime = originalXidWithTime
})
tests := []struct {
name string
ctx context.Context
want []slog.Attr
}{
{
name: "no request details in context",
ctx: context.Background(),
want: nil,
},
{
name: "request details in context",
ctx: WithRequestDetails(context.Background(), "instanceHost", "publicHost"),
want: []slog.Attr{
slog.Group("request",
slog.String("id", testID.String()),
slog.String("instance_host", "instanceHost"),
slog.String("public_host", "publicHost"),
),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := requestDetailsExtractor(tt.ctx, time.Time{}, 0, "")
assert.Equal(t, tt.want, got)
})
}
}
backend/v3/instrumentation/slog.go
+1 -52
View File
@@ -9,7 +9,6 @@ import (
"sync/atomic"
"time"
"github.com/rs/xid"
slogmulti "github.com/samber/slog-multi"
slogctx "github.com/veqryn/slog-context"
slogotel "github.com/veqryn/slog-context/otel"
@@ -158,9 +157,8 @@ func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
stdErrHandler,
&slogctx.HandlerOptions{
Prependers: []slogctx.AttrExtractor{
instanceExtractor,
requestIDExtractor,
causeExtractor,
requestDetailsExtractor,
slogotel.ExtractTraceSpanID,
slogctx.ExtractPrepended,
},
@@ -177,55 +175,6 @@ func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
slog.SetDefault(logger)
}
const (
ProtocolHttp = "http"
ProtocolConnect = "connect"
ProtocolGrpc = "grpc"
)
// Instance is a minimal interface for logging the instance ID.
type Instance interface {
InstanceID() string
}
// SetInstance adds the instance to the context for logging.
func SetInstance(ctx context.Context, instance Instance) context.Context {
return context.WithValue(ctx, ctxKeyInstance, instance)
}
// SetRequest generates a new [xid.ID] based on the passed request timestamp
// and adds it to the context.
func SetRequestID(ctx context.Context, ts time.Time) context.Context {
return context.WithValue(ctx, ctxKeyRequestID, xid.NewWithTime(ts))
}
type ctxKeyType int
const (
ctxKeyRequestID ctxKeyType = iota
ctxKeyInstance
)
// instanceExtractor sets the instance ID from [Instance] to a log entry.
func instanceExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
if instance, ok := ctx.Value(ctxKeyInstance).(Instance); ok {
return []slog.Attr{
slog.String("instance", instance.InstanceID()),
}
}
return nil
}
// requestIDExtractor sets the request XID to a log entry.
func requestIDExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
if r, ok := ctx.Value(ctxKeyRequestID).(xid.ID); ok {
return []slog.Attr{
slog.String("request_id", r.String()),
}
}
return nil
}
// causeExtractor sets the cause of a canceled context to a log entry.
func causeExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
err := context.Cause(ctx)