mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-18 05:52:51 +00:00
# Which Problems Are Solved - A projection that subscribes to several event types gets slower the further it falls behind. - The handler reads its events with one query: `aggregate_type = ANY(...) AND event_type = ANY(...)`, ordered by position and limited to the bulk limit. - The `es_projection` index returns each (aggregate type, event type) range in position order, but Postgres cannot merge those ranges. - So each batch reads every matching event after the projection's position, sorts them all, and keeps only the bulk limit. - A projection that is N events behind therefore reads about N rows per batch. Catching up costs about N² in total, and once events arrive faster than such batches can process them, the projection does not recover. - Every query that triggers the projection waits for that catch-up. Examples are `GetSession` and the notification handlers. # How the Problems Are Solved - **New opt-in, `SearchQueryBuilder.ScanEventTypesSeparately()`.** It applies to event queries for a single instance, ascending and limited, whose sub-queries filter only by aggregate and event types. - For such a query, the SQL layer reads each distinct (aggregate type, event type) combination in its own `ORDER BY … LIMIT` subquery, using `UNNEST … CROSS JOIN LATERAL`. - It then merges the results with the same sort key and limit. - Each subquery is one `es_projection` range read in position order, so it stops at the limit. A batch reads at most `limit × combinations` rows, however far behind the projection is. - Every other query renders exactly as before. - **The projection handler** now passes its real combinations, one sub-query per aggregate type instead of the cross product of all types, and sets the opt-in. - **Results are identical to the single query.** - The sort key (`position, in_tx_order, instance_id, aggregate_type, aggregate_id, sequence`) is a total order, and the global top n is contained in the union of each combination's top n. - The cursor from #12703 and the open-transaction cap apply inside every subquery. - A test against a real Postgres compares both shapes from every cursor and every position, for several limits. The test events include positions shared by several transactions, whose `in_tx_order` restarts. # Additional Changes This is a separate commit and can be split into its own PR: - `cmd/defaults.yaml` sets `BulkLimit: 1000` for the `sessions` projection. - `projection_state_latency` gets histogram buckets up to one week. Until now, every projection more than 30 minutes behind was reported in the same top bucket. # Additional Context These numbers come from a local, CPU-pinned sandbox, for comparing builds only. - **Database:** Postgres 17 in a VM with 8 vCPUs and 16 GiB. The `Eventstore.Autovacuum` and `Projections` settings mirror a production-like setup. - **Starting state:** `sessions8` 4.8M events behind, restored byte-identically for every run. Batch query for `sessions8`, `EXPLAIN ANALYZE`, median of 5 runs, parallel workers at the default: | | before | after | | --- | ---: | ---: | | at the backlog, limit 1000 | 832 ms | 4.0 ms | | at the backlog, limit 200 | 832 ms | 1.0 ms | | at the head (idle poll) | 0.14 ms | 0.19 ms | | generic plan (`plan_cache_mode=force_generic_plan`), at the backlog | 1,042 ms | 4.0 ms | Draining `sessions8` from that backlog for 600 s, 3 runs per build (median, with the range in brackets): | | before | after | | --- | ---: | ---: | | events/s | 975 (932-1,032) | 7,811 (7,523-7,955) | | mean batch time, including writes | 1.03 s | 0.13 s | | still behind after 600 s | 4.2M | 17k-277k | A new index `(instance_id, position, in_tx_order, aggregate_type, event_type)` with the old query was evaluated as an alternative and rejected. Under a generic plan, Postgres chose it for projections whose events are rare and walked the whole event stream: 430 ms instead of 0.15 ms. This is the behaviour that led to dropping `es_instance_position` in #10626. - Follow-up for #12703 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 <[email protected]> Co-authored-by: Copilot Autofix powered by AI <[email protected]>
169 lines
5.2 KiB
Go
169 lines
5.2 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/shopspring/decimal"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/zitadel/zitadel/internal/eventstore"
|
|
)
|
|
|
|
func TestHandler_eventQuery(t *testing.T) {
|
|
h := &Handler{
|
|
bulkLimit: 200,
|
|
eventTypes: map[eventstore.AggregateType][]eventstore.EventType{
|
|
"user": {"user.human.added", "user.human.password.changed"},
|
|
},
|
|
}
|
|
|
|
t.Run("scans every aggregate's event types separately", func(t *testing.T) {
|
|
h := &Handler{
|
|
bulkLimit: 200,
|
|
eventTypes: map[eventstore.AggregateType][]eventstore.EventType{
|
|
"user": {"user.human.added", "user.locked"},
|
|
"org": {"org.removed"},
|
|
},
|
|
}
|
|
builder := h.eventQuery(&state{instanceID: "inst"}, decimal.Decimal{})
|
|
assert.True(t, builder.GetScanEventTypesSeparately())
|
|
queries := builder.GetQueries()
|
|
require.Len(t, queries, 2)
|
|
assert.Equal(t, []eventstore.AggregateType{"org"}, queries[0].GetAggregateTypes())
|
|
assert.Equal(t, []eventstore.EventType{"org.removed"}, queries[0].GetEventTypes())
|
|
assert.Equal(t, []eventstore.AggregateType{"user"}, queries[1].GetAggregateTypes())
|
|
assert.Equal(t, []eventstore.EventType{"user.human.added", "user.locked"}, queries[1].GetEventTypes())
|
|
})
|
|
|
|
t.Run("aggregate without event types keeps the single query", func(t *testing.T) {
|
|
h := &Handler{
|
|
bulkLimit: 200,
|
|
eventTypes: map[eventstore.AggregateType][]eventstore.EventType{
|
|
"org": nil,
|
|
},
|
|
}
|
|
builder := h.eventQuery(&state{instanceID: "inst"}, decimal.Decimal{})
|
|
assert.False(t, builder.GetScanEventTypesSeparately())
|
|
queries := builder.GetQueries()
|
|
require.Len(t, queries, 1)
|
|
assert.Equal(t, []eventstore.AggregateType{"org"}, queries[0].GetAggregateTypes())
|
|
assert.Empty(t, queries[0].GetEventTypes())
|
|
})
|
|
|
|
t.Run("no cursor on empty position", func(t *testing.T) {
|
|
builder := h.eventQuery(&state{instanceID: "inst"}, decimal.Decimal{})
|
|
assert.Equal(t, uint32(0), builder.GetOffset())
|
|
assert.Nil(t, builder.GetEventSortKeyAfter())
|
|
assert.True(t, builder.GetPositionAtLeast().IsZero())
|
|
})
|
|
|
|
t.Run("resumes after event sort key", func(t *testing.T) {
|
|
cursor := eventstore.EventSortKey{
|
|
Position: decimal.NewFromFloat(1788336088.993079),
|
|
InTxOrder: 5,
|
|
AggregateType: "user",
|
|
AggregateID: "388963124960608862",
|
|
Sequence: 5,
|
|
}
|
|
builder := h.eventQuery(&state{
|
|
instanceID: "inst",
|
|
cursor: cursor,
|
|
}, decimal.Decimal{})
|
|
assert.Equal(t, uint32(0), builder.GetOffset())
|
|
assert.True(t, builder.GetPositionAtLeast().IsZero())
|
|
require.NotNil(t, builder.GetEventSortKeyAfter())
|
|
assert.Equal(t, cursor, *builder.GetEventSortKeyAfter())
|
|
})
|
|
|
|
t.Run("min position is an inclusive floor not a smashed cursor", func(t *testing.T) {
|
|
cursor := eventstore.EventSortKey{
|
|
Position: decimal.NewFromInt(1),
|
|
InTxOrder: 5,
|
|
AggregateType: "user",
|
|
AggregateID: "agg-a",
|
|
Sequence: 5,
|
|
}
|
|
minPosition := decimal.NewFromInt(10)
|
|
builder := h.eventQuery(&state{
|
|
instanceID: "inst",
|
|
cursor: cursor,
|
|
}, minPosition)
|
|
assert.Nil(t, builder.GetEventSortKeyAfter())
|
|
assert.True(t, builder.GetPositionAtLeast().Equal(minPosition))
|
|
assert.Equal(t, uint32(0), builder.GetOffset())
|
|
})
|
|
}
|
|
|
|
func TestHandler_eventsToStatements_inTxOrder(t *testing.T) {
|
|
h := &Handler{
|
|
projection: &projection{name: "users14"},
|
|
}
|
|
|
|
pos1 := decimal.NewFromFloat(1)
|
|
pos2 := decimal.NewFromFloat(1.5)
|
|
events := []eventstore.Event{
|
|
&eventstore.BaseEvent{
|
|
Agg: &eventstore.Aggregate{ID: "agg-a", Type: "user"},
|
|
EventType: "user.human.added",
|
|
Seq: 1,
|
|
Pos: pos1,
|
|
InTx: 1,
|
|
},
|
|
&eventstore.BaseEvent{
|
|
Agg: &eventstore.Aggregate{ID: "agg-a", Type: "user"},
|
|
EventType: "user.human.email.verified",
|
|
Seq: 2,
|
|
Pos: pos1,
|
|
InTx: 2,
|
|
},
|
|
&eventstore.BaseEvent{
|
|
Agg: &eventstore.Aggregate{ID: "agg-b", Type: "user"},
|
|
EventType: "user.human.password.changed",
|
|
Seq: 10,
|
|
Pos: pos2,
|
|
InTx: 1,
|
|
},
|
|
}
|
|
|
|
statements, err := h.eventsToStatements(context.Background(), nil, events)
|
|
require.NoError(t, err)
|
|
require.Len(t, statements, 3)
|
|
assert.Equal(t, uint32(1), statements[0].inTxOrder)
|
|
assert.Equal(t, uint32(2), statements[1].inTxOrder)
|
|
assert.Equal(t, uint32(1), statements[2].inTxOrder)
|
|
assert.Equal(t, "agg-b", statements[2].Aggregate.ID)
|
|
}
|
|
|
|
func TestHandler_eventsToStatements_samePositionInTxOrder(t *testing.T) {
|
|
h := &Handler{
|
|
projection: &projection{name: "users14"},
|
|
}
|
|
pos := decimal.NewFromInt(1)
|
|
events := []eventstore.Event{
|
|
&eventstore.BaseEvent{
|
|
Agg: &eventstore.Aggregate{ID: "agg-a", Type: "user"},
|
|
EventType: "user.human.added",
|
|
Seq: 1,
|
|
Pos: pos,
|
|
InTx: 1,
|
|
},
|
|
&eventstore.BaseEvent{
|
|
Agg: &eventstore.Aggregate{ID: "agg-b", Type: "user"},
|
|
EventType: "user.human.password.changed",
|
|
Seq: 1,
|
|
Pos: pos,
|
|
InTx: 1,
|
|
},
|
|
}
|
|
|
|
statements, err := h.eventsToStatements(context.Background(), nil, events)
|
|
require.NoError(t, err)
|
|
require.Len(t, statements, 2)
|
|
assert.Equal(t, "agg-a", statements[0].Aggregate.ID)
|
|
assert.Equal(t, "agg-b", statements[1].Aggregate.ID)
|
|
assert.Equal(t, uint32(1), statements[0].inTxOrder)
|
|
assert.Equal(t, uint32(1), statements[1].inTxOrder)
|
|
}
|