mirror of
https://github.com/andwati/zitadel.git
synced 2026-11-05 22:02:15 +00:00
perf(eventstore): read projection events per event type (#12753)
# Which Problems Are Solved - A projection that subscribes to several event types gets slower the further it falls behind. - The handler reads its events with one query: `aggregate_type = ANY(...) AND event_type = ANY(...)`, ordered by position and limited to the bulk limit. - The `es_projection` index returns each (aggregate type, event type) range in position order, but Postgres cannot merge those ranges. - So each batch reads every matching event after the projection's position, sorts them all, and keeps only the bulk limit. - A projection that is N events behind therefore reads about N rows per batch. Catching up costs about N² in total, and once events arrive faster than such batches can process them, the projection does not recover. - Every query that triggers the projection waits for that catch-up. Examples are `GetSession` and the notification handlers. # How the Problems Are Solved - **New opt-in, `SearchQueryBuilder.ScanEventTypesSeparately()`.** It applies to event queries for a single instance, ascending and limited, whose sub-queries filter only by aggregate and event types. - For such a query, the SQL layer reads each distinct (aggregate type, event type) combination in its own `ORDER BY … LIMIT` subquery, using `UNNEST … CROSS JOIN LATERAL`. - It then merges the results with the same sort key and limit. - Each subquery is one `es_projection` range read in position order, so it stops at the limit. A batch reads at most `limit × combinations` rows, however far behind the projection is. - Every other query renders exactly as before. - **The projection handler** now passes its real combinations, one sub-query per aggregate type instead of the cross product of all types, and sets the opt-in. - **Results are identical to the single query.** - The sort key (`position, in_tx_order, instance_id, aggregate_type, aggregate_id, sequence`) is a total order, and the global top n is contained in the union of each combination's top n. - The cursor from #12703 and the open-transaction cap apply inside every subquery. - A test against a real Postgres compares both shapes from every cursor and every position, for several limits. The test events include positions shared by several transactions, whose `in_tx_order` restarts. # Additional Changes This is a separate commit and can be split into its own PR: - `cmd/defaults.yaml` sets `BulkLimit: 1000` for the `sessions` projection. - `projection_state_latency` gets histogram buckets up to one week. Until now, every projection more than 30 minutes behind was reported in the same top bucket. # Additional Context These numbers come from a local, CPU-pinned sandbox, for comparing builds only. - **Database:** Postgres 17 in a VM with 8 vCPUs and 16 GiB. The `Eventstore.Autovacuum` and `Projections` settings mirror a production-like setup. - **Starting state:** `sessions8` 4.8M events behind, restored byte-identically for every run. Batch query for `sessions8`, `EXPLAIN ANALYZE`, median of 5 runs, parallel workers at the default: | | before | after | | --- | ---: | ---: | | at the backlog, limit 1000 | 832 ms | 4.0 ms | | at the backlog, limit 200 | 832 ms | 1.0 ms | | at the head (idle poll) | 0.14 ms | 0.19 ms | | generic plan (`plan_cache_mode=force_generic_plan`), at the backlog | 1,042 ms | 4.0 ms | Draining `sessions8` from that backlog for 600 s, 3 runs per build (median, with the range in brackets): | | before | after | | --- | ---: | ---: | | events/s | 975 (932-1,032) | 7,811 (7,523-7,955) | | mean batch time, including writes | 1.03 s | 0.13 s | | still behind after 600 s | 4.2M | 17k-277k | A new index `(instance_id, position, in_tx_order, aggregate_type, event_type)` with the old query was evaluated as an alternative and rejected. Under a generic plan, Postgres chose it for projections whose events are rare and walked the whole event stream: 430 ms instead of 0.15 ms. This is the behaviour that led to dropping `es_instance_position` in #10626. - Follow-up for #12703 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 <[email protected]> Co-authored-by: Copilot Autofix powered by AI <[email protected]>
This commit is contained in:
Tim Möhlmann
co-authored by
Claude Opus 5
Copilot Autofix powered by AI
Claude Opus 5
Copilot Autofix powered by AI
parent
98272ab5d1
commit
4bc484dac9
@@ -92,39 +92,44 @@ func query(ctx context.Context, criteria querier, searchQuery *eventstore.Search
|
||||
|
||||
where += awaitOpenTransactions(useV1)
|
||||
}
|
||||
query += where
|
||||
|
||||
// instead of using the max function of the database (which doesn't work for postgres)
|
||||
// we select the most recent row
|
||||
if q.Columns == eventstore.ColumnsMaxPosition {
|
||||
q.Limit = 1
|
||||
q.Desc = true
|
||||
}
|
||||
if canScanEventTypesSeparately(q, useV1) {
|
||||
query, values = prepareEventTypeScans(q)
|
||||
} else {
|
||||
query += where
|
||||
|
||||
// if there is only one subquery we can optimize the query ordering by sequence
|
||||
var shouldOrderBySequence bool
|
||||
if q.EventSortKeyAfter == nil && len(q.SubQueries) == 1 {
|
||||
for _, filter := range q.SubQueries[0] {
|
||||
if filter.Field == repository.FieldAggregateID {
|
||||
shouldOrderBySequence = filter.Operation == repository.OperationEquals
|
||||
// instead of using the max function of the database (which doesn't work for postgres)
|
||||
// we select the most recent row
|
||||
if q.Columns == eventstore.ColumnsMaxPosition {
|
||||
q.Limit = 1
|
||||
q.Desc = true
|
||||
}
|
||||
|
||||
// if there is only one subquery we can optimize the query ordering by sequence
|
||||
var shouldOrderBySequence bool
|
||||
if q.EventSortKeyAfter == nil && len(q.SubQueries) == 1 {
|
||||
for _, filter := range q.SubQueries[0] {
|
||||
if filter.Field == repository.FieldAggregateID {
|
||||
shouldOrderBySequence = filter.Operation == repository.OperationEquals
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch q.Columns {
|
||||
case eventstore.ColumnsEvent,
|
||||
eventstore.ColumnsMaxPosition:
|
||||
query += criteria.orderByEventSequence(q.Desc, shouldOrderBySequence, useV1)
|
||||
}
|
||||
switch q.Columns {
|
||||
case eventstore.ColumnsEvent,
|
||||
eventstore.ColumnsMaxPosition:
|
||||
query += criteria.orderByEventSequence(q.Desc, shouldOrderBySequence, useV1)
|
||||
}
|
||||
|
||||
if q.Limit > 0 {
|
||||
values = append(values, q.Limit)
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
if q.Limit > 0 {
|
||||
values = append(values, q.Limit)
|
||||
query += " LIMIT ?"
|
||||
}
|
||||
|
||||
if q.Offset > 0 {
|
||||
values = append(values, q.Offset)
|
||||
query += " OFFSET ?"
|
||||
if q.Offset > 0 {
|
||||
values = append(values, q.Offset)
|
||||
query += " OFFSET ?"
|
||||
}
|
||||
}
|
||||
|
||||
query = criteria.placeholder(query)
|
||||
@@ -147,6 +152,68 @@ func query(ctx context.Context, criteria querier, searchQuery *eventstore.Search
|
||||
return nil
|
||||
}
|
||||
|
||||
// canScanEventTypesSeparately reports if the events of the combinations in [repository.SearchQuery.EventTypeScans]
|
||||
// can be read one combination at a time: only the instance, the position or sort key and the await of open transactions
|
||||
// may restrict the query besides the combinations, and it must be ordered ascending with a limit.
|
||||
func canScanEventTypesSeparately(q *repository.SearchQuery, useV1 bool) bool {
|
||||
return !useV1 &&
|
||||
len(q.EventTypeScans) > 0 &&
|
||||
q.Columns == eventstore.ColumnsEvent &&
|
||||
q.Limit > 0 &&
|
||||
!q.Desc &&
|
||||
q.Offset == 0 &&
|
||||
q.InstanceID != nil &&
|
||||
q.InstanceIDs == nil &&
|
||||
q.ExcludedInstances == nil &&
|
||||
q.Creator == nil &&
|
||||
q.Owner == nil &&
|
||||
q.Sequence == nil &&
|
||||
q.CreatedAfter == nil &&
|
||||
q.CreatedBefore == nil &&
|
||||
len(q.ExcludeAggregateIDs) == 0
|
||||
}
|
||||
|
||||
// prepareEventTypeScans reads the events of every combination of aggregate type and event type
|
||||
// in sort key order up to the limit, and merges them.
|
||||
// Each combination is a range of the es_projection index (instance_id, aggregate_type, event_type, position),
|
||||
// which returns its events in position order, so every scan stops at the limit
|
||||
// and the query reads at most limit events per combination, however many events match after the position.
|
||||
func prepareEventTypeScans(q *repository.SearchQuery) (string, []any) {
|
||||
aggregateTypes := make(database.TextArray[eventstore.AggregateType], len(q.EventTypeScans))
|
||||
eventTypes := make(database.TextArray[eventstore.EventType], len(q.EventTypeScans))
|
||||
for i, typeScan := range q.EventTypeScans {
|
||||
aggregateTypes[i] = typeScan.AggregateType
|
||||
eventTypes[i] = typeScan.EventType
|
||||
}
|
||||
values := []any{aggregateTypes, eventTypes, q.InstanceID.Value}
|
||||
|
||||
conditions := "instance_id = ? AND aggregate_type = scans.scan_aggregate_type AND event_type = scans.scan_event_type"
|
||||
if q.Position != nil {
|
||||
conditions += ` AND "position" >= ?`
|
||||
values = append(values, q.Position.Value)
|
||||
}
|
||||
if q.EventSortKeyAfter != nil {
|
||||
conditions += ` AND (` + eventSortKeySQL + `) > (?, ?, ?, ?, ?, ?)`
|
||||
values = append(values,
|
||||
q.EventSortKeyAfter.Position,
|
||||
q.EventSortKeyAfter.InTxOrder,
|
||||
q.EventSortKeyAfter.InstanceID,
|
||||
q.EventSortKeyAfter.AggregateType,
|
||||
q.EventSortKeyAfter.AggregateID,
|
||||
q.EventSortKeyAfter.Sequence,
|
||||
)
|
||||
}
|
||||
if q.AwaitOpenTransactions {
|
||||
conditions += awaitOpenTransactions(false)
|
||||
}
|
||||
values = append(values, q.Limit, q.Limit)
|
||||
|
||||
return "SELECT events.* FROM UNNEST(?::TEXT[], ?::TEXT[]) AS scans(scan_aggregate_type, scan_event_type)" +
|
||||
" CROSS JOIN LATERAL (SELECT " + eventColumnsSQL + " FROM eventstore.events2 WHERE " + conditions +
|
||||
" ORDER BY " + eventSortKeySQL + " LIMIT ?) AS events" +
|
||||
" ORDER BY " + eventSortKeySQL + " LIMIT ?", values
|
||||
}
|
||||
|
||||
func prepareColumns(criteria querier, columns eventstore.Columns, useV1 bool) (string, func(s scan, dest interface{}) error) {
|
||||
switch columns {
|
||||
case eventstore.ColumnsMaxPosition:
|
||||
|
||||
Reference in New Issue
Block a user