mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-24 00:02:15 +00:00
0575f67e94
This PR overhauls our event projection system to make it more robust and prevent skipped events under high load. The core change replaces our custom, transaction-based locking with standard PostgreSQL advisory locks. We also introduce a worker pool to manage concurrency and prevent database connection exhaustion. ### Key Changes * **Advisory Locks for Projections:** Replaces exclusive row locks and inspection of `pg_stat_activity` with PostgreSQL advisory locks for managing projection state. This is a more reliable and standard approach to distributed locking. * **Simplified Await Logic:** Removes the complex logic for awaiting open transactions, simplifying it to a more straightforward time-based filtering of events. * **Projection Worker Pool:** Implements a worker pool to limit concurrent projection triggers, preventing connection exhaustion and improving stability under load. A new `MaxParallelTriggers` configuration option is introduced. ### Problem Solved Under high throughput, a race condition could cause projections to miss events from the eventstore. This led to inconsistent data in projection tables (e.g., a user grant might be missing). This PR fixes the underlying locking and concurrency issues to ensure all events are processed reliably. ### How it Works 1. **Event Writing:** When writing events, a *shared* advisory lock is taken. This signals that a write is in progress. 2. **Event Handling (Projections):** * A projection worker attempts to acquire an *exclusive* advisory lock for that specific projection. If the lock is already held, it means another worker is on the job, so the current one backs off. * Once the lock is acquired, the worker briefly acquires and releases the same *shared* lock used by event writers. This acts as a barrier, ensuring it waits for any in-flight writes to complete. * Finally, it processes all events that occurred before its transaction began. ### Additional Information * ZITADEL no longer modifies the `application_name` PostgreSQL variable during event writes. * The lock on the `current_states` table is now `FOR NO KEY UPDATE`. * Fixes https://github.com/zitadel/zitadel/issues/8509 --------- Co-authored-by: Copilot <[email protected]> Co-authored-by: Tim Möhlmann <[email protected]>
77 lines
2.9 KiB
TypeScript
77 lines
2.9 KiB
TypeScript
import { loginByUsernamePassword } from '../login_ui';
|
|
import { createOrg } from '../org';
|
|
import { User, createMachine } from '../user';
|
|
import { Config, MaxVUs } from '../config';
|
|
import { createProject, Project } from '../project';
|
|
import { addUserGrant } from '../user_grant';
|
|
|
|
export async function setup() {
|
|
const tokens = loginByUsernamePassword(Config.admin as User);
|
|
console.info('setup: admin signed in');
|
|
|
|
const org = await createOrg(tokens.accessToken!);
|
|
console.info(`setup: org (${org.organizationId}) created`);
|
|
|
|
const projects = await Promise.all(
|
|
Array.from({ length: 50 }, (_, i) => {
|
|
return createProject(`project-${i}`, org, tokens.accessToken!);
|
|
}),
|
|
);
|
|
console.log(`setup: ${projects.length} projects created`);
|
|
|
|
let machines = (
|
|
await Promise.all(
|
|
Array.from({ length: MaxVUs() }, async (_, i) => {
|
|
return await createMachine(`zitachine-${i}`, org, tokens.accessToken!);
|
|
}),
|
|
)
|
|
).map((machine) => {
|
|
return { userId: machine.userId, loginName: machine.loginNames[0] };
|
|
});
|
|
console.log(`setup: ${machines.length} machines created`);
|
|
|
|
return { tokens, org, machines, projects };
|
|
}
|
|
|
|
export default async function (data: any) {
|
|
const machine = await createMachine(`zitachine-${__VU}-${__ITER}`, data.org, data.tokens.accessToken!);
|
|
let userGrants = await Promise.all(
|
|
data.projects.map((project: Project) => {
|
|
return addUserGrant(data.org, machine.userId, project, [], data.tokens.accessToken!);
|
|
}),
|
|
);
|
|
|
|
return { userGrants };
|
|
}
|
|
|
|
export function teardown(data: any) {
|
|
// removeOrg(data.org, data.tokens.accessToken);
|
|
console.info('teardown: org is not removed to verify correctness of projections, do not forget to remove the org afterwards');
|
|
}
|
|
|
|
/**
|
|
* To verify the correctness of the projections you can use the following statements:
|
|
*
|
|
* set the owner of the events:
|
|
*
|
|
* set my.owner = '<org id of the created org>';
|
|
*
|
|
* check if the amount of events is the same as amount of objects
|
|
*
|
|
* select * from (
|
|
select 'projections.user_grants5', count(*) from projections.user_grants5 where resource_owner = (select current_setting('my.owner'))
|
|
union all
|
|
select 'projections.users14', count(*) from projections.users14 where resource_owner = (select current_setting('my.owner'))
|
|
union all
|
|
select 'projections.sessions8', count(*) from projections.sessions8 where user_resource_owner = (select current_setting('my.owner'))
|
|
union all
|
|
select aggregate_type, count(*) from eventstore.events2
|
|
where
|
|
aggregate_type in ('user', 'usergrant', 'session')
|
|
and event_type in ('user.machine.added', 'user.human.added', 'user.grant.added', 'session.user.checked')
|
|
and (owner = (select current_setting('my.owner'))
|
|
OR payload->>'userResourceOwner' = (select current_setting('my.owner'))
|
|
)
|
|
group by aggregate_type
|
|
) order by 2;
|
|
*/ |