fix(projections): overhaul the event projection system (#10560)

This PR overhauls our event projection system to make it more robust and
prevent skipped events under high load. The core change replaces our
custom, transaction-based locking with standard PostgreSQL advisory
locks. We also introduce a worker pool to manage concurrency and prevent
database connection exhaustion.

### Key Changes

* **Advisory Locks for Projections:** Replaces exclusive row locks and
inspection of `pg_stat_activity` with PostgreSQL advisory locks for
managing projection state. This is a more reliable and standard approach
to distributed locking.
* **Simplified Await Logic:** Removes the complex logic for awaiting
open transactions, simplifying it to a more straightforward time-based
filtering of events.
* **Projection Worker Pool:** Implements a worker pool to limit
concurrent projection triggers, preventing connection exhaustion and
improving stability under load. A new `MaxParallelTriggers`
configuration option is introduced.

### Problem Solved

Under high throughput, a race condition could cause projections to miss
events from the eventstore. This led to inconsistent data in projection
tables (e.g., a user grant might be missing). This PR fixes the
underlying locking and concurrency issues to ensure all events are
processed reliably.

### How it Works

1. **Event Writing:** When writing events, a *shared* advisory lock is
taken. This signals that a write is in progress.
2.  **Event Handling (Projections):**
* A projection worker attempts to acquire an *exclusive* advisory lock
for that specific projection. If the lock is already held, it means
another worker is on the job, so the current one backs off.
* Once the lock is acquired, the worker briefly acquires and releases
the same *shared* lock used by event writers. This acts as a barrier,
ensuring it waits for any in-flight writes to complete.
* Finally, it processes all events that occurred before its transaction
began.

### Additional Information

* ZITADEL no longer modifies the `application_name` PostgreSQL variable
during event writes.
*   The lock on the `current_states` table is now `FOR NO KEY UPDATE`.
*   Fixes https://github.com/zitadel/zitadel/issues/8509

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: Tim Möhlmann <[email protected]>

load-test/src/use_cases/session/add_session.ts

@@ -1,6 +1,6 @@
 import { loginByUsernamePassword } from '../../login_ui';
 import { createOrg, removeOrg } from '../../org';
-import { User, createHuman } from '../../user';
+import { User, createHuman, createMachine } from '../../user';
 import { Trend } from 'k6/metrics';
 import { Config, MaxVUs } from '../../config';
 import { createSession } from '../../session';

load-test/src/use_cases/verify_all_user_grants_exist.ts

@@ -0,0 +1,77 @@
+import { loginByUsernamePassword } from '../login_ui';
+import { createOrg } from '../org';
+import { User, createMachine } from '../user';
+import { Config, MaxVUs } from '../config';
+import { createProject, Project } from '../project';
+import { addUserGrant } from '../user_grant';
+
+export async function setup() {
+  const tokens = loginByUsernamePassword(Config.admin as User);
+  console.info('setup: admin signed in');
+
+  const org = await createOrg(tokens.accessToken!);
+  console.info(`setup: org (${org.organizationId}) created`);
+
+  const projects = await Promise.all(
+    Array.from({ length: 50 }, (_, i) => {
+      return createProject(`project-${i}`, org, tokens.accessToken!);
+    }),
+  );
+  console.log(`setup: ${projects.length} projects created`);
+
+  let machines = (
+    await Promise.all(
+      Array.from({ length: MaxVUs() }, async (_, i) => {
+        return await createMachine(`zitachine-${i}`, org, tokens.accessToken!);
+      }),
+    )
+  ).map((machine) => {
+    return { userId: machine.userId, loginName: machine.loginNames[0] };
+  });
+  console.log(`setup: ${machines.length} machines created`);
+
+  return { tokens, org, machines, projects };
+}
+
+export default async function (data: any) {
+  const machine = await createMachine(`zitachine-${__VU}-${__ITER}`, data.org, data.tokens.accessToken!);
+  let userGrants = await Promise.all(
+    data.projects.map((project: Project) => {
+      return addUserGrant(data.org, machine.userId, project, [], data.tokens.accessToken!);
+    }),
+  );
+
+  return { userGrants };
+}
+
+export function teardown(data: any) {
+  // removeOrg(data.org, data.tokens.accessToken);
+  console.info('teardown: org is not removed to verify correctness of projections, do not forget to remove the org afterwards');
+}
+
+/** 
+ * To verify the correctness of the projections you can use the following statements:
+ * 
+ * set the owner of the events:
+ * 
+ * set my.owner = '<org id of the created org>';
+ * 
+ * check if the amount of events is the same as amount of objects
+ * 
+ *  select * from (
+    select 'projections.user_grants5', count(*) from projections.user_grants5 where resource_owner = (select current_setting('my.owner'))
+    union all
+    select 'projections.users14', count(*) from projections.users14 where resource_owner = (select current_setting('my.owner'))
+    union all
+    select 'projections.sessions8', count(*) from projections.sessions8 where user_resource_owner = (select current_setting('my.owner'))
+    union all
+    select aggregate_type, count(*) from eventstore.events2
+    where
+        aggregate_type in ('user', 'usergrant', 'session')
+        and event_type in ('user.machine.added', 'user.human.added', 'user.grant.added', 'session.user.checked')
+        and (owner = (select current_setting('my.owner'))
+            OR payload->>'userResourceOwner' = (select current_setting('my.owner'))
+        )
+        group by aggregate_type
+) order by 2;
+ */