mirror of
https://github.com/andwati/zitadel.git
synced 2026-11-09 23:55:12 +00:00
perf(query): speed up ListUsers login name equality filters (#12460)
<!-- CURSOR_AGENT_PR_BODY_BEGIN --> ## Summary `ListUsers` with `LoginNameQuery` + equals/equals-ignore-case was very slow on large orgs (e.g. ~330k users) because the generated SQL filtered the `projections.login_names3` **view** on the computed `login_name_lower` expression and correlated that subquery on `users14.id`. Postgres nested-looped every user and never used `login_names3_users_search (instance_id, user_name_lower)`. This change rewrites the **query planner** for that hot path: when an equals/equals-ignore-case login-name filter is present (and not under `OR`/`NOT`), the user list query **INNER JOINs** an indexed matches subquery instead of filtering via the view expression. The matches SQL mirrors `user_by_login_name.sql` (`user_name_lower` / domain paths + `preferred` / `is_primary`). Non-equals methods and OR combinations keep the previous view-based filter so semantics stay unchanged. Also adds a k6 use case that mirrors login v2 discovery (`loginNameQuery` EQUALS_IGNORE_CASE + `organizationIdQuery`, `limit: 2`): ```bash cd benchmark make users_by_login_name USER_AMOUNT=100000 VUS=10 DURATION=60s ``` ## Approach 1. `NewLoginNameSearchQuery` for equals / equals-ignore-case returns a marker `loginNameEqualsFilter` (other methods unchanged). 2. `prepareUsersQuery` extracts that marker when safe, then: - builds the usual `sq.SelectBuilder` **without** the login-name view predicate - adds `JoinClause` to `user_login_name_matches(.sql)` / `_case_sensitive.sql` as `login_name_matches` - keeps metadata JOIN/`DISTINCT` only when metadata filters are present (same as before) 3. Embedded SQL files under `internal/query/` for the matches subquery. Local smoke against ~330k synthetic users: baseline ~1392ms → rewritten path ~0.4ms for a single equals-ignore-case lookup. ## Test plan - [x] `go test ./internal/query/ -run 'TestLoginName|TestUsers|TestUserByLoginName'` (after generate-stubs) - [x] Existing `user_test` expected SQL updated (no always-on metadata join; login-name equals uses JOIN) - [ ] Run k6 before/after on a large `USER_AMOUNT` (e.g. 50k–100k+) and compare `list_users_duration` p50/p95/p99 - [x] Manual login v2 username discovery against a large org - [x] Confirm OR / NOT / CONTAINS login-name queries still return expected results <!-- CURSOR_AGENT_PR_BODY_END --> <div><a href="https://cursor.com/agents/bc-252883bd-48d9-492e-b619-5ccfa93cf9c3"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-web-light.png"><img alt="Open in Web" width="114" height="28" src="https://cursor.com/assets/images/open-in-web-dark.png"></picture></a> <a href="https://cursor.com/background-agent?bcId=bc-252883bd-48d9-492e-b619-5ccfa93cf9c3"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-cursor-light.png"><img alt="Open in Cursor" width="131" height="28" src="https://cursor.com/assets/images/open-in-cursor-dark.png"></picture></a> </div> --------- Co-authored-by: Cursor Agent <[email protected]> Co-authored-by: Silvan <[email protected]>
This commit is contained in:
Silvan
co-authored by
Cursor Agent
Silvan
Cursor Agent
Silvan
parent
87538f29e1
commit
318acd4bc3
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Run async work over items with a fixed number of in-flight tasks.
|
||||
* Avoids unbounded Promise.all which can exhaust ephemeral ports
|
||||
* when seeding large USER_AMOUNT datasets.
|
||||
*/
|
||||
export async function mapPool<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
if (items.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const limit = Math.max(1, Math.min(concurrency, items.length));
|
||||
const results: R[] = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
async function worker(): Promise<void> {
|
||||
while (true) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (index >= items.length) {
|
||||
return;
|
||||
}
|
||||
results[index] = await fn(items[index], index);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({ length: limit }, () => worker()));
|
||||
return results;
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { options } from 'k6/http';
|
||||
// @ts-ignore Import module
|
||||
import { URL } from 'https://jslib.k6.io/url/1.0.0/index.js';
|
||||
|
||||
import { Config } from './config';
|
||||
|
||||
export type options = {
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { check } from 'k6';
|
||||
|
||||
import { Config } from '../config';
|
||||
import { loginByUsernamePassword } from '../login_ui';
|
||||
import { createOrg, Org, removeOrg } from '../org';
|
||||
import { mapPool } from '../pool';
|
||||
import { createHuman, listUsers, User } from '../user';
|
||||
|
||||
const userAmount = parseInt(__ENV.USER_AMOUNT) || 2500;
|
||||
// Unbounded Promise.all over large USER_AMOUNT exhausts ephemeral ports
|
||||
// (dial tcp ... can't assign requested address). Keep a modest in-flight cap.
|
||||
const setupConcurrency = parseInt(__ENV.SETUP_CONCURRENCY) || 50;
|
||||
|
||||
type SetupData = {
|
||||
tokens: { accessToken?: string };
|
||||
org: Org;
|
||||
targetLoginName: string;
|
||||
};
|
||||
|
||||
export async function setup(): Promise<SetupData> {
|
||||
const tokens = loginByUsernamePassword(Config.admin as User);
|
||||
console.info('setup: admin signed in');
|
||||
|
||||
const org = await createOrg(tokens.accessToken!);
|
||||
console.info(`setup: org (${org.organizationId}) created`);
|
||||
|
||||
const progressEvery = Math.max(1, Math.floor(userAmount / 100));
|
||||
const users = await mapPool(
|
||||
Array.from({ length: userAmount }, (_, i) => i),
|
||||
setupConcurrency,
|
||||
async (i) => {
|
||||
const user = await createHuman(`zitizen-${i}`, org, tokens.accessToken!);
|
||||
if (i % progressEvery === 0 || i === userAmount - 1) {
|
||||
console.log(`setup: ${i + 1} of ${userAmount} users setup`);
|
||||
}
|
||||
return user;
|
||||
},
|
||||
);
|
||||
console.info(`setup: ${users.length} users created (concurrency=${setupConcurrency})`);
|
||||
|
||||
const targetLoginName = users[0].loginNames[0];
|
||||
console.info(`setup: target login name ${targetLoginName}`);
|
||||
|
||||
return { tokens, org, targetLoginName };
|
||||
}
|
||||
|
||||
export default async function (data: SetupData) {
|
||||
const result = await listUsers(
|
||||
{
|
||||
query: { limit: 2 },
|
||||
queries: [
|
||||
{
|
||||
loginNameQuery: {
|
||||
loginName: data.targetLoginName,
|
||||
method: 'TEXT_QUERY_METHOD_EQUALS_IGNORE_CASE',
|
||||
},
|
||||
},
|
||||
{
|
||||
organizationIdQuery: {
|
||||
organizationId: data.org.organizationId,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
data.tokens.accessToken!,
|
||||
);
|
||||
|
||||
check(result, {
|
||||
'exact one user found': (res) => (res.result?.length ?? 0) === 1 || res.details.totalResult == 1,
|
||||
}) ||
|
||||
console.log(
|
||||
`unexpected list users result. expected 1 user but got result=${result.result?.length} totalResult=${result.details.totalResult}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function teardown(data: SetupData) {
|
||||
removeOrg(data.org, data.tokens.accessToken!);
|
||||
console.info('teardown: org removed');
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
import { loginByUsernamePassword } from '../login_ui';
|
||||
import { createOrg, removeOrg } from '../org';
|
||||
import { mapPool } from '../pool';
|
||||
import { createHuman, User, createMachine, setUserMetadata, listUsers } from '../user';
|
||||
import { Config } from '../config';
|
||||
import { check } from 'k6';
|
||||
import encoding from 'k6/encoding';
|
||||
|
||||
const userAmount = parseInt(__ENV.USER_AMOUNT) || 2500;
|
||||
const setupConcurrency = parseInt(__ENV.SETUP_CONCURRENCY) || 50;
|
||||
|
||||
export async function setup() {
|
||||
const tokens = loginByUsernamePassword(Config.admin as User);
|
||||
@@ -14,10 +16,11 @@ export async function setup() {
|
||||
const org = await createOrg(tokens.accessToken!);
|
||||
console.info(`setup: org (${org.organizationId}) created`);
|
||||
|
||||
const users: User[] = [];
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: userAmount }, async (_, i) => {
|
||||
const progressEvery = Math.max(1, Math.floor(userAmount / 100));
|
||||
const users = await mapPool(
|
||||
Array.from({ length: userAmount }, (_, i) => i),
|
||||
setupConcurrency,
|
||||
async (i) => {
|
||||
let user: User;
|
||||
let type: 'human' | 'machine';
|
||||
if (i % 2 === 0) {
|
||||
@@ -27,7 +30,6 @@ export async function setup() {
|
||||
user = await createMachine(`zitachine-${i}`, org, tokens.accessToken!);
|
||||
type = 'machine';
|
||||
}
|
||||
users.push(user);
|
||||
await setUserMetadata(
|
||||
[
|
||||
{ key: 'type', value: encoding.b64encode(type, 'rawurl') },
|
||||
@@ -37,12 +39,13 @@ export async function setup() {
|
||||
user.userId,
|
||||
tokens.accessToken!,
|
||||
);
|
||||
if (i % 10 === 0) {
|
||||
console.log(`setup: ${i} of ${userAmount} users setup`);
|
||||
if (i % progressEvery === 0 || i === userAmount - 1) {
|
||||
console.log(`setup: ${i + 1} of ${userAmount} users setup`);
|
||||
}
|
||||
}),
|
||||
return user;
|
||||
},
|
||||
);
|
||||
console.info(`setup: ${users.length} users created`);
|
||||
console.info(`setup: ${users.length} users created (concurrency=${setupConcurrency})`);
|
||||
|
||||
return { tokens, org, users };
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { loginByUsernamePassword } from '../login_ui';
|
||||
import { createOrg, removeOrg } from '../org';
|
||||
import { mapPool } from '../pool';
|
||||
import { createHuman, User, createMachine, setUserMetadata, listUsers } from '../user';
|
||||
import { Config } from '../config';
|
||||
import { check } from 'k6';
|
||||
import encoding from 'k6/encoding';
|
||||
|
||||
const userAmount = parseInt(__ENV.USER_AMOUNT) || 2500;
|
||||
const setupConcurrency = parseInt(__ENV.SETUP_CONCURRENCY) || 50;
|
||||
|
||||
export async function setup() {
|
||||
const tokens = loginByUsernamePassword(Config.admin as User);
|
||||
@@ -14,10 +16,11 @@ export async function setup() {
|
||||
const org = await createOrg(tokens.accessToken!);
|
||||
console.info(`setup: org (${org.organizationId}) created`);
|
||||
|
||||
const users: User[] = [];
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: userAmount }, async (_, i) => {
|
||||
const progressEvery = Math.max(1, Math.floor(userAmount / 100));
|
||||
const users = await mapPool(
|
||||
Array.from({ length: userAmount }, (_, i) => i),
|
||||
setupConcurrency,
|
||||
async (i) => {
|
||||
let user: User;
|
||||
let type: 'human' | 'machine';
|
||||
if (i % 2 === 0) {
|
||||
@@ -27,7 +30,6 @@ export async function setup() {
|
||||
user = await createMachine(`zitachine-${i}`, org, tokens.accessToken!);
|
||||
type = 'machine';
|
||||
}
|
||||
users.push(user);
|
||||
await setUserMetadata(
|
||||
[
|
||||
{ key: 'type', value: encoding.b64encode(type, 'rawurl') },
|
||||
@@ -37,12 +39,13 @@ export async function setup() {
|
||||
user.userId,
|
||||
tokens.accessToken!,
|
||||
);
|
||||
if (i % 10 === 0) {
|
||||
console.log(`setup: ${i} of ${userAmount} users setup`);
|
||||
if (i % progressEvery === 0 || i === userAmount - 1) {
|
||||
console.log(`setup: ${i + 1} of ${userAmount} users setup`);
|
||||
}
|
||||
}),
|
||||
return user;
|
||||
},
|
||||
);
|
||||
console.info(`setup: ${users.length} users created`);
|
||||
console.info(`setup: ${users.length} users created (concurrency=${setupConcurrency})`);
|
||||
|
||||
return { tokens, org, users };
|
||||
}
|
||||
|
||||
@@ -325,7 +325,26 @@ export function setUserMetadata(metadata: Metadata[], userId: string, accessToke
|
||||
}
|
||||
|
||||
export type ListUsersRequest = {
|
||||
query?: {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
};
|
||||
queries: {
|
||||
loginNameQuery?: {
|
||||
loginName: string;
|
||||
method:
|
||||
| 'TEXT_QUERY_METHOD_EQUALS'
|
||||
| 'TEXT_QUERY_METHOD_EQUALS_IGNORE_CASE'
|
||||
| 'TEXT_QUERY_METHOD_STARTS_WITH'
|
||||
| 'TEXT_QUERY_METHOD_STARTS_WITH_IGNORE_CASE'
|
||||
| 'TEXT_QUERY_METHOD_CONTAINS'
|
||||
| 'TEXT_QUERY_METHOD_CONTAINS_IGNORE_CASE'
|
||||
| 'TEXT_QUERY_METHOD_ENDS_WITH'
|
||||
| 'TEXT_QUERY_METHOD_ENDS_WITH_IGNORE_CASE';
|
||||
};
|
||||
organizationIdQuery?: {
|
||||
organizationId: string;
|
||||
};
|
||||
metadataKeyFilter?: {
|
||||
key: string;
|
||||
method: 'TEXT_FILTER_METHOD_EQUALS' | 'TEXT_FILTER_METHOD_CONTAINS' | 'TEXT_FILTER_METHOD_CONTAINS_IGNORE_CASE';
|
||||
@@ -341,6 +360,7 @@ export type ListUsersResult = {
|
||||
details: {
|
||||
totalResult: number;
|
||||
};
|
||||
result?: unknown[];
|
||||
};
|
||||
|
||||
const listUsersTrend = new Trend('list_users_duration', true);
|
||||
|
||||
Reference in New Issue
Block a user