mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-31 11:32:15 +00:00
perf(query): use UNION of index seeks for ListUsers login equality (#12701)
<!-- CURSOR_AGENT_PR_BODY_BEGIN --> # Which Problems Are Solved Login V2 `ListUsers` lookups of the form username OR email (and email OR phone) hash-join the full `users14` / `users14_humans` tables, then filter. That cost grows with instance size (~40ms at 10k users, ~650ms at 200k). Top-level login-name equals already used an indexed join, but the same login-name filter inside an `OrQuery` fell back to the slow view-based exists query. # How the Problems Are Solved **TL;DR:** equality lookups on username, email, phone, or login name become `INNER JOIN (UNION of indexed ID seeks) AS matches`. Mixed / contains / name searches keep the old WHERE shape. ### Reviewer map | Read first | Why | | --- | --- | | `internal/query/user_login_equality.go` | Typed filters + extractor + UNION join | | `internal/query/user.go` (`prepareUsersQuery`, constructors) | Wiring; username/email/phone equals return `lowerEqualsFilter` | | `cmd/setup/76.go` + `cmd/setup/76/*.sql` | Concurrent indexes for existing deployments; drop invalid leftovers | | `internal/query/projection/user.go` | Same indexes in `Init()` for new databases | | `internal/query/user_login_name_exists_test.go` | SQL-shape coverage | | `internal/api/grpc/user/v2/integration_test/query_test.go` | Nested username AND/OR user results | ### Query rewrite - Extract one equals leaf, a flat `OrQuery` of those leaves, or the same nested under `AndQuery`. Remaining AND conjuncts stay as WHERE. Equality nested under `OrQuery` / `NotQuery` is left as WHERE. - Login-name equals is a UNION arm that reuses `login_names3_users` SQL, so it works inside OR as well as at the top level. - EQUALS seeks use `LOWER(col)` for the index, then recheck `col = value` so case-sensitive searches do not match extra rows. EQUALS_IGNORE_CASE is unchanged. - Empty OR arms are not skipped (that would change OR meaning); mixed empty-OR falls back to WHERE. Login UI already omits empty arms. ### Indexes - Setup 76: `CREATE INDEX CONCURRENTLY IF NOT EXISTS` on `users14_username_lower_idx` `(instance_id, LOWER(username))` and `users14_humans_phone_lower_idx` `(instance_id, LOWER(phone))`. Not wrapped in a transaction. - Before create, drop any leftover `pg_index.indisvalid = false` indexes of those names (failed/cancelled `CREATE INDEX CONCURRENTLY`). `IF NOT EXISTS` would otherwise skip rebuild. - `Init()` creates the same indexes without `CONCURRENTLY` (empty tables). Existing `users14_username` and email lower indexes are unchanged. # Additional Changes - Unit tests assert UNION SQL for username OR email, email OR phone, login name OR email, username OR username, nested `AndQuery(loginName, org)`, and nested `AndQuery(Or(...), org)`; that contains / display-name / mixed OR / nested `OrQuery(AndQuery(...), ...)` do not UNION; that email EQUALS rechecks the raw column; and that empty-OR is not extracted. - Setup 76 sqlmock coverage: missing `users14` skips work; valid/absent indexes only create; invalid leftover username index is dropped then both creates run. - `TestServer_ListUsers_UsernameAndOr` checks live results: org AND (alice OR bob) returns both users, sibling username AND is empty, nested `AndQuery(Or(...), org)` returns both. # Additional Context - Base branch: `main` - Extends the login-name equals rewrite from #12460 to username/email/phone OR trees used by Login V2 search. <!-- CURSOR_AGENT_PR_BODY_END --> <div><a href="https://cursor.com/agents/bc-1f043b1c-6168-45ca-9e9e-c18451994207?cursor_ref=pr_footer&cursor_cta=open_in_web"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-web-light.png"><img alt="Open in Web" width="114" height="28" src="https://cursor.com/assets/images/open-in-web-dark.png"></picture></a> <a href="https://cursor.com/background-agent?bcId=bc-1f043b1c-6168-45ca-9e9e-c18451994207&cursor_ref=pr_footer&cursor_cta=open_in_cursor"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-cursor-light.png"><img alt="Open in Cursor" width="131" height="28" src="https://cursor.com/assets/images/open-in-cursor-dark.png"></picture></a> </div> --------- Co-authored-by: Cursor Agent <[email protected]> Co-authored-by: Silvan <[email protected]>
This commit is contained in:
Silvan
co-authored by
Cursor Agent
Silvan
Cursor Agent
Silvan
parent
95cbe6a796
commit
684de2e691
+10
-133
@@ -747,7 +747,7 @@ func NewUserResourceOwnerSearchQuery(value string, comparison TextComparison) (S
|
||||
}
|
||||
|
||||
func NewUserUsernameSearchQuery(value string, comparison TextComparison) (SearchQuery, error) {
|
||||
return NewTextQuery(UserUsernameCol, value, comparison)
|
||||
return newLowerEqualsSearchQuery(UserUsernameCol, userTable, UserIDCol, value, comparison)
|
||||
}
|
||||
|
||||
func NewUserFirstNameSearchQuery(value string, comparison TextComparison) (SearchQuery, error) {
|
||||
@@ -767,11 +767,11 @@ func NewUserDisplayNameSearchQuery(value string, comparison TextComparison) (Sea
|
||||
}
|
||||
|
||||
func NewUserEmailSearchQuery(value string, comparison TextComparison) (SearchQuery, error) {
|
||||
return NewTextQuery(HumanEmailCol, value, comparison)
|
||||
return newLowerEqualsSearchQuery(HumanEmailCol, humanTable, HumanUserIDCol, value, comparison)
|
||||
}
|
||||
|
||||
func NewUserPhoneSearchQuery(value string, comparison TextComparison) (SearchQuery, error) {
|
||||
return NewTextQuery(HumanPhoneCol, value, comparison)
|
||||
return newLowerEqualsSearchQuery(HumanPhoneCol, humanTable, HumanUserIDCol, value, comparison)
|
||||
}
|
||||
|
||||
func NewUserVerifiedEmailSearchQuery(value string) (SearchQuery, error) {
|
||||
@@ -801,8 +801,8 @@ var userLoginNameMatchesQuery string
|
||||
var userLoginNameMatchesCaseSensitiveQuery string
|
||||
|
||||
// NewUserLoginNameExistsQuery filters users by login name.
|
||||
// Equals / EqualsIgnoreCase use a planner marker rewritten in prepareUsersQuery
|
||||
// into an indexed join on login_names3_users. Other comparisons use the view.
|
||||
// Equals / EqualsIgnoreCase are rewritten in prepareUsersQuery into an indexed
|
||||
// ID-seek. Other comparisons use the login_names3 view.
|
||||
func NewUserLoginNameExistsQuery(value string, comparison TextComparison) (SearchQuery, error) {
|
||||
if comparison == TextEquals || comparison == TextEqualsIgnoreCase {
|
||||
return newLoginNameEqualsFilter(value, comparison == TextEqualsIgnoreCase)
|
||||
@@ -842,126 +842,6 @@ func newLoginNameExistsViewQuery(value string, comparison TextComparison) (Searc
|
||||
)
|
||||
}
|
||||
|
||||
// loginNameEqualsFilter marks a login-name equality filter for prepareUsersQuery
|
||||
// to rewrite as an indexed join. Unextracted markers (e.g. inside OrQuery) fall
|
||||
// back to the view-based exists query.
|
||||
type loginNameEqualsFilter struct {
|
||||
username string
|
||||
domain string
|
||||
loginName string
|
||||
ignoreCase bool
|
||||
}
|
||||
|
||||
func newLoginNameEqualsFilter(value string, ignoreCase bool) (*loginNameEqualsFilter, error) {
|
||||
if ignoreCase {
|
||||
value = strings.ToLower(value)
|
||||
}
|
||||
username := value
|
||||
domainIndex := strings.LastIndex(value, "@")
|
||||
var domainSuffix string
|
||||
// split between the last @ (so ignore it if the login name ends with it)
|
||||
if domainIndex > 0 && domainIndex != len(value)-1 {
|
||||
domainSuffix = value[domainIndex+1:]
|
||||
username = value[:domainIndex]
|
||||
}
|
||||
return &loginNameEqualsFilter{
|
||||
username: username,
|
||||
domain: domainSuffix,
|
||||
loginName: value,
|
||||
ignoreCase: ignoreCase,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) comparison() TextComparison {
|
||||
if q.ignoreCase {
|
||||
return TextEqualsIgnoreCase
|
||||
}
|
||||
return TextEquals
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) fallback() SearchQuery {
|
||||
// Equals / EqualsIgnoreCase construction cannot fail for valid columns.
|
||||
fallback, _ := newLoginNameExistsViewQuery(q.loginName, q.comparison())
|
||||
return fallback
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
|
||||
return q.fallback().toQuery(query)
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) Col() Column {
|
||||
return UserIDCol
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) comp() sq.Sqlizer {
|
||||
return q.fallback().comp()
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) matchesArgs(instanceID string) []interface{} {
|
||||
return []interface{}{
|
||||
instanceID,
|
||||
instanceID,
|
||||
q.domain,
|
||||
instanceID,
|
||||
q.username,
|
||||
q.loginName,
|
||||
q.username,
|
||||
q.domain,
|
||||
q.loginName,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *loginNameEqualsFilter) joinMatches(instanceID string) sq.Sqlizer {
|
||||
subQuery := userLoginNameMatchesQuery
|
||||
if !q.ignoreCase {
|
||||
subQuery = userLoginNameMatchesCaseSensitiveQuery
|
||||
}
|
||||
return sq.Expr(
|
||||
"INNER JOIN ("+subQuery+") AS login_name_matches ON "+UserIDCol.identifier()+" = login_name_matches.user_id",
|
||||
q.matchesArgs(instanceID)...,
|
||||
)
|
||||
}
|
||||
|
||||
// extractLoginNameEqualsFilter extracts one login-name equals marker from
|
||||
// top-level or AndQuery filters. Markers inside OrQuery / NotQuery are kept.
|
||||
func extractLoginNameEqualsFilter(queries []SearchQuery) (filter *loginNameEqualsFilter, remaining []SearchQuery, ok bool) {
|
||||
remaining = make([]SearchQuery, 0, len(queries))
|
||||
var found *loginNameEqualsFilter
|
||||
|
||||
for _, qry := range queries {
|
||||
switch v := qry.(type) {
|
||||
case *loginNameEqualsFilter:
|
||||
if found != nil {
|
||||
return nil, queries, false
|
||||
}
|
||||
found = v
|
||||
case *AndQuery:
|
||||
inner, rest, extracted := extractLoginNameEqualsFilter(v.queries)
|
||||
if !extracted {
|
||||
remaining = append(remaining, qry)
|
||||
continue
|
||||
}
|
||||
if found != nil {
|
||||
return nil, queries, false
|
||||
}
|
||||
found = inner
|
||||
if len(rest) == 1 {
|
||||
remaining = append(remaining, rest[0])
|
||||
} else if len(rest) > 1 {
|
||||
andQuery, _ := NewAndQuery(rest...)
|
||||
remaining = append(remaining, andQuery)
|
||||
}
|
||||
default:
|
||||
remaining = append(remaining, qry)
|
||||
}
|
||||
}
|
||||
|
||||
if found == nil {
|
||||
return nil, queries, false
|
||||
}
|
||||
return found, remaining, true
|
||||
}
|
||||
|
||||
func (q *UserSearchQueries) hasMetadataFilter() bool {
|
||||
return searchQueriesHaveMetadataFilter(q.Queries)
|
||||
}
|
||||
@@ -1416,18 +1296,15 @@ func prepareUserUniqueQuery() (sq.SelectBuilder, func(*sql.Row) (bool, error)) {
|
||||
// It is not possible to pass more filters to the returned query, as they need to be applied in the sub-select.
|
||||
//
|
||||
// Metadata JOIN and DISTINCT are only applied when a metadata filter is present.
|
||||
// Login-name equals markers are rewritten into an indexed join when extractable.
|
||||
// Login-equality filters (username, email, phone, login name EQUALS /
|
||||
// EQUALS_IGNORE_CASE) become an indexed UNION of ID seeks.
|
||||
func (q *UserSearchQueries) prepareUsersQuery(ctx context.Context, permissionCheckV2 bool) (sq.SelectBuilder, func(*sql.Rows) (*Users, error)) {
|
||||
if q.SortingColumn.isZero() {
|
||||
q.SortingColumn = UserIDCol
|
||||
}
|
||||
|
||||
instanceID := authz.GetInstance(ctx).InstanceID()
|
||||
loginNameFilter, remainingFilters, loginNameExtracted := extractLoginNameEqualsFilter(q.Queries)
|
||||
filters := q.Queries
|
||||
if loginNameExtracted {
|
||||
filters = remainingFilters
|
||||
}
|
||||
loginEqualitySeeks, filters, loginEqualityExtracted := extractLoginEqualitySeeks(instanceID, q.Queries)
|
||||
needsMetadataJoin := q.hasMetadataFilter()
|
||||
|
||||
// start building the sub-select
|
||||
@@ -1469,8 +1346,8 @@ func (q *UserSearchQueries) prepareUsersQuery(ctx context.Context, permissionChe
|
||||
JoinClause(joinLoginNames).
|
||||
Where(sq.Eq{UserInstanceIDCol.identifier(): instanceID})
|
||||
|
||||
if loginNameExtracted {
|
||||
query = query.JoinClause(loginNameFilter.joinMatches(instanceID))
|
||||
if loginEqualityExtracted {
|
||||
query = query.JoinClause(joinLoginEqualitySeeks(loginEqualitySeeks))
|
||||
}
|
||||
|
||||
if needsMetadataJoin {
|
||||
|
||||
Reference in New Issue
Block a user