diff --git a/apps/docs/content/self-hosting/manage/production.mdx b/apps/docs/content/self-hosting/manage/production.mdx index f6ebb4f54..6d9e6bdcf 100644 --- a/apps/docs/content/self-hosting/manage/production.mdx +++ b/apps/docs/content/self-hosting/manage/production.mdx @@ -149,51 +149,13 @@ Database: //highlight-end ``` -You also might want to configure how [projections](/concepts/eventstore/implementation#projections) are computed. These are the default values: - -```yaml -# The Projections section defines the behavior for the scheduled and synchronous events projections. -Projections: - # Time interval between scheduled projections - RequeueEvery: 60s - # Time between retried database statements resulting from projected events - RetryFailedAfter: 1s - # Retried execution number of database statements resulting from projected events - MaxFailureCount: 5 - # Number of concurrent projection routines. Values of 0 and below are overwritten to 1 - ConcurrentInstances: 1 - # Limit of returned events per query - BulkLimit: 200 - # Only instance are projected, for which at least a projection relevant event exists withing the timeframe - # from HandleActiveInstances duration in the past until the projections current time - # Defaults to twice the RequeueEvery duration - HandleActiveInstances: 120s - # In the Customizations section, all settings from above can be overwritten for each specific projection - Customizations: - Projects: - BulkLimit: 2000 - # The Notifications projection is used for sending emails and SMS to users - Notifications: - # As notification projections don't result in database statements, retries don't have an effect - MaxFailureCount: 0 - # The NotificationsQuotas projection is used for calling quota webhooks - NotificationsQuotas: - # Delivery guarantee requirements are probably higher for quota webhooks - # Defaults to 45 days - HandleActiveInstances: 1080h - # As quota notification projections don't result in database statements, retries don't have an effect - MaxFailureCount: 0 - # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much. - RequeueEvery: 300s -``` - ### Manage your data When designing your backup strategy, it is worth knowing that [ZITADEL is event sourced](/concepts/eventstore/overview). That means, ZITADEL itself is able to recompute its -whole state from the records in the table eventstore.events. +whole state from the records in the table `eventstore.events2`. The timestamp of your last record in the events table defines up to which point in time ZITADEL can restore its state. diff --git a/apps/docs/content/self-hosting/manage/tuning.mdx b/apps/docs/content/self-hosting/manage/tuning.mdx new file mode 100644 index 000000000..02cfd5680 --- /dev/null +++ b/apps/docs/content/self-hosting/manage/tuning.mdx @@ -0,0 +1,253 @@ +--- +title: ZITADEL Performance Tuning +description: "Tune ZITADEL's eventstore and projections for large-scale, high-throughput deployments." +sidebar_label: Performance Tuning +--- + +With [ZITADEL being event sourced](/concepts/eventstore/overview), a lot of the performance throughput depends on how the events table is accessed. +The events table is an append-only table which will grow over the lifetime of a deployment. +ZITADEL uses a [CQRS architecture](/concepts/architecture/software), which results in concurrent reads and writes against this table. +The eventstore is used to make consistent real-time decisions when executing "Commands". (For example: is a user not locked just before creating a token) +Projections store computed state of an object, which allows us to "Query" data more efficiently. (For example: finding a user by their email). +Because projections depend on events being reduced by a spooler, they are eventually consistent and may fall behind the state in the eventstore. +We counter eventual consistency by pub-sub and triggers. + +The defaults for the configuration sections discussed below typically work well for small to medium sized deployments and do not need tuning. +We try to provide some general guidelines here. +However we recommend reaching out to support if your installation runs at a scale that requires tuning these values, +so our engineers can provide you with proper guidance based on your workload. + +## Eventstore + +
+ +Eventstore default configuration + +```yaml +Eventstore: + # Sets the maximum duration of transactions pushing events + PushTimeout: 15s #ZITADEL_EVENTSTORE_PUSHTIMEOUT + # Maximum amount of push retries in case of primary key violation on the sequence + MaxRetries: 5 #ZITADEL_EVENTSTORE_MAXRETRIES + # Autovacuum tunes PostgreSQL's autovacuum and autoanalyze behavior of the + # eventstore.events2 table. As the table grows, PostgreSQL's default, + # percentage-based scale factors delay vacuum and analyze runs more and + # more, which can degrade read performance on large instances. Enabling + # this replaces the scale factors with static thresholds, so vacuum and + # analyze keep running at a predictable cadence regardless of table size. + # Settings are (re-)applied to the table as a repeatable step during `zitadel setup`. + Autovacuum: + # If enabled, the thresholds below are applied to the eventstore.events2 table + # and the default, percentage-based autovacuum scale factors are disabled. + # If disabled, the table is reset to the cluster's default autovacuum settings. + Enabled: false # ZITADEL_EVENTSTORE_AUTOVACUUM_ENABLED + # Number of modified rows that triggers an autovacuum run, regardless of table size (applied in both autovacuum_vacuum_threshold and autovacuum_vacuum_insert_threshold settings). + # Must be greater than 10000. + VacuumThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_VACUUMTHRESHOLD + # Number of changed rows that triggers an autoanalyze run, regardless of table size. + # Must be greater than 10000. + AnalyzeThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_ANALYZETHRESHOLD +``` + +
+ +### Pushing events + +Pushing events happens when a resource is created or modified in a "Command". +The sequence in which events are pushed for an object is important. +A push is typically sub-millisecond if the database has sufficient resources and disk I/O available, +but may become slower when operating close to the resource limits during traffic spikes. +We employ row-level locking and unique constraint checks to ensure the sequence value of events is never duplicated on a single object. +Events for objects of different resources or distinct IDs happen concurrently. Some example cases: + +- Two processes updating the metadata of the same organization at the same time: this cannot happen concurrently. One of two requests will take longer, waiting for the first to complete. +- Two independent users logging-in at the same time: happens concurrently. There is no lock or unique constraint stopping the two independent transactions. + +The following options control the behavior of the pusher when the database is under strain: + +- `Eventstore.PushTimeout`: How long the push transaction is allowed to run before it's canceled. + This protects against dead-locks, network timeouts or queries that simply run too long. + This acts as a circuit breaker in the case transactions freeze up. + Once the timeout expires the transaction is aborted, + and the underlying database connection discarded, instead of reused through the connection pool. + The push may be retried, or an error is returned to the caller / API client. + - A smaller value allows for fail-fast and faster retries. + However a value that's too small may result in unneeded retries, + exhausting the DBs resources even more. + - A larger value may result in unnecessarily long request lockups when the outcome is failure anyway. +- `Eventstore.MaxRetries`: If somehow, despite row-level locking, there is a duplicate sequence on an object's events, + the push transaction fails with a distinct unique constraint error. + ZITADEL will retry the push with an incremented sequence based on events previously inserted by the other transaction. + Retry is aggressive and does not use throttling! + Many concurrent retries will lead to resource starvation on the DB and ZITADEL. + When the pusher is out of retries, the last error is returned to the caller / API client. + - A smaller value protects the database against resource starvation. + However, running out of retries means an API request will fail, which may require the API client to retry instead. + - A larger value may result in resource starvation of the database. + API requests will take as long as the retries to execute. + This may result in a lot of back-pressure, making things only worse. + +In other words, **do not touch** these values unless dealing with some exotic issues or instructed so by ZITADEL engineers. + +### Reading events + +Events are read by both the projection spoolers and during "Command" execution. +During command execution ZITADEL first reads events to determine the latest state of an object (for example "is a user not locked") +and a second time during a [push](#pushing-events), to establish the latest sequence of an object. +Because the eventstore is append-only, the table can grow to 100s of millions of rows on large production systems. +Keeping the table and index statistics up-to-date is paramount for PostgreSQL to choose the correct indexes. +When the table and index statistics are stale, PostgreSQL will use suboptimal plans and will become slow. +This has major performance impact on ZITADEL: pushes will take longer and spoolers will become slower, +resulting in larger eventual consistency gaps and higher resource consumption overall. + +PostgreSQL can [`ANALYZE`](https://www.postgresql.org/docs/current/sql-analyze.html) the table to rebuild those statistics, +keeping the eventstore reads performant. PostgreSQL uses [routine vacuuming](https://www.postgresql.org/docs/current/routine-vacuuming.html) +jobs to automate analyzing of all tables. +However, PostgreSQL defaults to a percentage-based row count for it to trigger the vacuum and analyze routines, +with a default value of 10% for analyze routines. +Please consult your database admin or hosting platform, as they may use a different value for `autovacuum_analyze_scale_factor`. + +Keeping the default is usually fine for smaller deployments, and tuning the routines may even reduce performance. +However, for medium to large systems and hyperscalers, the routine intervals will progressively become longer as the table grows, +to the point where they almost never run. For example: + +1. With 100k existing events, 10k new events will trigger the analyze routine. +2. With 10M existing events, 1M new events will trigger the analyze routine. +3. With 100M existing events, 10M new events will trigger the analyze routine. + +Cases 1 and 2 will probably not experience any noticeable performance impact. +However, case 3 will notice a significant slowdown of the eventstore. +We found the tipping point lying between 2M-3M new rows inserted on a 250M row eventstore table. + +ZITADEL provides options to switch to static routine thresholds instead, +solving the problem of the routine intervals going to infinity. +- `Eventstore.Autovacuum.Enabled`: switches on the ZITADEL customization of threshold based routines. + This option defaults to `false`, as enabling this with the wrong threshold value will have adverse effects on performance. + Small systems don't need this as the ratio-based routines are actually better for small tables. +- `Eventstore.Autovacuum.VacuumThreshold`: Number of modified rows that triggers an autovacuum run. +- `Eventstore.Autovacuum.AnalyzeThreshold`: Number of changed rows that triggers an autoanalyze run. + +Both thresholds have the same performance consequences: + +- Pick a value that's too large and the routines never run, leading to stale planner statistics and bad performance. +- Pick a value that's too small and the routines will run too often, consuming valuable database resources, + even risking routines running back-to-back continuously. + +The correct value of these settings highly depends on the amount of events pushed over a certain timeframe, +not the current amount of events present in the table. +The amount of events pushed then again depends on the amount and type of requests served. For example: + +- Service account authentication pushes 2 events per request +- Human user authentication may push 5-10 events per successful flow, + depending on the amount of factors used and requested token types. + +As a rough target, aim for the routines to trigger every few minutes to a low tens-of-minutes at your expected peak traffic. +This is mainly relevant for Enterprise and Mega Enterprise scale deployments, where the eventstore table is large enough that stale +statistics or visibility maps have a noticeable performance impact within minutes. Medium and Homelab deployments naturally land on +longer intervals with the same threshold, and that's fine: their eventstore tables are much smaller, so infrequent maintenance doesn't +let statistics drift far enough to matter. +The following table provides example values for various deployment sizes: + +| Scale Profile | Events / Sec during peak traffic | Recommended Vacuum Threshold | Expected Trigger Frequency | Why This Setting? | +| :---- | :---- | :---- | :---- | :---- | +| **Homelab / Proof of Concept** | 0-10 | Keep switched off / use ratio based. | Every few hours | Keeps things snappy on weak CPUs without letting statistics get stale. | +| **Medium / Growth** | 10-100 | 50,000 | Every 8 to 80 minutes | Perfect balance of maintaining the visibility map for index-only scans without I/O pressure. | +| **Enterprise / High Scale** | 1,000 | 500,000 | Every 8 minutes | Prevents CPU thrashing while keeping index-only scans functional. | +| **Mega Enterprise** | 10,000 | 2,000,000 | Every 3.3 minutes | Keeps a massive 64-core PG instance highly optimized without queuing up concurrent autovacuum processes. | + +## Projections + +You also might want to configure how [projections](/concepts/eventstore/implementation#projections) are computed. +Real-time consistency is primarily driven by pub/sub: as soon as an event is pushed, subscribed projections are notified and reduce it immediately. +The settings below mostly tune a *secondary*, scheduled catch-up mechanism that acts as a safety net for cases where a pub/sub notification +is missed, for example due to a network hiccup or a restart. Most of these settings therefore affect how quickly a projection recovers +from a missed notification, not how fast a healthy system reacts to new events. + +
+ +Projections default configuration + +```yaml +# The Projections section defines the behavior for the scheduled and synchronous events projections. +Projections: + # The maximum duration a single projection transaction remains open while + # folding a batch of events into the projection table. + # 0 means no timeout is applied: the transaction runs until its batch (bounded by BulkLimit) finishes. + TransactionDuration: 1m # ZITADEL_PROJECTIONS_TRANSACTIONDURATION + # Time interval between scheduled catch-up sweeps + RequeueEvery: 60s # ZITADEL_PROJECTIONS_REQUEUEEVERY + # Time between retried database statements resulting from projected events + RetryFailedAfter: 1s # ZITADEL_PROJECTIONS_RETRYFAILEDAFTER + # Retried execution number of database statements resulting from projected events + MaxFailureCount: 5 # ZITADEL_PROJECTIONS_MAXFAILURECOUNT + # Limit of returned events per query + BulkLimit: 200 # ZITADEL_PROJECTIONS_BULKLIMIT + # Only instances are considered by the scheduled sweep, for which at least one API request was served within the + # timeframe from HandleActiveInstances duration in the past until now. + # If set to 0 (default), every instance ever seen is always considered active. + HandleActiveInstances: 0s # ZITADEL_PROJECTIONS_HANDLEACTIVEINSTANCES + # Maximum amount of instances cached as active. If set to 0 (default), every instance is always considered active. + MaxActiveInstances: 0 # ZITADEL_PROJECTIONS_MAXACTIVEINSTANCES + # Limits the amount of concurrently running projection triggers, shared across all projections and instances. + # If set to 0, 1/3 of Database.MaxOpenConns is used. Must be lower than Database.MaxOpenConns. + MaxParallelTriggers: 0 # ZITADEL_PROJECTIONS_MAXPARALLELTRIGGERS + # In the Customizations section, all settings from above (except HandleActiveInstances, MaxActiveInstances and + # MaxParallelTriggers, which are global) can be overwritten for each specific projection + Customizations: + # The Notifications projection prepares emails and SMS messages sent to users + Notifications: + MaxFailureCount: 10 # ZITADEL_PROJECTIONS_CUSTOMIZATIONS_NOTIFICATIONS_MAXFAILURECOUNT + # The NotificationsQuotas projection is used for calling quota webhooks + NotificationsQuotas: + MaxFailureCount: 10 # ZITADEL_PROJECTIONS_CUSTOMIZATIONS_NOTIFICATIONSQUOTAS_MAXFAILURECOUNT + # Quota notifications are not so time critical. Setting RequeueEvery every five minutes doesn't annoy the db too much. + RequeueEvery: 300s # ZITADEL_PROJECTIONS_CUSTOMIZATIONS_NOTIFICATIONSQUOTAS_REQUEUEEVERY + # Sending emails can take longer than 500ms + TransactionDuration: 5s # ZITADEL_PROJECTIONS_CUSTOMIZATIONS_NOTIFICATIONQUOTAS_TRANSACTIONDURATION +``` + +
+ +- `Projections.RequeueEvery`: Interval between scheduled catch-up sweeps. + - Too low: sweeps run needlessly often even though pub/sub has already kept every projection up to date, adding avoidable database load. + - Too high: if a pub/sub notification is ever missed, a projection can silently lag for up to this long before the next sweep corrects it. + Some projections override this to a much longer interval when their events are inherently low-urgency, for example + `Customizations.NotificationsQuotas.RequeueEvery: 300s` (5 minutes), since a delayed quota webhook doesn't justify sweeping every minute. +- `Projections.RetryFailedAfter` and `Projections.MaxFailureCount`: Control retries of an individual **statement** derived from + reducing one event, not of the whole projection or instance. + When a statement fails, ZITADEL waits `RetryFailedAfter` and retries; while retrying, it blocks further processing of the other + instances queued in the same catch-up batch, so a single stuck event can delay unrelated tenants. + Once a statement has failed `MaxFailureCount` times, it's recorded as permanently skipped and never retried again, letting the + projection move past it instead of getting stuck forever. + - Too small `RetryFailedAfter`: hammers the database with retries with no meaningful backoff. + - Too large `RetryFailedAfter`: a brief, transient error (deadlock, network blip) needlessly stalls the rest of that batch. + - Too small `MaxFailureCount`: a recoverable failure (e.g. a brief DB hiccup) may be abandoned before it would have succeeded. + - Too large `MaxFailureCount`: a truly broken event keeps retrying, and blocking its batch, for longer before being given up on. + As not every projection results in database statements, some override `MaxFailureCount`, for example `Customizations.Notifications` + and `Customizations.NotificationsQuotas` set it to `10`, since notification delivery only produces side effects (sending an + email, SMS, or webhook) rather than statements that could deadlock or violate a constraint. +- `Projections.BulkLimit`: Number of events fetched, and processed within one transaction, per catch-up iteration. + - Larger values: fewer round-trips when catching up a large backlog, but each transaction (and the locks it holds) runs longer. + - Smaller values: shorter transactions and more frequent progress checkpoints, at the cost of more round-trip overhead to consume the same backlog. +- `Projections.TransactionDuration`: Maximum time a single projection transaction may stay open while folding a batch of events. + - Too short: the transaction can be cut off mid-batch before it commits, wasting the work done so far and requiring that batch to be reprocessed. + - Too long / unbounded (`0`): a single slow or stuck batch can hold its transaction, and the row locks it acquired, open indefinitely. +- `Projections.HandleActiveInstances` and `Projections.MaxActiveInstances`: Limit which instances (ZITADEL tenants) the *scheduled* + sweep bothers with. An instance is considered "active" if it has served API traffic within the last `HandleActiveInstances` + duration — this is a cache of recently-seen tenants, not a scan of the events table, and it has no effect on the real-time + pub/sub trigger. + - The defaults (`0`/`0`) mean every instance ever seen stays "active" indefinitely and the cache never evicts. This is fine + for most deployments, but on an instance with heavy tenant churn, the periodic sweep keeps visiting every tenant it's + ever seen, including long-decommissioned ones. + - Setting `HandleActiveInstances` too short can drop a legitimately idle-but-still-in-use tenant out of the sweep list early; + this only matters if that tenant also happens to miss a pub/sub notification while considered inactive. + - `MaxActiveInstances` caps how many tenant IDs are cached (evicting the least-recently-used), bounding memory use on very + large multi-tenant deployments. +- `Projections.MaxParallelTriggers`: Size of the shared worker pool that processes projection triggers, across *all* projections + and *all* instances combined — not a per-projection setting. + - Must stay below `Database.MaxOpenConns`; a value of `0` auto-sizes it to a third of `Database.MaxOpenConns`. + - Too low: projection catch-up throughput becomes a bottleneck under load, widening eventual-consistency gaps. + - Too high: starves database connections needed for pushes, queries, and notifications. +- `Projections.Customizations`: Overrides `BulkLimit`, `MaxFailureCount`, `RequeueEvery`, `RetryFailedAfter`, and + `TransactionDuration` per named projection. `HandleActiveInstances`, `MaxActiveInstances`, and `MaxParallelTriggers` are + global settings and cannot be overridden per projection. diff --git a/apps/docs/lib/sidebar-data.ts b/apps/docs/lib/sidebar-data.ts index 51ac763e4..b7d529493 100644 --- a/apps/docs/lib/sidebar-data.ts +++ b/apps/docs/lib/sidebar-data.ts @@ -813,6 +813,7 @@ export const guidesSidebar: readonly SidebarItem[] = [ "self-hosting/manage/upgrade-v3-to-v4", "self-hosting/manage/database/database", "self-hosting/manage/cache", + "self-hosting/manage/tuning", ], }, diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index d7b151ce9..a4009463f 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -1066,6 +1066,24 @@ Eventstore: PushTimeout: 15s #ZITADEL_EVENTSTORE_PUSHTIMEOUT # Maximum amount of push retries in case of primary key violation on the sequence MaxRetries: 5 #ZITADEL_EVENTSTORE_MAXRETRIES + # Autovacuum tunes PostgreSQL's autovacuum and autoanalyze behavior of the + # eventstore.events2 table. As the table grows, PostgreSQL's default, + # percentage-based scale factors delay vacuum and analyze runs more and + # more, which can degrade read performance on large instances. Enabling + # this replaces the scale factors with static thresholds, so vacuum and + # analyze keep running at a predictable cadence regardless of table size. + # Settings are (re-)applied to the table as a repeatable step during `zitadel setup`. + Autovacuum: + # If enabled, the thresholds below are applied to the eventstore.events2 table + # and the default, percentage-based autovacuum scale factors are disabled. + # If disabled, the table is reset to the cluster's default autovacuum settings. + Enabled: false # ZITADEL_EVENTSTORE_AUTOVACUUM_ENABLED + # Number of modified rows that triggers an autovacuum run, regardless of table size (applied in both autovacuum_vacuum_threshold and autovacuum_vacuum_insert_threshold settings). + # Must be greater than 10000. + VacuumThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_VACUUMTHRESHOLD + # Number of changed rows that triggers an autoanalyze run, regardless of table size. + # Must be greater than 10000. + AnalyzeThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_ANALYZETHRESHOLD HTTPClient: # Sets the limit of the response body size in bytes, which will be processed by the HTTP client. diff --git a/cmd/setup/eventstore_autovacuum.go b/cmd/setup/eventstore_autovacuum.go new file mode 100644 index 000000000..759a2a3ff --- /dev/null +++ b/cmd/setup/eventstore_autovacuum.go @@ -0,0 +1,75 @@ +package setup + +import ( + "context" + "fmt" + + "github.com/zitadel/zitadel/internal/database" + "github.com/zitadel/zitadel/internal/eventstore" +) + +// eventstoreAutovacuum is a repeatable migration step that tunes PostgreSQL's +// autovacuum and autoanalyze behavior of the eventstore.events2 table, +// as configured by Eventstore.Autovacuum. +// +// It repeats whenever the configuration changed since the last run. +type eventstoreAutovacuum struct { + dbClient *database.DB + + Enabled bool `json:"enabled"` + VacuumThreshold uint32 `json:"vacuumThreshold"` + AnalyzeThreshold uint32 `json:"analyzeThreshold"` +} + +const eventstoreAutovacuumResetStmt = `ALTER TABLE eventstore.events2 RESET ( + autovacuum_vacuum_scale_factor, + autovacuum_analyze_scale_factor, + autovacuum_vacuum_insert_scale_factor, + autovacuum_vacuum_insert_threshold, + autovacuum_analyze_threshold, + autovacuum_vacuum_threshold +)` + +// minAutovacuumThreshold guards against configuring vacuum/analyze runs so +// frequently that they thrash the database with constant autovacuum activity. +const minAutovacuumThreshold = 10000 + +func (mig *eventstoreAutovacuum) Execute(ctx context.Context, _ eventstore.Event) error { + if !mig.Enabled { + _, err := mig.dbClient.ExecContext(ctx, eventstoreAutovacuumResetStmt) + return err + } + if mig.VacuumThreshold <= minAutovacuumThreshold || mig.AnalyzeThreshold <= minAutovacuumThreshold { + return fmt.Errorf("%s: VacuumThreshold and AnalyzeThreshold must be greater than %d", mig, minAutovacuumThreshold) + } + + // autovacuum_vacuum_threshold covers rows changed by updates and deletes, while + // autovacuum_vacuum_insert_threshold only covers inserts. events2 is append-only, but + // VacuumThreshold is applied to both, so a manual UPDATE/DELETE does not fall back to + // the effectively disabled default scale factor. + stmt := fmt.Sprintf(`ALTER TABLE eventstore.events2 SET ( + autovacuum_vacuum_scale_factor = 0.0, + autovacuum_analyze_scale_factor = 0.0, + autovacuum_vacuum_insert_scale_factor = 0.0, + autovacuum_vacuum_insert_threshold = %d, + autovacuum_analyze_threshold = %d, + autovacuum_vacuum_threshold = %d +)`, mig.VacuumThreshold, mig.AnalyzeThreshold, mig.VacuumThreshold) + _, err := mig.dbClient.ExecContext(ctx, stmt) + return err +} + +func (mig *eventstoreAutovacuum) String() string { + return "repeatable_eventstore_autovacuum" +} + +// Check implements [migration.RepeatableMigration]. +func (mig *eventstoreAutovacuum) Check(lastRun map[string]interface{}) bool { + enabled, _ := lastRun["enabled"].(bool) + vacuumThreshold, _ := lastRun["vacuumThreshold"].(float64) + analyzeThreshold, _ := lastRun["analyzeThreshold"].(float64) + + return enabled != mig.Enabled || + uint32(vacuumThreshold) != mig.VacuumThreshold || + uint32(analyzeThreshold) != mig.AnalyzeThreshold +} diff --git a/cmd/setup/eventstore_autovacuum_test.go b/cmd/setup/eventstore_autovacuum_test.go new file mode 100644 index 000000000..e60c79f97 --- /dev/null +++ b/cmd/setup/eventstore_autovacuum_test.go @@ -0,0 +1,231 @@ +package setup + +import ( + "context" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zitadel/zitadel/internal/database" +) + +func Test_eventstoreAutovacuum_Execute(t *testing.T) { + type fields struct { + Enabled bool + VacuumThreshold uint32 + AnalyzeThreshold uint32 + } + tests := []struct { + name string + fields fields + expects func(sqlmock.Sqlmock) + wantErr bool + }{ + { + name: "enabled, sets thresholds", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: 25000, + }, + expects: func(mock sqlmock.Sqlmock) { + mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET ( + autovacuum_vacuum_scale_factor = 0.0, + autovacuum_analyze_scale_factor = 0.0, + autovacuum_vacuum_insert_scale_factor = 0.0, + autovacuum_vacuum_insert_threshold = 50000, + autovacuum_analyze_threshold = 25000, + autovacuum_vacuum_threshold = 50000 +)`)).WillReturnResult(sqlmock.NewResult(0, 0)) + }, + }, + { + name: "enabled, db error", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: 50000, + }, + expects: func(mock sqlmock.Sqlmock) { + mock.ExpectExec(".*").WillReturnError(assert.AnError) + }, + wantErr: true, + }, + { + name: "enabled, vacuum threshold at minimum boundary fails", + fields: fields{ + Enabled: true, + VacuumThreshold: minAutovacuumThreshold, + AnalyzeThreshold: 50000, + }, + expects: func(sqlmock.Sqlmock) {}, + wantErr: true, + }, + { + name: "enabled, analyze threshold at minimum boundary fails", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: minAutovacuumThreshold, + }, + expects: func(sqlmock.Sqlmock) {}, + wantErr: true, + }, + { + name: "enabled, thresholds just above minimum succeed", + fields: fields{ + Enabled: true, + VacuumThreshold: minAutovacuumThreshold + 1, + AnalyzeThreshold: minAutovacuumThreshold + 1, + }, + expects: func(mock sqlmock.Sqlmock) { + mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET ( + autovacuum_vacuum_scale_factor = 0.0, + autovacuum_analyze_scale_factor = 0.0, + autovacuum_vacuum_insert_scale_factor = 0.0, + autovacuum_vacuum_insert_threshold = 10001, + autovacuum_analyze_threshold = 10001, + autovacuum_vacuum_threshold = 10001 +)`)).WillReturnResult(sqlmock.NewResult(0, 0)) + }, + }, + { + name: "disabled, resets to defaults", + fields: fields{ + Enabled: false, + }, + expects: func(mock sqlmock.Sqlmock) { + mock.ExpectExec(regexp.QuoteMeta(eventstoreAutovacuumResetStmt)). + WillReturnResult(sqlmock.NewResult(0, 0)) + }, + }, + { + name: "disabled, db error", + fields: fields{ + Enabled: false, + }, + expects: func(mock sqlmock.Sqlmock) { + mock.ExpectExec(".*").WillReturnError(assert.AnError) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { + require.NoError(t, mock.ExpectationsWereMet()) + }() + defer db.Close() + tt.expects(mock) + + mig := &eventstoreAutovacuum{ + dbClient: &database.DB{DB: db}, + Enabled: tt.fields.Enabled, + VacuumThreshold: tt.fields.VacuumThreshold, + AnalyzeThreshold: tt.fields.AnalyzeThreshold, + } + err = mig.Execute(context.Background(), nil) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } +} + +func Test_eventstoreAutovacuum_Check(t *testing.T) { + type fields struct { + Enabled bool + VacuumThreshold uint32 + AnalyzeThreshold uint32 + } + tests := []struct { + name string + fields fields + lastRun map[string]any + want bool + }{ + { + name: "no previous run", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: 50000, + }, + lastRun: nil, + want: true, + }, + { + name: "unchanged", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: 25000, + }, + lastRun: map[string]any{ + "enabled": true, + "vacuumThreshold": float64(50000), + "analyzeThreshold": float64(25000), + }, + want: false, + }, + { + name: "enabled changed", + fields: fields{ + Enabled: false, + VacuumThreshold: 50000, + AnalyzeThreshold: 50000, + }, + lastRun: map[string]any{ + "enabled": true, + "vacuumThreshold": float64(50000), + "analyzeThreshold": float64(50000), + }, + want: true, + }, + { + name: "vacuum threshold changed", + fields: fields{ + Enabled: true, + VacuumThreshold: 60000, + AnalyzeThreshold: 50000, + }, + lastRun: map[string]any{ + "enabled": true, + "vacuumThreshold": float64(50000), + "analyzeThreshold": float64(50000), + }, + want: true, + }, + { + name: "analyze threshold changed", + fields: fields{ + Enabled: true, + VacuumThreshold: 50000, + AnalyzeThreshold: 60000, + }, + lastRun: map[string]any{ + "enabled": true, + "vacuumThreshold": float64(50000), + "analyzeThreshold": float64(50000), + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mig := &eventstoreAutovacuum{ + Enabled: tt.fields.Enabled, + VacuumThreshold: tt.fields.VacuumThreshold, + AnalyzeThreshold: tt.fields.AnalyzeThreshold, + } + assert.Equal(t, tt.want, mig.Check(tt.lastRun)) + }) + } +} diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index 3e93eb201..d2831a581 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -354,6 +354,12 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) &RiverMigrateRepeatable{ client: dbClient, }, + &eventstoreAutovacuum{ + dbClient: dbClient, + Enabled: config.Eventstore.Autovacuum.Enabled, + VacuumThreshold: config.Eventstore.Autovacuum.VacuumThreshold, + AnalyzeThreshold: config.Eventstore.Autovacuum.AnalyzeThreshold, + }, } repeatableSteps = append(repeatableSteps, triggerSteps(dbClient)...) diff --git a/internal/eventstore/config.go b/internal/eventstore/config.go index 2e71dedf7..cc5280cb6 100644 --- a/internal/eventstore/config.go +++ b/internal/eventstore/config.go @@ -12,4 +12,34 @@ type Config struct { Querier Querier Searcher Searcher Queue ExecutionQueue + + // Autovacuum tunes PostgreSQL's autovacuum and autoanalyze behavior of the + // events2 table. + Autovacuum AutovacuumConfig +} + +// AutovacuumConfig configures table-level autovacuum and autoanalyze tuning +// for the eventstore's events2 table. +// +// PostgreSQL's default autovacuum uses a percentage-based scale factor, so as +// events2 grows, ever more rows need to change before a vacuum or analyze is +// triggered. This lets the table's visibility map and the query planner's +// statistics grow stale, which degrades read performance on large instances. +// Enabling this config replaces the scale factors with static, row-count +// based thresholds, so vacuum and analyze keep running at a predictable +// cadence regardless of table size. +type AutovacuumConfig struct { + // Enabled applies the thresholds below to the events2 table and disables + // the percentage-based autovacuum scale factors. When disabled, the table + // is reset to the cluster's default autovacuum settings. + Enabled bool + // VacuumThreshold is the number of modified rows that triggers an autovacuum run, + // regardless of table size (applied in both autovacuum_vacuum_threshold and + // autovacuum_vacuum_insert_threshold settings). + // Must be greater than 10000 to avoid thrashing the database with constant autovacuum activity. + VacuumThreshold uint32 + // AnalyzeThreshold is the number of changed rows that triggers an + // autoanalyze run, regardless of table size. Must be greater than 10000 + // to avoid thrashing the database with constant autoanalyze activity. + AnalyzeThreshold uint32 }