mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-18 05:52:51 +00:00
# Which Problems Are Solved
In Zitadel's append-only event-sourced architecture, the
`eventstore.events2` table grows indefinitely. PostgreSQL's default
autovacuum uses a percentage-based scale factor, so as the table grows,
the number of changed rows required to trigger a `VACUUM` or `ANALYZE`
drifts towards infinity. Without regular vacuums, the table's Visibility
Map becomes stale, disabling fast Index-Only Scans and forcing expensive
heap reads. Without regular analyzes, query planner statistics become
stale, leading to suboptimal execution plans.
This causes eventstore operations to progressively degrade as `events2`
grows, even without CPU, memory, or I/O saturation. A manual `VACUUM
ANALYZE` immediately restores performance, confirming the root cause.
- If `Eventstore.Autovacuum` is left at its default, `events2` keeps
using PostgreSQL's default, percentage-based autovacuum/autoanalyze
scale factors, which become impractically infrequent on large tables.
- There was previously no supported way to apply static, table-level
autovacuum tuning to `events2` through Zitadel's own configuration/setup
process.
# How the Problems Are Solved
- Added an `Eventstore.Autovacuum` runtime configuration block to
`cmd/defaults.yaml` (disabled by default):
```yaml
Eventstore:
Autovacuum:
Enabled: false # ZITADEL_EVENTSTORE_AUTOVACUUM_ENABLED
VacuumThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_VACUUMTHRESHOLD
AnalyzeThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_ANALYZETHRESHOLD
```
- Added a repeatable `zitadel setup` migration step
(`cmd/setup/eventstore_autovacuum.go`) that:
- When `Enabled: true`, disables the percentage-based
`autovacuum_vacuum_scale_factor`, `autovacuum_analyze_scale_factor`, and
`autovacuum_vacuum_insert_scale_factor` on `eventstore.events2`, and
applies static thresholds (`autovacuum_vacuum_insert_threshold`,
`autovacuum_vacuum_threshold`, `autovacuum_analyze_threshold`) from the
config instead.
- When `Enabled: false`, resets those storage parameters on
`eventstore.events2` back to the cluster defaults.
- Implements `Repeatable.Check()` so the step only re-runs when the
configuration actually changed since the last `zitadel setup` run.
- Added documentation in the new "Performance tuning" page.
# Additional Changes
- Move the projection documentation into performance tuning page
- Expand and update the projection documentation to the latest state in
zitadel. (contained some stale information)
# Additional Context
Several other issues describe read-performance symptoms consistent with
this same root cause (stale `events2` visibility map / planner
statistics at scale, without resource saturation). Since this PR
addresses the shared root cause:
- Closes #12448
- Closes #10754
- Closes #10260
- Closes #8585
- Closes #9239
---
_Generated by [Claude
Code](https://claude.ai/code/session_01HoKvEY7niCVgLCz7CajBwW)_
---------
Co-authored-by: Claude <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: Silvan <[email protected]>
232 lines
5.4 KiB
Go
232 lines
5.4 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"testing"
|
|
|
|
"github.com/DATA-DOG/go-sqlmock"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/zitadel/zitadel/internal/database"
|
|
)
|
|
|
|
func Test_eventstoreAutovacuum_Execute(t *testing.T) {
|
|
type fields struct {
|
|
Enabled bool
|
|
VacuumThreshold uint32
|
|
AnalyzeThreshold uint32
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
expects func(sqlmock.Sqlmock)
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "enabled, sets thresholds",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 25000,
|
|
},
|
|
expects: func(mock sqlmock.Sqlmock) {
|
|
mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET (
|
|
autovacuum_vacuum_scale_factor = 0.0,
|
|
autovacuum_analyze_scale_factor = 0.0,
|
|
autovacuum_vacuum_insert_scale_factor = 0.0,
|
|
autovacuum_vacuum_insert_threshold = 50000,
|
|
autovacuum_analyze_threshold = 25000,
|
|
autovacuum_vacuum_threshold = 50000
|
|
)`)).WillReturnResult(sqlmock.NewResult(0, 0))
|
|
},
|
|
},
|
|
{
|
|
name: "enabled, db error",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 50000,
|
|
},
|
|
expects: func(mock sqlmock.Sqlmock) {
|
|
mock.ExpectExec(".*").WillReturnError(assert.AnError)
|
|
},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "enabled, vacuum threshold at minimum boundary fails",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: minAutovacuumThreshold,
|
|
AnalyzeThreshold: 50000,
|
|
},
|
|
expects: func(sqlmock.Sqlmock) {},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "enabled, analyze threshold at minimum boundary fails",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: minAutovacuumThreshold,
|
|
},
|
|
expects: func(sqlmock.Sqlmock) {},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "enabled, thresholds just above minimum succeed",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: minAutovacuumThreshold + 1,
|
|
AnalyzeThreshold: minAutovacuumThreshold + 1,
|
|
},
|
|
expects: func(mock sqlmock.Sqlmock) {
|
|
mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET (
|
|
autovacuum_vacuum_scale_factor = 0.0,
|
|
autovacuum_analyze_scale_factor = 0.0,
|
|
autovacuum_vacuum_insert_scale_factor = 0.0,
|
|
autovacuum_vacuum_insert_threshold = 10001,
|
|
autovacuum_analyze_threshold = 10001,
|
|
autovacuum_vacuum_threshold = 10001
|
|
)`)).WillReturnResult(sqlmock.NewResult(0, 0))
|
|
},
|
|
},
|
|
{
|
|
name: "disabled, resets to defaults",
|
|
fields: fields{
|
|
Enabled: false,
|
|
},
|
|
expects: func(mock sqlmock.Sqlmock) {
|
|
mock.ExpectExec(regexp.QuoteMeta(eventstoreAutovacuumResetStmt)).
|
|
WillReturnResult(sqlmock.NewResult(0, 0))
|
|
},
|
|
},
|
|
{
|
|
name: "disabled, db error",
|
|
fields: fields{
|
|
Enabled: false,
|
|
},
|
|
expects: func(mock sqlmock.Sqlmock) {
|
|
mock.ExpectExec(".*").WillReturnError(assert.AnError)
|
|
},
|
|
wantErr: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
db, mock, err := sqlmock.New()
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
require.NoError(t, mock.ExpectationsWereMet())
|
|
}()
|
|
defer db.Close()
|
|
tt.expects(mock)
|
|
|
|
mig := &eventstoreAutovacuum{
|
|
dbClient: &database.DB{DB: db},
|
|
Enabled: tt.fields.Enabled,
|
|
VacuumThreshold: tt.fields.VacuumThreshold,
|
|
AnalyzeThreshold: tt.fields.AnalyzeThreshold,
|
|
}
|
|
err = mig.Execute(context.Background(), nil)
|
|
if tt.wantErr {
|
|
assert.Error(t, err)
|
|
return
|
|
}
|
|
assert.NoError(t, err)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_eventstoreAutovacuum_Check(t *testing.T) {
|
|
type fields struct {
|
|
Enabled bool
|
|
VacuumThreshold uint32
|
|
AnalyzeThreshold uint32
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
lastRun map[string]any
|
|
want bool
|
|
}{
|
|
{
|
|
name: "no previous run",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 50000,
|
|
},
|
|
lastRun: nil,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "unchanged",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 25000,
|
|
},
|
|
lastRun: map[string]any{
|
|
"enabled": true,
|
|
"vacuumThreshold": float64(50000),
|
|
"analyzeThreshold": float64(25000),
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "enabled changed",
|
|
fields: fields{
|
|
Enabled: false,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 50000,
|
|
},
|
|
lastRun: map[string]any{
|
|
"enabled": true,
|
|
"vacuumThreshold": float64(50000),
|
|
"analyzeThreshold": float64(50000),
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "vacuum threshold changed",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 60000,
|
|
AnalyzeThreshold: 50000,
|
|
},
|
|
lastRun: map[string]any{
|
|
"enabled": true,
|
|
"vacuumThreshold": float64(50000),
|
|
"analyzeThreshold": float64(50000),
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "analyze threshold changed",
|
|
fields: fields{
|
|
Enabled: true,
|
|
VacuumThreshold: 50000,
|
|
AnalyzeThreshold: 60000,
|
|
},
|
|
lastRun: map[string]any{
|
|
"enabled": true,
|
|
"vacuumThreshold": float64(50000),
|
|
"analyzeThreshold": float64(50000),
|
|
},
|
|
want: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
mig := &eventstoreAutovacuum{
|
|
Enabled: tt.fields.Enabled,
|
|
VacuumThreshold: tt.fields.VacuumThreshold,
|
|
AnalyzeThreshold: tt.fields.AnalyzeThreshold,
|
|
}
|
|
assert.Equal(t, tt.want, mig.Check(tt.lastRun))
|
|
})
|
|
}
|
|
}
|