Files
zitadel/cmd/setup/eventstore_autovacuum_test.go
75bc058bce feat(eventstore): autovacuum tuning for events2 table (#12449)
# Which Problems Are Solved

In Zitadel's append-only event-sourced architecture, the
`eventstore.events2` table grows indefinitely. PostgreSQL's default
autovacuum uses a percentage-based scale factor, so as the table grows,
the number of changed rows required to trigger a `VACUUM` or `ANALYZE`
drifts towards infinity. Without regular vacuums, the table's Visibility
Map becomes stale, disabling fast Index-Only Scans and forcing expensive
heap reads. Without regular analyzes, query planner statistics become
stale, leading to suboptimal execution plans.

This causes eventstore operations to progressively degrade as `events2`
grows, even without CPU, memory, or I/O saturation. A manual `VACUUM
ANALYZE` immediately restores performance, confirming the root cause.

- If `Eventstore.Autovacuum` is left at its default, `events2` keeps
using PostgreSQL's default, percentage-based autovacuum/autoanalyze
scale factors, which become impractically infrequent on large tables.
- There was previously no supported way to apply static, table-level
autovacuum tuning to `events2` through Zitadel's own configuration/setup
process.

# How the Problems Are Solved

- Added an `Eventstore.Autovacuum` runtime configuration block to
`cmd/defaults.yaml` (disabled by default):
  ```yaml
  Eventstore:
    Autovacuum:
      Enabled: false # ZITADEL_EVENTSTORE_AUTOVACUUM_ENABLED
VacuumThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_VACUUMTHRESHOLD
AnalyzeThreshold: 50000 # ZITADEL_EVENTSTORE_AUTOVACUUM_ANALYZETHRESHOLD
  ```
- Added a repeatable `zitadel setup` migration step
(`cmd/setup/eventstore_autovacuum.go`) that:
- When `Enabled: true`, disables the percentage-based
`autovacuum_vacuum_scale_factor`, `autovacuum_analyze_scale_factor`, and
`autovacuum_vacuum_insert_scale_factor` on `eventstore.events2`, and
applies static thresholds (`autovacuum_vacuum_insert_threshold`,
`autovacuum_vacuum_threshold`, `autovacuum_analyze_threshold`) from the
config instead.
- When `Enabled: false`, resets those storage parameters on
`eventstore.events2` back to the cluster defaults.
- Implements `Repeatable.Check()` so the step only re-runs when the
configuration actually changed since the last `zitadel setup` run.
  - Added documentation in the new "Performance tuning" page.
 
# Additional Changes

- Move the projection documentation into performance tuning page
- Expand and update the projection documentation to the latest state in
zitadel. (contained some stale information)

# Additional Context

Several other issues describe read-performance symptoms consistent with
this same root cause (stale `events2` visibility map / planner
statistics at scale, without resource saturation). Since this PR
addresses the shared root cause:

- Closes #12448
- Closes #10754
- Closes #10260
- Closes #8585
- Closes #9239


---
_Generated by [Claude
Code](https://claude.ai/code/session_01HoKvEY7niCVgLCz7CajBwW)_

---------

Co-authored-by: Claude <[email protected]>
Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: Silvan <[email protected]>
2026-07-16 14:42:11 +00:00

232 lines
5.4 KiB
Go

package setup
import (
"context"
"regexp"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zitadel/zitadel/internal/database"
)
func Test_eventstoreAutovacuum_Execute(t *testing.T) {
type fields struct {
Enabled bool
VacuumThreshold uint32
AnalyzeThreshold uint32
}
tests := []struct {
name string
fields fields
expects func(sqlmock.Sqlmock)
wantErr bool
}{
{
name: "enabled, sets thresholds",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: 25000,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET (
autovacuum_vacuum_scale_factor = 0.0,
autovacuum_analyze_scale_factor = 0.0,
autovacuum_vacuum_insert_scale_factor = 0.0,
autovacuum_vacuum_insert_threshold = 50000,
autovacuum_analyze_threshold = 25000,
autovacuum_vacuum_threshold = 50000
)`)).WillReturnResult(sqlmock.NewResult(0, 0))
},
},
{
name: "enabled, db error",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: 50000,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(".*").WillReturnError(assert.AnError)
},
wantErr: true,
},
{
name: "enabled, vacuum threshold at minimum boundary fails",
fields: fields{
Enabled: true,
VacuumThreshold: minAutovacuumThreshold,
AnalyzeThreshold: 50000,
},
expects: func(sqlmock.Sqlmock) {},
wantErr: true,
},
{
name: "enabled, analyze threshold at minimum boundary fails",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: minAutovacuumThreshold,
},
expects: func(sqlmock.Sqlmock) {},
wantErr: true,
},
{
name: "enabled, thresholds just above minimum succeed",
fields: fields{
Enabled: true,
VacuumThreshold: minAutovacuumThreshold + 1,
AnalyzeThreshold: minAutovacuumThreshold + 1,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(`ALTER TABLE eventstore.events2 SET (
autovacuum_vacuum_scale_factor = 0.0,
autovacuum_analyze_scale_factor = 0.0,
autovacuum_vacuum_insert_scale_factor = 0.0,
autovacuum_vacuum_insert_threshold = 10001,
autovacuum_analyze_threshold = 10001,
autovacuum_vacuum_threshold = 10001
)`)).WillReturnResult(sqlmock.NewResult(0, 0))
},
},
{
name: "disabled, resets to defaults",
fields: fields{
Enabled: false,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(regexp.QuoteMeta(eventstoreAutovacuumResetStmt)).
WillReturnResult(sqlmock.NewResult(0, 0))
},
},
{
name: "disabled, db error",
fields: fields{
Enabled: false,
},
expects: func(mock sqlmock.Sqlmock) {
mock.ExpectExec(".*").WillReturnError(assert.AnError)
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() {
require.NoError(t, mock.ExpectationsWereMet())
}()
defer db.Close()
tt.expects(mock)
mig := &eventstoreAutovacuum{
dbClient: &database.DB{DB: db},
Enabled: tt.fields.Enabled,
VacuumThreshold: tt.fields.VacuumThreshold,
AnalyzeThreshold: tt.fields.AnalyzeThreshold,
}
err = mig.Execute(context.Background(), nil)
if tt.wantErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
})
}
}
func Test_eventstoreAutovacuum_Check(t *testing.T) {
type fields struct {
Enabled bool
VacuumThreshold uint32
AnalyzeThreshold uint32
}
tests := []struct {
name string
fields fields
lastRun map[string]any
want bool
}{
{
name: "no previous run",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: 50000,
},
lastRun: nil,
want: true,
},
{
name: "unchanged",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: 25000,
},
lastRun: map[string]any{
"enabled": true,
"vacuumThreshold": float64(50000),
"analyzeThreshold": float64(25000),
},
want: false,
},
{
name: "enabled changed",
fields: fields{
Enabled: false,
VacuumThreshold: 50000,
AnalyzeThreshold: 50000,
},
lastRun: map[string]any{
"enabled": true,
"vacuumThreshold": float64(50000),
"analyzeThreshold": float64(50000),
},
want: true,
},
{
name: "vacuum threshold changed",
fields: fields{
Enabled: true,
VacuumThreshold: 60000,
AnalyzeThreshold: 50000,
},
lastRun: map[string]any{
"enabled": true,
"vacuumThreshold": float64(50000),
"analyzeThreshold": float64(50000),
},
want: true,
},
{
name: "analyze threshold changed",
fields: fields{
Enabled: true,
VacuumThreshold: 50000,
AnalyzeThreshold: 60000,
},
lastRun: map[string]any{
"enabled": true,
"vacuumThreshold": float64(50000),
"analyzeThreshold": float64(50000),
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mig := &eventstoreAutovacuum{
Enabled: tt.fields.Enabled,
VacuumThreshold: tt.fields.VacuumThreshold,
AnalyzeThreshold: tt.fields.AnalyzeThreshold,
}
assert.Equal(t, tt.want, mig.Check(tt.lastRun))
})
}
}