Files
zitadel/backend/v3/instrumentation/metric_test.go
T
Tim MöhlmannandGitHub 73de633c71 feat(database): publish pgxpool pool stats as metrics (#12737)
# Which Problems Are Solved

ZITADEL opens its Postgres connection pool via `pgxpool.NewWithConfig`
(`internal/database/postgres/pg.go`) but never reads
`pgxpool.Pool.Stat()`,
so operators have no way to observe pool saturation (acquired/idle/total
conns vs. `MaxConns`) or connection churn (acquire waits, canceled/empty
acquires, idle/lifetime destroys, new connections) through ZITADEL's
existing metrics pipeline.

# How the Problems Are Solved

- Adds `RegisterCounterObserver` (backed by an OTel
`Int64ObservableCounter`)
  to the `Metrics` interface, its implementation, and the test mock
  (`backend/v3/instrumentation/metric.go`,
  `backend/v3/instrumentation/metrics/metric.go`,
  `backend/v3/instrumentation/metrics/mock.go`), mirroring the existing
`RegisterValueObserver`/`RegisterUpDownSumObserver` — there was
previously
  no observable instrument for cumulative counters.
- Adds `database.RegisterPoolMetrics(ctx, pool)`
  (`internal/database/metrics.go`), which registers 5 observable gauges
  (`acquired_conns`, `constructing_conns`, `idle_conns`, `max_conns`,
  `total_conns`) and 7 observable counters (`acquire_count`,
  `acquire_duration_milliseconds`, `canceled_acquire_count`,
  `empty_acquire_count`, `max_idle_destroy_count`,
  `max_lifetime_destroy_count`, `new_conns_count`) sourced directly from
`pgxpool.Pool.Stat()`, published under `zitadel.database.pool.*` metric
  names.
- Wires the registration call into `cmd/start/start.go` right after the
  pool is created, so the running server process exposes these series on
  `/metrics`.

# Additional Changes

- Adds unit tests covering the new `RegisterCounterObserver` method
  (`backend/v3/instrumentation/metric_test.go`) and the pool metrics
  registration (`internal/database/metrics_test.go`).

# Additional Context

- Scope is intentionally limited to `cmd/start` (the long-running
process
  that serves `/metrics`); one-shot commands (`setup`, `mirror`,
`initialise`, `key`) are left untouched since an exporter running for a
  single command invocation wouldn't produce useful series.
- https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool#Stat
2026-09-15 12:13:30 +02:00

412 lines
9.5 KiB
Go

package instrumentation
import (
"context"
"slices"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/metric"
sdk_metric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/resource"
)
func TestMetricConfig_SetLegacyConfig(t *testing.T) {
tests := []struct {
name string // description of this test case
target MetricConfig
lc *LegacyMetricConfig
want MetricConfig
}{
{
name: "nil legacy config does not change target",
target: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
lc: nil,
want: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
},
{
name: "non-none target type does not change target",
target: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeStdOut,
},
},
lc: &LegacyMetricConfig{
Type: "otel",
},
want: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeStdOut,
},
},
},
{
name: "sets fields from legacy config (type none)",
target: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
lc: &LegacyMetricConfig{
Type: "otel",
},
want: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypePrometheus,
},
},
},
{
name: "sets fields from legacy config (type unspecified)",
target: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeUnspecified,
},
},
lc: &LegacyMetricConfig{
Type: "otel",
},
want: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypePrometheus,
},
},
},
{
name: "auto type is not overridden by legacy config",
target: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
lc: &LegacyMetricConfig{
Type: "otel",
},
want: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.target.SetLegacyConfig(tt.lc)
assert.Equal(t, tt.want, tt.target)
})
}
}
func Test_newMeterProvider(t *testing.T) {
resource, err := resource.New(t.Context())
require.NoError(t, err)
tests := []struct {
name string // description of this test case
// Named input parameters for target function.
cfg MetricConfig
wantErr bool
}{
{
name: "none exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
wantErr: false,
},
{
name: "unsupported exporter returns error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterType(-1),
},
},
wantErr: true,
},
{
name: "prometheus exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypePrometheus,
},
},
wantErr: false,
},
{
name: "stdout exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeStdOut,
},
},
wantErr: false,
},
{
name: "stderr exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeStdErr,
},
},
wantErr: false,
},
{
name: "http exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeHTTP,
Endpoint: "localhost:4318",
},
},
wantErr: false,
},
{
name: "google exporter return error (no credentials)",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeGoogle,
},
},
wantErr: true,
},
{
name: "grpc exporter returns no error",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeGRPC,
Endpoint: "localhost:4317",
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
if tt.wantErr {
assert.Error(t, gotErr)
assert.Nil(t, got)
} else {
assert.NoError(t, gotErr)
assert.NotNil(t, got)
}
})
}
}
func TestMeter_RegisterCounterObserver(t *testing.T) {
reader := sdk_metric.NewManualReader()
provider := sdk_metric.NewMeterProvider(sdk_metric.WithReader(reader))
m := &Meter{Meter: provider.Meter(t.Name())}
callback := func(_ context.Context, o metric.Int64Observer) error {
o.Observe(42)
return nil
}
err := m.RegisterCounterObserver("test.counter.observer", "test counter observer", callback)
require.NoError(t, err)
// registering the same name again is a no-op, not an error.
err = m.RegisterCounterObserver("test.counter.observer", "test counter observer", callback)
require.NoError(t, err)
_, exists := m.CounterObservers.Load("test.counter.observer")
assert.True(t, exists)
var rm metricdata.ResourceMetrics
require.NoError(t, reader.Collect(t.Context(), &rm))
got := findMetric(t, rm, "test.counter.observer")
sum, ok := got.Data.(metricdata.Sum[int64])
require.True(t, ok, "expected an observable counter (Sum[int64]), got %T", got.Data)
assert.True(t, sum.IsMonotonic)
require.Len(t, sum.DataPoints, 1)
assert.Equal(t, int64(42), sum.DataPoints[0].Value)
}
// findMetric locates a metric by name across all scopes collected from a reader.
func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics {
t.Helper()
for _, sm := range rm.ScopeMetrics {
if i := slices.IndexFunc(sm.Metrics, func(m metricdata.Metrics) bool { return m.Name == name }); i >= 0 {
return sm.Metrics[i]
}
}
t.Fatalf("metric %q not found in collected data", name)
return metricdata.Metrics{}
}
func Test_newMeterProvider_autoexport(t *testing.T) {
resource, err := resource.New(t.Context())
require.NoError(t, err)
tests := []struct {
name string
cfg MetricConfig
envVars map[string]string
wantErr bool
}{
{
name: "auto type with OTEL_METRICS_EXPORTER=console creates provider",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_METRICS_EXPORTER": "console",
},
},
{
name: "auto type with no OTEL env vars creates provider (noop fallback)",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
},
{
name: "auto type with OTEL_METRICS_EXPORTER=none creates provider (autoexport none)",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_METRICS_EXPORTER": "none",
},
},
{
name: "auto type with only OTEL_EXPORTER_OTLP_ENDPOINT uses OTLP default",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318",
},
},
{
name: "auto type with only OTEL_EXPORTER_OTLP_METRICS_ENDPOINT uses OTLP default",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://localhost:4318",
},
},
{
name: "auto type with only OTEL_EXPORTER_OTLP_PROTOCOL uses OTLP default",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
},
},
{
name: "auto type with only OTEL_EXPORTER_OTLP_METRICS_PROTOCOL uses OTLP default",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeAuto,
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf",
},
},
{
name: "none type ignores OTEL env vars (explicit disable)",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
envVars: map[string]string{
"OTEL_METRICS_EXPORTER": "invalid_exporter",
},
},
// Backward compatibility: ExporterTypeNone ignores global OTEL endpoint
{
name: "none type ignores OTEL_EXPORTER_OTLP_ENDPOINT",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318",
},
},
// Backward compatibility: ExporterTypeNone ignores per-signal OTEL var
{
name: "none type ignores OTEL_METRICS_EXPORTER=console",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeNone,
},
},
envVars: map[string]string{
"OTEL_METRICS_EXPORTER": "console",
},
},
// Backward compatibility: explicit ZITADEL types take priority over OTEL env vars
{
name: "stdout type ignores OTEL_METRICS_EXPORTER",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeStdOut,
},
},
envVars: map[string]string{
"OTEL_METRICS_EXPORTER": "grpc",
},
},
{
name: "grpc type ignores OTEL_EXPORTER_OTLP_ENDPOINT",
cfg: MetricConfig{
Exporter: ExporterConfig{
Type: ExporterTypeGRPC,
Endpoint: "localhost:4317",
},
},
envVars: map[string]string{
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://some-other-host:4318",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
for k, v := range tt.envVars {
t.Setenv(k, v)
}
got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
if tt.wantErr {
assert.Error(t, gotErr)
assert.Nil(t, got)
} else {
assert.NoError(t, gotErr)
assert.NotNil(t, got)
}
})
}
}