mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-18 05:52:51 +00:00
Tim MöhlmannandGitHub
73de633c71
# Which Problems Are Solved ZITADEL opens its Postgres connection pool via `pgxpool.NewWithConfig` (`internal/database/postgres/pg.go`) but never reads `pgxpool.Pool.Stat()`, so operators have no way to observe pool saturation (acquired/idle/total conns vs. `MaxConns`) or connection churn (acquire waits, canceled/empty acquires, idle/lifetime destroys, new connections) through ZITADEL's existing metrics pipeline. # How the Problems Are Solved - Adds `RegisterCounterObserver` (backed by an OTel `Int64ObservableCounter`) to the `Metrics` interface, its implementation, and the test mock (`backend/v3/instrumentation/metric.go`, `backend/v3/instrumentation/metrics/metric.go`, `backend/v3/instrumentation/metrics/mock.go`), mirroring the existing `RegisterValueObserver`/`RegisterUpDownSumObserver` — there was previously no observable instrument for cumulative counters. - Adds `database.RegisterPoolMetrics(ctx, pool)` (`internal/database/metrics.go`), which registers 5 observable gauges (`acquired_conns`, `constructing_conns`, `idle_conns`, `max_conns`, `total_conns`) and 7 observable counters (`acquire_count`, `acquire_duration_milliseconds`, `canceled_acquire_count`, `empty_acquire_count`, `max_idle_destroy_count`, `max_lifetime_destroy_count`, `new_conns_count`) sourced directly from `pgxpool.Pool.Stat()`, published under `zitadel.database.pool.*` metric names. - Wires the registration call into `cmd/start/start.go` right after the pool is created, so the running server process exposes these series on `/metrics`. # Additional Changes - Adds unit tests covering the new `RegisterCounterObserver` method (`backend/v3/instrumentation/metric_test.go`) and the pool metrics registration (`internal/database/metrics_test.go`). # Additional Context - Scope is intentionally limited to `cmd/start` (the long-running process that serves `/metrics`); one-shot commands (`setup`, `mirror`, `initialise`, `key`) are left untouched since an exporter running for a single command invocation wouldn't produce useful series. - https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool#Stat
67 lines
4.0 KiB
Go
67 lines
4.0 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.opentelemetry.io/otel/metric"
|
|
|
|
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
|
|
"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
|
|
)
|
|
|
|
const (
|
|
PoolAcquiredConns = "zitadel.database.pool.acquired_conns"
|
|
PoolConstructingConns = "zitadel.database.pool.constructing_conns"
|
|
PoolIdleConns = "zitadel.database.pool.idle_conns"
|
|
PoolMaxConns = "zitadel.database.pool.max_conns"
|
|
PoolTotalConns = "zitadel.database.pool.total_conns"
|
|
|
|
PoolAcquireCount = "zitadel.database.pool.acquire_count"
|
|
PoolAcquireDurationMs = "zitadel.database.pool.acquire_duration_milliseconds"
|
|
PoolCanceledAcquireCount = "zitadel.database.pool.canceled_acquire_count"
|
|
PoolEmptyAcquireCount = "zitadel.database.pool.empty_acquire_count"
|
|
PoolMaxIdleDestroyCount = "zitadel.database.pool.max_idle_destroy_count"
|
|
PoolMaxLifetimeDestroyCount = "zitadel.database.pool.max_lifetime_destroy_count"
|
|
PoolNewConnsCount = "zitadel.database.pool.new_conns_count"
|
|
)
|
|
|
|
// RegisterPoolMetrics registers observable gauges and counters reporting pool's
|
|
// underlying *pgxpool.Pool state (see pgxpool.Pool.Stat) with the global metrics
|
|
// pipeline.
|
|
func RegisterPoolMetrics(ctx context.Context, pool *pgxpool.Pool) {
|
|
registerPoolMetrics(ctx, metrics.GlobalMeter(), pool)
|
|
}
|
|
|
|
func registerPoolMetrics(ctx context.Context, m metrics.Metrics, pool *pgxpool.Pool) {
|
|
gauge := func(name, description string, value func(*pgxpool.Stat) int64) {
|
|
err := m.RegisterValueObserver(name, description, func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(value(pool.Stat()))
|
|
return nil
|
|
})
|
|
logging.OnError(ctx, err).Error("failed to register database pool gauge metric", slog.String("metric", name))
|
|
}
|
|
counter := func(name, description string, value func(*pgxpool.Stat) int64) {
|
|
err := m.RegisterCounterObserver(name, description, func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(value(pool.Stat()))
|
|
return nil
|
|
})
|
|
logging.OnError(ctx, err).Error("failed to register database pool counter metric", slog.String("metric", name))
|
|
}
|
|
|
|
gauge(PoolAcquiredConns, "Number of currently acquired connections in the pool", func(s *pgxpool.Stat) int64 { return int64(s.AcquiredConns()) })
|
|
gauge(PoolConstructingConns, "Number of connections currently being established", func(s *pgxpool.Stat) int64 { return int64(s.ConstructingConns()) })
|
|
gauge(PoolIdleConns, "Number of currently idle connections in the pool", func(s *pgxpool.Stat) int64 { return int64(s.IdleConns()) })
|
|
gauge(PoolMaxConns, "Maximum size of the pool", func(s *pgxpool.Stat) int64 { return int64(s.MaxConns()) })
|
|
gauge(PoolTotalConns, "Total number of connections currently in the pool", func(s *pgxpool.Stat) int64 { return int64(s.TotalConns()) })
|
|
|
|
counter(PoolAcquireCount, "Cumulative count of successful acquires from the pool", func(s *pgxpool.Stat) int64 { return s.AcquireCount() })
|
|
counter(PoolAcquireDurationMs, "Cumulative time spent waiting for successful acquires from the pool, in milliseconds", func(s *pgxpool.Stat) int64 { return s.AcquireDuration().Milliseconds() })
|
|
counter(PoolCanceledAcquireCount, "Cumulative count of acquires from the pool that were canceled by a context", func(s *pgxpool.Stat) int64 { return s.CanceledAcquireCount() })
|
|
counter(PoolEmptyAcquireCount, "Cumulative count of successful acquires that waited for a resource to be released or constructed because the pool was empty", func(s *pgxpool.Stat) int64 { return s.EmptyAcquireCount() })
|
|
counter(PoolMaxIdleDestroyCount, "Cumulative count of connections destroyed because they exceeded MaxConnIdleTime", func(s *pgxpool.Stat) int64 { return s.MaxIdleDestroyCount() })
|
|
counter(PoolMaxLifetimeDestroyCount, "Cumulative count of connections destroyed because they exceeded MaxConnLifetime", func(s *pgxpool.Stat) int64 { return s.MaxLifetimeDestroyCount() })
|
|
counter(PoolNewConnsCount, "Cumulative count of new connections opened", func(s *pgxpool.Stat) int64 { return s.NewConnsCount() })
|
|
}
|