mirror of
https://github.com/andwati/zitadel.git
synced 2026-10-18 05:52:51 +00:00
# Which Problems Are Solved ZITADEL opens its Postgres connection pool via `pgxpool.NewWithConfig` (`internal/database/postgres/pg.go`) but never reads `pgxpool.Pool.Stat()`, so operators have no way to observe pool saturation (acquired/idle/total conns vs. `MaxConns`) or connection churn (acquire waits, canceled/empty acquires, idle/lifetime destroys, new connections) through ZITADEL's existing metrics pipeline. # How the Problems Are Solved - Adds `RegisterCounterObserver` (backed by an OTel `Int64ObservableCounter`) to the `Metrics` interface, its implementation, and the test mock (`backend/v3/instrumentation/metric.go`, `backend/v3/instrumentation/metrics/metric.go`, `backend/v3/instrumentation/metrics/mock.go`), mirroring the existing `RegisterValueObserver`/`RegisterUpDownSumObserver` — there was previously no observable instrument for cumulative counters. - Adds `database.RegisterPoolMetrics(ctx, pool)` (`internal/database/metrics.go`), which registers 5 observable gauges (`acquired_conns`, `constructing_conns`, `idle_conns`, `max_conns`, `total_conns`) and 7 observable counters (`acquire_count`, `acquire_duration_milliseconds`, `canceled_acquire_count`, `empty_acquire_count`, `max_idle_destroy_count`, `max_lifetime_destroy_count`, `new_conns_count`) sourced directly from `pgxpool.Pool.Stat()`, published under `zitadel.database.pool.*` metric names. - Wires the registration call into `cmd/start/start.go` right after the pool is created, so the running server process exposes these series on `/metrics`. # Additional Changes - Adds unit tests covering the new `RegisterCounterObserver` method (`backend/v3/instrumentation/metric_test.go`) and the pool metrics registration (`internal/database/metrics_test.go`). # Additional Context - Scope is intentionally limited to `cmd/start` (the long-running process that serves `/metrics`); one-shot commands (`setup`, `mirror`, `initialise`, `key`) are left untouched since an exporter running for a single command invocation wouldn't produce useful series. - https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool#Stat
412 lines
9.5 KiB
Go
412 lines
9.5 KiB
Go
package instrumentation
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.opentelemetry.io/otel/metric"
|
|
sdk_metric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/metric/metricdata"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
)
|
|
|
|
func TestMetricConfig_SetLegacyConfig(t *testing.T) {
|
|
tests := []struct {
|
|
name string // description of this test case
|
|
target MetricConfig
|
|
lc *LegacyMetricConfig
|
|
want MetricConfig
|
|
}{
|
|
{
|
|
name: "nil legacy config does not change target",
|
|
target: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
lc: nil,
|
|
want: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "non-none target type does not change target",
|
|
target: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeStdOut,
|
|
},
|
|
},
|
|
lc: &LegacyMetricConfig{
|
|
Type: "otel",
|
|
},
|
|
want: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeStdOut,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "sets fields from legacy config (type none)",
|
|
target: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
lc: &LegacyMetricConfig{
|
|
Type: "otel",
|
|
},
|
|
want: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypePrometheus,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "sets fields from legacy config (type unspecified)",
|
|
target: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeUnspecified,
|
|
},
|
|
},
|
|
lc: &LegacyMetricConfig{
|
|
Type: "otel",
|
|
},
|
|
want: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypePrometheus,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "auto type is not overridden by legacy config",
|
|
target: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
lc: &LegacyMetricConfig{
|
|
Type: "otel",
|
|
},
|
|
want: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
tt.target.SetLegacyConfig(tt.lc)
|
|
assert.Equal(t, tt.want, tt.target)
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_newMeterProvider(t *testing.T) {
|
|
resource, err := resource.New(t.Context())
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string // description of this test case
|
|
// Named input parameters for target function.
|
|
cfg MetricConfig
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "none exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "unsupported exporter returns error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterType(-1),
|
|
},
|
|
},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "prometheus exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypePrometheus,
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "stdout exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeStdOut,
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "stderr exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeStdErr,
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "http exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeHTTP,
|
|
Endpoint: "localhost:4318",
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "google exporter return error (no credentials)",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeGoogle,
|
|
},
|
|
},
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "grpc exporter returns no error",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeGRPC,
|
|
Endpoint: "localhost:4317",
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
|
|
if tt.wantErr {
|
|
assert.Error(t, gotErr)
|
|
assert.Nil(t, got)
|
|
} else {
|
|
assert.NoError(t, gotErr)
|
|
assert.NotNil(t, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestMeter_RegisterCounterObserver(t *testing.T) {
|
|
reader := sdk_metric.NewManualReader()
|
|
provider := sdk_metric.NewMeterProvider(sdk_metric.WithReader(reader))
|
|
m := &Meter{Meter: provider.Meter(t.Name())}
|
|
|
|
callback := func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(42)
|
|
return nil
|
|
}
|
|
|
|
err := m.RegisterCounterObserver("test.counter.observer", "test counter observer", callback)
|
|
require.NoError(t, err)
|
|
|
|
// registering the same name again is a no-op, not an error.
|
|
err = m.RegisterCounterObserver("test.counter.observer", "test counter observer", callback)
|
|
require.NoError(t, err)
|
|
|
|
_, exists := m.CounterObservers.Load("test.counter.observer")
|
|
assert.True(t, exists)
|
|
|
|
var rm metricdata.ResourceMetrics
|
|
require.NoError(t, reader.Collect(t.Context(), &rm))
|
|
|
|
got := findMetric(t, rm, "test.counter.observer")
|
|
sum, ok := got.Data.(metricdata.Sum[int64])
|
|
require.True(t, ok, "expected an observable counter (Sum[int64]), got %T", got.Data)
|
|
assert.True(t, sum.IsMonotonic)
|
|
require.Len(t, sum.DataPoints, 1)
|
|
assert.Equal(t, int64(42), sum.DataPoints[0].Value)
|
|
}
|
|
|
|
// findMetric locates a metric by name across all scopes collected from a reader.
|
|
func findMetric(t *testing.T, rm metricdata.ResourceMetrics, name string) metricdata.Metrics {
|
|
t.Helper()
|
|
for _, sm := range rm.ScopeMetrics {
|
|
if i := slices.IndexFunc(sm.Metrics, func(m metricdata.Metrics) bool { return m.Name == name }); i >= 0 {
|
|
return sm.Metrics[i]
|
|
}
|
|
}
|
|
t.Fatalf("metric %q not found in collected data", name)
|
|
return metricdata.Metrics{}
|
|
}
|
|
|
|
func Test_newMeterProvider_autoexport(t *testing.T) {
|
|
resource, err := resource.New(t.Context())
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
cfg MetricConfig
|
|
envVars map[string]string
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "auto type with OTEL_METRICS_EXPORTER=console creates provider",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_METRICS_EXPORTER": "console",
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with no OTEL env vars creates provider (noop fallback)",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with OTEL_METRICS_EXPORTER=none creates provider (autoexport none)",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_METRICS_EXPORTER": "none",
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with only OTEL_EXPORTER_OTLP_ENDPOINT uses OTLP default",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318",
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with only OTEL_EXPORTER_OTLP_METRICS_ENDPOINT uses OTLP default",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://localhost:4318",
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with only OTEL_EXPORTER_OTLP_PROTOCOL uses OTLP default",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
|
|
},
|
|
},
|
|
{
|
|
name: "auto type with only OTEL_EXPORTER_OTLP_METRICS_PROTOCOL uses OTLP default",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeAuto,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf",
|
|
},
|
|
},
|
|
{
|
|
name: "none type ignores OTEL env vars (explicit disable)",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_METRICS_EXPORTER": "invalid_exporter",
|
|
},
|
|
},
|
|
// Backward compatibility: ExporterTypeNone ignores global OTEL endpoint
|
|
{
|
|
name: "none type ignores OTEL_EXPORTER_OTLP_ENDPOINT",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4318",
|
|
},
|
|
},
|
|
// Backward compatibility: ExporterTypeNone ignores per-signal OTEL var
|
|
{
|
|
name: "none type ignores OTEL_METRICS_EXPORTER=console",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeNone,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_METRICS_EXPORTER": "console",
|
|
},
|
|
},
|
|
// Backward compatibility: explicit ZITADEL types take priority over OTEL env vars
|
|
{
|
|
name: "stdout type ignores OTEL_METRICS_EXPORTER",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeStdOut,
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_METRICS_EXPORTER": "grpc",
|
|
},
|
|
},
|
|
{
|
|
name: "grpc type ignores OTEL_EXPORTER_OTLP_ENDPOINT",
|
|
cfg: MetricConfig{
|
|
Exporter: ExporterConfig{
|
|
Type: ExporterTypeGRPC,
|
|
Endpoint: "localhost:4317",
|
|
},
|
|
},
|
|
envVars: map[string]string{
|
|
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://some-other-host:4318",
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
for k, v := range tt.envVars {
|
|
t.Setenv(k, v)
|
|
}
|
|
got, gotErr := newMeterProvider(t.Context(), tt.cfg, resource)
|
|
if tt.wantErr {
|
|
assert.Error(t, gotErr)
|
|
assert.Nil(t, got)
|
|
} else {
|
|
assert.NoError(t, gotErr)
|
|
assert.NotNil(t, got)
|
|
}
|
|
})
|
|
}
|
|
}
|