From 471ed4a5d7f577ad3e2bb8c525271689a63550e6 Mon Sep 17 00:00:00 2001 From: Mridang Agarwalla Date: Sat, 21 Mar 2026 02:21:15 +1100 Subject: [PATCH] feat: add DSN/URL connection string support for PostgreSQL and Redis (#11729) --- .../deploy/kubernetes/caching.mdx | 102 +++----- .../deploy/kubernetes/configuration.mdx | 33 +-- .../deploy/kubernetes/database.mdx | 111 +++------ .../self-hosting/deploy/kubernetes/index.mdx | 33 +-- .../deploy/kubernetes/operations.mdx | 13 +- .../content/self-hosting/deploy/linux.mdx | 4 +- .../content/self-hosting/deploy/macos.mdx | 4 +- .../content/self-hosting/manage/cache.mdx | 35 ++- .../manage/configure/_linuxunix.mdx | 14 +- .../manage/configure/configure.mdx | 14 +- .../self-hosting/manage/database/index.mdx | 58 ++--- .../self-hosting/manage/production.mdx | 17 +- .../self-hosting/manage/updating_scaling.mdx | 12 +- cmd/defaults.yaml | 19 ++ cmd/mirror/defaults.yaml | 2 + deploy/compose/.env.example | 16 +- deploy/compose/.env.test | 7 +- deploy/compose/docker-compose.prodlike.yml | 10 +- deploy/compose/docker-compose.yml | 16 +- internal/cache/connector/connector.go | 6 +- internal/cache/connector/redis/connector.go | 93 ++++++- .../cache/connector/redis/connector_test.go | 187 ++++++++++++++ internal/cache/connector/redis/redis_test.go | 3 +- internal/database/postgres/pg.go | 72 +++++- internal/database/postgres/pg_test.go | 231 ++++++++++++++++++ 25 files changed, 810 insertions(+), 302 deletions(-) create mode 100644 internal/cache/connector/redis/connector_test.go create mode 100644 internal/database/postgres/pg_test.go diff --git a/apps/docs/content/self-hosting/deploy/kubernetes/caching.mdx b/apps/docs/content/self-hosting/deploy/kubernetes/caching.mdx index 1f6df20b7..5432f90db 100644 --- a/apps/docs/content/self-hosting/deploy/kubernetes/caching.mdx +++ b/apps/docs/content/self-hosting/deploy/kubernetes/caching.mdx @@ -15,94 +15,66 @@ Caching is an experimental beta feature. See the [Caches](/self-hosting/manage/c Zitadel supports Redis or any Redis-compatible store (for example, Valkey) in standalone mode. Sentinel and Redis Cluster are not supported because the cache connector expects a single endpoint and does not implement cluster routing or sentinel failover logic. -Zitadel supports server-auth TLS (`EnableTLS: true`) but does not support client certificates/mTLS for cache connections. +Zitadel supports server-auth TLS (`rediss://` scheme) but does not support client certificates/mTLS for cache connections. Each cache uses its own Redis database index. Set `DBOffset` so Zitadel-owned indexes do not overlap with other applications. Zitadel issues `FLUSHDB` on its cache databases, so sharing an index with other apps is unsafe. ## Connect without authentication (development only) +Pass the Redis URL as an environment variable: + ```yaml zitadel: - configmapConfig: - Caches: - Connectors: - Redis: - Enabled: true - Addr: "redis-master.caching.svc.cluster.local:6379" - DBOffset: 10 - Instance: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m - Organization: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m + env: + - name: ZITADEL_CACHES_CONNECTORS_REDIS_URL + value: "redis://redis-master.caching.svc.cluster.local:6379" + - name: ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED + value: "true" ``` Use this only on isolated clusters where the Redis endpoint does not require authentication. ## Connect with password from a Kubernetes Secret -Keep credentials in a secret while the rest of the cache configuration stays in values: - -```yaml -zitadel: - configSecretName: zitadel-cache-credentials - configmapConfig: - Caches: - Connectors: - Redis: - Enabled: true - Addr: "redis.database.svc.cluster.local:6379" - DBOffset: 10 - Instance: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m - Organization: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m -``` - -Create the secret with your Redis ACL user and password: +Store the Redis URL (with credentials) in a secret and pass it as an environment variable: ```bash kubectl create secret generic zitadel-cache-credentials \ - --from-literal=config.yaml=" -Caches: - Connectors: - Redis: - Username: cache-user # Omit for default user - Password: your-redis-password -" + --from-literal=url="redis://cache-user:your-redis-password@redis.database.svc.cluster.local:6379" +``` + +```yaml +zitadel: + env: + - name: ZITADEL_CACHES_CONNECTORS_REDIS_URL + valueFrom: + secretKeyRef: + name: zitadel-cache-credentials + key: url + - name: ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED + value: "true" ``` ## Connect with TLS (Redis or Valkey) -Enable TLS when your endpoint requires encryption. This uses server-auth TLS with the container's trust store. +Use the `rediss://` scheme (note the double `s`) when your endpoint requires encryption. This uses server-auth TLS with the container's trust store. + +```bash +kubectl create secret generic zitadel-cache-credentials \ + --from-literal=url="rediss://cache-user:your-redis-password@redis.example.com:6380" +``` ```yaml zitadel: - configSecretName: zitadel-cache-credentials - configmapConfig: - Caches: - Connectors: - Redis: - Enabled: true - Addr: "redis.example.com:6380" - EnableTLS: true - DBOffset: 10 - Instance: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m - Organization: - Connector: "redis" - MaxAge: 1h - LastUseAge: 10m + env: + - name: ZITADEL_CACHES_CONNECTORS_REDIS_URL + valueFrom: + secretKeyRef: + name: zitadel-cache-credentials + key: url + - name: ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED + value: "true" ``` -Reuse the secret from the previous section for credentials if needed. Custom certificate authorities must be added to the container's trust bundle (for example, by mounting a CA file into the pod); client certificate authentication is not supported. +Custom certificate authorities must be added to the container's trust bundle (for example, by mounting a CA file into the pod); client certificate authentication is not supported. diff --git a/apps/docs/content/self-hosting/deploy/kubernetes/configuration.mdx b/apps/docs/content/self-hosting/deploy/kubernetes/configuration.mdx index c69c11e5d..a75dcde95 100644 --- a/apps/docs/content/self-hosting/deploy/kubernetes/configuration.mdx +++ b/apps/docs/content/self-hosting/deploy/kubernetes/configuration.mdx @@ -162,36 +162,23 @@ kubectl create secret generic zitadel-masterkey \ --from-literal=masterkey="$(tr -dc A-Za-z0-9 @@ -62,7 +62,7 @@ ZITADEL_DATABASE_POSTGRES_HOST=localhost ZITADEL_DATABASE_POSTGRES_PORT=5432 ZIT ### Setup Zitadel with a service account ```bash -ZITADEL_DATABASE_POSTGRES_HOST=localhost ZITADEL_DATABASE_POSTGRES_PORT=5432 ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=zitadel ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=root ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable ZITADEL_EXTERNALSECURE=false ZITADEL_FIRSTINSTANCE_MACHINEKEYPATH=/tmp/zitadel-admin-sa.json ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME=zitadel-admin-sa ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME=Admin ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINEKEY_TYPE=1 zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled +ZITADEL_DATABASE_POSTGRES_DSN=postgresql://root:postgres@localhost:5432/postgres?sslmode=disable ZITADEL_EXTERNALSECURE=false ZITADEL_FIRSTINSTANCE_MACHINEKEYPATH=/tmp/zitadel-admin-sa.json ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME=zitadel-admin-sa ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME=Admin ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINEKEY_TYPE=1 zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled # then you can move your machine key mv /tmp/zitadel-admin-sa.json $HOME/zitadel-admin-sa.json diff --git a/apps/docs/content/self-hosting/deploy/macos.mdx b/apps/docs/content/self-hosting/deploy/macos.mdx index e14465fa0..c1e06bfee 100644 --- a/apps/docs/content/self-hosting/deploy/macos.mdx +++ b/apps/docs/content/self-hosting/deploy/macos.mdx @@ -45,7 +45,7 @@ LATEST=$(curl -i https://github.com/zitadel/zitadel/releases/latest | grep locat ## Run ZITADEL ```bash -ZITADEL_DATABASE_POSTGRES_HOST=localhost ZITADEL_DATABASE_POSTGRES_PORT=5432 ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=zitadel ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=$(whoami) ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD=postgres ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable ZITADEL_EXTERNALSECURE=false zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled +ZITADEL_DATABASE_POSTGRES_DSN="postgresql://root:postgres@localhost:5432/postgres?sslmode=disable" ZITADEL_EXTERNALSECURE=false zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled ``` @@ -57,7 +57,7 @@ ZITADEL_DATABASE_POSTGRES_HOST=localhost ZITADEL_DATABASE_POSTGRES_PORT=5432 ZIT ### Setup ZITADEL with a service account ```bash -ZITADEL_DATABASE_POSTGRES_HOST=localhost ZITADEL_DATABASE_POSTGRES_PORT=5432 ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=zitadel ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=$(whoami) ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable ZITADEL_EXTERNALSECURE=false ZITADEL_FIRSTINSTANCE_MACHINEKEYPATH=/tmp/zitadel-admin-sa.json ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME=zitadel-admin-sa ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME=Admin ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINEKEY_TYPE=1 zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled +ZITADEL_DATABASE_POSTGRES_DSN="postgresql://root:postgres@localhost:5432/postgres?sslmode=disable" ZITADEL_EXTERNALSECURE=false ZITADEL_FIRSTINSTANCE_MACHINEKEYPATH=/tmp/zitadel-admin-sa.json ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME=zitadel-admin-sa ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME=Admin ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINEKEY_TYPE=1 zitadel start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled # then you can move your machine key mv /tmp/zitadel-admin-sa.json $HOME/zitadel-admin-sa.json diff --git a/apps/docs/content/self-hosting/manage/cache.mdx b/apps/docs/content/self-hosting/manage/cache.mdx index f76e75cf5..13ef2566d 100644 --- a/apps/docs/content/self-hosting/manage/cache.mdx +++ b/apps/docs/content/self-hosting/manage/cache.mdx @@ -71,6 +71,25 @@ Drawbacks: - Increased operational overhead: need to run a Redis instance as part of your infrastructure. - When running multiple servers of ZITADEL in different regions, network roundtrip time might impact performance, neutralizing the benefit of a cache. +Connect to Redis by providing a URL via environment variable. Use `redis://` for plain connections or `rediss://` for TLS: + +```bash +export ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED=true +export ZITADEL_CACHES_CONNECTORS_REDIS_URL=redis://user:password@localhost:6379 +``` + +If you prefer configuration files, set the URL in your YAML config: + +```yaml +Caches: + Connectors: + Redis: + Enabled: true + URL: redis://user:password@localhost:6379 +``` + +Pool, timeout, retry, and circuit breaker settings can be configured alongside the URL as overlays. The DB number in the URL path is ignored for cache DB selection — ZITADEL derives the actual DB from `DBOffset` + purpose. + #### Circuit breaker A [circuit breaker](https://learn.microsoft.com/en-us/previous-versions/msp-n-p/dn589784(v=pandp.10)?redirectedfrom=MSDN) is provided for the Redis connector, to prevent a single point of failure in the case persistent errors. When the circuit breaker opens, the cache is temporary disabled and ignored. ZITADEL will continue to operate using queries to the database. @@ -80,7 +99,7 @@ Caches: Connectors: Redis: Enabled: true - Addr: localhost:6379 + URL: redis://user:password@localhost:6379 # Many other options... CircuitBreaker: # Interval when the counters are reset to 0. @@ -226,14 +245,24 @@ Caches: MaxAge: 1s ``` -When running many instances on multiple servers: +When running many instances on multiple servers, set the Redis URL and enable the connector: + +```bash +export ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED=true +export ZITADEL_CACHES_CONNECTORS_REDIS_URL=redis://user:password@redis.example.com:6379 +export ZITADEL_CACHES_INSTANCE_CONNECTOR=redis +export ZITADEL_CACHES_MILESTONES_CONNECTOR=redis +export ZITADEL_CACHES_ORGANIZATION_CONNECTOR=redis +``` + +Or in a configuration file: ```yaml Caches: Connectors: Redis: Enabled: true - # Other connection options + URL: redis://user:password@redis.example.com:6379 Instance: Connector: "redis" diff --git a/apps/docs/content/self-hosting/manage/configure/_linuxunix.mdx b/apps/docs/content/self-hosting/manage/configure/_linuxunix.mdx index 4c34f369e..4496023eb 100644 --- a/apps/docs/content/self-hosting/manage/configure/_linuxunix.mdx +++ b/apps/docs/content/self-hosting/manage/configure/_linuxunix.mdx @@ -51,16 +51,12 @@ zitadel start-from-init \ ### Configure by Environment Variables +The environment variable approach uses a single DSN string instead of the individual field-based configuration shown above. + ```bash -# Set runtime environment variables -export ZITADEL_DATABASE_POSTGRES_HOST=my.database -export ZITADEL_DATABASE_POSTGRES_PORT=5432 -export ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel -export ZITADEL_DATABASE_POSTGRES_USER_USERNAME=my_zitadel_db_user -export ZITADEL_DATABASE_POSTGRES_USER_PASSWORD=Secret_DB_User_Password -export ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=disable -export ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME=root -export ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE=disable +# Set the PostgreSQL connection via a single DSN +# Use the "postgres" maintenance database so ZITADEL can create its own database during initialization +export ZITADEL_DATABASE_POSTGRES_DSN=postgresql://my_zitadel_db_user:Secret_DB_User_Password@my.database:5432/postgres?sslmode=disable export ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME="root" export ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD="RootPassword1!" diff --git a/apps/docs/content/self-hosting/manage/configure/configure.mdx b/apps/docs/content/self-hosting/manage/configure/configure.mdx index 80439dd36..4a8ddefbe 100644 --- a/apps/docs/content/self-hosting/manage/configure/configure.mdx +++ b/apps/docs/content/self-hosting/manage/configure/configure.mdx @@ -93,22 +93,18 @@ There are three ways to pass the masterkey to the `zitadel` binary: - + - + - + diff --git a/apps/docs/content/self-hosting/manage/database/index.mdx b/apps/docs/content/self-hosting/manage/database/index.mdx index 0ce9fc4fd..8dc8521fd 100644 --- a/apps/docs/content/self-hosting/manage/database/index.mdx +++ b/apps/docs/content/self-hosting/manage/database/index.mdx @@ -22,35 +22,29 @@ If you are using Zitadel v2 and want to use a PostgreSQL database, you can [over Currently, versions 14 to 18 are supported. -Postgres can be configured as follows: +Connect ZITADEL to your PostgreSQL database by providing a DSN (connection URL) via the environment variable: + +```bash +export ZITADEL_DATABASE_POSTGRES_DSN=postgresql://zitadel:password@localhost:5432/zitadel?sslmode=disable +``` + +If you prefer configuration files, set the DSN in your YAML config: + ```yaml Database: postgres: - Host: localhost - Port: 5432 - Database: zitadel - AwaitInitialConn: 5m + DSN: postgresql://zitadel:password@localhost:5432/zitadel?sslmode=disable MaxOpenConns: 15 MaxIdleConns: 10 MaxConnLifetime: 1h MaxConnIdleTime: 5m - Options: - User: - Username: zitadel - Password: zitadel - SSL: - Mode: disable # Use 'require' or 'verify-full' for production - RootCert: - Cert: - Key: - Admin: - Username: postgres - Password: postgres - SSL: - Mode: disable # Use 'require' or 'verify-full' for production - RootCert: - Cert: - Key: +``` + +Pool and tuning settings can be configured alongside the DSN as overlays. +SSL is configured through DSN query parameters. For example, to require TLS with certificate verification: + +```bash +postgresql://zitadel:password@db.example.com:5432/zitadel?sslmode=verify-full&sslrootcert=/path/to/ca.crt ``` ### Managed PostgreSQL / No Admin Access @@ -83,17 +77,10 @@ Adjust `pg_hba.conf` if needed to allow the zitadel user to connect. **Schema bootstrapping step — Bootstrap schemas and tables (run as the service user)** -Set only the service user credentials — no `Admin.*` variables are needed for this command. - -For managed/remote PostgreSQL, set `SSL_MODE` to `require` or `verify-full` to encrypt the connection. `disable` is only appropriate for local development. +Provide the DSN with the service user credentials. For managed/remote PostgreSQL, set `sslmode` to `require` or `verify-full` to encrypt the connection. `disable` is only appropriate for local development. ```bash -ZITADEL_DATABASE_POSTGRES_HOST= \ - ZITADEL_DATABASE_POSTGRES_PORT=5432 \ - ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel \ - ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel \ - ZITADEL_DATABASE_POSTGRES_USER_PASSWORD= \ - ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=require \ +ZITADEL_DATABASE_POSTGRES_DSN=postgresql://zitadel:@:5432/zitadel?sslmode=require \ zitadel init schema ``` @@ -104,12 +91,7 @@ For production deployments, configure TLS to encrypt authentication traffic. See ```bash -ZITADEL_DATABASE_POSTGRES_HOST= \ - ZITADEL_DATABASE_POSTGRES_PORT=5432 \ - ZITADEL_DATABASE_POSTGRES_DATABASE=zitadel \ - ZITADEL_DATABASE_POSTGRES_USER_USERNAME=zitadel \ - ZITADEL_DATABASE_POSTGRES_USER_PASSWORD= \ - ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE=require \ +ZITADEL_DATABASE_POSTGRES_DSN=postgresql://zitadel:@:5432/zitadel?sslmode=require \ zitadel start-from-setup --masterkey "<32-character-key>" ``` @@ -117,7 +99,7 @@ See the [phases guide](/self-hosting/manage/updating_scaling#separating-init-and ## Zitadel Credentials -The [init phase](/self-hosting/manage/updating_scaling#separating-init-and-setup-from-the-runtime) of ZITADEL creates the ZITADEL database user (`Database.*.User.Username` & `Database.*.User.Password`) if it does not exist and admin credentials are provided. **Init does not update or deprecate existing users or passwords.** If you need to change the ZITADEL database user or rotate its password, you must do this manually. +The [init phase](/self-hosting/manage/updating_scaling#separating-init-and-setup-from-the-runtime) of ZITADEL creates the ZITADEL database user if it does not exist and admin credentials are provided. **Init does not update or deprecate existing users or passwords.** If you need to change the ZITADEL database user or rotate its password, you must do this manually. The `init` command is designed to be run **only once**, during the initial setup of ZITADEL. Running `init` again after changing the database user or credentials does **not** migrate database object ownerships, nor does it reassign permissions on schemas, tables, or other objects to a new user. You must update these permissions manually if you switch users. diff --git a/apps/docs/content/self-hosting/manage/production.mdx b/apps/docs/content/self-hosting/manage/production.mdx index 5411e5772..f6ebb4f54 100644 --- a/apps/docs/content/self-hosting/manage/production.mdx +++ b/apps/docs/content/self-hosting/manage/production.mdx @@ -125,19 +125,28 @@ please create an issue in our [github repository](https://github.com/zitadel/zit Depending on your environment, you maybe would want to tweak some settings about how ZITADEL interacts with the database in the database section of your ZITADEL configuration. Read more about your [database configuration options](/self-hosting/manage/database). +Set the database connection via environment variable: + +```bash +export ZITADEL_DATABASE_POSTGRES_DSN=postgresql://zitadel:password@localhost:5432/zitadel?sslmode=disable +export ZITADEL_DATABASE_POSTGRES_MAXOPENCONNS=10 +export ZITADEL_DATABASE_POSTGRES_MAXIDLECONNS=5 +export ZITADEL_DATABASE_POSTGRES_MAXCONNLIFETIME=30m +export ZITADEL_DATABASE_POSTGRES_MAXCONNIDLETIME=5m +``` + +Or in a configuration file: + ```yaml Database: postgres: - Host: localhost - Port: 5432 - Database: zitadel + DSN: postgresql://zitadel:password@localhost:5432/zitadel?sslmode=disable //highlight-start MaxOpenConns: 10 MaxIdleConns: 5 MaxConnLifetime: 30m MaxConnIdleTime: 5m //highlight-end - Options: "" ``` You also might want to configure how [projections](/concepts/eventstore/implementation#projections) are computed. These are the default values: diff --git a/apps/docs/content/self-hosting/manage/updating_scaling.mdx b/apps/docs/content/self-hosting/manage/updating_scaling.mdx index 2488473c0..b4c9fbe27 100644 --- a/apps/docs/content/self-hosting/manage/updating_scaling.mdx +++ b/apps/docs/content/self-hosting/manage/updating_scaling.mdx @@ -56,17 +56,17 @@ when you install ZITADEL from scratch. `zitadel init` performs two steps that require different privileges: -**Provisioning step** (requires a PostgreSQL superuser via `Database.postgres.Admin.Username`): -- If it doesn't exist already, it creates the unprivileged user configured in `Database.postgres.User.Username`. - Subsequent phases connect to the database with this user's credentials only. -- If it doesn't exist already, it creates a database with the configured database name. +**Provisioning step** (requires a PostgreSQL superuser): +- If it doesn’t exist already, it creates the unprivileged database user. + Subsequent phases connect to the database with this user’s credentials only. +- If it doesn’t exist already, it creates a database with the configured database name. - If not already done, it grants the necessary permissions to the unprivileged user. **Schema bootstrapping step** (requires database owner privileges, uses the service user): -- If they don't exist already, it creates all schemas (`eventstore`, `projections`, `system`) and base tables. +- If they don’t exist already, it creates all schemas (`eventstore`, `projections`, `system`) and base tables. -**Manual provisioning:** If you handle the provisioning step yourself (e.g. on a managed PostgreSQL service where you don't have superuser access), +**Manual provisioning:** If you handle the provisioning step yourself (e.g. on a managed PostgreSQL service where you don’t have superuser access), you can skip it entirely by running only the schema bootstrapping step: ```bash diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index ee525ac2c..8e0497977 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -231,6 +231,16 @@ WebAuthNName: ZITADEL # ZITADEL_WEBAUTHNNAME Database: # Postgres is the default database of ZITADEL postgres: + # DSN is a full PostgreSQL connection URL (see + # https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). + # When set, individual connection fields (Host, Port, Database, User, Admin, + # Options) are ignored. + # Format: postgresql://user:password@host:port/dbname?sslmode=disable + # Note: In DSN mode, zitadel init cannot use the Admin connection to create + # a separate target database/user. The DB and user from the DSN must already + # exist and have sufficient privileges, or the DSN must directly point to + # the final ZITADEL database. + DSN: "" # ZITADEL_DATABASE_POSTGRES_DSN Host: localhost # ZITADEL_DATABASE_POSTGRES_HOST Port: 5432 # ZITADEL_DATABASE_POSTGRES_PORT Database: zitadel # ZITADEL_DATABASE_POSTGRES_DATABASE @@ -286,6 +296,15 @@ Caches: TimeOut: 30s Redis: Enabled: false + # URL is a full Redis connection URL (see + # https://redis.io/docs/latest/develop/tools/cli/#host-port-password-and-database). + # When set, individual connection fields + # (Addr, Username, Password, Network, EnableTLS) are ignored. + # Pool, timeout, retry, and circuit breaker settings can still be used as overlays. + # Format: redis://user:password@host:port/db or rediss://... for TLS. + # Note: ZITADEL ignores the `/db` path when selecting Redis databases; DBs are chosen + # internally via DBOffset + purpose. Use `/0` in the URL to avoid confusion. + URL: "" # ZITADEL_CACHES_CONNECTORS_REDIS_URL # The network type, either tcp or unix. # Default is tcp. # Network string diff --git a/cmd/mirror/defaults.yaml b/cmd/mirror/defaults.yaml index 929a13d0c..109e4cd22 100644 --- a/cmd/mirror/defaults.yaml +++ b/cmd/mirror/defaults.yaml @@ -136,6 +136,7 @@ Source: # Postgres is used as soon as a value is set # The values describe the possible fields to set values postgres: + DSN: "" # ZITADEL_SOURCE_POSTGRES_DSN Host: # ZITADEL_SOURCE_POSTGRES_HOST Port: # ZITADEL_SOURCE_POSTGRES_PORT Database: # ZITADEL_SOURCE_POSTGRES_DATABASE @@ -155,6 +156,7 @@ Source: Destination: postgres: + DSN: "" # ZITADEL_DESTINATION_POSTGRES_DSN Host: localhost # ZITADEL_DESTINATION_POSTGRES_HOST Port: 5432 # ZITADEL_DESTINATION_POSTGRES_PORT Database: zitadel # ZITADEL_DESTINATION_POSTGRES_DATABASE diff --git a/deploy/compose/.env.example b/deploy/compose/.env.example index 89f40c3c7..5cea718d6 100644 --- a/deploy/compose/.env.example +++ b/deploy/compose/.env.example @@ -53,8 +53,14 @@ LETSENCRYPT_EMAIL=ops@example.com POSTGRES_DB=zitadel POSTGRES_ADMIN_USER=postgres POSTGRES_ADMIN_PASSWORD=postgres -POSTGRES_ZITADEL_USER=zitadel -POSTGRES_ZITADEL_PASSWORD=zitadel +# DSN used by ZITADEL to connect to PostgreSQL. +# The start-from-init command uses this DSN to create the database schema and for +# normal operation. When a DSN is configured, ZITADEL does not create or switch +# to a separate unprivileged database user; the user in this DSN is used directly +# and must already exist. For production, configure this DSN to use a non-superuser +# role with only the permissions ZITADEL needs. +# See https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING +ZITADEL_DATABASE_POSTGRES_DSN=postgresql://postgres:postgres@postgres:5432/zitadel?sslmode=disable # ----------------------------- # Access logs @@ -67,9 +73,9 @@ ZITADEL_ACCESS_LOG_STDOUT_ENABLED=true # Enable with --profile cache and switch connectors to redis. # ----------------------------- ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED=false -ZITADEL_CACHES_CONNECTORS_REDIS_ADDR=redis:6379 -ZITADEL_CACHES_CONNECTORS_REDIS_USERNAME= -ZITADEL_CACHES_CONNECTORS_REDIS_PASSWORD= +# DSN used by ZITADEL to connect to Redis. +# See https://redis.io/docs/latest/develop/tools/cli/#host-port-password-and-database +ZITADEL_CACHES_CONNECTORS_REDIS_URL=redis://redis:6379/0 ZITADEL_CACHES_INSTANCE_CONNECTOR= ZITADEL_CACHES_MILESTONES_CONNECTOR= ZITADEL_CACHES_ORGANIZATION_CONNECTOR= diff --git a/deploy/compose/.env.test b/deploy/compose/.env.test index 487492e15..1ec4d9f94 100644 --- a/deploy/compose/.env.test +++ b/deploy/compose/.env.test @@ -32,8 +32,7 @@ TRAEFIK_TRUSTED_IPS=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 POSTGRES_DB=zitadel POSTGRES_ADMIN_USER=postgres POSTGRES_ADMIN_PASSWORD=postgres -POSTGRES_ZITADEL_USER=zitadel -POSTGRES_ZITADEL_PASSWORD=zitadel +ZITADEL_DATABASE_POSTGRES_DSN=postgresql://postgres:postgres@postgres:5432/zitadel?sslmode=disable # Disable access logs to reduce noise in test output ZITADEL_ACCESS_LOG_STDOUT_ENABLED=false @@ -46,9 +45,7 @@ ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE=false # Cache disabled (no Redis profile) ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED=false -ZITADEL_CACHES_CONNECTORS_REDIS_ADDR=redis:6379 -ZITADEL_CACHES_CONNECTORS_REDIS_USERNAME= -ZITADEL_CACHES_CONNECTORS_REDIS_PASSWORD= +ZITADEL_CACHES_CONNECTORS_REDIS_URL=redis://redis:6379/0 ZITADEL_CACHES_INSTANCE_CONNECTOR= ZITADEL_CACHES_MILESTONES_CONNECTOR= ZITADEL_CACHES_ORGANIZATION_CONNECTOR= diff --git a/deploy/compose/docker-compose.prodlike.yml b/deploy/compose/docker-compose.prodlike.yml index b23d30e44..99d4db463 100644 --- a/deploy/compose/docker-compose.prodlike.yml +++ b/deploy/compose/docker-compose.prodlike.yml @@ -1,13 +1,5 @@ x-zitadel-db-env: &zitadel-db-env - ZITADEL_DATABASE_POSTGRES_HOST: postgres - ZITADEL_DATABASE_POSTGRES_PORT: 5432 - ZITADEL_DATABASE_POSTGRES_DATABASE: ${POSTGRES_DB} - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: ${POSTGRES_ADMIN_USER} - ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: ${POSTGRES_ADMIN_PASSWORD} - ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_USER_USERNAME: ${POSTGRES_ZITADEL_USER} - ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: ${POSTGRES_ZITADEL_PASSWORD} - ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable + ZITADEL_DATABASE_POSTGRES_DSN: ${ZITADEL_DATABASE_POSTGRES_DSN} services: zitadel-init: diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index 0d35b8f25..93a53d5f3 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -40,15 +40,7 @@ services: ZITADEL_EXTERNALSECURE: ${ZITADEL_EXTERNALSECURE} ZITADEL_TLS_ENABLED: false - ZITADEL_DATABASE_POSTGRES_HOST: postgres - ZITADEL_DATABASE_POSTGRES_PORT: 5432 - ZITADEL_DATABASE_POSTGRES_DATABASE: ${POSTGRES_DB} - ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: ${POSTGRES_ADMIN_USER} - ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: ${POSTGRES_ADMIN_PASSWORD} - ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable - ZITADEL_DATABASE_POSTGRES_USER_USERNAME: ${POSTGRES_ZITADEL_USER} - ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: ${POSTGRES_ZITADEL_PASSWORD} - ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable + ZITADEL_DATABASE_POSTGRES_DSN: ${ZITADEL_DATABASE_POSTGRES_DSN} ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED: false ZITADEL_FIRSTINSTANCE_LOGINCLIENTPATPATH: /zitadel/bootstrap/login-client.pat @@ -70,9 +62,7 @@ services: ZITADEL_INSTRUMENTATION_SERVICENAME: ${ZITADEL_INSTRUMENTATION_SERVICENAME} ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED: ${ZITADEL_CACHES_CONNECTORS_REDIS_ENABLED} - ZITADEL_CACHES_CONNECTORS_REDIS_ADDR: ${ZITADEL_CACHES_CONNECTORS_REDIS_ADDR} - ZITADEL_CACHES_CONNECTORS_REDIS_USERNAME: ${ZITADEL_CACHES_CONNECTORS_REDIS_USERNAME} - ZITADEL_CACHES_CONNECTORS_REDIS_PASSWORD: ${ZITADEL_CACHES_CONNECTORS_REDIS_PASSWORD} + ZITADEL_CACHES_CONNECTORS_REDIS_URL: ${ZITADEL_CACHES_CONNECTORS_REDIS_URL} ZITADEL_CACHES_INSTANCE_CONNECTOR: ${ZITADEL_CACHES_INSTANCE_CONNECTOR} ZITADEL_CACHES_MILESTONES_CONNECTOR: ${ZITADEL_CACHES_MILESTONES_CONNECTOR} ZITADEL_CACHES_ORGANIZATION_CONNECTOR: ${ZITADEL_CACHES_ORGANIZATION_CONNECTOR} @@ -149,7 +139,7 @@ services: - CMD - /bin/sh - -c - - node /app/healthcheck.js http://localhost:3000/ui/v2/login/healthy + - node /app/healthcheck.mjs http://localhost:3000/ui/v2/login/healthy interval: 10s timeout: 30s retries: 12 diff --git a/internal/cache/connector/connector.go b/internal/cache/connector/connector.go index 9e25f445c..c3a413342 100644 --- a/internal/cache/connector/connector.go +++ b/internal/cache/connector/connector.go @@ -38,11 +38,15 @@ func StartConnectors(conf *CachesConfig, client *database.DB) (Connectors, error if conf == nil { return Connectors{}, nil } + redisConnector, err := redis.NewConnector(conf.Connectors.Redis) + if err != nil { + return Connectors{}, fmt.Errorf("start redis connector: %w", err) + } return Connectors{ Config: *conf, Memory: gomap.NewConnector(conf.Connectors.Memory), Postgres: pg.NewConnector(conf.Connectors.Postgres, client), - Redis: redis.NewConnector(conf.Connectors.Redis), + Redis: redisConnector, }, nil } diff --git a/internal/cache/connector/redis/connector.go b/internal/cache/connector/redis/connector.go index a10a0c25d..073efbf36 100644 --- a/internal/cache/connector/redis/connector.go +++ b/internal/cache/connector/redis/connector.go @@ -2,6 +2,7 @@ package redis import ( "crypto/tls" + "fmt" "time" "github.com/redis/go-redis/v9" @@ -10,6 +11,14 @@ import ( type Config struct { Enabled bool + // URL is a full Redis connection URL. When set, individual connection + // fields (Addr, Username, Password, Network, EnableTLS) are ignored. + // Pool, timeout, retry, and circuit breaker settings can still be used as overlays. + // Format: redis://user:password@host:port[/db] or rediss://... for TLS. + // Note: the DB number in the URL path is ignored for cache DB selection. + // ZITADEL derives the actual DB from DBOffset + purpose; use /0 or omit the /db path. + URL string + // The network type, either tcp or unix. // Default is tcp. Network string @@ -114,14 +123,91 @@ type Connector struct { Config Config } -func NewConnector(config Config) *Connector { +func NewConnector(config Config) (*Connector, error) { if !config.Enabled { - return nil + return nil, nil } + + var opts *redis.Options + if config.URL != "" { + var err error + opts, err = optionsFromURL(config) + if err != nil { + return nil, err + } + } else { + opts = optionsFromConfig(config) + } + return &Connector{ - Client: redis.NewClient(optionsFromConfig(config)), + Client: redis.NewClient(opts), Config: config, + }, nil +} + +func optionsFromURL(c Config) (*redis.Options, error) { + opts, err := redis.ParseURL(c.URL) + if err != nil { + return nil, fmt.Errorf("invalid Redis URL: %w", err) } + + opts.Protocol = 3 + opts.ContextTimeoutEnabled = true + + if c.ClientName != "" { + opts.ClientName = c.ClientName + } + if c.MaxRetries != 0 { + opts.MaxRetries = c.MaxRetries + } + if c.MinRetryBackoff != 0 { + opts.MinRetryBackoff = c.MinRetryBackoff + } + if c.MaxRetryBackoff != 0 { + opts.MaxRetryBackoff = c.MaxRetryBackoff + } + if c.DialTimeout != 0 { + opts.DialTimeout = c.DialTimeout + } + if c.ReadTimeout != 0 { + opts.ReadTimeout = c.ReadTimeout + } + if c.WriteTimeout != 0 { + opts.WriteTimeout = c.WriteTimeout + } + if c.PoolFIFO { + opts.PoolFIFO = c.PoolFIFO + } + if c.PoolSize != 0 { + opts.PoolSize = c.PoolSize + } + if c.PoolTimeout != 0 { + opts.PoolTimeout = c.PoolTimeout + } + if c.MinIdleConns != 0 { + opts.MinIdleConns = c.MinIdleConns + } + if c.MaxIdleConns != 0 { + opts.MaxIdleConns = c.MaxIdleConns + } + if c.MaxActiveConns != 0 { + opts.MaxActiveConns = c.MaxActiveConns + } + if c.ConnMaxIdleTime != 0 { + opts.ConnMaxIdleTime = c.ConnMaxIdleTime + } + if c.ConnMaxLifetime != 0 { + opts.ConnMaxLifetime = c.ConnMaxLifetime + } + + opts.DisableIdentity = c.DisableIndentity + if c.IdentitySuffix != "" { + opts.IdentitySuffix = c.IdentitySuffix + } + + opts.Limiter = newLimiter(c.CircuitBreaker, c.MaxActiveConns) + + return opts, nil } func optionsFromConfig(c Config) *redis.Options { @@ -140,6 +226,7 @@ func optionsFromConfig(c Config) *redis.Options { WriteTimeout: c.WriteTimeout, ContextTimeoutEnabled: true, PoolFIFO: c.PoolFIFO, + PoolSize: c.PoolSize, PoolTimeout: c.PoolTimeout, MinIdleConns: c.MinIdleConns, MaxIdleConns: c.MaxIdleConns, diff --git a/internal/cache/connector/redis/connector_test.go b/internal/cache/connector/redis/connector_test.go new file mode 100644 index 000000000..8bd2f6266 --- /dev/null +++ b/internal/cache/connector/redis/connector_test.go @@ -0,0 +1,187 @@ +package redis + +import ( + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOptionsFromURL(t *testing.T) { + t.Run("basic URL", func(t *testing.T) { + c := Config{URL: "redis://localhost:6379/0"} + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, "localhost:6379", opts.Addr) + assert.Equal(t, 0, opts.DB) + assert.Equal(t, 3, opts.Protocol) + assert.True(t, opts.ContextTimeoutEnabled) + }) + + t.Run("with auth", func(t *testing.T) { + c := Config{URL: "redis://myuser:mypass@host:6380/2"} + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, "host:6380", opts.Addr) + assert.Equal(t, "myuser", opts.Username) + assert.Equal(t, "mypass", opts.Password) + assert.Equal(t, 2, opts.DB) + }) + + t.Run("TLS via rediss scheme", func(t *testing.T) { + c := Config{URL: "rediss://host:6380/0"} + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.NotNil(t, opts.TLSConfig) + }) + + t.Run("password only auth", func(t *testing.T) { + c := Config{URL: "redis://:secret@host:6379/0"} + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, "", opts.Username) + assert.Equal(t, "secret", opts.Password) + }) + + t.Run("URL without DB number", func(t *testing.T) { + c := Config{URL: "redis://localhost:6379"} + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, "localhost:6379", opts.Addr) + assert.Equal(t, 0, opts.DB) + }) + + t.Run("overlays pool settings", func(t *testing.T) { + c := Config{ + URL: "redis://localhost:6379/0", + PoolSize: 42, + MaxActiveConns: 100, + MinIdleConns: 5, + } + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, 42, opts.PoolSize) + assert.Equal(t, 100, opts.MaxActiveConns) + assert.Equal(t, 5, opts.MinIdleConns) + }) + + t.Run("overlays timeout settings", func(t *testing.T) { + c := Config{ + URL: "redis://localhost:6379/0", + DialTimeout: 10 * time.Second, + ReadTimeout: 5 * time.Second, + } + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, 10*time.Second, opts.DialTimeout) + assert.Equal(t, 5*time.Second, opts.ReadTimeout) + }) + + t.Run("overlays retry settings", func(t *testing.T) { + c := Config{ + URL: "redis://localhost:6379/0", + MaxRetries: 5, + } + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, 5, opts.MaxRetries) + }) + + t.Run("overlays client name", func(t *testing.T) { + c := Config{ + URL: "redis://localhost:6379/0", + ClientName: "zitadel", + } + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.Equal(t, "zitadel", opts.ClientName) + }) + + t.Run("overlays circuit breaker", func(t *testing.T) { + c := Config{ + URL: "redis://localhost:6379/0", + CircuitBreaker: &CBConfig{ + MaxConsecutiveFailures: 10, + }, + MaxActiveConns: 50, + } + opts, err := optionsFromURL(c) + require.NoError(t, err) + assert.NotNil(t, opts.Limiter) + }) + + t.Run("invalid URL returns error", func(t *testing.T) { + c := Config{URL: "garbage://???"} + _, err := optionsFromURL(c) + require.Error(t, err) + }) +} + +func TestOptionsFromConfig(t *testing.T) { + t.Run("maps all fields", func(t *testing.T) { + c := Config{ + Addr: "myhost:6380", + Username: "user", + Password: "pass", + PoolSize: 42, + MaxRetries: 5, + } + opts := optionsFromConfig(c) + assert.Equal(t, "myhost:6380", opts.Addr) + assert.Equal(t, "user", opts.Username) + assert.Equal(t, "pass", opts.Password) + assert.Equal(t, 42, opts.PoolSize) + assert.Equal(t, 5, opts.MaxRetries) + assert.Equal(t, 3, opts.Protocol) + assert.True(t, opts.ContextTimeoutEnabled) + }) + + t.Run("EnableTLS sets TLSConfig", func(t *testing.T) { + c := Config{Addr: "host:6379", EnableTLS: true} + opts := optionsFromConfig(c) + assert.NotNil(t, opts.TLSConfig) + }) +} + +func TestNewConnector(t *testing.T) { + t.Run("disabled returns nil without error", func(t *testing.T) { + c, err := NewConnector(Config{Enabled: false, URL: "redis://localhost:6379/0"}) + require.NoError(t, err) + assert.Nil(t, c) + }) + + t.Run("URL mode connects to miniredis", func(t *testing.T) { + server := miniredis.RunT(t) + url := "redis://" + server.Addr() + "/0" + c, err := NewConnector(Config{ + Enabled: true, + URL: url, + DisableIndentity: true, + }) + require.NoError(t, err) + require.NotNil(t, c) + t.Cleanup(func() { c.Close() }) + }) + + t.Run("legacy mode connects to miniredis", func(t *testing.T) { + server := miniredis.RunT(t) + c, err := NewConnector(Config{ + Enabled: true, + Addr: server.Addr(), + DisableIndentity: true, + }) + require.NoError(t, err) + require.NotNil(t, c) + t.Cleanup(func() { c.Close() }) + }) + + t.Run("invalid URL errors", func(t *testing.T) { + _, err := NewConnector(Config{ + Enabled: true, + URL: "garbage://???", + }) + require.Error(t, err) + }) +} diff --git a/internal/cache/connector/redis/redis_test.go b/internal/cache/connector/redis/redis_test.go index ff6034cfa..b1461964d 100644 --- a/internal/cache/connector/redis/redis_test.go +++ b/internal/cache/connector/redis/redis_test.go @@ -705,7 +705,8 @@ func prepareCache(t *testing.T, conf cache.Config, options ...func(*Config)) (ca for _, option := range options { option(&connConfig) } - connector := NewConnector(connConfig) + connector, err := NewConnector(connConfig) + require.NoError(t, err) t.Cleanup(func() { connector.Close() server.Close() diff --git a/internal/database/postgres/pg.go b/internal/database/postgres/pg.go index da32f8cc4..18411137f 100644 --- a/internal/database/postgres/pg.go +++ b/internal/database/postgres/pg.go @@ -3,8 +3,10 @@ package postgres import ( "context" "database/sql" + "fmt" "strconv" "strings" + "sync" "time" "github.com/jackc/pgx/v5" @@ -29,6 +31,17 @@ const ( ) type Config struct { + // DSN is a full PostgreSQL connection URL. When set, individual connection + // fields (Host, Port, Database, User, Admin, Options) are ignored, and the + // database name and user are taken exclusively from the DSN. + // Format: postgresql://user:password@host:port/dbname?sslmode=disable + // Note: In DSN mode, ZITADEL will not create the target database or user. + // The referenced database/user must already exist, and admin-creation + // semantics used in non-DSN mode (e.g. via useAdmin) do not apply. + DSN string + parsedDSN *pgxpool.Config `mapstructure:"-"` + parseOnce sync.Once `mapstructure:"-"` + Host string Port int32 Database string @@ -70,15 +83,35 @@ func (_ *Config) Decode(configs []interface{}) (dialect.Connector, error) { } } + if connector.DSN != "" { + parsed, err := pgxpool.ParseConfig(connector.DSN) + if err != nil { + return nil, fmt.Errorf("invalid PostgreSQL DSN: %w", err) + } + connector.parsedDSN = parsed + } + return connector, nil } func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) { connConfig := dialect.NewConnectionConfig(c.MaxOpenConns, c.MaxIdleConns) - config, err := pgxpool.ParseConfig(c.String(useAdmin)) - if err != nil { - return nil, nil, err + var config *pgxpool.Config + var err error + if parsed := c.ensureParsedDSN(); parsed != nil { + config = parsed.Copy() + if config.ConnConfig.RuntimeParams == nil { + config.ConnConfig.RuntimeParams = map[string]string{} + } + if _, ok := config.ConnConfig.RuntimeParams["application_name"]; !ok { + config.ConnConfig.RuntimeParams["application_name"] = dialect.DefaultAppName + } + } else { + config, err = pgxpool.ParseConfig(c.String(useAdmin)) + if err != nil { + return nil, nil, err + } } if len(connConfig.AfterConnect) > 0 { @@ -146,15 +179,43 @@ func (c *Config) Connect(useAdmin bool) (*sql.DB, *pgxpool.Pool, error) { return stdlib.OpenDBFromPool(pool), pool, nil } +// ensureParsedDSN lazily parses the DSN on first access so that accessors +// return correct values even if Decode() was not called. +func (c *Config) ensureParsedDSN() *pgxpool.Config { + if c.DSN == "" { + return nil + } + c.parseOnce.Do(func() { + if c.parsedDSN == nil { + parsed, err := pgxpool.ParseConfig(c.DSN) + if err != nil { + logging.WithError(err).Warn("failed to parse PostgreSQL DSN") + return + } + c.parsedDSN = parsed + } + }) + return c.parsedDSN +} + func (c *Config) DatabaseName() string { + if parsed := c.ensureParsedDSN(); parsed != nil { + return parsed.ConnConfig.Database + } return c.Database } func (c *Config) Username() string { + if parsed := c.ensureParsedDSN(); parsed != nil { + return parsed.ConnConfig.User + } return c.User.Username } func (c *Config) Password() string { + if parsed := c.ensureParsedDSN(); parsed != nil { + return parsed.ConnConfig.Password + } return c.User.Password } @@ -204,7 +265,10 @@ func (s *Config) checkSSL(user User) { } } -func (c Config) String(useAdmin bool) string { +func (c *Config) String(useAdmin bool) string { + if c.DSN != "" { + return c.DSN + } user := c.User if useAdmin { user = c.Admin.User diff --git a/internal/database/postgres/pg_test.go b/internal/database/postgres/pg_test.go new file mode 100644 index 000000000..b268af4ec --- /dev/null +++ b/internal/database/postgres/pg_test.go @@ -0,0 +1,231 @@ +package postgres + +import ( + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// parseDSN is a test helper that parses a DSN and returns a Config +// with parsedDSN populated, mimicking what Decode() does. +func parseDSN(t *testing.T, dsn string) Config { + t.Helper() + parsed, err := pgxpool.ParseConfig(dsn) + require.NoError(t, err) + return Config{DSN: dsn, parsedDSN: parsed} +} + +func TestConfig_parseDSN_parsesDSN(t *testing.T) { + c := parseDSN(t, "postgresql://myuser:mypass@dbhost:5433/mydb?sslmode=require") + require.NotNil(t, c.parsedDSN) + assert.Equal(t, "dbhost", c.parsedDSN.ConnConfig.Host) + assert.Equal(t, uint16(5433), c.parsedDSN.ConnConfig.Port) + assert.Equal(t, "mydb", c.parsedDSN.ConnConfig.Database) + assert.Equal(t, "myuser", c.parsedDSN.ConnConfig.User) + assert.Equal(t, "mypass", c.parsedDSN.ConnConfig.Password) +} + +func TestConfig_DatabaseName(t *testing.T) { + t.Run("from DSN", func(t *testing.T) { + c := parseDSN(t, "postgresql://u:p@h:5432/fromdsn?sslmode=disable") + assert.Equal(t, "fromdsn", c.DatabaseName()) + }) + + t.Run("from legacy field", func(t *testing.T) { + c := Config{Database: "fromlegacy"} + assert.Equal(t, "fromlegacy", c.DatabaseName()) + }) +} + +func TestConfig_Username(t *testing.T) { + t.Run("from DSN", func(t *testing.T) { + c := parseDSN(t, "postgresql://dsnuser:p@h:5432/db?sslmode=disable") + assert.Equal(t, "dsnuser", c.Username()) + }) + + t.Run("from legacy field", func(t *testing.T) { + c := Config{User: User{Username: "legacyuser"}} + assert.Equal(t, "legacyuser", c.Username()) + }) +} + +func TestConfig_Password(t *testing.T) { + t.Run("from DSN", func(t *testing.T) { + c := parseDSN(t, "postgresql://u:dsnpass@h:5432/db?sslmode=disable") + assert.Equal(t, "dsnpass", c.Password()) + }) + + t.Run("from legacy field", func(t *testing.T) { + c := Config{User: User{Password: "legacypass"}} + assert.Equal(t, "legacypass", c.Password()) + }) +} + +func TestConfig_String(t *testing.T) { + t.Run("returns DSN when set", func(t *testing.T) { + dsn := "postgresql://u:p@h:5432/db?sslmode=disable" + c := Config{DSN: dsn} + assert.Equal(t, dsn, c.String(false)) + }) + + t.Run("builds legacy string", func(t *testing.T) { + c := Config{ + Host: "myhost", + Port: 5433, + Database: "mydb", + User: User{ + Username: "myuser", + Password: "mypass", + SSL: SSL{Mode: "disable"}, + }, + } + s := c.String(false) + assert.Contains(t, s, "host=myhost") + assert.Contains(t, s, "port=5433") + assert.Contains(t, s, "user=myuser") + assert.Contains(t, s, "password=mypass") + assert.Contains(t, s, "dbname=mydb") + assert.Contains(t, s, "sslmode=disable") + }) +} + +func TestConfig_DSN_fullSpec(t *testing.T) { + tests := []struct { + name string + dsn string + wantHost string + wantPort uint16 + wantDB string + wantUser string + wantPass string + }{ + { + name: "standard URL format", + dsn: "postgresql://myuser:mypass@myhost:5433/mydb?sslmode=disable", + wantHost: "myhost", + wantPort: 5433, + wantDB: "mydb", + wantUser: "myuser", + wantPass: "mypass", + }, + { + name: "postgres:// scheme", + dsn: "postgres://u:p@h:5432/db?sslmode=disable", + wantHost: "h", + wantPort: 5432, + wantDB: "db", + wantUser: "u", + wantPass: "p", + }, + { + name: "URL-encoded password", + dsn: "postgresql://user:p%40ss%23word@host:5432/db?sslmode=disable", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "p@ss#word", + }, + { + name: "with SSL parameters", + dsn: "postgresql://user:pass@host:5432/db?sslmode=require", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "pass", + }, + { + name: "with application_name", + dsn: "postgresql://user:pass@host:5432/db?sslmode=disable&application_name=myapp", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "pass", + }, + { + name: "with options parameter", + dsn: "postgresql://user:pass@host:5432/db?sslmode=disable&options=-c%20search_path%3Dpublic", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "pass", + }, + { + name: "key-value format", + dsn: "host=kvhost port=5434 user=kvuser password=kvpass dbname=kvdb sslmode=disable", + wantHost: "kvhost", + wantPort: 5434, + wantDB: "kvdb", + wantUser: "kvuser", + wantPass: "kvpass", + }, + { + name: "default port (omitted)", + dsn: "postgresql://user:pass@host/db?sslmode=disable", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "pass", + }, + { + name: "without password", + dsn: "postgresql://user@host:5432/db?sslmode=disable", + wantHost: "host", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "", + }, + { + name: "without database", + dsn: "postgresql://user:pass@host:5432?sslmode=disable", + wantHost: "host", + wantPort: 5432, + wantDB: "", + wantUser: "user", + wantPass: "pass", + }, + { + name: "IPv6 host", + dsn: "postgresql://user:pass@[::1]:5432/db?sslmode=disable", + wantHost: "::1", + wantPort: 5432, + wantDB: "db", + wantUser: "user", + wantPass: "pass", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := parseDSN(t, tt.dsn) + + assert.Equal(t, tt.wantDB, c.DatabaseName()) + assert.Equal(t, tt.wantUser, c.Username()) + assert.Equal(t, tt.wantPass, c.Password()) + + assert.Equal(t, tt.wantHost, c.parsedDSN.ConnConfig.Host) + assert.Equal(t, tt.wantPort, c.parsedDSN.ConnConfig.Port) + }) + } +} + +func TestConfig_DSN_ignores_legacy_fields(t *testing.T) { + // When DSN is set and parsedDSN is populated, accessors return + // values from the DSN, not from the legacy fields. + c := parseDSN(t, "postgresql://dsnuser:dsnpass@dsnhost:5433/dsndb?sslmode=disable") + c.Host = "legacyhost" + c.Port = 9999 + c.Database = "legacydb" + c.User = User{Username: "legacyuser", Password: "legacypass"} + + assert.Equal(t, "dsndb", c.DatabaseName()) + assert.Equal(t, "dsnuser", c.Username()) + assert.Equal(t, "dsnpass", c.Password()) +}