From ce85d8e8ed6a49db79fc100389e9cfc2e726a23c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Racedo?= Date: Fri, 2 Jan 2026 12:58:18 -0300 Subject: [PATCH] docs(guide): Using Prometheus to scrape Zitadel metrics (#11255) Resolves #6062 --- .../self-hosting/manage/metrics/overview.mdx | 71 ++++ .../self-hosting/manage/metrics/prometheus.md | 322 ++++++++++++++++++ docs/sidebars.js | 10 + 3 files changed, 403 insertions(+) create mode 100644 docs/docs/self-hosting/manage/metrics/overview.mdx create mode 100644 docs/docs/self-hosting/manage/metrics/prometheus.md diff --git a/docs/docs/self-hosting/manage/metrics/overview.mdx b/docs/docs/self-hosting/manage/metrics/overview.mdx new file mode 100644 index 000000000..340c48154 --- /dev/null +++ b/docs/docs/self-hosting/manage/metrics/overview.mdx @@ -0,0 +1,71 @@ +--- +title: Metrics +description: Overview of ZITADEL metrics, exposure endpoint, and compatibility with monitoring systems. +--- + +# Metrics + +ZITADEL exposes operational and performance metrics to support monitoring, alerting, and capacity planning. + +Metrics are **enabled by default** in standard deployments and are exposed via an HTTP endpoint that can be consumed by Prometheus and other monitoring systems that understand the Prometheus exposition format or use an OpenTelemetry pipeline. + + +## Metrics endpoint + +ZITADEL exposes metrics at the following endpoint: +`/debug/metrics` + +This endpoint returns a snapshot of the current metrics in **Prometheus-compatible exposition format**. + +If you changed ports, paths, or network exposure as part of your deployment (for example via Helm values, reverse proxies, or ingress), adjust the endpoint accordingly. + + +## Instrumentation model + +Internally, ZITADEL instruments metrics using **OpenTelemetry (OTel)**. + +The exposed metrics include: + +- **Process and runtime metrics** + (for example Go runtime, memory usage, goroutines) + +- **HTTP and gRPC server metrics** + (request counts, latencies, error rates) + +- **ZITADEL-specific metrics** + (server behavior and internal components available in your version) + +The exact set of metrics and labels may evolve between ZITADEL releases. + + +## Collection and compatibility + +The `/debug/metrics` endpoint can be consumed in multiple ways: + +- **Prometheus** can scrape the endpoint directly. +- **OpenTelemetry Collector** can scrape the endpoint using a Prometheus receiver and forward metrics to other backends. +- **Managed monitoring platforms** (for example Grafana Cloud, Amazon Managed Service for Prometheus, Datadog, New Relic) can ingest the same endpoint via Prometheus-compatible agents or collectors. + +ZITADEL does **not** require a specific metrics backend. Any system capable of scraping or ingesting Prometheus-format metrics can be used. + + +## Configuration + +Metrics are enabled by default in standard ZITADEL deployments. + +If metrics were explicitly disabled in your configuration, re-enable them before proceeding. The default configuration is defined in the project's `defaults.yaml` file: + +- https://github.com/zitadel/zitadel/blob/main/cmd/defaults.yaml + +Refer to the configuration documentation for details on enabling or adjusting metrics behavior. + + +## Next steps + +Choose how you want to collect and store metrics: + +- **Prometheus** — scrape metrics directly from ZITADEL +- **OpenTelemetry Collector** — aggregate and forward metrics to another backend +- **Managed observability platforms** — use their Prometheus-compatible agents + +The following sections describe concrete setups, starting with **Prometheus**. \ No newline at end of file diff --git a/docs/docs/self-hosting/manage/metrics/prometheus.md b/docs/docs/self-hosting/manage/metrics/prometheus.md new file mode 100644 index 000000000..4a0fa2016 --- /dev/null +++ b/docs/docs/self-hosting/manage/metrics/prometheus.md @@ -0,0 +1,322 @@ +--- +title: Prometheus +description: Configure Prometheus to scrape ZITADEL metrics from /debug/metrics on Kubernetes and Docker. +--- + +# Prometheus + +This page shows platform-agnostic ways to scrape ZITADEL metrics using: + +- Kubernetes **annotation-based autodiscovery** (no Operator required) +- Kubernetes with **Prometheus Operator** using **ServiceMonitor** +- Standalone Prometheus on a VM or using **Docker** +- How to verify scraping, plus a minimal alert example + +> ZITADEL’s endpoint path is `/debug/metrics`. If you changed ports or paths via your deployment tooling, adjust the examples accordingly. + + + +## Before you begin + +- You have a running ZITADEL instance reachable from your monitoring stack. +- You know the base URL and port to reach ZITADEL (e.g., `http://zitadel.zitadel.svc:8080` in Kubernetes or `http://localhost:8080` locally). +- Metrics are enabled in your runtime [configuration](https://zitadel.com/docs/self-hosting/manage/configure) (they are enabled by default in standard setups). If you explicitly disabled metrics in your configuration, re-enable them before proceeding. The (default) configuration is located in the [defaults.yaml](https://github.com/zitadel/zitadel/blob/main/cmd/defaults.yaml). + + + +## Option A — Kubernetes (annotation-based autodiscovery) + +This approach is common when you **don’t use the Prometheus Operator**. Vanilla Prometheus can **auto-discover** scrape targets by reading standard annotations on Pods/Services: + +```yaml +metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/debug/metrics" + prometheus.io/port: "8080" +``` + +When your Prometheus server is configured with **Kubernetes service discovery** and **relabeling rules** that honor these annotations, it will automatically discover and scrape ZITADEL without any per-target `scrape_configs`. + +### 1) Ensure Prometheus has Kubernetes discovery and relabeling + +Your Prometheus configuration (often installed via Helm) should include jobs like the following. These are canonical examples that keep annotated Pods and map the annotated path/port to the actual metrics endpoint: + +```yaml +scrape_configs: + - job_name: "kubernetes-pods" + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Only keep pods with prometheus.io/scrape: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + # Use prometheus.io/path for the metrics path + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + # Replace the address with : + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ +``` + +> Most Prometheus Helm charts already ship with similar discovery jobs and relabeling rules. If you installed Prometheus via Helm, you likely already have these in place. + +### 2) Annotate ZITADEL Pods (or the Service) + +If you deploy ZITADEL via Helm and the chart emits scrape annotations on the Deployment/Pods, no extra work is needed. Otherwise, add the annotations yourself (via values override or a strategic patch): + +```yaml +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/debug/metrics" + prometheus.io/port: "8080" +``` + +If you prefer annotating the Service and your Prometheus config uses `role: service` discovery, add the same keys to the Service metadata. + +### 3) RBAC + +Prometheus must have permission to **list/watch** Pods/Endpoints in the target namespaces. Ensure its ServiceAccount has the standard ClusterRole/ClusterRoleBinding for discovery. Missing RBAC typically shows up as discovery errors in the Prometheus logs. + + +## Option B — Kubernetes with Prometheus Operator (ServiceMonitor) + +If you run **kube-prometheus-stack** or the **Prometheus Operator**, use a **ServiceMonitor** (or **PodMonitor**). ZITADEL’s Helm chart provides **out-of-the-box** [ServiceMonitor](https://github.com/zitadel/zitadel-charts/blob/main/charts/zitadel/templates/servicemonitor.yaml) support that you can enable via values—no manual YAML is required. + +### Enable the built-in ServiceMonitor (recommended) + +```bash +helm repo add zitadel https://charts.zitadel.com +helm upgrade --install zitadel zitadel/zitadel \ + --namespace zitadel --create-namespace \ + --set metrics.enabled=true \ + --set metrics.serviceMonitor.enabled=true +``` + +**Optional but common settings:** + +* Make the ServiceMonitor discoverable by your Prometheus (Operator) instance (many stacks match a label like `release=kube-prometheus-stack`): + + ```bash + --set metrics.serviceMonitor.additionalLabels.release=kube-prometheus-stack + ``` +* Place the ServiceMonitor in a central monitoring namespace (default is the ZITADEL release namespace): + + ```bash + --set metrics.serviceMonitor.namespace=monitoring + ``` +* Tune scraping: + + ```bash + --set metrics.serviceMonitor.scrapeInterval=15s \ + --set metrics.serviceMonitor.scrapeTimeout=10s + ``` +* Network/TLS customization (use only if you need them): + + ```bash + --set metrics.serviceMonitor.scheme=https \ + --set metrics.serviceMonitor.tlsConfig.insecureSkipVerify=true + ``` +* Relabeling: + + ```bash + --set metrics.serviceMonitor.relabellings[0].action=replace \ + --set metrics.serviceMonitor.metricRelabellings[0].action=drop + ``` + +### What the chart generates (explained) + +The chart renders a `ServiceMonitor` roughly equivalent to: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: # {{ include "zitadel.fullname" . }} + # namespace: # only if you set it + labels: + # Standard chart labels + any you add: + # {{- include "zitadel.start.labels" . | nindent 4 }} + # {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} +spec: + jobLabel: # {{ include "zitadel.fullname" . }} + namespaceSelector: + matchNames: + - "" # defaults to the Helm release namespace + selector: + matchLabels: + # Matches the ZITADEL Service created by the chart + # {{- include "zitadel.service.selectorLabels" . | nindent 6 }} + endpoints: + - port: "-server" # e.g., "http-server" or "https-server" + path: /debug/metrics + # Optional tunables below are included only if set: + interval: + scrapeTimeout: + scheme: # http|https + tlsConfig: # metrics.serviceMonitor.tlsConfig + # ... + proxyUrl: + honorLabels: + honorTimestamps: + relabelings: # metrics.serviceMonitor.relabellings + # ... + metricRelabelings: # metrics.serviceMonitor.metricRelabellings + # ... +``` + +**Details that matter:** + +* **Port name**: The chart uses + `port: {{ regexReplaceAll "\\W+" .Values.service.protocol "-" }}-server` + which resolves to **`http-server`** when `service.protocol=http` (default) or **`https-server`** when `service.protocol=https`. You do **not** need to edit this—just make sure you didn’t rename the ZITADEL Service port. + +* **Namespace selection**: By default, the ServiceMonitor targets the **ZITADEL release namespace** via: + + ```yaml + namespaceSelector: + matchNames: + - "" + ``` + + Set `metrics.serviceMonitor.namespace` if you want the ServiceMonitor object itself to live elsewhere (e.g., `monitoring`). The `selector.matchLabels` still points to the ZITADEL **Service** labels. + +* **Labels for discovery**: If your Prometheus (Operator) instance selects ServiceMonitors by label (common in kube-prometheus-stack), add those under `metrics.serviceMonitor.additionalLabels`—for example: + + ```yaml + metrics: + serviceMonitor: + additionalLabels: + release: kube-prometheus-stack + ``` + +* **Path**: The chart fixes the metrics path to **`/debug/metrics`** (matches ZITADEL’s endpoint). + +### Manual ServiceMonitor (only if you don’t want Helm to render it) + +If you prefer to manage the `ServiceMonitor` yourself, keep it aligned with the chart’s conventions: + +* Target the ZITADEL **Service** (not pods) using the same selector labels the chart adds. +* Use the correct **port name** (`http-server` or `https-server`) and **path** (`/debug/metrics`). +* Ensure your Prometheus (Operator) selects this ServiceMonitor by label/namespace. + +> **Tip** +> If you already run the **Prometheus Operator**, prefer this ServiceMonitor approach. If you run **vanilla Prometheus without the Operator**, consider the **annotation-based discovery** method instead (Option A). + + + +## Option C — Standalone Prometheus (VM / Docker) + +If you run Prometheus outside of Kubernetes, add a static job pointing at ZITADEL’s metrics endpoint: + +```yaml +global: + scrape_interval: 15s + +scrape_configs: + - job_name: "zitadel" + metrics_path: "/debug/metrics" + scheme: "http" # use https if TLS is enabled for Zitadel + static_configs: + - targets: [":8080"] # e.g., "localhost:8080", "zitadel.internal:8080" or "host.docker.internal:8080" +``` + +In this snippet, replace `:8080` with the appropriate address. This could be localhost:8080 for local deployments, or a DNS name / IP of the server or Kubernetes service where ZITADEL is running. If ZITADEL is behind a reverse proxy or ingress, ensure that `/debug/metrics` is reachable (you might expose it internally only). The metrics_path is set to /debug/metrics to match ZITADEL’s endpoint. We use http scheme assuming an internal/non-TLS endpoint; if you have enabled TLS on ZITADEL, use https and the appropriate port (e.g., `443`) and adjust any hostname (like zitadel.yourdomain.com). + +> When running **Prometheus in Docker** on your workstation: +> +> - **macOS/Windows**: if ZITADEL runs on your host, use `host.docker.internal:8080`. +> - **Linux**: either add `--add-host=host.docker.internal:host-gateway` to `docker run`, attach Prometheus to the same Docker network as ZITADEL and use the service name (e.g., `zitadel:8080`), or run Prometheus with `--network host` (Linux only). + + + +## Verify scraping + +Use Prometheus’s built-in UI to confirm your target is up: + +1. Open the UI (default `http://localhost:9090`). +2. Go to **Status → Targets** and find the `zitadel` target. It should be **UP**. +3. Go to **Graph** and run: + ```promql + up{job="zitadel"} + ``` + Value **1** indicates successful scrapes. + +To explore ZITADEL metrics, type `zitadel` in the expression box and pick from auto-complete. Common families include: + +- **Go runtime / process** (e.g., `go_goroutines`, `process_resident_memory_bytes`) +- **HTTP / gRPC** request counters/latencies (names vary by build and exporters) +- **ZITADEL server / DB** metrics (e.g., connection pool counts or other instrumentation available in your version) + +> Metric names and labels can change between releases. Use the **Graph → Insert metric at cursor** dropdown to discover what your instance exposes. + + + +## Minimal alerting example + +Create a basic rule to alert when ZITADEL stops scraping: + +```yaml +# /etc/prometheus/alerts/zitadel.yml +groups: + - name: zitadel-basic + rules: + - alert: ZitadelTargetDown + expr: up{job="zitadel"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "ZITADEL metrics target is down" + description: "Prometheus has not scraped ZITADEL successfully for 5 minutes." +``` + +Reference the rule file in your Prometheus config: + +```yaml +rule_files: + - "/etc/prometheus/alerts/*.yml" +``` + +(Configure Alertmanager routing according to your environment.) + + + +## Troubleshooting + +**Target is DOWN / connection refused** +- In Docker: remember **`localhost` inside the Prometheus container is the container itself**, not your host. Use `host.docker.internal:8080` (plus `--add-host=host.docker.internal:host-gateway` on Linux), or join Prometheus to the **same Docker network** as ZITADEL and use the **service name** (e.g., `zitadel:8080`), or run with `--network host` on Linux. +- In Kubernetes: verify the Service port name (for example, `http-server` or `https-server`) and path (`/debug/metrics`) match your ServiceMonitor (or annotations). Check that Prometheus has RBAC to list Pods/Endpoints. + +**Metrics path mismatch** +- ZITADEL uses **`/debug/metrics`**. If you see 404s, confirm your Prometheus job or annotations aren’t still using `/metrics`. + +**No targets discovered (Kubernetes)** +- If using annotations, make sure your Prometheus config has **Kubernetes discovery & relabeling** rules that honor `prometheus.io/*` annotations and that the Pods/Service are annotated. +- If using ServiceMonitor, ensure your Prometheus Operator instance **selects** the ServiceMonitor by label/namespace. + +**Nothing shows up in the Graph dropdown** +- First confirm `up{job="zitadel"}` returns **1**. If yes, metrics are being scraped—start typing generic prefixes like `go_` or `process_` to explore. ZITADEL’s exported metric set can evolve; check the raw output at `/debug/metrics` to see exactly what is exposed by your version. + + + +## Alternatives that understand OTEL/Prometheus + +While Prometheus is the most common choice, other collectors and services can ingest the same endpoint: + +- **Amazon Managed Service for Prometheus (AMP)** — managed, Prometheus-compatible backend on AWS. +- **AWS CloudWatch via ADOT/OTel Collector** — scrape with the OpenTelemetry Collector and export to CloudWatch Metrics. +- **Grafana Cloud / VictoriaMetrics / Thanos** — remote-write targets or managed TSDBs for Prometheus data. +- **Datadog / New Relic / Splunk Observability** — agents or OTel pipelines can ingest Prometheus-format metrics. + +If you already operate one of these platforms, you can point their agents/collectors at `/debug/metrics` or use an OTel Collector with a **Prometheus receiver** and the appropriate exporter. + diff --git a/docs/sidebars.js b/docs/sidebars.js index cabeff173..1dc6b94b8 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -1165,6 +1165,16 @@ module.exports = { "self-hosting/manage/database/database", "self-hosting/manage/cache", "self-hosting/manage/service_ping", + { + type: "category", + label: "Metrics", + collapsed: false, + link: { + type: "doc", + id: "self-hosting/manage/metrics/overview", + }, + items: ["self-hosting/manage/metrics/prometheus"], + }, "self-hosting/manage/updating_scaling", "self-hosting/manage/usage_control", {