diff --git a/apps/login/src/app/ready/route.test.ts b/apps/login/src/app/ready/route.test.ts index 792644fb7..9fdc18464 100644 --- a/apps/login/src/app/ready/route.test.ts +++ b/apps/login/src/app/ready/route.test.ts @@ -1,21 +1,30 @@ -import { createServiceForHost } from "@/lib/service"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { GET } from "./route"; -vi.mock("@/lib/service", () => ({ - createServiceForHost: vi.fn(), +vi.mock("@/lib/logger", () => ({ + createLogger: () => ({ error: vi.fn(), warn: vi.fn() }), })); -vi.mock("@/lib/logger", () => ({ - createLogger: () => ({ error: vi.fn() }), -})); +function mockApiReady(status = 200) { + const fetchMock = vi.fn().mockResolvedValue( + new Response(status === 200 ? '"ok"' : "{}", { + status, + headers: { "Content-Type": "application/json" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +} describe("GET /ready", () => { let savedApiUrl: string | undefined; + let savedCustomHeaders: string | undefined; beforeEach(() => { savedApiUrl = process.env.ZITADEL_API_URL; + savedCustomHeaders = process.env.CUSTOM_REQUEST_HEADERS; process.env.ZITADEL_API_URL = "http://localhost:8080"; + delete process.env.CUSTOM_REQUEST_HEADERS; }); afterEach(() => { @@ -24,25 +33,29 @@ describe("GET /ready", () => { } else { process.env.ZITADEL_API_URL = savedApiUrl; } + if (savedCustomHeaders === undefined) { + delete process.env.CUSTOM_REQUEST_HEADERS; + } else { + process.env.CUSTOM_REQUEST_HEADERS = savedCustomHeaders; + } + vi.unstubAllGlobals(); vi.restoreAllMocks(); }); - test("should return 200 when gRPC call succeeds", async () => { - vi.mocked(createServiceForHost).mockResolvedValue({ - getGeneralSettings: vi.fn().mockResolvedValue({ allowedLanguages: ["en"] }), - } as any); + test("should return 200 when the API is ready", async () => { + const fetchMock = mockApiReady(); const response = await GET(); expect(response.status).toBe(200); expect(response.headers.get("Content-Type")).toBe("text/plain"); expect(await response.text()).toBe("OK"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0].toString()).toBe("http://localhost:8080/debug/ready"); }); - test("should return 503 when gRPC call fails", async () => { - vi.mocked(createServiceForHost).mockResolvedValue({ - getGeneralSettings: vi.fn().mockRejectedValue(new Error("connection refused")), - } as any); + test("should return 503 when the API is not ready", async () => { + mockApiReady(412); const response = await GET(); @@ -51,17 +64,17 @@ describe("GET /ready", () => { expect(await response.text()).toBe("Service unavailable"); }); - test("should return 503 when authentication fails", async () => { - vi.mocked(createServiceForHost).mockRejectedValue(new Error("No authentication credentials found")); + test("should return 503 when the API is unreachable", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("connection refused"))); const response = await GET(); expect(response.status).toBe(503); - expect(response.headers.get("Content-Type")).toBe("text/plain"); expect(await response.text()).toBe("Service unavailable"); }); test("should return 503 when ZITADEL_API_URL is not set", async () => { + const fetchMock = mockApiReady(); delete process.env.ZITADEL_API_URL; const response = await GET(); @@ -69,5 +82,17 @@ describe("GET /ready", () => { expect(response.status).toBe(503); expect(response.headers.get("Content-Type")).toBe("text/plain"); expect(await response.text()).toBe("Service unavailable"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + test("should forward CUSTOM_REQUEST_HEADERS to the readiness request", async () => { + process.env.CUSTOM_REQUEST_HEADERS = "Host: zitadel.internal, X-Forwarded-Host: login.example.com"; + const fetchMock = mockApiReady(); + + await GET(); + + const headers = fetchMock.mock.calls[0][1].headers as Headers; + expect(headers.get("Host")).toBe("zitadel.internal"); + expect(headers.get("X-Forwarded-Host")).toBe("login.example.com"); }); }); diff --git a/apps/login/src/app/ready/route.ts b/apps/login/src/app/ready/route.ts index e608a732c..8864348ac 100644 --- a/apps/login/src/app/ready/route.ts +++ b/apps/login/src/app/ready/route.ts @@ -1,33 +1,61 @@ +import { applyCustomHeaders } from "@/lib/custom-headers"; import { createLogger } from "@/lib/logger"; -import { createServiceForHost } from "@/lib/service"; -import { Client } from "@zitadel/client"; -import { SettingsService } from "@zitadel/proto/zitadel/settings/v2/settings_service_pb"; import { NextResponse } from "next/server"; const logger = createLogger("readiness"); +const READINESS_TIMEOUT_MS = 5_000; + +/** + * Readiness probe. + * + * Reports whether the ZITADEL API is reachable and ready by calling its own + * unauthenticated readiness endpoint. This endpoint is served outside of the + * API's access-log and quota interceptors, so probe traffic is never counted + * as authenticated requests of the instance. + * + * The configured API credentials are validated once at process startup + * (see instrumentation.ts), not on every probe. + */ export async function GET() { - if (!process.env.ZITADEL_API_URL) { - return new NextResponse("Service unavailable", { - status: 503, - headers: { "Content-Type": "text/plain", "Cache-Control": "no-store" }, - }); + const apiUrl = process.env.ZITADEL_API_URL; + if (!apiUrl) { + return unavailable(); } try { - const settingsService: Client = await createServiceForHost(SettingsService, { - baseUrl: process.env.ZITADEL_API_URL, + const headers = new Headers(); + applyCustomHeaders({ + set: (key, value) => headers.set(key, value), + remove: (key) => headers.delete(key), }); - await settingsService.getGeneralSettings({}); + + const response = await fetch(new URL("/debug/ready", apiUrl), { + method: "GET", + headers, + cache: "no-store", + signal: AbortSignal.timeout(READINESS_TIMEOUT_MS), + }); + // consume the body so the connection can be reused + await response.arrayBuffer().catch(() => undefined); + + if (!response.ok) { + throw new Error(`ZITADEL API readiness check returned HTTP ${response.status}`); + } + return new NextResponse("OK", { status: 200, headers: { "Content-Type": "text/plain", "Cache-Control": "no-store" }, }); } catch (e) { logger.error("Readiness check failed", { error: e }); - return new NextResponse("Service unavailable", { - status: 503, - headers: { "Content-Type": "text/plain", "Cache-Control": "no-store" }, - }); + return unavailable(); } } + +function unavailable() { + return new NextResponse("Service unavailable", { + status: 503, + headers: { "Content-Type": "text/plain", "Cache-Control": "no-store" }, + }); +} diff --git a/apps/login/src/instrumentation.test.ts b/apps/login/src/instrumentation.test.ts new file mode 100644 index 000000000..61645b647 --- /dev/null +++ b/apps/login/src/instrumentation.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const verifyApiCredentials = vi.fn(); +const registerNode = vi.fn(); + +// Keep the real FATAL_CREDENTIAL_CHECK_RESULTS, but replace the check itself +// and cut the import chain into the API client. +vi.mock("./lib/verify-credentials", async (importOriginal) => ({ + ...(await importOriginal()), + verifyApiCredentials, +})); +vi.mock("@/lib/service", () => ({ createServiceForHost: vi.fn() })); +vi.mock("./instrumentation.node", () => ({ registerNode })); + +async function loadRegister() { + vi.resetModules(); + const { register } = await import("./instrumentation"); + return register; +} + +describe("register", () => { + const savedEnv = { + NEXT_RUNTIME: process.env.NEXT_RUNTIME, + NODE_ENV: process.env.NODE_ENV, + OTEL_SDK_DISABLED: process.env.OTEL_SDK_DISABLED, + }; + let exitSpy: ReturnType; + + beforeEach(() => { + process.env.NEXT_RUNTIME = "nodejs"; + process.env.OTEL_SDK_DISABLED = "true"; + verifyApiCredentials.mockReset().mockResolvedValue("ok"); + registerNode.mockReset().mockResolvedValue(null); + exitSpy = vi.spyOn(process, "exit").mockImplementation((() => undefined) as never); + }); + + afterEach(() => { + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + vi.restoreAllMocks(); + }); + + test("verifies credentials on startup and continues when they are accepted", async () => { + const register = await loadRegister(); + + await register(); + + expect(verifyApiCredentials).toHaveBeenCalledTimes(1); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + test.each(["rejected", "invalid", "missing"])("exits the process when the credential check returns %s", async (result) => { + verifyApiCredentials.mockResolvedValue(result); + const register = await loadRegister(); + + await register(); + + expect(exitSpy).toHaveBeenCalledWith(1); + }); + + test.each(["unreachable", "skipped"])("continues when the credential check returns %s", async (result) => { + verifyApiCredentials.mockResolvedValue(result); + const register = await loadRegister(); + + await register(); + + expect(exitSpy).not.toHaveBeenCalled(); + }); + + test("does nothing outside the nodejs runtime", async () => { + process.env.NEXT_RUNTIME = "edge"; + const register = await loadRegister(); + + await register(); + + expect(verifyApiCredentials).not.toHaveBeenCalled(); + expect(registerNode).not.toHaveBeenCalled(); + }); + + test("registers OpenTelemetry before the credential check when enabled", async () => { + delete process.env.OTEL_SDK_DISABLED; + const calls: string[] = []; + registerNode.mockImplementation(async () => { + calls.push("otel"); + return null; + }); + verifyApiCredentials.mockImplementation(async () => { + calls.push("credentials"); + return "ok"; + }); + const register = await loadRegister(); + + await register(); + + expect(calls).toEqual(["otel", "credentials"]); + }); + + test("still verifies credentials when OpenTelemetry is disabled", async () => { + const register = await loadRegister(); + + await register(); + + expect(registerNode).not.toHaveBeenCalled(); + expect(verifyApiCredentials).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/login/src/instrumentation.ts b/apps/login/src/instrumentation.ts index 7e1de9792..a202881b1 100644 --- a/apps/login/src/instrumentation.ts +++ b/apps/login/src/instrumentation.ts @@ -29,19 +29,31 @@ import type { LoggerProvider } from "@opentelemetry/api-logs"; let _loggerProvider: LoggerProvider | null = null; export async function register(): Promise { - // Only run OpenTelemetry in the Node.js environment - if (process.env.NEXT_RUNTIME === "nodejs") { - // Disable by default in local development to avoid unnecessary overhead - if (process.env.NODE_ENV === "development" && process.env.OTEL_SDK_DISABLED !== "false") { - return; - } - - // Explicit check for disabled env variable - if (process.env.OTEL_SDK_DISABLED === "true") return; + // Only run in the Node.js environment + if (process.env.NEXT_RUNTIME !== "nodejs") return; + if (isOtelEnabled()) { const { registerNode } = await import("./instrumentation.node"); _loggerProvider = await registerNode(); } + + // Verify the configured API credentials once at startup (fail fast). + // Connectivity problems are only logged; reachability is covered by the + // readiness probe (/ready) and ZITADEL_API_AWAITINITIALCONN. + const { verifyApiCredentials, FATAL_CREDENTIAL_CHECK_RESULTS } = await import("./lib/verify-credentials"); + const result = await verifyApiCredentials(); + if (FATAL_CREDENTIAL_CHECK_RESULTS.has(result)) { + process.exit(1); + } +} + +function isOtelEnabled(): boolean { + // Disable by default in local development to avoid unnecessary overhead + if (process.env.NODE_ENV === "development" && process.env.OTEL_SDK_DISABLED !== "false") { + return false; + } + // Explicit check for disabled env variable + return process.env.OTEL_SDK_DISABLED !== "true"; } export function getLoggerProvider(): LoggerProvider | null { diff --git a/apps/login/src/lib/verify-credentials.test.ts b/apps/login/src/lib/verify-credentials.test.ts new file mode 100644 index 000000000..e88e5842e --- /dev/null +++ b/apps/login/src/lib/verify-credentials.test.ts @@ -0,0 +1,130 @@ +import { createServiceForHost } from "@/lib/service"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { verifyApiCredentials } from "./verify-credentials"; + +vi.mock("@/lib/service", () => ({ + createServiceForHost: vi.fn(), +})); + +vi.mock("@/lib/logger", () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})); + +const ENV_KEYS = [ + "ZITADEL_API_URL", + "ZITADEL_SERVICE_USER_TOKEN", + "ZITADEL_LOGINCLIENT_KEYFILE", + "AUDIENCE", + "SYSTEM_USER_ID", + "SYSTEM_USER_PRIVATE_KEY", + "SYSTEM_USER_PRIVATE_KEY_FILE", +] as const; + +describe("verifyApiCredentials", () => { + const savedEnv: Partial> = {}; + + beforeEach(() => { + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + process.env.ZITADEL_API_URL = "http://localhost:8080"; + process.env.ZITADEL_SERVICE_USER_TOKEN = "token"; + vi.mocked(createServiceForHost).mockReset(); + }); + + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = savedEnv[key]; + } + } + vi.restoreAllMocks(); + }); + + test("returns ok when the API accepts the credentials", async () => { + const getGeneralSettings = vi.fn().mockResolvedValue({ allowedLanguages: ["en"] }); + vi.mocked(createServiceForHost).mockResolvedValue({ getGeneralSettings } as any); + + await expect(verifyApiCredentials()).resolves.toBe("ok"); + + expect(createServiceForHost).toHaveBeenCalledWith(expect.anything(), { baseUrl: "http://localhost:8080" }); + expect(getGeneralSettings).toHaveBeenCalledTimes(1); + }); + + test("bounds the startup RPC with a timeout", async () => { + const getGeneralSettings = vi.fn().mockResolvedValue({}); + vi.mocked(createServiceForHost).mockResolvedValue({ getGeneralSettings } as any); + + await verifyApiCredentials(); + + expect(getGeneralSettings).toHaveBeenCalledWith({}, { timeoutMs: expect.any(Number) }); + expect(getGeneralSettings.mock.calls[0][1].timeoutMs).toBeGreaterThan(0); + }); + + test("returns skipped when ZITADEL_API_URL is not set", async () => { + delete process.env.ZITADEL_API_URL; + + await expect(verifyApiCredentials()).resolves.toBe("skipped"); + + expect(createServiceForHost).not.toHaveBeenCalled(); + }); + + test("returns missing when no credentials are configured", async () => { + delete process.env.ZITADEL_SERVICE_USER_TOKEN; + + await expect(verifyApiCredentials()).resolves.toBe("missing"); + + expect(createServiceForHost).not.toHaveBeenCalled(); + }); + + test("returns invalid when the credentials cannot be loaded or signed", async () => { + vi.mocked(createServiceForHost).mockRejectedValue( + new Error('Failed to read login client key file "/keys/login.json": ENOENT: no such file or directory'), + ); + + await expect(verifyApiCredentials()).resolves.toBe("invalid"); + }); + + test.each([ + ["Unauthenticated", Code.Unauthenticated], + ["PermissionDenied", Code.PermissionDenied], + ])("returns rejected when the API responds with %s", async (_, code) => { + vi.mocked(createServiceForHost).mockResolvedValue({ + getGeneralSettings: vi.fn().mockRejectedValue(new ConnectError("invalid token", code)), + } as any); + + await expect(verifyApiCredentials()).resolves.toBe("rejected"); + }); + + test("returns skipped when ZITADEL_API_URL does not resolve to an instance", async () => { + vi.mocked(createServiceForHost).mockResolvedValue({ + getGeneralSettings: vi.fn().mockRejectedValue(new ConnectError("no instanceHost specified", Code.NotFound)), + } as any); + + await expect(verifyApiCredentials()).resolves.toBe("skipped"); + }); + + test.each([ + ["Unavailable", Code.Unavailable], + ["DeadlineExceeded", Code.DeadlineExceeded], + ["Internal", Code.Internal], + ])("returns unreachable when the API responds with %s", async (_, code) => { + vi.mocked(createServiceForHost).mockResolvedValue({ + getGeneralSettings: vi.fn().mockRejectedValue(new ConnectError("failed", code)), + } as any); + + await expect(verifyApiCredentials()).resolves.toBe("unreachable"); + }); + + test("returns unreachable on non-Connect errors", async () => { + vi.mocked(createServiceForHost).mockResolvedValue({ + getGeneralSettings: vi.fn().mockRejectedValue(new Error("ECONNRESET")), + } as any); + + await expect(verifyApiCredentials()).resolves.toBe("unreachable"); + }); +}); diff --git a/apps/login/src/lib/verify-credentials.ts b/apps/login/src/lib/verify-credentials.ts new file mode 100644 index 000000000..f5fa5ba76 --- /dev/null +++ b/apps/login/src/lib/verify-credentials.ts @@ -0,0 +1,99 @@ +import { hasLoginClientKey, hasServiceUserToken, hasSystemUserCredentials } from "@/lib/deployment"; +import { createLogger } from "@/lib/logger"; +import { createServiceForHost } from "@/lib/service"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { Client } from "@zitadel/client"; +import { SettingsService } from "@zitadel/proto/zitadel/settings/v2/settings_service_pb"; + +const logger = createLogger("startup"); + +/** Upper bound for the startup RPC so a hanging API cannot block process startup. */ +const CREDENTIAL_CHECK_TIMEOUT_MS = 10_000; + +/** gRPC codes that mean the API received the request but rejected the credentials. */ +const CREDENTIAL_ERROR_CODES: ReadonlySet = new Set([Code.Unauthenticated, Code.PermissionDenied]); + +export type CredentialCheckResult = + /** The API accepted the configured credentials. */ + | "ok" + /** The API rejected the configured credentials. */ + | "rejected" + /** Credentials are configured but could not be loaded or signed (e.g. missing or malformed key file). */ + | "invalid" + /** No credentials are configured at all. */ + | "missing" + /** The API could not be reached (or did not answer in time), so the credentials could not be verified. */ + | "unreachable" + /** No check was performed: ZITADEL_API_URL is not set, or it does not resolve to an instance. */ + | "skipped"; + +/** Results that indicate a configuration error the process cannot recover from. */ +export const FATAL_CREDENTIAL_CHECK_RESULTS: ReadonlySet = new Set([ + "rejected", + "invalid", + "missing", +]); + +/** + * Verifies once, at process startup, that the configured API credentials are + * accepted by the ZITADEL API, using the same auth and transport path as real + * requests. + * + * This deliberately runs at startup and not in the readiness probe: probes are + * executed every few seconds per replica and every authenticated API call is + * recorded as a billable request of the instance. Verifying once per process + * catches misconfigured credentials early (fail fast) without generating + * continuous authenticated traffic. + * + * The check is scoped to the instance that ZITADEL_API_URL resolves to. In + * multi-tenant deployments, where the instance is only known per request, the + * API answers with NotFound and the check is skipped. + * + * The function never throws; the caller decides what to do with the result. + */ +export async function verifyApiCredentials(): Promise { + const apiUrl = process.env.ZITADEL_API_URL; + if (!apiUrl) { + logger.warn("ZITADEL_API_URL is not set, skipping API credential check"); + return "skipped"; + } + + if (!hasSystemUserCredentials() && !hasLoginClientKey() && !hasServiceUserToken()) { + logger.error( + "No API credentials configured. Set ZITADEL_LOGINCLIENT_KEYFILE, system user credentials (AUDIENCE, SYSTEM_USER_ID, SYSTEM_USER_PRIVATE_KEY or SYSTEM_USER_PRIVATE_KEY_FILE), or ZITADEL_SERVICE_USER_TOKEN", + ); + return "missing"; + } + + // Loading and signing the credentials happens locally, before any request is + // sent. Failures here are configuration errors, not connectivity problems. + let settingsService: Client; + try { + settingsService = await createServiceForHost(SettingsService, { baseUrl: apiUrl }); + } catch (e) { + logger.error("The configured API credentials could not be loaded", { apiUrl, error: e }); + return "invalid"; + } + + try { + await settingsService.getGeneralSettings({}, { timeoutMs: CREDENTIAL_CHECK_TIMEOUT_MS }); + logger.info("API credentials verified", { apiUrl }); + return "ok"; + } catch (e) { + if (e instanceof ConnectError) { + if (CREDENTIAL_ERROR_CODES.has(e.code)) { + logger.error("The ZITADEL API rejected the configured credentials", { apiUrl, error: e }); + return "rejected"; + } + if (e.code === Code.NotFound) { + logger.warn( + "ZITADEL_API_URL does not resolve to an instance, skipping API credential check (expected for multi-tenant deployments)", + { apiUrl, error: e }, + ); + return "skipped"; + } + } + logger.warn("Could not verify API credentials because the ZITADEL API is not reachable", { apiUrl, error: e }); + return "unreachable"; + } +}