feat: integrate vault provider management and environment variable resolution

- Added support for managing various vault providers (HashiCorp, AWS, Azure, Doppler, Infisical) in the dashboard.
- Implemented environment variable resolution using vault references, allowing seamless integration of secrets into application environments.
- Enhanced UI components to display and manage vault providers effectively.
- Introduced tests for vault reference resolution and environment variable preparation.
- Updated related components to utilize new vault management features.

packages/server/src/utils/builders/compose.ts

@@ -9,13 +9,15 @@ import {
 	getEnvironmentVariablesObject,
 	prepareEnvironmentVariablesForFile,
 } from "../docker/utils";
+import { withResolvedVaultRefs } from "../vault";
 
 export type ComposeNested = InferResultType<
 	"compose",
 	{ environment: { with: { project: true } }; mounts: true; domains: true }
 >;
 
-export const getBuildComposeCommand = async (compose: ComposeNested) => {
+export const getBuildComposeCommand = async (rawCompose: ComposeNested) => {
+	const compose = await withResolvedVaultRefs(rawCompose);
 	const { COMPOSE_PATH } = paths(!!compose.serverId);
 	const { sourceType, appName, mounts, composeType, domains } = compose;
 	const command = createCommand(compose);

packages/server/src/utils/builders/index.ts

@@ -12,6 +12,7 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 import { getDockerCommand } from "./docker-file";
 import { getHerokuCommand } from "./heroku";
 import { getNixpacksCommand } from "./nixpacks";
@@ -38,7 +39,8 @@ export type ApplicationNested = InferResultType<
 	}
 >;
 
-export const getBuildCommand = async (application: ApplicationNested) => {
+export const getBuildCommand = async (rawApplication: ApplicationNested) => {
+	const application = await withResolvedVaultRefs(rawApplication);
 	let command = "";
 
 	if (application.sourceType !== "docker") {
@@ -77,8 +79,9 @@ export const getBuildCommand = async (application: ApplicationNested) => {
 };
 
 export const mechanizeDockerContainer = async (
-	application: ApplicationNested,
+	rawApplication: ApplicationNested,
 ) => {
+	const application = await withResolvedVaultRefs(rawApplication);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/libsql.ts

@@ -10,6 +10,7 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type LibsqlNested = InferResultType<
 	"libsql",
@@ -18,7 +19,8 @@ export type LibsqlNested = InferResultType<
 		environment: { with: { project: true } };
 	}
 >;
-export const buildLibsql = async (libsql: LibsqlNested) => {
+export const buildLibsql = async (rawLibsql: LibsqlNested) => {
+	const libsql = await withResolvedVaultRefs(rawLibsql);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/mariadb.ts

@@ -10,12 +10,14 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type MariadbNested = InferResultType<
 	"mariadb",
 	{ mounts: true; environment: { with: { project: true } } }
 >;
-export const buildMariadb = async (mariadb: MariadbNested) => {
+export const buildMariadb = async (rawMariadb: MariadbNested) => {
+	const mariadb = await withResolvedVaultRefs(rawMariadb);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/mongo.ts

@@ -10,13 +10,15 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type MongoNested = InferResultType<
 	"mongo",
 	{ mounts: true; environment: { with: { project: true } } }
 >;
 
-export const buildMongo = async (mongo: MongoNested) => {
+export const buildMongo = async (rawMongo: MongoNested) => {
+	const mongo = await withResolvedVaultRefs(rawMongo);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/mysql.ts

@@ -10,13 +10,15 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type MysqlNested = InferResultType<
 	"mysql",
 	{ mounts: true; environment: { with: { project: true } } }
 >;
 
-export const buildMysql = async (mysql: MysqlNested) => {
+export const buildMysql = async (rawMysql: MysqlNested) => {
+	const mysql = await withResolvedVaultRefs(rawMysql);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/postgres.ts

@@ -10,12 +10,14 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type PostgresNested = InferResultType<
 	"postgres",
 	{ mounts: true; environment: { with: { project: true } } }
 >;
-export const buildPostgres = async (postgres: PostgresNested) => {
+export const buildPostgres = async (rawPostgres: PostgresNested) => {
+	const postgres = await withResolvedVaultRefs(rawPostgres);
 	const {
 		appName,
 		env,

packages/server/src/utils/databases/redis.ts

@@ -10,12 +10,14 @@ import {
 	prepareEnvironmentVariables,
 } from "../docker/utils";
 import { getRemoteDocker } from "../servers/remote-docker";
+import { withResolvedVaultRefs } from "../vault";
 
 export type RedisNested = InferResultType<
 	"redis",
 	{ mounts: true; environment: { with: { project: true } } }
 >;
-export const buildRedis = async (redis: RedisNested) => {
+export const buildRedis = async (rawRedis: RedisNested) => {
+	const redis = await withResolvedVaultRefs(rawRedis);
 	const {
 		appName,
 		env,

packages/server/src/utils/docker/utils.ts

@@ -401,6 +401,13 @@ export const prepareEnvironmentVariables = (
 	projectEnv?: string | null,
 	environmentEnv?: string | null,
 ) => {
+	for (const source of [serviceEnv, projectEnv, environmentEnv]) {
+		if (source?.includes("${{vault.")) {
+			throw new Error(
+				"Unresolved vault reference: call withResolvedVaultRefs() on the entity before preparing environment variables",
+			);
+		}
+	}
 	const projectVars = parse(projectEnv ?? "");
 	const environmentVars = parse(environmentEnv ?? "");
 	const serviceVars = parse(serviceEnv ?? "");

packages/server/src/utils/vault/aws.ts

@@ -0,0 +1,107 @@
+import {
+	GetSecretValueCommand,
+	ListSecretsCommand,
+	SecretsManagerClient,
+} from "@aws-sdk/client-secrets-manager";
+import type { awsVaultConfigSchema } from "@dokploy/server/db/schema";
+import type { z } from "zod";
+import type { VaultClient } from "./types";
+
+type AwsConfig = z.infer<typeof awsVaultConfigSchema>;
+
+const parseRef = (ref: string) => {
+	if (ref.startsWith("arn:")) {
+		throw new Error(
+			`Invalid AWS Secrets Manager reference "${ref}": use the secret name, not the ARN`,
+		);
+	}
+	const separatorIndex = ref.lastIndexOf(":");
+	if (separatorIndex === -1) {
+		return { secretId: ref, field: null };
+	}
+	return {
+		secretId: ref.slice(0, separatorIndex),
+		field: ref.slice(separatorIndex + 1),
+	};
+};
+
+const createClient = (config: AwsConfig) =>
+	new SecretsManagerClient({
+		region: config.region,
+		credentials: {
+			accessKeyId: config.accessKeyId,
+			secretAccessKey: config.secretAccessKey,
+		},
+		...(config.endpoint && { endpoint: config.endpoint }),
+	});
+
+export const awsClient: VaultClient<AwsConfig> = {
+	async getSecrets(config, refs) {
+		const client = createClient(config);
+		const secretIds = [...new Set(refs.map((ref) => parseRef(ref).secretId))];
+
+		const secretStrings = new Map<string, string>();
+		await Promise.all(
+			secretIds.map(async (secretId) => {
+				const response = await client.send(
+					new GetSecretValueCommand({ SecretId: secretId }),
+				);
+				if (response.SecretString === undefined) {
+					throw new Error(
+						`AWS Secrets Manager: secret "${secretId}" has no string value (binary secrets are not supported)`,
+					);
+				}
+				secretStrings.set(secretId, response.SecretString);
+			}),
+		);
+
+		const result: Record<string, string> = {};
+		for (const ref of refs) {
+			const { secretId, field } = parseRef(ref);
+			const secretString = secretStrings.get(secretId) as string;
+			if (field === null) {
+				result[ref] = secretString;
+				continue;
+			}
+			let parsed: Record<string, unknown>;
+			try {
+				parsed = JSON.parse(secretString);
+			} catch {
+				throw new Error(
+					`AWS Secrets Manager: secret "${secretId}" is not JSON, cannot extract field "${field}"`,
+				);
+			}
+			const value = parsed[field];
+			if (value === undefined || value === null) {
+				throw new Error(
+					`AWS Secrets Manager: field "${field}" not found in secret "${secretId}"`,
+				);
+			}
+			result[ref] = typeof value === "string" ? value : JSON.stringify(value);
+		}
+		return result;
+	},
+
+	async testConnection(config) {
+		const client = createClient(config);
+		await client.send(new ListSecretsCommand({ MaxResults: 1 }));
+	},
+
+	async listSecretNames(config) {
+		const client = createClient(config);
+		const names: string[] = [];
+		let nextToken: string | undefined;
+		do {
+			const response = await client.send(
+				new ListSecretsCommand({ MaxResults: 100, NextToken: nextToken }),
+			);
+			for (const secret of response.SecretList ?? []) {
+				if (secret.Name) {
+					names.push(secret.Name);
+				}
+			}
+			nextToken = response.NextToken;
+		} while (nextToken && names.length < 500);
+		return names;
+	},
+};

packages/server/src/utils/vault/azure.ts

@@ -0,0 +1,123 @@
+import type { azureVaultConfigSchema } from "@dokploy/server/db/schema";
+import type { z } from "zod";
+import { type VaultClient, vaultFetch } from "./types";
+
+type AzureConfig = z.infer<typeof azureVaultConfigSchema>;
+
+const API_VERSION = "7.4";
+
+const baseUrl = (config: AzureConfig) => config.vaultUri.replace(/\/+$/, "");
+
+const getAccessToken = async (config: AzureConfig) => {
+	const body = new URLSearchParams({
+		grant_type: "client_credentials",
+		client_id: config.clientId,
+		client_secret: config.clientSecret,
+		scope: "https://vault.azure.net/.default",
+	});
+	const response = await vaultFetch(
+		`https://login.microsoftonline.com/${encodeURIComponent(config.tenantId)}/oauth2/v2.0/token`,
+		{
+			method: "POST",
+			headers: { "Content-Type": "application/x-www-form-urlencoded" },
+			body: body.toString(),
+		},
+	);
+
+	if (!response.ok) {
+		let detail = "";
+		try {
+			const body = (await response.json()) as { error_description?: string };
+			detail = (body.error_description ?? "").split("\n")[0] ?? "";
+		} catch {}
+		throw new Error(
+			`Azure Key Vault: authentication failed (status ${response.status}${detail ? `: ${detail}` : ""})`,
+		);
+	}
+
+	const data = (await response.json()) as { access_token?: string };
+	if (!data.access_token) {
+		throw new Error("Azure Key Vault: no access token returned");
+	}
+	return data.access_token;
+};
+
+const readSecret = async (
+	config: AzureConfig,
+	token: string,
+	name: string,
+) => {
+	const response = await vaultFetch(
+		`${baseUrl(config)}/secrets/${encodeURIComponent(name)}?api-version=${API_VERSION}`,
+		{ headers: { Authorization: `Bearer ${token}` } },
+	);
+
+	if (response.status === 404) {
+		throw new Error(`Azure Key Vault: secret "${name}" not found`);
+	}
+	if (!response.ok) {
+		throw new Error(
+			`Azure Key Vault: failed to read secret "${name}" (status ${response.status})`,
+		);
+	}
+
+	const data = (await response.json()) as { value?: string };
+	if (data.value === undefined) {
+		throw new Error(`Azure Key Vault: secret "${name}" has no value`);
+	}
+	return data.value;
+};
+
+export const azureClient: VaultClient<AzureConfig> = {
+	async getSecrets(config, refs) {
+		const token = await getAccessToken(config);
+		const result: Record<string, string> = {};
+		await Promise.all(
+			refs.map(async (ref) => {
+				result[ref] = await readSecret(config, token, ref);
+			}),
+		);
+		return result;
+	},
+
+	async testConnection(config) {
+		const token = await getAccessToken(config);
+		const response = await vaultFetch(
+			`${baseUrl(config)}/secrets?api-version=${API_VERSION}&maxresults=1`,
+			{ headers: { Authorization: `Bearer ${token}` } },
+		);
+		if (!response.ok) {
+			throw new Error(
+				`Azure Key Vault: cannot list secrets (status ${response.status})`,
+			);
+		}
+	},
+
+	async listSecretNames(config) {
+		const token = await getAccessToken(config);
+		const names: string[] = [];
+		let url: string | null =
+			`${baseUrl(config)}/secrets?api-version=${API_VERSION}&maxresults=25`;
+
+		while (url && names.length < 200) {
+			const response = await vaultFetch(url, {
+				headers: { Authorization: `Bearer ${token}` },
+			});
+			if (!response.ok) {
+				break;
+			}
+			const data = (await response.json()) as {
+				value?: { id: string }[];
+				nextLink?: string | null;
+			};
+			for (const item of data.value ?? []) {
+				const name = item.id.split("/").pop();
+				if (name) {
+					names.push(name);
+				}
+			}
+			url = data.nextLink ?? null;
+		}
+		return names.slice(0, 200);
+	},
+};

packages/server/src/utils/vault/doppler.ts

@@ -0,0 +1,61 @@
+import type { dopplerVaultConfigSchema } from "@dokploy/server/db/schema";
+import type { z } from "zod";
+import { type VaultClient, vaultFetch } from "./types";
+
+type DopplerConfig = z.infer<typeof dopplerVaultConfigSchema>;
+
+const downloadUrl = (config: DopplerConfig) => {
+	const params = new URLSearchParams({ format: "json" });
+	if (config.project) {
+		params.set("project", config.project);
+	}
+	if (config.config) {
+		params.set("config", config.config);
+	}
+	return `https://api.doppler.com/v3/configs/config/secrets/download?${params.toString()}`;
+};
+
+const downloadSecrets = async (config: DopplerConfig) => {
+	const response = await vaultFetch(downloadUrl(config), {
+		headers: {
+			Authorization: `Bearer ${config.serviceToken}`,
+			Accept: "application/json",
+		},
+	});
+
+	if (!response.ok) {
+		let detail = "";
+		try {
+			const body = (await response.json()) as { messages?: string[] };
+			detail = (body.messages ?? []).join(", ");
+		} catch {}
+		throw new Error(
+			`Doppler: failed to fetch secrets (status ${response.status}${detail ? `: ${detail}` : ""})`,
+		);
+	}
+
+	return (await response.json()) as Record<string, string>;
+};
+
+export const dopplerClient: VaultClient<DopplerConfig> = {
+	async getSecrets(config, refs) {
+		const secrets = await downloadSecrets(config);
+		const result: Record<string, string> = {};
+		for (const ref of refs) {
+			if (secrets[ref] === undefined) {
+				throw new Error(`Doppler: secret "${ref}" not found in this config`);
+			}
+			result[ref] = secrets[ref];
+		}
+		return result;
+	},
+
+	async testConnection(config) {
+		await downloadSecrets(config);
+	},
+
+	async listSecretNames(config) {
+		const secrets = await downloadSecrets(config);
+		return Object.keys(secrets);
+	},
+};

packages/server/src/utils/vault/hashicorp.ts

@@ -0,0 +1,140 @@
+import type { hashicorpVaultConfigSchema } from "@dokploy/server/db/schema";
+import type { z } from "zod";
+import { type VaultClient, vaultFetch } from "./types";
+
+type HashicorpConfig = z.infer<typeof hashicorpVaultConfigSchema>;
+
+const parseRef = (ref: string) => {
+	const separatorIndex = ref.lastIndexOf(":");
+	if (separatorIndex <= 0 || separatorIndex === ref.length - 1) {
+		throw new Error(
+			`Invalid HashiCorp Vault reference "${ref}": expected format <path>:<field> (e.g. myapp/prod:DB_PASSWORD)`,
+		);
+	}
+	return {
+		path: ref.slice(0, separatorIndex),
+		field: ref.slice(separatorIndex + 1),
+	};
+};
+
+const buildHeaders = (config: HashicorpConfig) => {
+	const headers: Record<string, string> = {
+		"X-Vault-Token": config.token,
+	};
+	if (config.namespace) {
+		headers["X-Vault-Namespace"] = config.namespace;
+	}
+	return headers;
+};
+
+const baseUrl = (config: HashicorpConfig) => config.url.replace(/\/+$/, "");
+
+const encodePath = (path: string) =>
+	path.split("/").map(encodeURIComponent).join("/");
+
+const readSecret = async (config: HashicorpConfig, path: string) => {
+	const url = `${baseUrl(config)}/v1/${encodeURIComponent(config.mount)}/data/${encodePath(path)}`;
+	const response = await vaultFetch(url, { headers: buildHeaders(config) });
+
+	if (!response.ok) {
+		throw new Error(
+			`HashiCorp Vault: failed to read secret at "${path}" (status ${response.status})`,
+		);
+	}
+
+	const body = (await response.json()) as {
+		data?: { data?: Record<string, unknown> };
+	};
+	return body.data?.data ?? {};
+};
+
+export const hashicorpClient: VaultClient<HashicorpConfig> = {
+	async getSecrets(config, refs) {
+		const byPath = new Map<string, string[]>();
+		for (const ref of refs) {
+			const { path } = parseRef(ref);
+			byPath.set(path, [...(byPath.get(path) ?? []), ref]);
+		}
+
+		const result: Record<string, string> = {};
+		await Promise.all(
+			[...byPath.entries()].map(async ([path, pathRefs]) => {
+				const data = await readSecret(config, path);
+				for (const ref of pathRefs) {
+					const { field } = parseRef(ref);
+					const value = data[field];
+					if (value === undefined || value === null) {
+						throw new Error(
+							`HashiCorp Vault: field "${field}" not found in secret "${path}"`,
+						);
+					}
+					result[ref] =
+						typeof value === "string" ? value : JSON.stringify(value);
+				}
+			}),
+		);
+		return result;
+	},
+
+	async testConnection(config) {
+		const response = await vaultFetch(
+			`${baseUrl(config)}/v1/auth/token/lookup-self`,
+			{ headers: buildHeaders(config) },
+		);
+		if (!response.ok) {
+			throw new Error(
+				`HashiCorp Vault: token validation failed (status ${response.status})`,
+			);
+		}
+	},
+
+	async listSecretNames(config) {
+		const MAX_DEPTH = 4;
+		const MAX_ENTRIES = 200;
+		const names: string[] = [];
+
+		const listKeys = async (path: string) => {
+			const cleanPath = path.replace(/\/+$/, "");
+			const suffix = cleanPath ? `/${encodePath(cleanPath)}` : "";
+			const response = await vaultFetch(
+				`${baseUrl(config)}/v1/${encodeURIComponent(config.mount)}/metadata${suffix}?list=true`,
+				{ headers: buildHeaders(config) },
+			);
+			if (!response.ok) {
+				return [];
+			}
+			const body = (await response.json()) as { data?: { keys?: string[] } };
+			return body.data?.keys ?? [];
+		};
+
+		const walk = async (path: string, depth: number) => {
+			if (depth > MAX_DEPTH || names.length >= MAX_ENTRIES) {
+				return;
+			}
+			for (const key of await listKeys(path)) {
+				if (names.length >= MAX_ENTRIES) {
+					return;
+				}
+				const fullPath = `${path}${key}`;
+				if (key.endsWith("/")) {
+					await walk(fullPath, depth + 1);
+					continue;
+				}
+				const fields = await readSecret(config, fullPath).catch(
+					() => ({}) as Record<string, unknown>,
+				);
+				const fieldNames = Object.keys(fields);
+				if (fieldNames.length === 0) {
+					names.push(fullPath);
+					continue;
+				}
+				for (const field of fieldNames) {
+					names.push(`${fullPath}:${field}`);
+				}
+			}
+		};
+
+		await walk("", 0);
+		return names.slice(0, MAX_ENTRIES);
+	},
+};

packages/server/src/utils/vault/infisical.ts

@@ -0,0 +1,87 @@
+import type { infisicalVaultConfigSchema } from "@dokploy/server/db/schema";
+import type { z } from "zod";
+import { type VaultClient, vaultFetch } from "./types";
+
+type InfisicalConfig = z.infer<typeof infisicalVaultConfigSchema>;
+
+const baseUrl = (config: InfisicalConfig) => config.siteUrl.replace(/\/+$/, "");
+
+const login = async (config: InfisicalConfig) => {
+	const response = await vaultFetch(
+		`${baseUrl(config)}/api/v1/auth/universal-auth/login`,
+		{
+			method: "POST",
+			headers: { "Content-Type": "application/json" },
+			body: JSON.stringify({
+				clientId: config.clientId,
+				clientSecret: config.clientSecret,
+			}),
+		},
+	);
+
+	if (!response.ok) {
+		throw new Error(
+			`Infisical: authentication failed (status ${response.status})`,
+		);
+	}
+
+	const body = (await response.json()) as { accessToken?: string };
+	if (!body.accessToken) {
+		throw new Error("Infisical: no access token returned");
+	}
+	return body.accessToken;
+};
+
+const fetchSecrets = async (config: InfisicalConfig) => {
+	const accessToken = await login(config);
+	const params = new URLSearchParams({
+		workspaceId: config.projectId,
+		environment: config.environmentSlug,
+		secretPath: config.secretPath,
+	});
+	const response = await vaultFetch(
+		`${baseUrl(config)}/api/v3/secrets/raw?${params.toString()}`,
+		{ headers: { Authorization: `Bearer ${accessToken}` } },
+	);
+
+	if (!response.ok) {
+		throw new Error(
+			`Infisical: failed to fetch secrets (status ${response.status})`,
+		);
+	}
+
+	const body = (await response.json()) as {
+		secrets?: { secretKey: string; secretValue: string }[];
+	};
+
+	const secrets: Record<string, string> = {};
+	for (const secret of body.secrets ?? []) {
+		secrets[secret.secretKey] = secret.secretValue;
+	}
+	return secrets;
+};
+
+export const infisicalClient: VaultClient<InfisicalConfig> = {
+	async getSecrets(config, refs) {
+		const secrets = await fetchSecrets(config);
+		const result: Record<string, string> = {};
+		for (const ref of refs) {
+			if (secrets[ref] === undefined) {
+				throw new Error(
+					`Infisical: secret "${ref}" not found in environment "${config.environmentSlug}"`,
+				);
+			}
+			result[ref] = secrets[ref];
+		}
+		return result;
+	},
+
+	async testConnection(config) {
+		await fetchSecrets(config);
+	},
+
+	async listSecretNames(config) {
+		const secrets = await fetchSecrets(config);
+		return Object.keys(secrets);
+	},
+};

packages/server/src/utils/vault/types.ts

@@ -0,0 +1,18 @@
+import type { VaultProviderConfig } from "@dokploy/server/db/schema";
+
+export interface VaultClient<
+	C extends VaultProviderConfig = VaultProviderConfig,
+> {
+	getSecrets(config: C, refs: string[]): Promise<Record<string, string>>;
+	testConnection(config: C): Promise<void>;
+	listSecretNames?(config: C): Promise<string[]>;
+}
+
+export const VAULT_REQUEST_TIMEOUT_MS = 15_000;
+
+export const vaultFetch = async (url: string, init: RequestInit = {}) => {
+	return await fetch(url, {
+		...init,
+		signal: AbortSignal.timeout(VAULT_REQUEST_TIMEOUT_MS),
+	});
+};