feat: add Docker Health diagnostics tab

Server diagnostics tab (Docker dashboard) with:
- Per-network IP usage: subnet capacity vs. containers in use, including
  reserved networks like dokploy-network which are excluded from the
  managed Networks tab. Surfaces read errors instead of silently showing
  zeroes when the Docker API call fails.
- Broadened daemon-error detection and failure classification for Swarm
  network failures: IP pool exhaustion on an existing network
  ("could not find an available IP", "task allocation failure") and
  network attach timeouts ("attaching to network failed", "context
  deadline exceeded") — both previously fell through as Unclassified.
  Verified end-to-end against a real systemd-managed dockerd.
- Failed-deployment rows now carry the raw log tail used for
  classification, shown on hover and linked to their service/project,
  so a guess (or lack of one) can be verified against the actual error
  and followed straight to the app.
- Daemon-error panel is a read-only code editor, always visible (not
  hidden when empty), prefixed with the actual "Logs from X to Y"
  window passed to journalctl so the --since window can be verified.
- "Download report" exports everything on screen (metrics + raw logs)
  as Markdown, so a customer can send one file instead of a screen-share.
- Health check now IS_CLOUD-gated the same way network sync already is:
  a cloud org can no longer omit serverId and have the check silently
  run against the shared platform host instead of their own server.

Includes an inotify diagnostics card (current vs. max_user_instances/
watches/queued_events) and a failed-deployments log with automatic
cause classification based on daemon/log error patterns.
This commit is contained in:
Narciso
2026-08-13 10:00:54 -04:00
parent 59ebf809aa
commit 4bf6dfff13
13 changed files with 1561 additions and 7 deletions
@@ -233,3 +233,10 @@ export const apiFindAllByType = z.object({
"volumeBackup",
]),
});
export const apiFindFailedDeployments = z.object({
dateFrom: z.string().datetime().optional(),
dateTo: z.string().datetime().optional(),
projectId: z.string().optional(),
environmentId: z.string().optional(),
});
+2
View File
@@ -50,6 +50,7 @@ export * from "./services/rollbacks";
export * from "./services/schedule";
export * from "./services/security";
export * from "./services/server";
export * from "./services/server-health";
export * from "./services/settings";
export * from "./services/ssh-key";
export * from "./services/user";
@@ -95,6 +96,7 @@ export * from "./utils/builders/utils";
export * from "./utils/cluster/upload";
export * from "./utils/crons/enterprise";
export * from "./utils/databases/rebuild";
export * from "./utils/deployment-failure-classifier";
export * from "./utils/docker/collision";
export * from "./utils/docker/compose";
export * from "./utils/docker/compose/configs";
+182 -1
View File
@@ -10,12 +10,14 @@ import {
type apiCreateDeploymentSchedule,
type apiCreateDeploymentServer,
type apiCreateDeploymentVolumeBackup,
type apiFindFailedDeployments,
applications,
compose,
deployments,
environments,
projects,
} from "@dokploy/server/db/schema";
import { classifyFailure } from "@dokploy/server/utils/deployment-failure-classifier";
import { removeDirectoryIfExistsContent } from "@dokploy/server/utils/filesystem/directory";
import {
execAsync,
@@ -23,7 +25,7 @@ import {
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { format } from "date-fns";
import { and, desc, eq, inArray, or, sql } from "drizzle-orm";
import { and, desc, eq, gte, inArray, lte, or, sql } from "drizzle-orm";
import type { z } from "zod";
import {
type Application,
@@ -942,6 +944,185 @@ export const findAllDeploymentsCentralized = async (
});
};
// Kept separate from getApplicationIdsInOrg/getComposeIdsInOrg above to avoid
// touching that existing code path; adds projectId/environmentId filtering.
type FailedDeploymentIdFilters = { projectId?: string; environmentId?: string };
async function getApplicationIdsForFailedDeployments(
orgId: string,
accessedServices: string[] | null,
filters: FailedDeploymentIdFilters,
): Promise<string[]> {
const conditions = [
eq(projects.organizationId, orgId),
...(accessedServices !== null
? [inArray(applications.applicationId, accessedServices)]
: []),
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
...(filters.environmentId
? [eq(environments.environmentId, filters.environmentId)]
: []),
];
const rows = await db
.select({ applicationId: applications.applicationId })
.from(applications)
.innerJoin(
environments,
eq(applications.environmentId, environments.environmentId),
)
.innerJoin(projects, eq(environments.projectId, projects.projectId))
.where(and(...conditions));
return rows.map((r) => r.applicationId);
}
async function getComposeIdsForFailedDeployments(
orgId: string,
accessedServices: string[] | null,
filters: FailedDeploymentIdFilters,
): Promise<string[]> {
const conditions = [
eq(projects.organizationId, orgId),
...(accessedServices !== null
? [inArray(compose.composeId, accessedServices)]
: []),
...(filters.projectId ? [eq(projects.projectId, filters.projectId)] : []),
...(filters.environmentId
? [eq(environments.environmentId, filters.environmentId)]
: []),
];
const rows = await db
.select({ composeId: compose.composeId })
.from(compose)
.innerJoin(
environments,
eq(compose.environmentId, environments.environmentId),
)
.innerJoin(projects, eq(environments.projectId, projects.projectId))
.where(and(...conditions));
return rows.map((r) => r.composeId);
}
const readLogTail = async (
logPath: string | null,
serverId: string | null | undefined,
) => {
if (!logPath) return "";
const command = `tail -c 4096 "${logPath}" 2>/dev/null || echo ""`;
const { stdout } = serverId
? await execAsyncRemote(serverId, command)
: await execAsync(command);
return stdout;
};
// Caps simultaneous SSH connections when classifying a batch of failures.
const LOG_TAIL_CONCURRENCY = 8;
async function mapWithConcurrency<T, R>(
items: T[],
limit: number,
fn: (item: T) => Promise<R>,
): Promise<R[]> {
const results = new Array<R>(items.length);
let nextIndex = 0;
const worker = async () => {
while (nextIndex < items.length) {
const current = nextIndex++;
results[current] = await fn(items[current] as T);
}
};
await Promise.all(
Array.from({ length: Math.min(limit, items.length) }, worker),
);
return results;
}
export type FailedDeploymentFilters = z.infer<typeof apiFindFailedDeployments>;
// Same org-wide scope as findAllDeploymentsCentralized, narrowed to failed
// deployments, each tagged with a classifyFailure() guess from its log tail.
export const findFailedDeploymentsCentralized = async (
orgId: string,
accessedServices: string[] | null,
filters: FailedDeploymentFilters = {},
) => {
if (accessedServices !== null && accessedServices.length === 0) {
return [];
}
const idFilters: FailedDeploymentIdFilters = {
projectId: filters.projectId,
environmentId: filters.environmentId,
};
const [appIds, compIds] = await Promise.all([
getApplicationIdsForFailedDeployments(orgId, accessedServices, idFilters),
getComposeIdsForFailedDeployments(orgId, accessedServices, idFilters),
]);
if (appIds.length === 0 && compIds.length === 0) {
return [];
}
// createdAt is stored as text via toISOString(); normalize the filters the
// same way so the string range comparison below stays valid.
const dateFrom = filters.dateFrom
? new Date(filters.dateFrom).toISOString()
: undefined;
const dateTo = filters.dateTo
? new Date(filters.dateTo).toISOString()
: undefined;
const serviceConditions = [
...(appIds.length > 0 ? [inArray(deployments.applicationId, appIds)] : []),
...(compIds.length > 0 ? [inArray(deployments.composeId, compIds)] : []),
];
const conditions = [
eq(deployments.status, "error"),
serviceConditions.length === 1
? serviceConditions[0]
: or(...serviceConditions),
...(dateFrom ? [gte(deployments.createdAt, dateFrom)] : []),
...(dateTo ? [lte(deployments.createdAt, dateTo)] : []),
];
const failed = await db.query.deployments.findMany({
where: and(...conditions),
orderBy: desc(deployments.createdAt),
with: centralizedDeploymentsWith,
limit: 200,
});
// Skip further attempts to a server once it's timed out once, so a batch
// of failures on one unreachable host doesn't pay the SSH timeout N times.
const deadServers = new Set<string>();
return mapWithConcurrency(
failed,
LOG_TAIL_CONCURRENCY,
async (deployment) => {
const serverId =
deployment.buildServer?.serverId ||
deployment.server?.serverId ||
deployment.application?.server?.serverId ||
deployment.compose?.server?.serverId ||
null;
let logTail = "";
if (!serverId || !deadServers.has(serverId)) {
try {
logTail = await readLogTail(deployment.logPath, serverId);
} catch {
if (serverId) deadServers.add(serverId);
}
}
const combined = `${deployment.errorMessage ?? ""}\n${logTail}`;
const classification = classifyFailure(combined);
// Tail end only: the actual failure line is almost always near the end of the log.
const rawError = combined.trim().slice(-1000) || null;
return { ...deployment, classification, rawError };
},
);
};
export const updateDeployment = async (
deploymentId: string,
deploymentData: Partial<Deployment>,
@@ -0,0 +1,386 @@
import { db } from "@dokploy/server/db";
import {
applications,
compose,
environments,
projects,
} from "@dokploy/server/db/schema";
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { and, eq, isNull } from "drizzle-orm";
import { IS_CLOUD } from "../constants";
import { getRemoteDocker } from "../utils/servers/remote-docker";
export interface NetworkIpUsage {
name: string;
driver: string;
subnet: string | null;
containersInUse: number;
capacity: number | null;
percentUsed: number | null;
}
export interface ServerHealthResult {
checkedAt: string;
containers: {
containerCount: number;
serviceCount: number;
};
resources: {
memTotalBytes: number;
memUsedBytes: number;
cpuCount: number;
};
inotify: {
maxWatches: number;
maxInstances: number;
maxQueuedEvents: number;
currentInstances: number;
persisted: boolean;
};
disk: {
totalBytes: number;
usedBytes: number;
};
dockerNetworks: {
count: number;
addressPools: unknown | null;
/** Per-network IP utilization, including reserved networks like dokploy-network */
usage: NetworkIpUsage[];
/** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */
usageError?: string;
};
daemonErrors: string[];
/** The window actually passed to `journalctl --since`, computed on the target machine's clock */
daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null;
reservation: {
memoryReservedBytes: number;
cpuReservedNanoCpus: number;
appCount: number;
unsupportedComposeCount: number;
} | null;
error?: string;
}
interface RawHealthOutput {
containerCount?: number | string;
serviceCount?: number | string;
memTotalBytes?: number | string;
memUsedBytes?: number | string;
cpuCount?: number | string;
inotifyMaxWatches?: number | string;
inotifyMaxInstances?: number | string;
inotifyMaxQueuedEvents?: number | string;
inotifyCurrentInstances?: number | string;
inotifyPersistedCount?: number | string;
diskTotalBytes?: number | string;
diskUsedBytes?: number | string;
networkCount?: number | string;
daemonConfigBase64?: string;
daemonErrorsBase64?: string;
daemonLogsFromEpoch?: number | string;
daemonLogsToEpoch?: number | string;
}
const toInt = (value: unknown): number => {
const n = Number.parseInt(String(value ?? "0"), 10);
return Number.isFinite(n) ? n : 0;
};
const b64Decode = (value?: string): string => {
if (!value) return "";
try {
return Buffer.from(value, "base64").toString("utf-8");
} catch {
return "";
}
};
// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own.
const buildHealthScript = (sinceHours: number) => `
containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ')
serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ')
memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0
memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0
cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0
inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0
inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0
inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0
inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ')
inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ')
diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0
diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0
networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ')
daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n')
daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0
daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600))
daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n')
printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch"
`;
const emptyResult = (error: unknown): ServerHealthResult => ({
checkedAt: new Date().toISOString(),
containers: { containerCount: 0, serviceCount: 0 },
resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 },
inotify: {
maxWatches: 0,
maxInstances: 0,
maxQueuedEvents: 0,
currentInstances: 0,
persisted: false,
},
disk: { totalBytes: 0, usedBytes: 0 },
dockerNetworks: { count: 0, addressPools: null, usage: [] },
daemonErrors: [],
daemonLogsWindow: null,
reservation: null,
error:
error instanceof Error ? error.message : "Could not read server health",
});
/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */
export const getSubnetCapacity = (
subnet: string | undefined,
): number | null => {
if (!subnet) return null;
const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10);
if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null;
const hostBits = 32 - prefix;
if (hostBits <= 1) return 0;
return 2 ** hostBits - 2;
};
type DockerNetworkSummary = {
Name: string;
Driver: string;
IPAM?: { Config?: Array<{ Subnet?: string }> | null };
};
type InspectableNetwork = {
inspect: (opts?: { verbose?: boolean }) => Promise<{
Containers?: Record<string, unknown>;
}>;
};
// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI.
const getNetworksIpUsage = async (
serverId: string | undefined,
): Promise<NetworkIpUsage[]> => {
const docker = await getRemoteDocker(serverId);
const dockerNetworks =
(await docker.listNetworks()) as DockerNetworkSummary[];
const relevant = dockerNetworks.filter(
(n) => n.Driver === "bridge" || n.Driver === "overlay",
);
const rows = await Promise.all(
relevant.map(async (n): Promise<NetworkIpUsage> => {
// IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first.
const subnet = n.IPAM?.Config?.find(
(c) => c.Subnet && !c.Subnet.includes(":"),
)?.Subnet;
const capacity = getSubnetCapacity(subnet);
let containersInUse = 0;
try {
// Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call.
const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork;
// verbose: true for full swarm-wide container list on overlay networks
const info = await net.inspect(
n.Driver === "overlay" ? { verbose: true } : undefined,
);
containersInUse = Object.keys(info.Containers ?? {}).length;
} catch {
// Network may have disappeared between list and inspect; report 0
}
return {
name: n.Name,
driver: n.Driver,
subnet: subnet ?? null,
containersInUse,
capacity,
percentUsed:
capacity && capacity > 0
? Math.round((containersInUse / capacity) * 100)
: null,
};
}),
);
return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1));
};
// Sums Application memoryReservation/cpuReservation (bytes/NanoCPUs, see
// calculateResources in utils/docker/utils.ts). Compose has no such columns.
export const getReservationSummary = async (
orgId: string,
serverId?: string,
) => {
const appRows = await db
.select({
memoryReservation: applications.memoryReservation,
cpuReservation: applications.cpuReservation,
})
.from(applications)
.innerJoin(
environments,
eq(applications.environmentId, environments.environmentId),
)
.innerJoin(projects, eq(environments.projectId, projects.projectId))
.where(
and(
eq(projects.organizationId, orgId),
serverId
? eq(applications.serverId, serverId)
: isNull(applications.serverId),
),
);
let memoryReservedBytes = 0;
let cpuReservedNanoCpus = 0;
for (const row of appRows) {
if (row.memoryReservation) {
memoryReservedBytes += toInt(row.memoryReservation);
}
if (row.cpuReservation) {
cpuReservedNanoCpus += toInt(row.cpuReservation);
}
}
const composeRows = await db
.select({ composeId: compose.composeId })
.from(compose)
.innerJoin(
environments,
eq(compose.environmentId, environments.environmentId),
)
.innerJoin(projects, eq(environments.projectId, projects.projectId))
.where(
and(
eq(projects.organizationId, orgId),
serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId),
),
);
return {
memoryReservedBytes,
cpuReservedNanoCpus,
appCount: appRows.length,
unsupportedComposeCount: composeRows.length,
};
};
export const getServerHealth = async (
orgId: string,
serverId?: string,
sinceHours = 24,
): Promise<ServerHealthResult> => {
if (IS_CLOUD && !serverId) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Server is required",
});
}
const hours = Math.max(
1,
Math.min(168, Math.floor(Number(sinceHours) || 24)),
);
const script = buildHealthScript(hours);
let stdout = "";
try {
const result = serverId
? await execAsyncRemote(serverId, script)
: await execAsync(script);
stdout = result.stdout;
} catch (error) {
return emptyResult(error);
}
let parsed: RawHealthOutput;
try {
parsed = JSON.parse(stdout.trim());
} catch {
return emptyResult(new Error("Could not parse server health output"));
}
const daemonErrors = b64Decode(parsed.daemonErrorsBase64)
.split("\n")
.map((line) => line.trim())
.filter(Boolean);
let addressPools: unknown = null;
const daemonConfigText = b64Decode(parsed.daemonConfigBase64);
if (daemonConfigText) {
try {
addressPools =
JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null;
} catch {
addressPools = null;
}
}
// Not part of the shell script above: needs structured per-network data from the Docker API.
const [reservation, networkUsageResult] = await Promise.all([
getReservationSummary(orgId, serverId).catch(() => null),
getNetworksIpUsage(serverId).then(
(usage) => ({ usage, error: undefined }),
(error: unknown) => ({
usage: [] as NetworkIpUsage[],
error:
error instanceof Error
? error.message
: "Failed to read network usage",
}),
),
]);
return {
checkedAt: new Date().toISOString(),
containers: {
containerCount: toInt(parsed.containerCount),
serviceCount: toInt(parsed.serviceCount),
},
resources: {
memTotalBytes: toInt(parsed.memTotalBytes),
memUsedBytes: toInt(parsed.memUsedBytes),
cpuCount: toInt(parsed.cpuCount),
},
inotify: {
maxWatches: toInt(parsed.inotifyMaxWatches),
maxInstances: toInt(parsed.inotifyMaxInstances),
maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents),
currentInstances: toInt(parsed.inotifyCurrentInstances),
persisted: toInt(parsed.inotifyPersistedCount) > 0,
},
disk: {
totalBytes: toInt(parsed.diskTotalBytes),
usedBytes: toInt(parsed.diskUsedBytes),
},
dockerNetworks: {
count: toInt(parsed.networkCount),
addressPools,
usage: networkUsageResult.usage,
usageError: networkUsageResult.error,
},
daemonErrors,
daemonLogsWindow:
toInt(parsed.daemonLogsToEpoch) > 0
? {
fromEpoch: toInt(parsed.daemonLogsFromEpoch),
toEpoch: toInt(parsed.daemonLogsToEpoch),
}
: null,
reservation,
};
};
@@ -0,0 +1,69 @@
export interface FailurePattern {
id: string;
label: string;
pattern: RegExp;
}
export const FAILURE_PATTERNS: FailurePattern[] = [
{
id: "inotify-limit",
label: "Inotify limit reached",
pattern: /max_user_instances|max_user_watches|inotify watch limit reached/i,
},
{
id: "oom",
label: "Out of memory (OOM)",
pattern: /\boom[-_ ]?kill(er|ed)?\b|killed process|out of memory/i,
},
{
id: "address-pool-exhausted",
label: "Docker network address pool exhausted",
pattern:
/could not find an available[,]? non-overlapping|available.*address pool/i,
},
{
id: "network-ip-exhausted",
label: "Network out of IP addresses",
pattern:
/could not find an available ip|no available ip addresses|task allocation failure/i,
},
{
id: "network-attach-timeout",
label: "Network attach timed out",
pattern:
/attaching to network failed|failed to set up container networking/i,
},
{
id: "disk-full",
label: "Disk space exhausted",
pattern: /no space left on device/i,
},
{
id: "conntrack-full",
label: "Connection tracking table full",
pattern: /nf_conntrack:?\s*table full/i,
},
{
id: "fd-limit",
label: "File descriptor limit reached",
pattern: /too many open files/i,
},
{
id: "pid-limit",
label: "Process/PID limit reached",
pattern:
/pids\.max|cannot allocate memory|fork:.*resource temporarily unavailable/i,
},
];
export const classifyFailure = (
text: string | null | undefined,
): { id: string; label: string } | null => {
if (!text) return null;
for (const entry of FAILURE_PATTERNS) {
if (entry.pattern.test(text)) {
return { id: entry.id, label: entry.label };
}
}
return null;
};