Merge pull request #5072 from Dokploy/feat/docker-health-dashboard

feat: add Docker Health diagnostics tab

apps/dokploy/__test__/server/server-health-subnet.test.ts

@@ -0,0 +1,24 @@
+import { getSubnetCapacity } from "@dokploy/server/services/server-health";
+import { describe, expect, test } from "vitest";
+
+describe("getSubnetCapacity", () => {
+	test("returns null for missing/invalid input", () => {
+		expect(getSubnetCapacity(undefined)).toBeNull();
+		expect(getSubnetCapacity("")).toBeNull();
+		expect(getSubnetCapacity("10.0.0.0")).toBeNull();
+		expect(getSubnetCapacity("not-a-subnet")).toBeNull();
+		expect(getSubnetCapacity("10.0.0.0/33")).toBeNull();
+		expect(getSubnetCapacity("10.0.0.0/-1")).toBeNull();
+	});
+
+	test("excludes network and broadcast addresses", () => {
+		expect(getSubnetCapacity("10.0.1.0/24")).toBe(254);
+		expect(getSubnetCapacity("10.0.0.0/16")).toBe(65534);
+		expect(getSubnetCapacity("10.99.99.0/30")).toBe(2);
+	});
+
+	test("returns 0 for subnets too small to hold a host", () => {
+		expect(getSubnetCapacity("10.0.0.0/31")).toBe(0);
+		expect(getSubnetCapacity("10.0.0.0/32")).toBe(0);
+	});
+});

apps/dokploy/components/dashboard/docker/health/show-health.tsx

@@ -0,0 +1,454 @@
+import {
+	AlertTriangle,
+	Cpu,
+	Download,
+	HardDrive,
+	Loader2,
+	Network,
+	RefreshCw,
+	Server as ServerIcon,
+} from "lucide-react";
+import { useState } from "react";
+import { CodeEditor } from "@/components/shared/code-editor";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Card } from "@/components/ui/card";
+import { Progress } from "@/components/ui/progress";
+import {
+	Select,
+	SelectContent,
+	SelectItem,
+	SelectTrigger,
+	SelectValue,
+} from "@/components/ui/select";
+import {
+	Table,
+	TableBody,
+	TableCell,
+	TableHead,
+	TableHeader,
+	TableRow,
+} from "@/components/ui/table";
+import { api } from "@/utils/api";
+
+interface Props {
+	serverId?: string;
+}
+
+const bytesToGb = (bytes: number) => (bytes / 1024 ** 3).toFixed(1);
+const nanoCpusToCores = (nanoCpus: number) => (nanoCpus / 1e9).toFixed(2);
+const pct = (used: number, total: number) =>
+	Math.min(100, Math.round((used / (total || 1)) * 100));
+
+const SINCE_HOURS_OPTIONS = [
+	{ label: "Last hour", value: 1 },
+	{ label: "Last 6 hours", value: 6 },
+	{ label: "Last 24 hours", value: 24 },
+	{ label: "Last 7 days", value: 168 },
+];
+
+export const ShowHealth = ({ serverId }: Props) => {
+	const [sinceHours, setSinceHours] = useState(24);
+	const {
+		data: health,
+		isFetching,
+		refetch,
+		isFetched,
+	} = api.docker.getServerHealth.useQuery(
+		{ serverId, sinceHours },
+		{ refetchOnMount: false, refetchOnWindowFocus: false },
+	);
+
+	const memUsedPct = health
+		? pct(health.resources.memUsedBytes, health.resources.memTotalBytes)
+		: 0;
+	const diskUsedPct = health
+		? pct(health.disk.usedBytes, health.disk.totalBytes)
+		: 0;
+	const inotifyUsedPct = health
+		? pct(health.inotify.currentInstances, health.inotify.maxInstances)
+		: 0;
+
+	const daemonLogsWindowText = health?.daemonLogsWindow
+		? `Logs from ${new Date(health.daemonLogsWindow.fromEpoch * 1000).toLocaleString()} to ${new Date(health.daemonLogsWindow.toEpoch * 1000).toLocaleString()}`
+		: null;
+	const daemonErrorsText = [
+		daemonLogsWindowText,
+		health && health.daemonErrors.length > 0
+			? health.daemonErrors.join("\n")
+			: "(no daemon errors matched in this window)",
+	]
+		.filter(Boolean)
+		.join("\n\n");
+
+	const buildHealthReport = () => {
+		if (!health) return "";
+		const lines: string[] = [];
+		lines.push("# Dokploy server health report");
+		lines.push(`Generated: ${new Date(health.checkedAt).toLocaleString()}`);
+		lines.push(
+			`Window: ${SINCE_HOURS_OPTIONS.find((o) => o.value === sinceHours)?.label ?? `${sinceHours}h`}`,
+		);
+		lines.push("");
+
+		lines.push("## Containers & services");
+		lines.push(`Containers: ${health.containers.containerCount}`);
+		lines.push(`Swarm services: ${health.containers.serviceCount}`);
+		lines.push("");
+
+		lines.push("## Host resources");
+		lines.push(
+			`Memory: ${bytesToGb(health.resources.memUsedBytes)} / ${bytesToGb(health.resources.memTotalBytes)} GB`,
+		);
+		lines.push(`CPU cores: ${health.resources.cpuCount}`);
+		lines.push("");
+
+		lines.push("## Disk (/)");
+		lines.push(
+			`${bytesToGb(health.disk.usedBytes)} / ${bytesToGb(health.disk.totalBytes)} GB`,
+		);
+		lines.push("");
+
+		lines.push("## Inotify");
+		lines.push(
+			`max_user_instances: ${health.inotify.currentInstances} / ${health.inotify.maxInstances}`,
+		);
+		lines.push(`max_user_watches: ${health.inotify.maxWatches}`);
+		lines.push(`max_queued_events: ${health.inotify.maxQueuedEvents}`);
+		lines.push(
+			`Persisted in sysctl: ${health.inotify.persisted ? "yes" : "no"}`,
+		);
+		lines.push("");
+
+		lines.push("## Docker networks");
+		lines.push(`Total networks: ${health.dockerNetworks.count}`);
+		if (health.dockerNetworks.addressPools) {
+			lines.push(
+				`default-address-pools: ${JSON.stringify(health.dockerNetworks.addressPools)}`,
+			);
+		}
+		if (health.dockerNetworks.usageError) {
+			lines.push(
+				`Could not read network IP usage: ${health.dockerNetworks.usageError}`,
+			);
+		} else if (health.dockerNetworks.usage.length > 0) {
+			lines.push("");
+			lines.push(
+				"| Network | Driver | Subnet | IPs in use | Capacity | Usage |",
+			);
+			lines.push("|---|---|---|---|---|---|");
+			for (const n of health.dockerNetworks.usage) {
+				lines.push(
+					`| ${n.name} | ${n.driver} | ${n.subnet ?? "auto"} | ${n.containersInUse} | ${n.capacity ?? "—"} | ${n.percentUsed ?? "—"}% |`,
+				);
+			}
+		}
+		lines.push("");
+
+		if (health.reservation) {
+			lines.push("## Memory/CPU reservation");
+			lines.push(
+				`Reserved memory: ${bytesToGb(health.reservation.memoryReservedBytes)} GB`,
+			);
+			lines.push(
+				`Reserved CPU: ${nanoCpusToCores(health.reservation.cpuReservedNanoCpus)} cores`,
+			);
+			lines.push(`Across ${health.reservation.appCount} Application services`);
+			if (health.reservation.unsupportedComposeCount > 0) {
+				lines.push(
+					`${health.reservation.unsupportedComposeCount} Compose service(s) not included (not tracked per-service)`,
+				);
+			}
+			lines.push("");
+		}
+
+		lines.push("## Docker daemon errors (raw)");
+		lines.push("```");
+		lines.push(daemonErrorsText);
+		lines.push("```");
+
+		return lines.join("\n");
+	};
+
+	const handleDownload = () => {
+		const report = buildHealthReport();
+		if (!report) return;
+		const blob = new Blob([report], { type: "text/markdown" });
+		const url = URL.createObjectURL(blob);
+		const a = document.createElement("a");
+		a.href = url;
+		a.download = `dokploy-health-report-${new Date().toISOString().replace(/[:.]/g, "-")}.md`;
+		a.click();
+		URL.revokeObjectURL(url);
+	};
+
+	return (
+		<div className="flex flex-col gap-4 w-full">
+			<Card className="bg-sidebar p-2.5 rounded-xl w-full">
+				<div className="rounded-xl bg-background shadow-md p-6 flex flex-col gap-4">
+					<div className="flex items-center justify-between gap-4 flex-wrap">
+						<div>
+							<h3 className="text-lg font-medium">Server diagnostics</h3>
+							<p className="text-sm text-muted-foreground max-w-xl">
+								Runs a read-only check over SSH (inotify limits, disk, Docker
+								network pool, daemon errors). Runs automatically when you open
+								this tab — click Re-check to refresh.
+							</p>
+						</div>
+						<div className="flex items-center gap-2">
+							<Select
+								value={String(sinceHours)}
+								onValueChange={(v) => setSinceHours(Number(v))}
+							>
+								<SelectTrigger className="w-[160px]">
+									<SelectValue />
+								</SelectTrigger>
+								<SelectContent>
+									{SINCE_HOURS_OPTIONS.map((opt) => (
+										<SelectItem key={opt.value} value={String(opt.value)}>
+											{opt.label}
+										</SelectItem>
+									))}
+								</SelectContent>
+							</Select>
+							<Button onClick={() => refetch()} disabled={isFetching}>
+								{isFetching ? (
+									<Loader2 className="size-4 animate-spin mr-2" />
+								) : (
+									<RefreshCw className="size-4 mr-2" />
+								)}
+								{isFetched ? "Re-check" : "Check server health"}
+							</Button>
+							{health && !health.error && (
+								<Button variant="outline" onClick={handleDownload}>
+									<Download className="size-4 mr-2" />
+									Download report
+								</Button>
+							)}
+						</div>
+					</div>
+
+					{isFetching && !health && (
+						<div className="flex flex-col items-center justify-center gap-2 py-12 text-muted-foreground border border-dashed rounded-xl">
+							<Loader2 className="size-8 animate-spin" />
+							<span>Checking server health…</span>
+						</div>
+					)}
+
+					{health?.error && (
+						<div className="text-sm text-destructive border border-destructive/30 bg-destructive/10 rounded-lg p-3">
+							Couldn't read server health: {health.error}
+						</div>
+					)}
+
+					{health && !health.error && (
+						<div
+							className={`grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4 transition-opacity ${isFetching ? "opacity-50" : ""}`}
+						>
+							<Card className="p-4">
+								<div className="flex items-center gap-2 text-sm font-medium mb-2">
+									<ServerIcon className="size-4 text-muted-foreground" />
+									Containers &amp; services
+								</div>
+								<div className="text-2xl font-semibold">
+									{health.containers.containerCount}
+								</div>
+								<div className="text-xs text-muted-foreground">
+									containers · {health.containers.serviceCount} swarm services
+								</div>
+							</Card>
+
+							<Card className="p-4">
+								<div className="flex items-center gap-2 text-sm font-medium mb-2">
+									<Cpu className="size-4 text-muted-foreground" />
+									Host resources
+								</div>
+								<div className="text-sm">
+									Memory: {bytesToGb(health.resources.memUsedBytes)} /{" "}
+									{bytesToGb(health.resources.memTotalBytes)} GB
+								</div>
+								<Progress value={memUsedPct} className="h-2 mt-1" />
+								<div className="text-xs text-muted-foreground mt-2">
+									{health.resources.cpuCount} CPU cores
+								</div>
+							</Card>
+
+							<Card className="p-4">
+								<div className="flex items-center gap-2 text-sm font-medium mb-2">
+									<HardDrive className="size-4 text-muted-foreground" />
+									Disk (/)
+								</div>
+								<div className="text-sm">
+									{bytesToGb(health.disk.usedBytes)} /{" "}
+									{bytesToGb(health.disk.totalBytes)} GB
+								</div>
+								<Progress value={diskUsedPct} className="h-2 mt-1" />
+							</Card>
+
+							<Card className="p-4">
+								<div className="flex items-center justify-between gap-2 text-sm font-medium mb-2">
+									<span className="flex items-center gap-2">
+										<AlertTriangle className="size-4 text-muted-foreground" />
+										Inotify
+									</span>
+									<Badge
+										variant={health.inotify.persisted ? "default" : "secondary"}
+									>
+										{health.inotify.persisted ? "persisted" : "runtime only"}
+									</Badge>
+								</div>
+								<div className="text-sm">
+									max_user_instances: {health.inotify.currentInstances} /{" "}
+									{health.inotify.maxInstances.toLocaleString()} (
+									{inotifyUsedPct}%)
+								</div>
+								<Progress value={inotifyUsedPct} className="h-2 mt-1" />
+								<div className="text-xs text-muted-foreground mt-2">
+									max_user_watches: {health.inotify.maxWatches.toLocaleString()}{" "}
+									· max_queued_events:{" "}
+									{health.inotify.maxQueuedEvents.toLocaleString()}
+								</div>
+							</Card>
+
+							<Card className="p-4">
+								<div className="flex items-center gap-2 text-sm font-medium mb-2">
+									<Network className="size-4 text-muted-foreground" />
+									Docker networks
+								</div>
+								<div className="text-2xl font-semibold">
+									{health.dockerNetworks.count}
+								</div>
+								<div className="text-xs text-muted-foreground">
+									The default Docker address pool is exhausted around ~30
+									networks unless <code>default-address-pools</code> is
+									configured in <code>/etc/docker/daemon.json</code>.
+									{health.dockerNetworks.addressPools
+										? " Custom pool detected on this server."
+										: ""}
+								</div>
+							</Card>
+
+							{health.reservation && (
+								<Card className="p-4">
+									<div className="flex items-center gap-2 text-sm font-medium mb-2">
+										<Cpu className="size-4 text-muted-foreground" />
+										Memory/CPU reservation
+									</div>
+									<div className="text-sm">
+										{bytesToGb(health.reservation.memoryReservedBytes)} GB
+										reserved ·{" "}
+										{nanoCpusToCores(health.reservation.cpuReservedNanoCpus)}{" "}
+										CPUs reserved
+									</div>
+									<div className="text-xs text-muted-foreground mt-2">
+										Across {health.reservation.appCount} Application services.
+										{health.reservation.unsupportedComposeCount > 0
+											? ` ${health.reservation.unsupportedComposeCount} Compose service(s) on this server aren't included — reservations aren't tracked per-service for Compose.`
+											: ""}
+									</div>
+								</Card>
+							)}
+						</div>
+					)}
+
+					{health &&
+						!health.error &&
+						(health.dockerNetworks.usage.length > 0 ||
+							health.dockerNetworks.usageError) && (
+							<div className="flex flex-col gap-2">
+								<h4 className="text-sm font-medium flex items-center gap-2">
+									<Network className="size-4 text-muted-foreground" />
+									Network IP usage
+								</h4>
+								<p className="text-xs text-muted-foreground">
+									IPs assigned to containers vs. the subnet's usable capacity,
+									per network — including reserved networks like{" "}
+									<code>dokploy-network</code>. A network stuck near 100% blocks
+									new containers from starting even when the host has plenty of
+									other resources free.
+								</p>
+								{health.dockerNetworks.usageError ? (
+									<div className="text-sm text-destructive border border-destructive/30 bg-destructive/10 rounded-lg p-3">
+										Couldn't read network IP usage:{" "}
+										{health.dockerNetworks.usageError}
+									</div>
+								) : (
+									<div className="rounded-lg border overflow-x-auto">
+										<Table>
+											<TableHeader>
+												<TableRow>
+													<TableHead>Network</TableHead>
+													<TableHead>Driver</TableHead>
+													<TableHead>Subnet</TableHead>
+													<TableHead>IPs in use</TableHead>
+													<TableHead>Usage</TableHead>
+												</TableRow>
+											</TableHeader>
+											<TableBody>
+												{health.dockerNetworks.usage.map((n) => (
+													<TableRow key={n.name}>
+														<TableCell className="font-medium">
+															{n.name}
+														</TableCell>
+														<TableCell className="text-muted-foreground">
+															{n.driver}
+														</TableCell>
+														<TableCell className="text-muted-foreground">
+															{n.subnet ?? "Auto"}
+														</TableCell>
+														<TableCell className="text-muted-foreground">
+															{n.capacity !== null
+																? `${n.containersInUse} / ${n.capacity}`
+																: n.containersInUse}
+														</TableCell>
+														<TableCell>
+															{n.percentUsed !== null ? (
+																<Badge
+																	variant={
+																		n.percentUsed >= 90
+																			? "red"
+																			: n.percentUsed >= 70
+																				? "yellow"
+																				: "green"
+																	}
+																>
+																	{n.percentUsed}%
+																</Badge>
+															) : (
+																<span className="text-muted-foreground text-xs">
+																	—
+																</span>
+															)}
+														</TableCell>
+													</TableRow>
+												))}
+											</TableBody>
+										</Table>
+									</div>
+								)}
+							</div>
+						)}
+
+					{health && !health.error && (
+						<div className="flex flex-col gap-2">
+							<h4 className="text-sm font-medium">
+								Docker daemon errors (
+								{SINCE_HOURS_OPTIONS.find(
+									(o) => o.value === sinceHours,
+								)?.label.toLowerCase()}
+								)
+							</h4>
+							<CodeEditor
+								value={daemonErrorsText}
+								language="shell"
+								disabled
+								lineWrapping
+								wrapperClassName="h-64 rounded-lg border"
+							/>
+						</div>
+					)}
+				</div>
+			</Card>
+		</div>
+	);
+};

apps/dokploy/components/dashboard/settings/servers/show-health-modal.tsx

@@ -0,0 +1,28 @@
+import { Stethoscope } from "lucide-react";
+import { useState } from "react";
+import { ShowHealth } from "@/components/dashboard/docker/health/show-health";
+import { Button } from "@/components/ui/button";
+import { Dialog, DialogContent, DialogTrigger } from "@/components/ui/dialog";
+
+interface Props {
+	serverId: string;
+}
+
+export const ShowHealthModal = ({ serverId }: Props) => {
+	const [isOpen, setIsOpen] = useState(false);
+
+	return (
+		<Dialog open={isOpen} onOpenChange={setIsOpen}>
+			<DialogTrigger asChild>
+				<Button variant="outline" size="icon" className="h-9 w-9">
+					<Stethoscope className="h-4 w-4" />
+				</Button>
+			</DialogTrigger>
+			<DialogContent className="sm:max-w-5xl max-h-[85vh] overflow-y-auto">
+				<div className="flex gap-4 py-4 w-full">
+					<ShowHealth serverId={serverId} />
+				</div>
+			</DialogContent>
+		</Dialog>
+	);
+};

apps/dokploy/components/dashboard/settings/servers/show-servers.tsx

@@ -35,6 +35,7 @@ import { TerminalModal } from "../web-server/terminal-modal";
 import { ShowServerActions } from "./actions/show-server-actions";
 import { HandleServers } from "./handle-servers";
 import { SetupServer } from "./setup-server";
+import { ShowHealthModal } from "./show-health-modal";
 import { ShowMonitoringModal } from "./show-monitoring-modal";
 import { WelcomeSubscription } from "./welcome-stripe/welcome-subscription";
 
@@ -324,6 +325,24 @@ export const ShowServers = () => {
 																						</Tooltip>
 																					)}
 
+																				{permissions?.docker.read &&
+																					permissions?.server.read &&
+																					server.sshKeyId &&
+																					!isBuildServer && (
+																						<Tooltip>
+																							<TooltipTrigger asChild>
+																								<div>
+																									<ShowHealthModal
+																										serverId={server.serverId}
+																									/>
+																								</div>
+																							</TooltipTrigger>
+																							<TooltipContent>
+																								<p>Health</p>
+																							</TooltipContent>
+																						</Tooltip>
+																					)}
+
 																				<div className="flex-1" />
 
 																				{permissions?.server.delete && (

apps/dokploy/pages/dashboard/docker.tsx

@@ -6,6 +6,7 @@ import type { ReactElement } from "react";
 import superjson from "superjson";
 import { ShowDiskUsage } from "@/components/dashboard/docker/disk-usage/show-disk-usage";
 import { ShowDockerEvents } from "@/components/dashboard/docker/events/show-docker-events";
+import { ShowHealth } from "@/components/dashboard/docker/health/show-health";
 import { ShowImages } from "@/components/dashboard/docker/images/show-images";
 import { ShowContainers } from "@/components/dashboard/docker/show/show-containers";
 import { ShowVolumes } from "@/components/dashboard/docker/volumes/show-volumes";
@@ -18,15 +19,21 @@ import { ServerFilter } from "@/components/shared/server-filter";
 import { Card } from "@/components/ui/card";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { appRouter } from "@/server/api/root";
+import { api } from "@/utils/api";
 
 const DEFAULT_TAB = "containers";
 const DEFAULT_SWARM_TAB = "overview";
 
 const Dashboard = () => {
 	const router = useRouter();
+	const { data: permissions } = api.user.getPermissions.useQuery();
+	// Host-level diagnostics, so this needs server.read on top of docker.read.
+	const canSeeHealth = !!permissions?.docker.read && !!permissions?.server.read;
 
-	const activeTab =
+	const queryTab =
 		typeof router.query.tab === "string" ? router.query.tab : DEFAULT_TAB;
+	const activeTab =
+		queryTab === "health" && !canSeeHealth ? DEFAULT_TAB : queryTab;
 
 	const activeSwarmTab =
 		typeof router.query.subtab === "string"
@@ -70,6 +77,7 @@ const Dashboard = () => {
 						<TabsTrigger value="networks">Networks</TabsTrigger>
 						<TabsTrigger value="events">Events</TabsTrigger>
 						<TabsTrigger value="disk-usage">Disk Usage</TabsTrigger>
+						{canSeeHealth && <TabsTrigger value="health">Health</TabsTrigger>}
 					</TabsList>
 					<TabsContent value="containers">
 						<ShowContainers serverId={serverId} />
@@ -111,6 +119,11 @@ const Dashboard = () => {
 					<TabsContent value="disk-usage">
 						<ShowDiskUsage serverId={serverId} />
 					</TabsContent>
+					{canSeeHealth && (
+						<TabsContent value="health">
+							<ShowHealth serverId={serverId} />
+						</TabsContent>
+					)}
 				</Tabs>
 			)}
 		</ServerFilter>

apps/dokploy/server/api/routers/docker.ts

@@ -11,6 +11,7 @@ import {
 	getContainersByAppLabel,
 	getContainersByAppNameMatch,
 	getDockerEvents,
+	getServerHealth as getServerHealthData,
 	getServiceContainersByAppName,
 	getStackContainersByAppName,
 	listContainerFiles,
@@ -18,11 +19,12 @@ import {
 	uploadFileToContainer,
 	writeContainerFile,
 } from "@dokploy/server";
+import { checkPermission } from "@dokploy/server/services/permission";
 import { TRPCError } from "@trpc/server";
 import { z } from "zod";
 import { audit } from "@/server/api/utils/audit";
 import { uploadFileToContainerSchema } from "@/utils/schema";
-import { createTRPCRouter, withPermission } from "../trpc";
+import { createTRPCRouter, protectedProcedure, withPermission } from "../trpc";
 
 export const containerIdRegex = /^[a-zA-Z0-9.\-_]+$/;
 
@@ -52,6 +54,29 @@ export const dockerRouter = createTRPCRouter({
 			return await getContainers(input.serverId);
 		}),
 
+	// Host-level diagnostics, so this requires docker.read AND server.read.
+	getServerHealth: protectedProcedure
+		.input(
+			z.object({
+				serverId: z.string().optional(),
+				sinceHours: z.number().int().min(1).max(168).optional(),
+			}),
+		)
+		.query(async ({ input, ctx }) => {
+			await checkPermission(ctx, { docker: ["read"], server: ["read"] });
+			if (input.serverId) {
+				const server = await findServerById(input.serverId);
+				if (server.organizationId !== ctx.session?.activeOrganizationId) {
+					throw new TRPCError({ code: "UNAUTHORIZED" });
+				}
+			}
+			return await getServerHealthData(
+				ctx.session.activeOrganizationId,
+				input.serverId,
+				input.sinceHours,
+			);
+		}),
+
 	restartContainer: withPermission("docker", "read")
 		.input(
 			z.object({

packages/server/src/index.ts

@@ -50,6 +50,7 @@ export * from "./services/rollbacks";
 export * from "./services/schedule";
 export * from "./services/security";
 export * from "./services/server";
+export * from "./services/server-health";
 export * from "./services/settings";
 export * from "./services/ssh-key";
 export * from "./services/user";

packages/server/src/services/server-health.ts

@@ -0,0 +1,385 @@
+import { db } from "@dokploy/server/db";
+import {
+	applications,
+	compose,
+	environments,
+	projects,
+} from "@dokploy/server/db/schema";
+import {
+	execAsync,
+	execAsyncRemote,
+} from "@dokploy/server/utils/process/execAsync";
+import { TRPCError } from "@trpc/server";
+import { and, eq, isNull } from "drizzle-orm";
+import { IS_CLOUD } from "../constants";
+import { getRemoteDocker } from "../utils/servers/remote-docker";
+
+export interface NetworkIpUsage {
+	name: string;
+	driver: string;
+	subnet: string | null;
+	containersInUse: number;
+	capacity: number | null;
+	percentUsed: number | null;
+}
+
+export interface ServerHealthResult {
+	checkedAt: string;
+	containers: {
+		containerCount: number;
+		serviceCount: number;
+	};
+	resources: {
+		memTotalBytes: number;
+		memUsedBytes: number;
+		cpuCount: number;
+	};
+	inotify: {
+		maxWatches: number;
+		maxInstances: number;
+		maxQueuedEvents: number;
+		currentInstances: number;
+		persisted: boolean;
+	};
+	disk: {
+		totalBytes: number;
+		usedBytes: number;
+	};
+	dockerNetworks: {
+		count: number;
+		addressPools: unknown | null;
+		/** Per-network IP utilization, including reserved networks like dokploy-network */
+		usage: NetworkIpUsage[];
+		/** Set when `usage` couldn't be read, so the UI can tell that apart from "no networks" */
+		usageError?: string;
+	};
+	daemonErrors: string[];
+	/** The window actually passed to `journalctl --since`, computed on the target machine's clock */
+	daemonLogsWindow: { fromEpoch: number; toEpoch: number } | null;
+	reservation: {
+		memoryReservedBytes: number;
+		cpuReservedNanoCpus: number;
+		appCount: number;
+		unsupportedComposeCount: number;
+	} | null;
+	error?: string;
+}
+
+interface RawHealthOutput {
+	containerCount?: number | string;
+	serviceCount?: number | string;
+	memTotalBytes?: number | string;
+	memUsedBytes?: number | string;
+	cpuCount?: number | string;
+	inotifyMaxWatches?: number | string;
+	inotifyMaxInstances?: number | string;
+	inotifyMaxQueuedEvents?: number | string;
+	inotifyCurrentInstances?: number | string;
+	inotifyPersistedCount?: number | string;
+	diskTotalBytes?: number | string;
+	diskUsedBytes?: number | string;
+	networkCount?: number | string;
+	daemonConfigBase64?: string;
+	daemonErrorsBase64?: string;
+	daemonLogsFromEpoch?: number | string;
+	daemonLogsToEpoch?: number | string;
+}
+
+const toInt = (value: unknown): number => {
+	const n = Number.parseInt(String(value ?? "0"), 10);
+	return Number.isFinite(n) ? n : 0;
+};
+
+const b64Decode = (value?: string): string => {
+	if (!value) return "";
+	try {
+		return Buffer.from(value, "base64").toString("utf-8");
+	} catch {
+		return "";
+	}
+};
+
+// Single read-only script, one SSH/exec round-trip; each check falls back to 0/empty on its own.
+const buildHealthScript = (sinceHours: number) => `
+containerCount=$(docker ps -a -q 2>/dev/null | wc -l | tr -d ' ')
+serviceCount=$(docker service ls --format '{{.Name}}' 2>/dev/null | wc -l | tr -d ' ')
+
+memTotal=$(free -b 2>/dev/null | awk '/^Mem:/{print $2}'); [ -z "$memTotal" ] && memTotal=0
+memUsed=$(free -b 2>/dev/null | awk '/^Mem:/{print $3}'); [ -z "$memUsed" ] && memUsed=0
+cpuCount=$(nproc 2>/dev/null); [ -z "$cpuCount" ] && cpuCount=0
+
+inotifyMaxWatches=$(cat /proc/sys/fs/inotify/max_user_watches 2>/dev/null); [ -z "$inotifyMaxWatches" ] && inotifyMaxWatches=0
+inotifyMaxInstances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null); [ -z "$inotifyMaxInstances" ] && inotifyMaxInstances=0
+inotifyMaxQueued=$(cat /proc/sys/fs/inotify/max_queued_events 2>/dev/null); [ -z "$inotifyMaxQueued" ] && inotifyMaxQueued=0
+inotifyCurrentInstances=$(find /proc/*/fd -lname 'anon_inode:inotify*' 2>/dev/null | wc -l | tr -d ' ')
+inotifyPersistedCount=$(grep -rl inotify /etc/sysctl.conf /etc/sysctl.d/ 2>/dev/null | wc -l | tr -d ' ')
+
+diskTotal=$(df -B1 / 2>/dev/null | awk 'NR==2{print $2}'); [ -z "$diskTotal" ] && diskTotal=0
+diskUsed=$(df -B1 / 2>/dev/null | awk 'NR==2{print $3}'); [ -z "$diskUsed" ] && diskUsed=0
+
+networkCount=$(docker network ls -q 2>/dev/null | wc -l | tr -d ' ')
+daemonConfigB64=$(cat /etc/docker/daemon.json 2>/dev/null | base64 2>/dev/null | tr -d '\\n')
+
+daemonLogsToEpoch=$(date +%s 2>/dev/null); [ -z "$daemonLogsToEpoch" ] && daemonLogsToEpoch=0
+daemonLogsFromEpoch=$((daemonLogsToEpoch - ${sinceHours} * 3600))
+
+daemonErrorsB64=$(journalctl -u docker --no-pager --since "${sinceHours} hours ago" 2>/dev/null | grep -iE "inotify|too many open|cannot allocate|oom|conntrack|no space|pids.max|fork:|resource temporarily unavailable|could not find an available ip|no available ip|task allocation failure|address already in use" | tail -n 50 | base64 2>/dev/null | tr -d '\\n')
+
+printf '{"containerCount":%s,"serviceCount":%s,"memTotalBytes":%s,"memUsedBytes":%s,"cpuCount":%s,"inotifyMaxWatches":%s,"inotifyMaxInstances":%s,"inotifyMaxQueuedEvents":%s,"inotifyCurrentInstances":%s,"inotifyPersistedCount":%s,"diskTotalBytes":%s,"diskUsedBytes":%s,"networkCount":%s,"daemonConfigBase64":"%s","daemonErrorsBase64":"%s","daemonLogsFromEpoch":%s,"daemonLogsToEpoch":%s}' "$containerCount" "$serviceCount" "$memTotal" "$memUsed" "$cpuCount" "$inotifyMaxWatches" "$inotifyMaxInstances" "$inotifyMaxQueued" "$inotifyCurrentInstances" "$inotifyPersistedCount" "$diskTotal" "$diskUsed" "$networkCount" "$daemonConfigB64" "$daemonErrorsB64" "$daemonLogsFromEpoch" "$daemonLogsToEpoch"
+`;
+
+const emptyResult = (error: unknown): ServerHealthResult => ({
+	checkedAt: new Date().toISOString(),
+	containers: { containerCount: 0, serviceCount: 0 },
+	resources: { memTotalBytes: 0, memUsedBytes: 0, cpuCount: 0 },
+	inotify: {
+		maxWatches: 0,
+		maxInstances: 0,
+		maxQueuedEvents: 0,
+		currentInstances: 0,
+		persisted: false,
+	},
+	disk: { totalBytes: 0, usedBytes: 0 },
+	dockerNetworks: { count: 0, addressPools: null, usage: [] },
+	daemonErrors: [],
+	daemonLogsWindow: null,
+	reservation: null,
+	error:
+		error instanceof Error ? error.message : "Could not read server health",
+});
+
+/** Usable host addresses in a subnet, excluding network + broadcast. Null if the subnet is missing/invalid. */
+export const getSubnetCapacity = (
+	subnet: string | undefined,
+): number | null => {
+	if (!subnet) return null;
+	const prefix = Number.parseInt(subnet.split("/")[1] ?? "", 10);
+	if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return null;
+	const hostBits = 32 - prefix;
+	if (hostBits <= 1) return 0;
+	return 2 ** hostBits - 2;
+};
+
+type DockerNetworkSummary = {
+	Name: string;
+	Driver: string;
+	IPAM?: { Config?: Array<{ Subnet?: string }> | null };
+};
+
+type InspectableNetwork = {
+	inspect: (opts?: { verbose?: boolean }) => Promise<{
+		Containers?: Record<string, unknown>;
+	}>;
+};
+
+// Includes reserved networks like dokploy-network, which are excluded from the `network` table/UI.
+const getNetworksIpUsage = async (
+	serverId: string | undefined,
+): Promise<NetworkIpUsage[]> => {
+	const docker = await getRemoteDocker(serverId);
+	const dockerNetworks =
+		(await docker.listNetworks()) as DockerNetworkSummary[];
+	const relevant = dockerNetworks.filter(
+		(n) => n.Driver === "bridge" || n.Driver === "overlay",
+	);
+
+	const rows = await Promise.all(
+		relevant.map(async (n): Promise<NetworkIpUsage> => {
+			// IPv4 entry specifically: dual-stack networks list an IPv6 config too, and it may come first.
+			const subnet = n.IPAM?.Config?.find(
+				(c) => c.Subnet && !c.Subnet.includes(":"),
+			)?.Subnet;
+			const capacity = getSubnetCapacity(subnet);
+			let containersInUse = 0;
+			try {
+				// Cast the object (not the detached method) so `net.inspect(...)` stays a bound method call.
+				const net = docker.getNetwork(n.Name) as unknown as InspectableNetwork;
+				// verbose: true for full swarm-wide container list on overlay networks
+				const info = await net.inspect(
+					n.Driver === "overlay" ? { verbose: true } : undefined,
+				);
+				containersInUse = Object.keys(info.Containers ?? {}).length;
+			} catch {
+				// Network may have disappeared between list and inspect; report 0
+			}
+			return {
+				name: n.Name,
+				driver: n.Driver,
+				subnet: subnet ?? null,
+				containersInUse,
+				capacity,
+				percentUsed:
+					capacity && capacity > 0
+						? Math.round((containersInUse / capacity) * 100)
+						: null,
+			};
+		}),
+	);
+
+	return rows.sort((a, b) => (b.percentUsed ?? -1) - (a.percentUsed ?? -1));
+};
+
+// Sums Application memoryReservation/cpuReservation; Compose has no such columns.
+export const getReservationSummary = async (
+	orgId: string,
+	serverId?: string,
+) => {
+	const appRows = await db
+		.select({
+			memoryReservation: applications.memoryReservation,
+			cpuReservation: applications.cpuReservation,
+		})
+		.from(applications)
+		.innerJoin(
+			environments,
+			eq(applications.environmentId, environments.environmentId),
+		)
+		.innerJoin(projects, eq(environments.projectId, projects.projectId))
+		.where(
+			and(
+				eq(projects.organizationId, orgId),
+				serverId
+					? eq(applications.serverId, serverId)
+					: isNull(applications.serverId),
+			),
+		);
+
+	let memoryReservedBytes = 0;
+	let cpuReservedNanoCpus = 0;
+	for (const row of appRows) {
+		if (row.memoryReservation) {
+			memoryReservedBytes += toInt(row.memoryReservation);
+		}
+		if (row.cpuReservation) {
+			cpuReservedNanoCpus += toInt(row.cpuReservation);
+		}
+	}
+
+	const composeRows = await db
+		.select({ composeId: compose.composeId })
+		.from(compose)
+		.innerJoin(
+			environments,
+			eq(compose.environmentId, environments.environmentId),
+		)
+		.innerJoin(projects, eq(environments.projectId, projects.projectId))
+		.where(
+			and(
+				eq(projects.organizationId, orgId),
+				serverId ? eq(compose.serverId, serverId) : isNull(compose.serverId),
+			),
+		);
+
+	return {
+		memoryReservedBytes,
+		cpuReservedNanoCpus,
+		appCount: appRows.length,
+		unsupportedComposeCount: composeRows.length,
+	};
+};
+
+export const getServerHealth = async (
+	orgId: string,
+	serverId?: string,
+	sinceHours = 24,
+): Promise<ServerHealthResult> => {
+	if (IS_CLOUD && !serverId) {
+		throw new TRPCError({
+			code: "BAD_REQUEST",
+			message: "Server is required",
+		});
+	}
+
+	const hours = Math.max(
+		1,
+		Math.min(168, Math.floor(Number(sinceHours) || 24)),
+	);
+	const script = buildHealthScript(hours);
+
+	let stdout = "";
+	try {
+		const result = serverId
+			? await execAsyncRemote(serverId, script)
+			: await execAsync(script);
+		stdout = result.stdout;
+	} catch (error) {
+		return emptyResult(error);
+	}
+
+	let parsed: RawHealthOutput;
+	try {
+		parsed = JSON.parse(stdout.trim());
+	} catch {
+		return emptyResult(new Error("Could not parse server health output"));
+	}
+
+	const daemonErrors = b64Decode(parsed.daemonErrorsBase64)
+		.split("\n")
+		.map((line) => line.trim())
+		.filter(Boolean);
+
+	let addressPools: unknown = null;
+	const daemonConfigText = b64Decode(parsed.daemonConfigBase64);
+	if (daemonConfigText) {
+		try {
+			addressPools =
+				JSON.parse(daemonConfigText)?.["default-address-pools"] ?? null;
+		} catch {
+			addressPools = null;
+		}
+	}
+
+	// Not part of the shell script above: needs structured per-network data from the Docker API.
+	const [reservation, networkUsageResult] = await Promise.all([
+		getReservationSummary(orgId, serverId).catch(() => null),
+		getNetworksIpUsage(serverId).then(
+			(usage) => ({ usage, error: undefined }),
+			(error: unknown) => ({
+				usage: [] as NetworkIpUsage[],
+				error:
+					error instanceof Error
+						? error.message
+						: "Failed to read network usage",
+			}),
+		),
+	]);
+
+	return {
+		checkedAt: new Date().toISOString(),
+		containers: {
+			containerCount: toInt(parsed.containerCount),
+			serviceCount: toInt(parsed.serviceCount),
+		},
+		resources: {
+			memTotalBytes: toInt(parsed.memTotalBytes),
+			memUsedBytes: toInt(parsed.memUsedBytes),
+			cpuCount: toInt(parsed.cpuCount),
+		},
+		inotify: {
+			maxWatches: toInt(parsed.inotifyMaxWatches),
+			maxInstances: toInt(parsed.inotifyMaxInstances),
+			maxQueuedEvents: toInt(parsed.inotifyMaxQueuedEvents),
+			currentInstances: toInt(parsed.inotifyCurrentInstances),
+			persisted: toInt(parsed.inotifyPersistedCount) > 0,
+		},
+		disk: {
+			totalBytes: toInt(parsed.diskTotalBytes),
+			usedBytes: toInt(parsed.diskUsedBytes),
+		},
+		dockerNetworks: {
+			count: toInt(parsed.networkCount),
+			addressPools,
+			usage: networkUsageResult.usage,
+			usageError: networkUsageResult.error,
+		},
+		daemonErrors,
+		daemonLogsWindow:
+			toInt(parsed.daemonLogsToEpoch) > 0
+				? {
+						fromEpoch: toInt(parsed.daemonLogsFromEpoch),
+						toEpoch: toInt(parsed.daemonLogsToEpoch),
+					}
+				: null,
+		reservation,
+	};
+};