Merge pull request #5082 from bestmaa/fix/volume-backup-restart

fix(volume-backups): restart services after backup failure

(cherry picked from commit 3054cf53be)

[skip ci]

apps/dokploy/__test__/backups/volume-backup-restart.test.ts

@@ -0,0 +1,82 @@
+import { spawnSync } from "node:child_process";
+import { createRestartSafeBackupCommand } from "@dokploy/server/utils/volume-backups/backup";
+import { describe, expect, it } from "vitest";
+
+const runCommand = (command: string) =>
+	spawnSync("bash", ["-c", command], {
+		encoding: "utf8",
+	});
+
+const outputLines = (stdout: string) =>
+	stdout
+		.trim()
+		.split("\n")
+		.filter((line) => line.length > 0);
+
+describe("createRestartSafeBackupCommand", () => {
+	it("restarts the service and preserves the backup error status", () => {
+		const result = runCommand(
+			createRestartSafeBackupCommand({
+				stopCommand: 'echo "stop"',
+				backupCommand: 'echo "backup"; exit 23',
+				startCommand: 'echo "start"',
+				uploadCommand: 'echo "upload"',
+			}),
+		);
+
+		expect(result.status).toBe(23);
+		expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]);
+	});
+
+	it("preserves the backup error status when the restart also fails", () => {
+		const result = runCommand(
+			createRestartSafeBackupCommand({
+				stopCommand: 'echo "stop"',
+				backupCommand: 'echo "backup"; exit 23',
+				startCommand: 'echo "start"; exit 17',
+				uploadCommand: 'echo "upload"',
+			}),
+		);
+
+		expect(result.status).toBe(23);
+		expect(outputLines(result.stdout)).toEqual([
+			"stop",
+			"backup",
+			"start",
+			"Service restart also failed with exit code 17",
+		]);
+	});
+
+	it("returns the restart error status when the backup succeeds", () => {
+		const result = runCommand(
+			createRestartSafeBackupCommand({
+				stopCommand: 'echo "stop"',
+				backupCommand: 'echo "backup"',
+				startCommand: 'echo "start"; exit 17',
+				uploadCommand: 'echo "upload"',
+			}),
+		);
+
+		expect(result.status).toBe(17);
+		expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]);
+	});
+
+	it("uploads only after a successful backup and service restart", () => {
+		const result = runCommand(
+			createRestartSafeBackupCommand({
+				stopCommand: 'echo "stop"',
+				backupCommand: 'echo "backup"',
+				startCommand: 'echo "start"',
+				uploadCommand: 'echo "upload"',
+			}),
+		);
+
+		expect(result.status).toBe(0);
+		expect(outputLines(result.stdout)).toEqual([
+			"stop",
+			"backup",
+			"start",
+			"upload",
+		]);
+	});
+});

packages/server/src/utils/volume-backups/backup.ts

@@ -9,6 +9,43 @@ import {
 	normalizeS3Path,
 } from "../backups/utils";
 
+interface RestartSafeBackupCommandOptions {
+	stopCommand: string;
+	backupCommand: string;
+	startCommand: string;
+	uploadCommand: string;
+}
+
+export const createRestartSafeBackupCommand = ({
+	stopCommand,
+	backupCommand,
+	startCommand,
+	uploadCommand,
+}: RestartSafeBackupCommandOptions) => `
+	${stopCommand}
+	set +e
+	(
+		${backupCommand}
+	)
+	DOKPLOY_VOLUME_BACKUP_STATUS=$?
+	(
+		set -e
+		${startCommand}
+	)
+	DOKPLOY_VOLUME_RESTART_STATUS=$?
+	set -e
+	if [ "$DOKPLOY_VOLUME_BACKUP_STATUS" -ne 0 ]; then
+		if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then
+			echo "Service restart also failed with exit code $DOKPLOY_VOLUME_RESTART_STATUS"
+		fi
+		exit "$DOKPLOY_VOLUME_BACKUP_STATUS"
+	fi
+	if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then
+		exit "$DOKPLOY_VOLUME_RESTART_STATUS"
+	fi
+	${uploadCommand}
+`;
+
 export const getVolumeServiceAppName = (
 	volumeBackup: Awaited<ReturnType<typeof findVolumeBackupById>>,
 ): string => {
@@ -117,16 +154,20 @@ export const backupVolume = async (
 	);
 
 	if (serviceType === "application") {
-		return lockWrapper(`
+		return lockWrapper(
-		echo "Stopping application to 0 replicas"
+			createRestartSafeBackupCommand({
-		ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
+				stopCommand: `
-		echo "Actual replicas: $ACTUAL_REPLICAS"
+				echo "Stopping application to 0 replicas"
-		docker service update --replicas=0 ${volumeBackup.application?.appName}
+				ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
-        ${backupCommand}
+				echo "Actual replicas: $ACTUAL_REPLICAS"
-		echo "Starting application to $ACTUAL_REPLICAS replicas"
+				docker service update --replicas=0 ${volumeBackup.application?.appName}`,
-        docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}
+				backupCommand,
-		${uploadCommand}
+				startCommand: `
-  `);
+				echo "Starting application to $ACTUAL_REPLICAS replicas"
+				docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}`,
+				uploadCommand,
+			}),
+		);
 	}
 	if (serviceType === "compose") {
 		const compose = await findComposeById(
@@ -158,11 +199,13 @@ export const backupVolume = async (
 			echo "Compose container started"
 			`;
 		}
-		return lockWrapper(`
+		return lockWrapper(
-        ${stopCommand}
+			createRestartSafeBackupCommand({
-        ${backupCommand}
+				stopCommand,
-        ${startCommand}
+				backupCommand,
-		${uploadCommand}
+				startCommand,
-  `);
+				uploadCommand,
+			}),
+		);
 	}
 };