Merge pull request #3637 from Dokploy/3392-conflict-between-daily-docker-cleanup-and-volume-backup-turn-off-container-during-backup

3392 conflict between daily docker cleanup and volume backup turn off container during backup

apps/dokploy/server/api/routers/settings.ts

@@ -1,4 +1,5 @@
 import {
+	CLEANUP_CRON_JOB,
 	canAccessToTraefikFiles,
 	checkGPUStatus,
 	checkPortInUse,
@@ -298,12 +299,12 @@ export const settingsRouter = createTRPCRouter({
 					}
 					if (IS_CLOUD) {
 						await schedule({
-							cronSchedule: "0 0 * * *",
+							cronSchedule: CLEANUP_CRON_JOB,
 							serverId: input.serverId,
 							type: "server",
 						});
 					} else {
-						scheduleJob(server.serverId, "0 0 * * *", async () => {
+						scheduleJob(server.serverId, CLEANUP_CRON_JOB, async () => {
 							console.log(
 								`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
 							);
@@ -316,7 +317,7 @@ export const settingsRouter = createTRPCRouter({
 				} else {
 					if (IS_CLOUD) {
 						await removeJob({
-							cronSchedule: "0 0 * * *",
+							cronSchedule: CLEANUP_CRON_JOB,
 							serverId: input.serverId,
 							type: "server",
 						});
@@ -331,7 +332,7 @@ export const settingsRouter = createTRPCRouter({
 				});
 
 				if (settingsUpdated?.enableDockerCleanup) {
-					scheduleJob("docker-cleanup", "0 0 * * *", async () => {
+					scheduleJob("docker-cleanup", CLEANUP_CRON_JOB, async () => {
 						console.log(
 							`Docker Cleanup ${new Date().toLocaleString()}] Running...`,
 						);

apps/dokploy/server/api/routers/volume-backups.ts

@@ -21,7 +21,7 @@ import {
 } from "@dokploy/server/utils/process/execAsync";
 import { TRPCError } from "@trpc/server";
 import { observable } from "@trpc/server/observable";
-import { eq } from "drizzle-orm";
+import { desc, eq } from "drizzle-orm";
 import { z } from "zod";
 import { removeJob, schedule, updateJob } from "@/server/utils/backup";
 import { createTRPCRouter, protectedProcedure } from "../trpc";
@@ -54,6 +54,7 @@ export const volumeBackupsRouter = createTRPCRouter({
 					redis: true,
 					compose: true,
 				},
+				orderBy: [desc(volumeBackups.createdAt)],
 			});
 		}),
 	create: protectedProcedure

apps/schedules/src/utils.ts

@@ -1,4 +1,5 @@
 import {
+	CLEANUP_CRON_JOB,
 	cleanupAll,
 	findBackupById,
 	findScheduleById,
@@ -125,7 +126,7 @@ export const initializeJobs = async () => {
 		scheduleJob({
 			serverId,
 			type: "server",
-			cronSchedule: "0 0 * * *",
+			cronSchedule: CLEANUP_CRON_JOB,
 		});
 	}
 

packages/server/src/constants/index.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import Docker from "dockerode";
 
 export const IS_CLOUD = process.env.IS_CLOUD === "true";
+export const CLEANUP_CRON_JOB = "50 23 * * *";
 export const docker = new Docker();
 
 export const BETTER_AUTH_SECRET =
@@ -29,5 +30,6 @@ export const paths = (isServer = false) => {
 		REGISTRY_PATH: `${BASE_PATH}/registry`,
 		SCHEDULES_PATH: `${BASE_PATH}/schedules`,
 		VOLUME_BACKUPS_PATH: `${BASE_PATH}/volume-backups`,
+		VOLUME_BACKUP_LOCK_PATH: `${BASE_PATH}/volume-backup-lock`,
 	};
 };

packages/server/src/utils/backups/index.ts

@@ -1,4 +1,5 @@
 import path from "node:path";
+import { CLEANUP_CRON_JOB } from "@dokploy/server/constants";
 import { member } from "@dokploy/server/db/schema";
 import type { BackupSchedule } from "@dokploy/server/services/backup";
 import { getAllServers } from "@dokploy/server/services/server";
@@ -29,7 +30,7 @@ export const initCronJobs = async () => {
 	const webServerSettings = await getWebServerSettings();
 
 	if (webServerSettings?.enableDockerCleanup) {
-		scheduleJob("docker-cleanup", "0 0 * * *", async () => {
+		scheduleJob("docker-cleanup", CLEANUP_CRON_JOB, async () => {
 			console.log(
 				`Docker Cleanup ${new Date().toLocaleString()}]  Running docker cleanup`,
 			);
@@ -45,7 +46,7 @@ export const initCronJobs = async () => {
 	for (const server of servers) {
 		const { serverId, enableDockerCleanup, name } = server;
 		if (enableDockerCleanup) {
-			scheduleJob(serverId, "0 0 * * *", async () => {
+			scheduleJob(serverId, CLEANUP_CRON_JOB, async () => {
 				console.log(
 					`SERVER-BACKUP[${new Date().toLocaleString()}] Running Cleanup ${name}`,
 				);

packages/server/src/utils/volume-backups/backup.ts

@@ -10,7 +10,7 @@ export const backupVolume = async (
 	const { serviceType, volumeName, turnOff, prefix } = volumeBackup;
 	const serverId =
 		volumeBackup.application?.serverId || volumeBackup.compose?.serverId;
-	const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
+	const { VOLUME_BACKUPS_PATH, VOLUME_BACKUP_LOCK_PATH } = paths(!!serverId);
 	const destination = volumeBackup.destination;
 	const backupFileName = `${volumeName}-${new Date().toISOString()}.tar`;
 	const bucketDestination = `${normalizeS3Path(prefix)}${backupFileName}`;
@@ -45,8 +45,48 @@ export const backupVolume = async (
 		return baseCommand;
 	}
 
+	const serviceLockId =
+		serviceType === "application"
+			? volumeBackup.application?.appName
+			: `${volumeBackup.compose?.appName}_${volumeBackup.serviceName}`;
+
+	const lockPath = `${VOLUME_BACKUP_LOCK_PATH}-${serviceLockId}`;
+
+	const lockWrapper = (body: string) => `
+		set -e
+
+		LOCK_PATH="${lockPath}"
+
+		echo "Waiting for volume backup lock: $LOCK_PATH"
+
+		if command -v flock >/dev/null 2>&1; then
+			exec 9>"$LOCK_PATH"
+			flock 9
+		else
+			LOCK_DIR="$LOCK_PATH.dir"
+			while ! mkdir "$LOCK_DIR" 2>/dev/null; do
+				echo "Waiting for volume backup lock: $LOCK_PATH"
+				sleep 5
+			done
+			trap 'rm -rf "$LOCK_DIR"' EXIT
+		fi
+
+		echo "Volume backup lock acquired"
+
+		${body}
+
+		echo "Volume backup lock released"
+	`;
+
+	console.log(
+		lockWrapper(`
+		echo "Volume backup lock acquired"
+		echo "Volume backup lock released"
+	`),
+	);
+
 	if (serviceType === "application") {
-		return `
+		return lockWrapper(`
 		echo "Stopping application to 0 replicas"
 		ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}")
 		echo "Actual replicas: $ACTUAL_REPLICAS"
@@ -54,7 +94,7 @@ export const backupVolume = async (
         ${baseCommand}
 		echo "Starting application to $ACTUAL_REPLICAS replicas"
         docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}
-  `;
+  `);
 	}
 	if (serviceType === "compose") {
 		const compose = await findComposeById(
@@ -70,6 +110,7 @@ export const backupVolume = async (
             ACTUAL_REPLICAS=$(docker service inspect ${compose.appName}_${volumeBackup.serviceName} --format "{{.Spec.Mode.Replicated.Replicas}}")
             echo "Actual replicas: $ACTUAL_REPLICAS"
             docker service update --replicas=0 ${compose.appName}_${volumeBackup.serviceName}`;
+
 			startCommand = `
 			echo "Starting compose to $ACTUAL_REPLICAS replicas"
 			docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${compose.appName}_${volumeBackup.serviceName}`;
@@ -78,16 +119,17 @@ export const backupVolume = async (
 			echo "Stopping compose container"
             ID=$(docker ps -q --filter "label=com.docker.compose.project=${compose.appName}" --filter "label=com.docker.compose.service=${volumeBackup.serviceName}")
             docker stop $ID`;
+
 			startCommand = `
             echo "Starting compose container"
             docker start $ID
 			echo "Compose container started"
 			`;
 		}
-		return `
+		return lockWrapper(`
         ${stopCommand}
         ${baseCommand}
         ${startCommand}
-  `;
+  `);
 	}
 };