Add a retry feature to the backup and clean commands

config/backup.php

@@ -139,6 +139,17 @@
          * available on your system.
          */
         'encryption' => 'default',
+
+        /**
+         * The number of attempts, in case the backup command encounters an exception
+         */
+        'tries' => 1,
+
+        /**
+         * The number of seconds to wait before attempting a new backup if the previous try failed
+         * Set to `0` for none
+         */
+        'retry_delay' => 0,
     ],
 
     /*
@@ -275,6 +286,17 @@
              */
             'delete_oldest_backups_when_using_more_megabytes_than' => 5000,
         ],
+
+        /**
+         * The number of attempts, in case the cleanup command encounters an exception
+         */
+        'tries' => 1,
+
+        /**
+         * The number of seconds to wait before attempting a new cleanup if the previous try failed
+         * Set to `0` for none
+         */
+        'retry_delay' => 0,
     ],
 
 ];

src/Commands/BackupCommand.php

@@ -3,19 +3,22 @@
 namespace Spatie\Backup\Commands;
 
 use Exception;
+use Spatie\Backup\Traits\Retryable;
 use Spatie\Backup\Events\BackupHasFailed;
 use Spatie\Backup\Exceptions\InvalidCommand;
 use Spatie\Backup\Tasks\Backup\BackupJobFactory;
 
 class BackupCommand extends BaseCommand
 {
-    protected $signature = 'backup:run {--filename=} {--only-db} {--db-name=*} {--only-files} {--only-to-disk=} {--disable-notifications} {--timeout=}';
+    use Retryable;
+
+    protected $signature = 'backup:run {--filename=} {--only-db} {--db-name=*} {--only-files} {--only-to-disk=} {--disable-notifications} {--timeout=} {--tries=}';
 
     protected $description = 'Run the backup.';
 
     public function handle()
     {
-        consoleOutput()->comment('Starting backup...');
+        consoleOutput()->comment($this->currentTry > 1 ? sprintf('Attempt n°%d...', $this->currentTry) : 'Starting backup...');
 
         $disableNotifications = $this->option('disable-notifications');
 
@@ -47,6 +50,8 @@ public function handle()
                 $backupJob->setFilename($this->option('filename'));
             }
 
+            $this->setTries('backup');
+
             if ($disableNotifications) {
                 $backupJob->disableNotifications();
             }
@@ -59,6 +64,15 @@ public function handle()
 
             consoleOutput()->comment('Backup completed!');
         } catch (Exception $exception) {
+            if ($this->shouldRetry()) {
+                if ($this->hasRetryDelay('backup')) {
+                    $this->sleepFor($this->getRetryDelay('backup'));
+                }
+
+                $this->currentTry += 1;
+                return $this->handle();
+            }
+
             consoleOutput()->error("Backup failed because: {$exception->getMessage()}.");
 
             report($exception);

src/Commands/CleanupCommand.php

@@ -3,15 +3,18 @@
 namespace Spatie\Backup\Commands;
 
 use Exception;
-use Spatie\Backup\BackupDestination\BackupDestinationFactory;
+use Spatie\Backup\Traits\Retryable;
 use Spatie\Backup\Events\CleanupHasFailed;
 use Spatie\Backup\Tasks\Cleanup\CleanupJob;
 use Spatie\Backup\Tasks\Cleanup\CleanupStrategy;
+use Spatie\Backup\BackupDestination\BackupDestinationFactory;
 
 class CleanupCommand extends BaseCommand
 {
+    use Retryable;
+
     /** @var string */
-    protected $signature = 'backup:clean {--disable-notifications}';
+    protected $signature = 'backup:clean {--disable-notifications} {--tries=}';
 
     /** @var string */
     protected $description = 'Remove all backups older than specified number of days in config.';
@@ -27,10 +30,12 @@ public function __construct(CleanupStrategy $strategy)
 
     public function handle()
     {
-        consoleOutput()->comment('Starting cleanup...');
+        consoleOutput()->comment($this->currentTry > 1 ? sprintf('Attempt n°%d...', $this->currentTry) : 'Starting cleanup...');
 
         $disableNotifications = $this->option('disable-notifications');
 
+        $this->setTries('cleanup');
+
         try {
             $config = config('backup');
 
@@ -42,6 +47,15 @@ public function handle()
 
             consoleOutput()->comment('Cleanup completed!');
         } catch (Exception $exception) {
+            if ($this->shouldRetry()) {
+                if ($this->hasRetryDelay('cleanup')) {
+                    $this->sleepFor($this->getRetryDelay('cleanup'));
+                }
+
+                $this->currentTry += 1;
+                return $this->handle();
+            }
+
             if (! $disableNotifications) {
                 event(new CleanupHasFailed($exception));
             }

src/Helpers/functions.php

@@ -6,3 +6,8 @@ function consoleOutput(): ConsoleOutput
 {
     return app(ConsoleOutput::class);
 }
+
+function isSleepHelperAvailable()
+{
+    return class_exists('Illuminate\Support\Sleep');
+}

src/Traits/Retryable.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Spatie\Backup\Traits;
+
+use Illuminate\Support\Sleep;
+
+trait Retryable
+{
+    protected int $tries = 1;
+
+    protected int $currentTry = 1;
+
+    protected function shouldRetry()
+    {
+        if ($this->tries <= 1) {
+            return false;
+        }
+
+        return $this->currentTry < $this->tries;
+    }
+
+    protected function hasRetryDelay(string $type)
+    {
+        return !empty($this->getRetryDelay($type));
+    }
+
+    protected function sleepFor(int $seconds = 0)
+    {
+        if (isSleepHelperAvailable()) {
+            Sleep::for($seconds)->seconds();
+            return;
+        }
+
+        sleep($seconds);
+    }
+
+    protected function setTries(string $type)
+    {
+        if ($this->option('tries')) {
+            $this->tries = (int)$this->option('tries');
+            return;
+        }
+
+        $this->tries = (int)config('backup.' . $type . '.tries', 1);
+    }
+
+    protected function getRetryDelay(string $type)
+    {
+        return (int)config('backup.' . $type . '.retry_delay', 0);
+    }
+}

tests/Commands/BackupCommandTest.php

@@ -1,8 +1,10 @@
 <?php
 
 use Carbon\Carbon;
-use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Sleep;
 use Illuminate\Support\Facades\Event;
+use Carbon\CarbonInterval as Duration;
+use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Storage;
 use Spatie\Backup\Events\BackupHasFailed;
 use Spatie\Backup\Events\BackupZipWasCreated;
@@ -354,3 +356,49 @@
 
     $this->assertFileDoesntExistsInZip('local', $this->expectedZipPath, '2016-01-01-20-01-01.zip');
 });
+
+it('should try again after encountering an exception when using the tries argument', function () {
+    // Use an invalid dbname to trigger failure
+    $exitCode = Artisan::call('backup:run --only-db --db-name=wrongName --tries=3');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+});
+
+it('should try again after encountering an exception when using the tries configuration option', function () {
+    config()->set('backup.backup.tries', 3);
+
+    // Use an invalid dbname to trigger failure
+    $exitCode = Artisan::call('backup:run --only-db --db-name=wrongName');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+});
+
+it('should wait before trying again when retry_delay is configured (with Sleep helper)', function () {
+    Sleep::fake();
+
+    config()->set('backup.backup.tries', 3);
+    config()->set('backup.backup.retry_delay', 3);
+
+    // Use an invalid dbname to trigger failure
+    $exitCode = Artisan::call('backup:run --only-db --db-name=wrongName');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+
+    Sleep::assertSleptTimes(2);
+    Sleep::assertSequence([
+        Sleep::for(3)->seconds(),
+        Sleep::for(3)->seconds(),
+    ]);
+})->skip(!isSleepHelperAvailable(), 'requires the Sleep helper (Laravel >= 10)');

tests/Commands/CleanupCommandTest.php

@@ -1,8 +1,9 @@
 <?php
 
+use Illuminate\Support\Sleep;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Storage;
 use Spatie\Backup\Events\CleanupWasSuccessful;
 
@@ -151,3 +152,54 @@
 
     $this->seeInConsoleOutput('after cleanup: 4 MB.');
 });
+
+it('should try again after encountering an exception when using the tries argument', function () {
+    // Use an invalid destination disk to trigger an exception
+    config()->set('backup.backup.destination.disks', ['wrong']);
+
+    $exitCode = Artisan::call('backup:clean --tries=3');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+});
+
+it('should try again after encountering an exception when using the tries configuration option', function () {
+    config()->set('backup.cleanup.tries', 3);
+    // Use an invalid destination disk to trigger an exception
+    config()->set('backup.backup.destination.disks', ['wrong']);
+
+    $exitCode = Artisan::call('backup:clean');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+});
+
+it('should wait before trying again when retry_delay is configured (with Sleep helper)', function () {
+    Sleep::fake();
+
+    // Use an invalid destination disk to trigger an exception
+    config()->set('backup.backup.destination.disks', ['wrong']);
+
+    config()->set('backup.cleanup.tries', 3);
+    config()->set('backup.cleanup.retry_delay', 3);
+
+    $exitCode = Artisan::call('backup:clean');
+    $output = Artisan::output();
+
+    expect($exitCode)->toEqual(1);
+
+    $this->assertStringContainsString('Attempt n°2...', $output);
+    $this->assertStringContainsString('Attempt n°3...', $output);
+
+    Sleep::assertSleptTimes(2);
+    Sleep::assertSequence([
+        Sleep::for(3)->seconds(),
+        Sleep::for(3)->seconds(),
+    ]);
+})->skip(!isSleepHelperAvailable(), 'requires the Sleep helper (Laravel >= 10)');