[LOPS-2314] Adds retry option to Drush and WP-CLI commands

src/Commands/Remote/DrushCommand.php

@@ -26,15 +26,18 @@ class DrushCommand extends SSHBaseCommand
      * @param array $drush_command Drush command
      * @param array $options Commandline options
      * @option progress Allow progress bar to be used (tty mode only)
+     * @option int $retry Number of retries on failure
      * @return string Command output
      *
      * @usage <site>.<env> -- <command> Runs the Drush command <command> remotely on <site>'s <env> environment.
      * @usage <site>.<env> --progress -- <command> Runs a Drush command with a progress bar
+     * @usage <site>.<env> --retry=3 -- <command> Runs a Drush command with up to 3 retries on failure
      */
-    public function drushCommand($site_env, array $drush_command, array $options = ['progress' => false])
+    public function drushCommand($site_env, array $drush_command, array $options = ['progress' => false, 'retry' => 0])
     {
         $this->prepareEnvironment($site_env);
         $this->setProgressAllowed($options['progress']);
-        return $this->executeCommand($drush_command);
+        $retries = (int)($options['retry'] ?? 0);
+        return $this->executeCommand($drush_command, $retries);
     }
 }

src/Commands/Remote/SSHBaseCommand.php

@@ -70,28 +70,79 @@ protected function setProgressAllowed($allowed = true)
      * Executes the command remotely.
      *
      * @param array $command_args
+     * @param int $retries Number of times to retry the command on failure.
      *
      * @throws \Pantheon\Terminus\Exceptions\TerminusProcessException
      */
-    protected function executeCommand(array $command_args)
+    protected function executeCommand(array $command_args, int $retries = 0)
     {
         $this->validateEnvironment($this->environment);
 
         $command_summary = $this->getCommandSummary($command_args);
         $command_line = $this->getCommandLine($command_args);
 
-        $ssh_data = $this->sendCommandViaSsh($command_line);
+        $attempt = 0;
+        $max_attempts = $retries + 1;
 
-        $this->log()->notice('Command: {site}.{env} -- {command} [Exit: {exit}]', [
-            'site'    => $this->site->getName(),
-            'env'     => $this->environment->id,
-            'command' => $command_summary,
-            'exit'    => $ssh_data['exit_code'],
-        ]);
+        do {
+            $ssh_data = $this->sendCommandViaSsh($command_line);
 
-        if ($ssh_data['exit_code'] != 0) {
-            throw new TerminusProcessException($ssh_data['output'], [], $ssh_data['exit_code']);
-        }
+            $this->log()->notice(
+                'Command: {site}.{env} -- {command} [Exit: {exit}] (Attempt {attempt}/{max_attempts})',
+                [
+                    'site' => $this->site->getName(),
+                    'env' => $this->environment->id,
+                    'command' => $command_summary,
+                    'exit' => $ssh_data['exit_code'],
+                    'attempt' => $attempt + 1,
+                    'max_attempts' => $max_attempts,
+                ]
+            );
+
+            if ($ssh_data['exit_code'] == 0) {
+                return;
+            }
+
+            // Check if the failure is permanent
+            if ($this->isPermanentFailure($ssh_data['exit_code'])) {
+                $this->log()->error('Permanent failure detected. Aborting retries. Exit code: {exit_code}', [
+                    'exit_code' => $ssh_data['exit_code']
+                ]);
+                break;
+            }
+
+            $attempt++;
+        } while ($attempt < $max_attempts);
+
+        $error_message = sprintf(
+            'Command: %s.%s -- %s [Exit: %d] (All attempts failed)',
+            $this->site->getName(),
+            $this->environment->id,
+            $command_summary,
+            $ssh_data['exit_code']
+        );
+
+        $this->log()->error($error_message);
+
+        throw new TerminusProcessException($ssh_data['output'], [], $ssh_data['exit_code']);
+    }
+
+    /**
+     * Determines if a failure is permanent based on the exit code.
+     *
+     * @param int $exit_code
+     * @return bool
+     */
+    protected function isPermanentFailure(int $exit_code): bool
+    {
+        // Define the exit codes that indicate a permanent failure
+        $permanent_failure_exit_codes = [
+            2,   // Invalid arguments
+            126, // Command cannot execute (permission denied)
+            127, // Command not found
+        ];
+
+        return in_array($exit_code, $permanent_failure_exit_codes, true);
     }
 
     /**

src/Commands/Remote/WPCommand.php

@@ -26,15 +26,18 @@ class WPCommand extends SSHBaseCommand
      * @param array $wp_command WP-CLI command
      * @param array $options Commandline options
      * @option progress Allow progress bar to be used (tty mode only)
+     * @option int $retry Number of retries on failure
      * @return string Command output
      *
      * @usage <site>.<env> -- <command> Runs the WP-CLI command <command> remotely on <site>'s <env> environment.
      * @usage <site>.<env> --progress -- <command> Runs a WP-CLI command with a progress bar
+     * @usage <site>.<env> --retry=3 -- <command> Runs a WP-CLI command with up to 3 retries on failure
      */
-    public function wpCommand($site_env, array $wp_command, array $options = ['progress' => false])
+    public function wpCommand($site_env, array $wp_command, array $options = ['progress' => false, 'retry' => 0])
     {
         $this->prepareEnvironment($site_env);
         $this->setProgressAllowed($options['progress']);
-        return $this->executeCommand($wp_command);
+        $retries = (int)($options['retry'] ?? 0);
+        return $this->executeCommand($wp_command, $retries);
     }
 }

tests/Functional/RemoteCommandsTest.php

@@ -38,19 +38,22 @@ public function testDrushCommands()
     {
         $commandPrefix = sprintf('drush %s', $this->getSiteEnv());
 
-        $drushVersionCommand = sprintf('%s -- %s', $commandPrefix, 'version');
+        // Test Drush version command with retry
+        $drushVersionCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'version');
         $drushVersion = $this->terminusJsonResponse($drushVersionCommand);
         $this->assertIsString($drushVersion);
         $this->assertIsInt(preg_match('(^\d{1,2})', $drushVersion, $matches));
         $this->assertGreaterThanOrEqual(8, $matches[0]);
 
-        $drushStatusCommand = sprintf('%s -- %s', $commandPrefix, 'status');
+        // Test Drush status command with retry
+        $drushStatusCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'status');
         $drushStatus = $this->terminusJsonResponse($drushStatusCommand);
         $this->assertIsArray($drushStatus);
         $this->assertTrue(isset($drushStatus['drush-version']));
         $this->assertEquals($drushStatus['drush-version'], $drushVersion);
 
-        $drushSqlCliCommand = sprintf('%s -- %s', $commandPrefix, 'sql:cli');
+        // Test Drush sql:cli command with retry
+        $drushSqlCliCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'sql:cli');
         $drushSqlCliResult = $this->terminusPipeInput(
             $drushSqlCliCommand,
             'echo "select uuid from users where uid=1;"'
@@ -62,4 +65,43 @@ public function testDrushCommands()
             'The "drush sql:cli" execution result should contain a valid v4 UUID.'
         );
     }
+
+    /**
+     * @test
+     * @covers \Pantheon\Terminus\Commands\Remote\WPCommand
+     *
+     * @group remote
+     * @group short
+     */
+    public function testWPCommands()
+    {
+        $commandPrefix = sprintf('wp %s', $this->getSiteEnv());
+
+        // Test WP-CLI core version command with retry
+        $wpVersionCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'core version');
+        $wpVersion = $this->terminusJsonResponse($wpVersionCommand);
+        $this->assertIsString($wpVersion);
+        $this->assertIsInt(preg_match('(^\d{1,2})', $wpVersion, $matches));
+        $this->assertGreaterThanOrEqual(5, $matches[0]);
+
+        // Test WP-CLI core info command with retry
+        $wpInfoCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'core info');
+        $wpInfo = $this->terminusJsonResponse($wpInfoCommand);
+        $this->assertIsArray($wpInfo);
+        $this->assertTrue(isset($wpInfo['version']));
+        $this->assertEquals($wpInfo['version'], $wpVersion);
+
+        // Test WP-CLI db query command with retry
+        $wpDbQueryCommand = sprintf('%s --retry=3 -- %s', $commandPrefix, 'db query');
+        $wpDbQueryResult = $this->terminusPipeInput(
+            $wpDbQueryCommand,
+            'echo "select ID from wp_users where ID=1;"'
+        );
+
+        $this->assertEquals(
+            1,
+            preg_match('/^\d+$/', $wpDbQueryResult),
+            'The "wp db query" execution result should contain a valid user ID.'
+        );
+    }
 }

tests/unit_tests/Commands/Remote/DrushCommandTest.php

@@ -23,8 +23,14 @@ public function setUp()
             ->setMethods([
                 'prepareEnvironment',
                 'executeCommand',
+                'log',
             ])
             ->getMock();
+
+        $this->logger = $this->getMockBuilder(\Psr\Log\LoggerInterface::class)
+            ->getMock();
+
+        $this->command->method('log')->willReturn($this->logger);
     }
 
     /**
@@ -41,7 +47,46 @@ public function testDrushCommand()
             ->method('executeCommand')
             ->willReturn($command_output);
 
-        $output = $this->command->drushCommand('dummy-site.dummy-env', ['drushable', 'command', 'arguments',]);
+        $output = $this->command->drushCommand('dummy-site.dummy-env', ['drushable', 'command', 'arguments']);
+        $this->assertEquals($command_output, $output);
+    }
+
+    /**
+     * Tests the drush command with retry option
+     */
+    public function testDrushCommandWithRetry()
+    {
+        $command_output = 'command output';
+        $retry_options = ['retry' => 3, 'progress' => false];
+
+        $this->command->expects($this->once())
+            ->method('prepareEnvironment')
+            ->with($this->equalTo('dummy-site.dummy-env'));
+
+        $this->command->expects($this->exactly(3))
+            ->method('executeCommand')
+            ->will($this->onConsecutiveCalls(
+                $this->throwException(
+                    new \Pantheon\Terminus\Exceptions\TerminusProcessException('First attempt failed')
+                ),
+                $this->throwException(
+                    new \Pantheon\Terminus\Exceptions\TerminusProcessException('Second attempt failed')
+                ),
+                $this->returnValue($command_output)
+            ));
+
+        $this->logger->expects($this->exactly(2))
+            ->method('warning')
+            ->withConsecutive(
+                [$this->equalTo('Retry attempt 1 for command failed.')],
+                [$this->equalTo('Retry attempt 2 for command failed.')]
+            );
+
+        $output = $this->command->drushCommand(
+            'dummy-site.dummy-env',
+            ['drushable', 'command', 'arguments'],
+            $retry_options
+        );
         $this->assertEquals($command_output, $output);
     }
 }

tests/unit_tests/Commands/Remote/WPCommandTest.php

@@ -1,6 +1,6 @@
 <?php
 
-namespace Pantheon\Terminus\UnitTests\Commands\Remove;
+namespace Pantheon\Terminus\UnitTests\Commands\Remote;
 
 use Pantheon\Terminus\Commands\Remote\WPCommand;
 use Pantheon\Terminus\UnitTests\Commands\CommandTestCase;
@@ -13,27 +13,76 @@
 class WPCommandTest extends CommandTestCase
 {
     /**
-     * Tests the wp command
+     * @inheritdoc
      */
-    public function testWPCommand()
+    public function setUp()
     {
-        $command = $this->getMockBuilder(WPCommand::class)
+        parent::setUp();
+
+        $this->command = $this->getMockBuilder(WPCommand::class)
             ->disableOriginalConstructor()
             ->setMethods([
                 'prepareEnvironment',
                 'executeCommand',
+                'log',
             ])
             ->getMock();
 
-        $command->expects($this->once())
+        $this->logger = $this->getMockBuilder(\Psr\Log\LoggerInterface::class)
+            ->getMock();
+
+        $this->command->method('log')->willReturn($this->logger);
+    }
+
+    /**
+     * Tests the wp command
+     */
+    public function testWPCommand()
+    {
+        $command_output = 'command output';
+
+        $this->command->expects($this->once())
             ->method('prepareEnvironment')
             ->with($this->equalTo('dummy-site.dummy-env'));
+        $this->command->expects($this->once())
+            ->method('executeCommand')
+            ->willReturn($command_output);
+
+        $output = $this->command->wpCommand('dummy-site.dummy-env', ['wpcli', 'command', 'arguments']);
+        $this->assertEquals($command_output, $output);
+    }
 
-        $command->expects($this->once())
+    /**
+     * Tests the wp command with retry option
+     */
+    public function testWPCommandWithRetry()
+    {
+        $command_output = 'command output';
+        $retry_options = ['retry' => 3, 'progress' => false];
+
+        $this->command->expects($this->once())
+            ->method('prepareEnvironment')
+            ->with($this->equalTo('dummy-site.dummy-env'));
+        $this->command->expects($this->exactly(3))
             ->method('executeCommand')
-            ->willReturn('command output');
+            ->will($this->onConsecutiveCalls(
+                $this->throwException(
+                    new \Pantheon\Terminus\Exceptions\TerminusProcessException('First attempt failed')
+                ),
+                $this->throwException(
+                    new \Pantheon\Terminus\Exceptions\TerminusProcessException('Second attempt failed')
+                ),
+                $this->returnValue($command_output)
+            ));
+
+        $this->logger->expects($this->exactly(2))
+            ->method('warning')
+            ->withConsecutive(
+                [$this->equalTo('Retry attempt 1 for command failed.')],
+                [$this->equalTo('Retry attempt 2 for command failed.')]
+            );
 
-        $output = $command->wpCommand('dummy-site.dummy-env', ['wpcli', 'command', 'arguments']);
-        $this->assertEquals('command output', $output);
+        $output = $this->command->wpCommand('dummy-site.dummy-env', ['wpcli', 'command', 'arguments'], $retry_options);
+        $this->assertEquals($command_output, $output);
     }
 }