New option '--retry-total' as a circuit breaker for option '--retry'

CHANGELOG.md

@@ -10,6 +10,7 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo
 
 ## [Unreleased]
 ### Added
+* Add option `--retry-total` ([PR#](https://github.com/cucumber/cucumber-ruby/pull/1669))
 
 ### Changed
 

features/docs/cli/retry_failing_tests.feature

@@ -95,3 +95,31 @@ Feature: Retry failing tests
 
       3 scenarios (2 flaky, 1 passed)
       """
+
+  Scenario: Retry each test but suspend retrying after 2 failing tests, so later tests are not retried
+    Given a scenario "Fails-forever-1" that fails
+    And a scenario "Fails-forever-2" that fails
+    When I run `cucumber -q --retry 1 --retry-total 2 --format summary`
+    Then it should fail with:
+      """
+      5 scenarios (4 failed, 1 passed)
+      """
+    And it should fail with:
+      """
+      Fails-forever-1
+        Fails-forever-1 ✗
+        Fails-forever-1 ✗
+
+      Fails-forever-2
+        Fails-forever-2 ✗
+        Fails-forever-2 ✗
+
+      Fails-once feature
+        Fails-once ✗
+
+      Fails-twice feature
+        Fails-twice ✗
+
+      Solid
+        Solid ✓
+      """

features/lib/step_definitions/scenarios_and_steps.rb

@@ -30,26 +30,26 @@ def snake_case(name)
 Given('a scenario {string} that passes') do |name|
   create_feature(name) do
     create_scenario(name) do
-      '  Given it passes'
+      "  Given it passes in #{name}"
     end
   end
 
   write_file(
     "features/step_definitions/#{name}_steps.rb",
-    step_definition('/^it passes$/', 'expect(true).to be true')
+    step_definition("/^it passes in #{name}$/", 'expect(true).to be true')
   )
 end
 
 Given('a scenario {string} that fails') do |name|
   create_feature(name) do
     create_scenario(name) do
-      '  Given it fails'
+      "  Given it fails in #{name}"
     end
   end
 
   write_file(
     "features/step_definitions/#{name}_steps.rb",
-    step_definition('/^it fails$/', 'expect(false).to be true')
+    step_definition("/^it fails in #{name}$/", 'expect(false).to be true')
   )
 end
 
@@ -58,14 +58,14 @@ def snake_case(name)
 
   create_feature("#{full_name} feature") do
     create_scenario(full_name) do
-      '  Given it fails once, then passes'
+      "  Given it fails once, then passes in #{full_name}"
     end
   end
 
   write_file(
     "features/step_definitions/#{name}_steps.rb",
     step_definition(
-      '/^it fails once, then passes$/',
+      "/^it fails once, then passes in #{full_name}$/",
       [
         "$#{name} += 1",
         "expect($#{name}).to be > 1"
@@ -84,14 +84,14 @@ def snake_case(name)
 
   create_feature("#{full_name} feature") do
     create_scenario(full_name) do
-      '  Given it fails twice, then passes'
+      "  Given it fails twice, then passes in #{full_name}"
     end
   end
 
   write_file(
     "features/step_definitions/#{name}_steps.rb",
     step_definition(
-      '/^it fails twice, then passes$/',
+      "/^it fails twice, then passes in #{full_name}$/",
       [
         "$#{name} ||= 0",
         "$#{name} += 1",

lib/cucumber/cli/options.rb

@@ -56,11 +56,12 @@ class Options
       NO_PROFILE_LONG_FLAG = '--no-profile'.freeze
       FAIL_FAST_FLAG = '--fail-fast'.freeze
       RETRY_FLAG = '--retry'.freeze
+      RETRY_TOTAL_FLAG = '--retry-total'.freeze
       OPTIONS_WITH_ARGS = [
         '-r', '--require', '--i18n-keywords', '-f', '--format', '-o',
         '--out', '-t', '--tags', '-n', '--name', '-e', '--exclude',
-        PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, '-l',
-        '--lines', '--port', '-I', '--snippet-type'
+        PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, RETRY_TOTAL_FLAG,
+        '-l', '--lines', '--port', '-I', '--snippet-type'
       ].freeze
       ORDER_TYPES = %w[defined random].freeze
       TAG_LIMIT_MATCHER = /(?<tag_name>@\w+):(?<limit>\d+)/x
@@ -108,6 +109,7 @@ def parse!(args) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           opts.on('-j DIR', '--jars DIR', 'Load all the jars under DIR') { |jars| load_jars(jars) } if Cucumber::JRUBY
 
           opts.on("#{RETRY_FLAG} ATTEMPTS", *retry_msg) { |v| set_option :retry, v.to_i }
+          opts.on("#{RETRY_TOTAL_FLAG} TESTS", *retry_total_msg) { |v| set_option :retry_total, v.to_i }
           opts.on('--i18n-languages', *i18n_languages_msg) { list_languages_and_exit }
           opts.on('--i18n-keywords LANG', *i18n_keywords_msg) { |lang| language lang }
           opts.on(FAIL_FAST_FLAG, 'Exit immediately following the first failing scenario') { set_option :fail_fast }
@@ -271,6 +273,13 @@ def retry_msg
         ['Specify the number of times to retry failing tests (default: 0)']
       end
 
+      def retry_total_msg
+        [
+          'The total number of failing test after which retrying of tests is suspended.',
+          'Example: --retry-total 10 -> Will stop retrying tests after 10 failing tests.'
+        ]
+      end
+
       def name_msg
         [
           'Only execute the feature elements which match part of the given name.',
@@ -543,6 +552,7 @@ def reverse_merge(other_options) # rubocop:disable Metrics/AbcSize
         end
 
         @options[:retry] = other_options[:retry] if @options[:retry].zero?
+        @options[:retry_total] = other_options[:retry_total] if @options[:retry_total].infinite?
 
         self
       end
@@ -616,7 +626,8 @@ def default_options
           snippets: true,
           source: true,
           duration: true,
-          retry: 0
+          retry: 0,
+          retry_total: Float::INFINITY
         }
       end
     end

lib/cucumber/configuration.rb

@@ -78,6 +78,10 @@ def retry_attempts
       @options[:retry]
     end
 
+    def retry_total_tests
+      @options[:retry_total]
+    end
+
     def guess?
       @options[:guess]
     end
@@ -273,7 +277,8 @@ def default_options
         snippets: true,
         source: true,
         duration: true,
-        event_bus: Cucumber::Events.make_event_bus
+        event_bus: Cucumber::Events.make_event_bus,
+        retry_total: Float::INFINITY
       }
     end
 

lib/cucumber/filters/retry.rb

@@ -7,6 +7,11 @@
 module Cucumber
   module Filters
     class Retry < Core::Filter.new(:configuration)
+      def initialize(*_args)
+        super
+        @total_permanently_failed = 0
+      end
+
       def test_case(test_case)
         configuration.on_event(:test_case_finished) do |event|
           next unless retry_required?(test_case, event)
@@ -21,7 +26,21 @@ def test_case(test_case)
       private
 
       def retry_required?(test_case, event)
-        event.test_case == test_case && event.result.failed? && test_case_counts[test_case] < configuration.retry_attempts
+        return false unless event.test_case == test_case
+
+        return false unless event.result.failed?
+
+        return false if @total_permanently_failed >= configuration.retry_total_tests
+
+        retry_required = test_case_counts[test_case] < configuration.retry_attempts
+        if retry_required
+          # retry test
+          true
+        else
+          # test failed after max. attempts
+          @total_permanently_failed += 1
+          false
+        end
       end
 
       def test_case_counts

spec/cucumber/cli/options_spec.rb

@@ -379,6 +379,26 @@ def with_env(name, value)
               end
             end
           end
+
+          context '--retry-total TESTS' do
+            context '--retry-total option not defined on the command line' do
+              it 'uses the --retry-total option from the profile' do
+                given_cucumber_yml_defined_as('foo' => '--retry-total 2')
+                options.parse!(%w[-p foo])
+
+                expect(options[:retry_total]).to be 2
+              end
+            end
+
+            context '--retry-total option defined on the command line' do
+              it 'ignores the --retry-total option from the profile' do
+                given_cucumber_yml_defined_as('foo' => '--retry-total 2')
+                options.parse!(%w[--retry-total 1 -p foo])
+
+                expect(options[:retry_total]).to be 1
+              end
+            end
+          end
         end
 
         context '-P or --no-profile' do
@@ -441,6 +461,20 @@ def with_env(name, value)
           end
         end
 
+        context '--retry-total TESTS' do
+          it 'is INFINITY by default' do
+            after_parsing('') do
+              expect(options[:retry_total]).to eql Float::INFINITY
+            end
+          end
+
+          it 'sets the options[:retry_total] value' do
+            after_parsing('--retry 3 --retry-total 10') do
+              expect(options[:retry_total]).to eql 10
+            end
+          end
+        end
+
         it 'assigns any extra arguments as paths to features' do
           after_parsing('-f pretty my_feature.feature my_other_features') do
             expect(options[:paths]).to eq ['my_feature.feature', 'my_other_features']

spec/cucumber/filters/retry_spec.rb

@@ -13,7 +13,8 @@
   include Cucumber::Core
   include Cucumber::Events
 
-  let(:configuration) { Cucumber::Configuration.new(retry: 2) }
+  let(:configuration) { Cucumber::Configuration.new(retry: 2, retry_total: retry_total) }
+  let(:retry_total) { Float::INFINITY }
   let(:id) { double }
   let(:name) { double }
   let(:location) { double }
@@ -55,12 +56,32 @@
   context 'consistently failing test case' do
     let(:result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) }
 
-    it 'describes the test case the specified number of times' do
-      expect(receiver).to receive(:test_case) { |test_case|
-        configuration.notify :test_case_finished, test_case, result
-      }.exactly(3).times
+    shared_examples 'retries the test case the specified number of times' do |expected_nr_of_times|
+      it 'describes the test case the specified number of times' do
+        expect(receiver).to receive(:test_case) { |test_case|
+          configuration.notify :test_case_finished, test_case, result
+        }.exactly(expected_nr_of_times).times
 
-      filter.test_case(test_case)
+        filter.test_case(test_case)
+      end
+    end
+
+    context 'when retry_total infinit' do
+      let(:retry_total) { Float::INFINITY }
+
+      include_examples 'retries the test case the specified number of times', 3
+    end
+
+    context 'when retry_total 1' do
+      let(:retry_total) { 1 }
+
+      include_examples 'retries the test case the specified number of times', 3
+    end
+
+    context 'when retry_total 0' do
+      let(:retry_total) { 0 }
+
+      include_examples 'retries the test case the specified number of times', 1
     end
   end
 
@@ -100,4 +121,28 @@
       end
     end
   end
+
+  context 'too many failing tests' do
+    let(:retry_total) { 1 }
+    let(:always_failing_test_case1) do
+      Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:1', tags, language)
+    end
+    let(:always_failing_test_case2) do
+      Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:9', tags, language)
+    end
+    let(:fail_result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) }
+
+    it 'stops retrying tests' do
+      expect(receiver).to receive(:test_case).with(always_failing_test_case1) { |test_case|
+        configuration.notify :test_case_finished, test_case, fail_result
+      }.ordered.exactly(3).times
+
+      expect(receiver).to receive(:test_case).with(always_failing_test_case2) { |test_case|
+        configuration.notify :test_case_finished, test_case, fail_result
+      }.ordered.exactly(1).times
+
+      filter.test_case(always_failing_test_case1)
+      filter.test_case(always_failing_test_case2)
+    end
+  end
 end