[ci.rb] Add attempts variable to allow CI steps to retry

Some CI steps might flake and currently that fails the whole local CI run.

To work around that, there's a new `attempts: 1` optional variable that can be set on a step to retry. By default it's 1, meaning no retries. If set to a higher number, the step will be retried up to that many times before failing. If it's not positive, there will be an ArgumentError. New test cases have been added for this feature.

Author: harsh183

activesupport/CHANGELOG.md

@@ -16,5 +16,14 @@
 
     *Gannon McGibbon*
 
+*   Add `attempts` parameter to `ActiveSupport::ContinuousIntegration#step` to
+    allow retrying flaky CI steps a specified number of times before marking
+    them as failed.
+    ```ruby
+    step "Flaky test", "bin/rails test test/models/flaky_test.rb", attempts: 3
+    ```
+
+    *Harsh Deep*
+
 
 Please check [8-1-stable](https://github.com/rails/rails/blob/8-1-stable/activesupport/CHANGELOG.md) for previous changes.

activesupport/lib/active_support/continuous_integration.rb

@@ -72,9 +72,17 @@ def initialize
     #
     #   step "Setup", "bin/setup"
     #   step "Single test", "bin/rails", "test", "--name", "test_that_is_one"
-    def step(title, *command)
+    #   step "Flaky test", "bin/rails test test/models/flaky_test.rb", attempts: 3
+    def step(title, *command, attempts: 1)
+      raise ArgumentError, "attempts must be a positive integer" unless attempts.positive?
+
       heading title, command.join(" "), type: :title
-      report(title) { results << [ system(*command), title ] }
+
+      successful = retry_until_success(attempts) do
+        system(*command)
+      end
+
+      report(title) { results << [ successful, title ] }
     end
 
     # Returns true if all steps were successful.
@@ -159,5 +167,13 @@ def timing
       def colorize(text, type)
         "#{COLORS.fetch(type)}#{text}\033[0m"
       end
+
+      def retry_until_success(attempts)
+        attempts.times do |attempt_number|
+          return true if yield
+          echo "Attempt #{attempt_number + 1} failed. Retrying...", type: :error
+        end
+        false
+      end
   end
 end

activesupport/test/continuous_integration_test.rb

@@ -18,6 +18,35 @@ class ContinuousIntegrationTest < ActiveSupport::TestCase
     assert_not @CI.success?
   end
 
+  test "step retries on failures" do
+    output = capture_io {
+      @CI.step "Failed!", "false", attempts: 2
+    }.to_s
+
+    assert_not @CI.success?
+    assert_match(/Attempt 1 failed. Retrying.../, output)
+    assert_match(/Attempt 2 failed. Retrying.../, output)
+  end
+
+  test "successful step does not retry" do
+    output = capture_io {
+      @CI.step "Success!", "true", attempts: 2
+    }.to_s
+
+    assert @CI.success?
+    assert_no_match(/Attempt \d+ failed. Retrying.../, output)
+  end
+
+  test "step raises error for invalid attempts" do
+    assert_raises ArgumentError, match: "attempts must be a positive integer" do
+      @CI.step "Invalid attempts", "true", attempts: 0
+    end
+
+    assert_raises ArgumentError, match: "attempts must be a positive integer" do
+      @CI.step "Invalid attempts", "true", attempts: -1
+    end
+  end
+
   test "report with only successful steps combined gives success" do
     output = capture_io do
       @CI.report("CI") do