Skip to content

Commit

Permalink
Remove default seed value
Browse files Browse the repository at this point in the history 
Changes
* Remove the default `seed` value (ref: #33)
* Use exit status from RSpec calls (ref: #20)
  • Loading branch information
@ilyazub
ilyazub committed Nov 22, 2023
1 parent b9e812e commit e0dc49f
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 95 deletions.
9 changes: 3 additions & 6 deletions lib/turbo_tests/cli.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def run
end
end

success = TurboTests::Runner.run(
exitstatus = TurboTests::Runner.run(
formatters: formatters,
tags: tags,
files: @argv.empty? ? ["spec"] : @argv,
Expand All @@ -109,11 +109,8 @@ def run
seed: seed
)

if success
exit 0
else
exit 1
end
# From https://github.com/serpapi/turbo_tests/pull/20/
exit exitstatus
end
end
end
46 changes: 36 additions & 10 deletions lib/turbo_tests/reporter.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ module TurboTests
class Reporter
attr_writer :load_time

def self.from_config(formatter_config, start_time)
reporter = new(start_time)
def self.from_config(formatter_config, start_time, seed, seed_used)
reporter = new(start_time, seed, seed_used)

formatter_config.each do |config|
name, outputs = config.values_at(:name, :outputs)
Expand All @@ -23,13 +23,15 @@ def self.from_config(formatter_config, start_time)
attr_reader :pending_examples
attr_reader :failed_examples

def initialize(start_time)
def initialize(start_time, seed, seed_used)
@formatters = []
@pending_examples = []
@failed_examples = []
@all_examples = []
@messages = []
@start_time = start_time
@seed = seed
@seed_used = seed_used
@load_time = 0
@errors_outside_of_examples_count = 0
end
Expand All @@ -50,6 +52,31 @@ def add(name, outputs)
end
end

def report(expected_example_count)
start(expected_example_count)
begin
yield self
ensure
finish
end
end

def start(example_groups)
delegate_to_formatters(:seed, RSpec::Core::Notifications::SeedNotification.new(@seed, @seed_used))

report_number_of_tests(example_groups)
end

def report_number_of_tests(groups)
name = ParallelTests::RSpec::Runner.test_file_name

num_processes = groups.size
num_tests = groups.map(&:size).sum
tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round

puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
end

def group_started(notification)
delegate_to_formatters(:example_group_started, notification)
end
Expand Down Expand Up @@ -88,8 +115,7 @@ def error_outside_of_examples
end

def finish
# SEE: https://bit.ly/2NP87Cz
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
end_time = RSpec::Core::Time.now

delegate_to_formatters(:start_dump,
RSpec::Core::Notifications::NullNotification)
Expand All @@ -112,11 +138,11 @@ def finish
))
delegate_to_formatters(:close,
RSpec::Core::Notifications::NullNotification)
end

def seed_notification(seed, seed_used)
puts RSpec::Core::Notifications::SeedNotification.new(seed, seed_used).fully_formatted
puts
delegate_to_formatters(:seed,
RSpec::Core::Notifications::SeedNotification.new(
@seed,
@seed_used,
))
end

protected
Expand Down
65 changes: 31 additions & 34 deletions lib/turbo_tests/runner.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,19 @@ def self.run(opts = {})
formatters = opts[:formatters]
tags = opts[:tags]

# SEE: https://bit.ly/2NP87Cz
start_time = opts.fetch(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
start_time = opts.fetch(:start_time) { RSpec::Core::Time.now }
runtime_log = opts.fetch(:runtime_log, nil)
verbose = opts.fetch(:verbose, false)
fail_fast = opts.fetch(:fail_fast, nil)
count = opts.fetch(:count, nil)
seed = opts.fetch(:seed) || rand(0xFFFF).to_s
seed_used = !opts[:seed].nil?
seed = opts.fetch(:seed)
seed_used = !seed.nil?

if verbose
STDERR.puts "VERBOSE"
end

reporter = Reporter.from_config(formatters, start_time)
reporter = Reporter.from_config(formatters, start_time, seed, seed_used)

new(
reporter: reporter,
Expand All @@ -38,7 +37,7 @@ def self.run(opts = {})
fail_fast: fail_fast,
count: count,
seed: seed,
seed_used: seed_used
seed_used: seed_used,
).run
end

Expand All @@ -50,11 +49,12 @@ def initialize(opts)
@verbose = opts[:verbose]
@fail_fast = opts[:fail_fast]
@count = opts[:count]
@seed = opts[:seed]
@seed_used = opts[:seed_used]

@load_time = 0
@load_count = 0
@failure_count = 0
@seed = opts[:seed]
@seed_used = opts[:seed_used]

@messages = Thread::Queue.new
@threads = []
Expand Down Expand Up @@ -87,26 +87,25 @@ def run
setup_tmp_dir

subprocess_opts = {
record_runtime: use_runtime_info
record_runtime: use_runtime_info,
}

report_number_of_tests(tests_in_groups)

@reporter.seed_notification(@seed, @seed_used)

wait_threads = tests_in_groups.map.with_index do |tests, process_id|
start_regular_subprocess(tests, process_id + 1, **subprocess_opts)
end

handle_messages

@reporter.finish
@reporter.report(tests_in_groups) do |reporter|
wait_threads = tests_in_groups.map.with_index do |tests, process_id|
start_regular_subprocess(tests, process_id + 1, **subprocess_opts)
end

@reporter.seed_notification(@seed, @seed_used)
handle_messages

@threads.each(&:join)
@threads.each(&:join)

@reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?)
if @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?)
0
else
# From https://github.com/serpapi/turbo_tests/pull/20/
wait_threads.map { |thread| thread.value.exitstatus }.max
end
end
end

private
Expand Down Expand Up @@ -157,10 +156,18 @@ def start_subprocess(env, extra_args, tests, process_id, record_runtime:)
[]
end

seed_option = if @seed_used
[
"--seed", @seed,
]
else
[]
end

command = [
"rspec",
*extra_args,
"--seed", @seed,
*seed_option,
"--format", "TurboTests::JsonRowsFormatter",
"--out", tmp_filename,
*record_runtime_options,
Expand Down Expand Up @@ -273,15 +280,5 @@ def handle_messages
def fail_fast_met
!@fail_fast.nil? && @failure_count >= @fail_fast
end

def report_number_of_tests(groups)
name = ParallelTests::RSpec::Runner.test_file_name

num_processes = groups.size
num_tests = groups.map(&:size).sum
tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round

puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
end
end
end
130 changes: 85 additions & 45 deletions spec/cli_spec.rb
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
RSpec.describe TurboTests::CLI do
subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed 1234`.strip }

before { output }

context "errors outside of examples" do
let(:expected_start_of_output) {
%(
1 processes for 1 specs, ~ 1 specs per process
subject(:output) { `bundle exec turbo_tests -f d #{fixture}`.strip }

Randomized with seed 1234
context "when the 'seed' parameter was used" do
let(:seed) { 1234 }

subject(:output) { `bundle exec turbo_tests -f d #{fixture} --seed #{seed}`.strip }

context "errors outside of examples" do
let(:expected_start_of_output) {
%(
Randomized with seed #{seed}
1 processes for 1 specs, ~ 1 specs per process
An error occurred while loading #{fixture}.
\e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m
Expand All @@ -20,61 +23,98 @@
\e[36m# #{fixture}:4:in `block in <top (required)>'\e[0m
\e[36m# #{fixture}:1:in `<top (required)>'\e[0m
).strip
}
let(:expected_end_of_output) do
"0 examples, 0 failures\n"\
}

let(:expected_end_of_output) do
"0 examples, 0 failures\n"\
"\n\n"\
"Randomized with seed 1234"
"Randomized with seed #{seed}"
end

let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" }

it "reports" do
expect($?.exitstatus).to eql(1)

expect(output).to start_with(expected_start_of_output)
expect(output).to end_with(expected_end_of_output)
end
end

let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" }
context "pending exceptions", :aggregate_failures do
let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" }

it "reports" do
expect($?.exitstatus).to eql(1)
it "reports" do
expect($?.exitstatus).to eql(0)

[
"is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')",
"is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')",
"is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)",

expect(output).to start_with(expected_start_of_output)
expect(output).to end_with(expected_end_of_output)
"Pending: (Failures listed here are expected and do not affect your suite's status)",
].each do |part|
expect(output).to include(part)
end

expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed #{seed}")
end
end
end

context "pending exceptions", :aggregate_failures do
let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" }
context "when 'seed' parameter was not used" do
context "errors outside of examples" do
let(:expected_start_of_output) {
%(
1 processes for 1 specs, ~ 1 specs per process
An error occurred while loading #{fixture}.
\e[31mFailure/Error: \e[0m\e[1;34m1\e[0m / \e[1;34m0\e[0m\e[0m
\e[31m\e[0m
\e[31mZeroDivisionError:\e[0m
\e[31m divided by 0\e[0m
\e[36m# #{fixture}:4:in `/'\e[0m
\e[36m# #{fixture}:4:in `block in <top (required)>'\e[0m
\e[36m# #{fixture}:1:in `<top (required)>'\e[0m
).strip
}

let(:expected_end_of_output) do
"0 examples, 0 failures"
end

let(:fixture) { "./fixtures/rspec/errors_outside_of_examples_spec.rb" }

it "reports" do
expect($?.exitstatus).to eql(0)
it "reports" do
expect($?.exitstatus).to eql(1)

[
"is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')",
"is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')",
"is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)",
expect(output).to start_with(expected_start_of_output)
expect(output).to end_with(expected_end_of_output)
end

"Pending: (Failures listed here are expected and do not affect your suite's status)",
it "exludes the seed message from the output" do
expect(output).to_not include("seed")
end
end

%{
Fixture of spec file with pending failed examples is implemented but skipped with 'pending'
# TODO: skipped with 'pending'
Failure/Error: DEFAULT_FAILURE_NOTIFIER = lambda { |failure, _opts| raise failure }
context "pending exceptions", :aggregate_failures do
let(:fixture) { "./fixtures/rspec/pending_exceptions_spec.rb" }

expected: 3
got: 2
it "reports" do
expect($?.exitstatus).to eql(0)

(compared using ==)
}.strip,
[
"is implemented but skipped with 'pending' (PENDING: TODO: skipped with 'pending')",
"is implemented but skipped with 'skip' (PENDING: TODO: skipped with 'skip')",
"is implemented but skipped with 'xit' (PENDING: Temporarily skipped with xit)",

%(
Fixture of spec file with pending failed examples is implemented but skipped with 'skip'
# TODO: skipped with 'skip'
).strip,
"Pending: (Failures listed here are expected and do not affect your suite's status)",
].each do |part|
expect(output).to include(part)
end

%(
Fixture of spec file with pending failed examples is implemented but skipped with 'xit'
# Temporarily skipped with xit
).strip
].each do |part|
expect(output).to include(part)
expect(output).to end_with("3 examples, 0 failures, 3 pending")
end

expect(output).to end_with("3 examples, 0 failures, 3 pending\n\n\nRandomized with seed 1234")
end
end

Expand Down

0 comments on commit e0dc49f

Please sign in to comment.