Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallelise test suite #1041

Merged
merged 9 commits into from
Sep 23, 2016
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -13,11 +13,13 @@
/Library/Homebrew/cask/bin
/Library/Homebrew/cask/vendor
/Library/Homebrew/cask/coverage
/Library/Homebrew/cask/tmp
/Library/Homebrew/test/.bundle
/Library/Homebrew/test/bin
/Library/Homebrew/test/vendor
/Library/Homebrew/test/coverage
/Library/Homebrew/test/fs_leak_log
/Library/Homebrew/tmp
/Library/LinkedKegs
/Library/Locks
/Library/PinnedKegs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/usr/bin/env ruby

SimpleCov.start do
coverage_dir File.expand_path("../coverage", File.realpath(__FILE__))
root File.expand_path("../..", File.realpath(__FILE__))
coverage_dir File.expand_path("../test/coverage", File.realpath(__FILE__))
root File.expand_path("..", File.realpath(__FILE__))

# We manage the result cache ourselves and the default of 10 minutes can be
# too low (particularly on Travis CI), causing results from some integration
2 changes: 1 addition & 1 deletion Library/Homebrew/cask/.simplecov
1 change: 1 addition & 0 deletions Library/Homebrew/cask/Gemfile
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@ group :test do
gem "minitest", "5.4.1"
gem "minitest-reporters"
gem "mocha", "1.1.0", require: false
gem "parallel_tests"
gem "rspec", "~> 3.0.0"
gem "rspec-its", require: false
gem "rspec-wait", require: false
4 changes: 4 additions & 0 deletions Library/Homebrew/cask/Gemfile.lock
Original file line number Diff line number Diff line change
@@ -23,6 +23,9 @@ GEM
ruby-progressbar
mocha (1.1.0)
metaclass (~> 0.0.1)
parallel (1.9.0)
parallel_tests (2.9.0)
parallel
parser (2.3.1.2)
ast (~> 2.2)
powerpack (0.1.1)
@@ -80,6 +83,7 @@ DEPENDENCIES
minitest (= 5.4.1)
minitest-reporters
mocha (= 1.1.0)
parallel_tests
pry
pry-byebug
rake
25 changes: 4 additions & 21 deletions Library/Homebrew/cask/Rakefile
Original file line number Diff line number Diff line change
@@ -7,23 +7,9 @@ $LOAD_PATH.unshift(File.expand_path("#{homebrew_repo}/Library/Homebrew"))
$LOAD_PATH.unshift(File.expand_path("../lib", __FILE__))

namespace :test do
Rake::TestTask.new(:minitest) do |t|
# TODO: setting the --seed here is an ugly temporary hack, to remain only
# until test-suite glitches are fixed.
ENV["TESTOPTS"] = "--seed=14830" if ENV["TRAVIS"]
t.pattern = "test/**/*_test.rb"
t.libs << "test"
end

RSpec::Core::RakeTask.new(:rspec)

desc "Run tests for minitest and RSpec with coverage"
task :coverage do
ENV["HOMEBREW_TESTS_COVERAGE"] = "1"

Rake::Task[:test].invoke

if ENV["CODECOV_TOKEN"]
namespace :coverage do
desc "Upload coverage to Codecov"
task :upload do
require "simplecov"
require "codecov"
formatter = SimpleCov::Formatter::Codecov.new
@@ -32,14 +18,11 @@ namespace :test do
end
end

desc "Run tests for minitest and RSpec"
task test: ["test:minitest", "test:rspec"]

RuboCop::RakeTask.new(:rubocop) do |t|
t.options = ["--force-exclusion"]
end

task default: [:test, :rubocop]
task default: [:rubocop]

desc "Open a REPL for debugging and experimentation"
task :console do
31 changes: 26 additions & 5 deletions Library/Homebrew/cask/cmd/brew-cask-tests.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
require "English"

def run_tests(executable, files, args = [])
opts = []
opts << "--serialize-stdout" if ENV["CI"]

system "bundle", "exec", executable, *opts, "--", *args, "--", *files
end

repo_root = Pathname(__FILE__).realpath.parent.parent
repo_root.cd do
ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1"
@@ -9,12 +16,26 @@
system "bundle", "install", "--path", "vendor/bundle"
end

test_task = "test"
%w[rspec minitest coverage].each do |subtask|
next unless ARGV.flag?("--#{subtask}")
test_task = "test:#{subtask}"
rspec = ARGV.flag?("--rspec") || !ARGV.flag?("--minitest")
minitest = ARGV.flag?("--minitest") || !ARGV.flag?("--rspec")

ENV["HOMEBREW_TESTS_COVERAGE"] = "1" if ARGV.flag?("--coverage")

if rspec
run_tests "parallel_rspec", Dir["spec/**/*_spec.rb"], %w[
--format progress
--format ParallelTests::RSpec::RuntimeLogger
--out tmp/parallel_runtime_rspec.log
]
end

if minitest
run_tests "parallel_test", Dir["test/**/*_test.rb"]
end

if ENV["CODECOV_TOKEN"]
system "bundle", "exec", "rake", "test:coverage:upload"
end

system "bundle", "exec", "rake", test_task
Homebrew.failed = !$CHILD_STATUS.success?
end
2 changes: 2 additions & 0 deletions Library/Homebrew/cask/test/test_helper.rb
Original file line number Diff line number Diff line change
@@ -39,6 +39,8 @@ def sudo(*args)
require "minitest/reporters"
Minitest::Reporters.use! Minitest::Reporters::DefaultReporter.new(color: true)

require "parallel_tests/test/runtime_logger"

# Force mocha to patch MiniTest since we have both loaded thanks to homebrew's testing_env
require "mocha/api"
require "mocha/integration/mini_test"
22 changes: 17 additions & 5 deletions Library/Homebrew/dev-cmd/tests.rb
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@

module Homebrew
def tests
(HOMEBREW_LIBRARY/"Homebrew/test").cd do
(HOMEBREW_LIBRARY/"Homebrew").cd do
ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "1"
ENV["HOMEBREW_DEVELOPER"] = "1"
ENV["TESTOPTS"] = "-v" if ARGV.verbose?
@@ -19,9 +19,11 @@ def tests

if ARGV.include? "--coverage"
ENV["HOMEBREW_TESTS_COVERAGE"] = "1"
FileUtils.rm_f "coverage/.resultset.json"
FileUtils.rm_f "test/coverage/.resultset.json"
end

ENV["BUNDLE_GEMFILE"] = "#{Dir.pwd}/test/Gemfile"

# Override author/committer as global settings might be invalid and thus
# will cause silent failure during the setup of dummy Git repositories.
%w[AUTHOR COMMITTER].each do |role|
@@ -37,16 +39,26 @@ def tests
# Make it easier to reproduce test runs.
ENV["SEED"] = ARGV.next if ARGV.include? "--seed"

files = Dir["test/test_*.rb"]
files -= Dir["test/test_os_mac_*.rb"] unless OS.mac?

opts = []
opts << "--serialize-stdout" if ENV["CI"]

args = []
args << "--trace" if ARGV.include? "--trace"

if ARGV.value("only")
ENV["HOMEBREW_TESTS_ONLY"] = "1"
test_name, test_method = ARGV.value("only").split("/", 2)
args << "TEST=test_#{test_name}.rb"
args << "TESTOPTS=--name=test_#{test_method}" if test_method
files = ["test/test_#{test_name}.rb"]
args << "--name=test_#{test_method}" if test_method
end

args += ARGV.named.select { |v| v[/^TEST(OPTS)?=/] }
system "bundle", "exec", "rake", "test", *args

system "bundle", "exec", "parallel_test", *opts,
"--", *args, "--", *files

Homebrew.failed = !$?.success?

1 change: 1 addition & 0 deletions Library/Homebrew/test/Gemfile
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ source "https://rubygems.org"
gem "mocha", "~> 1.1"
gem "minitest", "~> 5.3"
gem "rake", "~> 10.3"
gem "parallel_tests", "~> 2.9"

group :coverage do
# This is SimpleCov v0.12.0 with one PR merged on top, that finally resolves
6 changes: 5 additions & 1 deletion Library/Homebrew/test/Gemfile.lock
Original file line number Diff line number Diff line change
@@ -21,6 +21,9 @@ GEM
minitest (5.9.0)
mocha (1.1.0)
metaclass (~> 0.0.1)
parallel (1.9.0)
parallel_tests (2.9.0)
parallel
rake (10.5.0)
simplecov-html (0.10.0)
url (0.3.2)
@@ -32,8 +35,9 @@ DEPENDENCIES
codecov
minitest (~> 5.3)
mocha (~> 1.1)
parallel_tests (~> 2.9)
rake (~> 10.3)
simplecov (= 0.12.0)!

BUNDLED WITH
1.12.5
1.13.1
29 changes: 0 additions & 29 deletions Library/Homebrew/test/Rakefile

This file was deleted.

13 changes: 8 additions & 5 deletions Library/Homebrew/test/test_inreplace.rb
Original file line number Diff line number Diff line change
@@ -91,26 +91,29 @@ def test_sub_gsub
end

def test_inreplace_errors
require "tempfile"
extend(Utils::Inreplace)

open("test", "w") { |f| f.write "a\nb\nc\n" }
file = Tempfile.new("test")

file.write "a\nb\nc\n"

assert_raises(Utils::InreplaceError) do
inreplace "test", "d", "f"
inreplace file.path, "d", "f"
end

assert_raises(Utils::InreplaceError) do
# Under current context, we are testing `String#gsub!`, so let's disable rubocop temporarily.
inreplace("test") { |s| s.gsub!("d", "f") } # rubocop:disable Performance/StringReplacement
inreplace(file.path) { |s| s.gsub!("d", "f") } # rubocop:disable Performance/StringReplacement
end

assert_raises(Utils::InreplaceError) do
inreplace("test") do |s|
inreplace(file.path) do |s|
s.change_make_var! "VAR", "value"
s.remove_make_var! "VAR2"
end
end
ensure
File.unlink("test")
file.unlink
end
end
3 changes: 2 additions & 1 deletion Library/Homebrew/test/testing_env.rb
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@
begin
require "rubygems"
require "minitest/autorun"
require "parallel_tests/test/runtime_logger"
require "mocha/setup"
rescue LoadError
abort "Run `bundle install` or install the mocha and minitest gems before running the tests"
@@ -42,7 +43,7 @@ def assert_version_nil(url)
module FSLeakLogger
def self.included(klass)
require "find"
@@log = File.open("fs_leak_log", "w")
@@log = File.open("#{__dir__}/fs_leak_log", "w")
klass.make_my_diffs_pretty!
end