diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95638773..5467afe4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,5 @@ name: Build -on: [push, pull_request] +on: [ push, pull_request ] jobs: rubocop: @@ -16,14 +16,25 @@ jobs: - name: Run rubocop run: | bundle exec rubocop --parallel + tests: name: Tests runs-on: ubuntu-latest strategy: fail-fast: false matrix: - ruby-version: [3.3.5] - database: [mysql, postgres, sqlite] + ruby-version: + - 3.2.0 + - 3.2.4 + - 3.3.0 + - 3.3.1 + - 3.3.2 + - 3.3.4 + - 3.3.5 + - 3.3.6 + - 3.4.0 + - 3.4.1 + database: [ mysql, postgres, sqlite ] services: mysql: image: mysql:8.0.31 @@ -53,3 +64,42 @@ jobs: bin/rails db:setup - name: Run tests run: bin/rails test + + tests-ruby-3-1-6: + name: Tests Ruby 3.1.6 + runs-on: ubuntu-latest + strategy: + matrix: + database: [ mysql, postgres, sqlite ] + services: + mysql: + image: mysql:8.0.31 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + ports: + - 33060:3306 + options: --health-cmd "mysql -h localhost -e \"select now()\"" --health-interval 1s --health-timeout 5s --health-retries 30 + postgres: + image: postgres:15.1 + env: + POSTGRES_HOST_AUTH_METHOD: "trust" + ports: + - 55432:5432 + env: + TARGET_DB: ${{ matrix.database }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Ruby and install specific gems for 3.1.6 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.1.6 + - name: Install dependencies with specific Gemfile.lock + run: | + cp Gemfile.lock.ruby_3_1_6 Gemfile.lock + bundle install + - name: Setup test database + run: | + bin/rails db:setup + - name: Run tests + run: bin/rails test \ No newline at end of file diff --git a/Gemfile.lock.ruby_3_1_6 b/Gemfile.lock.ruby_3_1_6 new file mode 100644 index 00000000..d95cdea9 --- /dev/null +++ b/Gemfile.lock.ruby_3_1_6 @@ -0,0 +1,205 @@ +PATH + remote: . + specs: + solid_queue (1.1.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) + +GEM + remote: https://rubygems.org/ + specs: + actionpack (7.1.5.1) + actionview (= 7.1.5.1) + activesupport (= 7.1.5.1) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + actionview (7.1.5.1) + activesupport (= 7.1.5.1) + builder (~> 3.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activejob (7.1.5.1) + activesupport (= 7.1.5.1) + globalid (>= 0.3.6) + activemodel (7.1.5.1) + activesupport (= 7.1.5.1) + activerecord (7.1.5.1) + activemodel (= 7.1.5.1) + activesupport (= 7.1.5.1) + timeout (>= 0.4.0) + activesupport (7.1.5.1) + base64 + benchmark (>= 0.3) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + logger (>= 1.4.2) + minitest (>= 5.1) + mutex_m + securerandom (>= 0.3) + tzinfo (~> 2.0) + ast (2.4.2) + base64 (0.2.0) + benchmark (0.4.0) + bigdecimal (3.1.9) + builder (3.3.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + crass (1.0.6) + date (3.4.1) + debug (1.9.2) + irb (~> 1.10) + reline (>= 0.3.8) + drb (2.2.1) + erubi (1.13.1) + et-orbi (1.2.11) + tzinfo + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) + globalid (1.2.1) + activesupport (>= 6.1) + i18n (1.14.6) + concurrent-ruby (~> 1.0) + io-console (0.8.0) + irb (1.14.3) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.9.1) + language_server-protocol (3.17.0.3) + logger (1.6.2) + loofah (2.23.1) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + minitest (5.25.4) + mocha (2.1.0) + ruby2_keywords (>= 0.0.5) + mutex_m (0.3.0) + mysql2 (0.5.6) + nio4r (2.7.4) + nokogiri (1.18.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.26.3) + parser (3.3.6.0) + ast (~> 2.4.1) + racc + pg (1.5.4) + psych (5.2.2) + date + stringio + puma (6.4.3) + nio4r (~> 2.0) + raabro (1.4.0) + racc (1.8.1) + rack (3.1.8) + rack-session (2.0.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rackup (2.2.1) + rack (>= 3) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.2) + loofah (~> 2.21) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.1.5.1) + actionpack (= 7.1.5.1) + activesupport (= 7.1.5.1) + irb + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) + rainbow (3.1.1) + rake (13.2.1) + rdoc (6.8.1) + psych (>= 4.0.0) + regexp_parser (2.10.0) + reline (0.6.0) + io-console (~> 0.5) + rubocop (1.69.2) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.37.0) + parser (>= 3.3.1.0) + rubocop-minitest (0.36.0) + rubocop (>= 1.61, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-performance (1.23.0) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.28.0) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.52.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails-omakase (1.0.0) + rubocop + rubocop-minitest + rubocop-performance + rubocop-rails + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + securerandom (0.4.1) + sqlite3 (1.5.4-arm64-darwin) + sqlite3 (1.5.4-x86_64-darwin) + sqlite3 (1.5.4-x86_64-linux) + stringio (3.1.2) + thor (1.3.2) + timeout (0.4.3) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (3.1.3) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + zeitwerk (2.6.0) + +PLATFORMS + arm64-darwin-22 + arm64-darwin-23 + arm64-darwin-24 + x86_64-darwin-21 + x86_64-darwin-23 + x86_64-linux + +DEPENDENCIES + debug (~> 1.9) + logger + mocha + mysql2 + pg + puma + rdoc + rubocop-rails-omakase + solid_queue! + sqlite3 + zeitwerk (= 2.6.0) + +BUNDLED WITH + 2.5.9 diff --git a/README.md b/README.md index dbd0fd3b..99820af3 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Solid Queue is configured by default in new Rails 8 applications. But if you're 1. `bundle add solid_queue` 2. `bin/rails solid_queue:install` +(Note: The minimum supported version of Rails is 7.1 and Ruby is 3.1.6.) + This will configure Solid Queue as the production Active Job backend, create the configuration files `config/queue.yml` and `config/recurring.yml`, and create the `db/queue_schema.rb`. It'll also create a `bin/jobs` executable wrapper that you can use to start Solid Queue. Once you've done that, you will then have to add the configuration for the queue database in `config/database.yml`. If you're using SQLite, it'll look like this: diff --git a/app/models/solid_queue/scheduled_execution.rb b/app/models/solid_queue/scheduled_execution.rb index 72aafd53..0f45626b 100644 --- a/app/models/solid_queue/scheduled_execution.rb +++ b/app/models/solid_queue/scheduled_execution.rb @@ -14,7 +14,8 @@ class << self def dispatch_next_batch(batch_size) transaction do job_ids = next_batch(batch_size).non_blocking_lock.pluck(:job_id) - if job_ids.empty? then [] + if job_ids.empty? + 0 else SolidQueue.instrument(:dispatch_scheduled, batch_size: batch_size) do |payload| payload[:size] = dispatch_jobs(job_ids) diff --git a/lib/solid_queue/dispatcher.rb b/lib/solid_queue/dispatcher.rb index 62e4294d..c60602ed 100644 --- a/lib/solid_queue/dispatcher.rb +++ b/lib/solid_queue/dispatcher.rb @@ -25,7 +25,7 @@ def metadata def poll batch = dispatch_next_batch - batch.size.zero? ? polling_interval : 0.seconds + batch.zero? ? polling_interval : 0.seconds end def dispatch_next_batch diff --git a/lib/solid_queue/engine.rb b/lib/solid_queue/engine.rb index d10997c7..99e14150 100644 --- a/lib/solid_queue/engine.rb +++ b/lib/solid_queue/engine.rb @@ -37,5 +37,13 @@ class Engine < ::Rails::Engine include ActiveJob::ConcurrencyControls end end + + initializer "solid_queue.include_interruptible_concern" do + if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2") + SolidQueue::Processes::Base.include SolidQueue::Processes::Interruptible + else + SolidQueue::Processes::Base.include SolidQueue::Processes::OgInterruptible + end + end end end diff --git a/lib/solid_queue/processes/base.rb b/lib/solid_queue/processes/base.rb index 6069a90d..59ec9f1a 100644 --- a/lib/solid_queue/processes/base.rb +++ b/lib/solid_queue/processes/base.rb @@ -4,7 +4,7 @@ module SolidQueue module Processes class Base include Callbacks # Defines callbacks needed by other concerns - include AppExecutor, Registrable, Interruptible, Procline + include AppExecutor, Registrable, Procline attr_reader :name diff --git a/lib/solid_queue/processes/interruptible.rb b/lib/solid_queue/processes/interruptible.rb index 3bff1dd9..b7755f1f 100644 --- a/lib/solid_queue/processes/interruptible.rb +++ b/lib/solid_queue/processes/interruptible.rb @@ -2,6 +2,8 @@ module SolidQueue::Processes module Interruptible + include SolidQueue::AppExecutor + def wake_up interrupt end @@ -13,17 +15,19 @@ def interrupt end # Sleeps for 'time'. Can be interrupted asynchronously and return early via wake_up. - # @param time [Numeric] the time to sleep. 0 returns immediately. - # @return [true, nil] - # * returns `true` if an interrupt was requested via #wake_up between the - # last call to `interruptible_sleep` and now, resulting in an early return. - # * returns `nil` if it slept the full `time` and was not interrupted. + # @param time [Numeric, Duration] the time to sleep. 0 returns immediately. def interruptible_sleep(time) # Invoking this from the main thread may result in significant slowdown. # Utilizing asynchronous execution (Futures) addresses this performance issue. Concurrent::Promises.future(time) do |timeout| - queue.pop(timeout:).tap { queue.clear } + queue.clear unless queue.pop(timeout:).nil? + end.on_rejection! do |e| + wrapped_exception = RuntimeError.new("Interruptible#interruptible_sleep - #{e.class}: #{e.message}") + wrapped_exception.set_backtrace(e.backtrace) + handle_thread_error(wrapped_exception) end.value + + nil end def queue diff --git a/lib/solid_queue/processes/og_interruptible.rb b/lib/solid_queue/processes/og_interruptible.rb new file mode 100644 index 00000000..d3b6e390 --- /dev/null +++ b/lib/solid_queue/processes/og_interruptible.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# frozen_string_literal: true + +module SolidQueue::Processes + # The original implementation of Interruptible that works + # with Ruby 3.1 and earlier + module OgInterruptible + def wake_up + interrupt + end + + private + SELF_PIPE_BLOCK_SIZE = 11 + + def interrupt + self_pipe[:writer].write_nonblock(".") + rescue Errno::EAGAIN, Errno::EINTR + # Ignore writes that would block and retry + # if another signal arrived while writing + retry + end + + def interruptible_sleep(time) + if time > 0 && self_pipe[:reader].wait_readable(time) + loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) } + end + rescue Errno::EAGAIN, Errno::EINTR + end + + # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html) + def self_pipe + @self_pipe ||= create_self_pipe + end + + def create_self_pipe + reader, writer = IO.pipe + { reader: reader, writer: writer } + end + end +end diff --git a/solid_queue.gemspec b/solid_queue.gemspec index 91472454..8bccc770 100644 --- a/solid_queue.gemspec +++ b/solid_queue.gemspec @@ -39,4 +39,8 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rubocop-rails-omakase" spec.add_development_dependency "rdoc" spec.add_development_dependency "logger" + + if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + spec.add_development_dependency "zeitwerk", "2.6.0" + end end diff --git a/test/models/solid_queue/claimed_execution_test.rb b/test/models/solid_queue/claimed_execution_test.rb index 4e99fd04..b7892b21 100644 --- a/test/models/solid_queue/claimed_execution_test.rb +++ b/test/models/solid_queue/claimed_execution_test.rb @@ -31,7 +31,7 @@ class SolidQueue::ClaimedExecutionTest < ActiveSupport::TestCase assert job.failed? assert_equal "RuntimeError", job.failed_execution.exception_class assert_equal "This is a RuntimeError exception", job.failed_execution.message - assert_match /app\/jobs\/raising_job\.rb:\d+:in `perform'/, job.failed_execution.backtrace.first + assert_match /\/app\/jobs\/raising_job\.rb:\d+:in [`'](RaisingJob#)?perform'/, job.failed_execution.backtrace.first assert_equal @process, claimed_execution.process end diff --git a/test/unit/dispatcher_test.rb b/test/unit/dispatcher_test.rb index 5bca7743..9aa2196e 100644 --- a/test/unit/dispatcher_test.rb +++ b/test/unit/dispatcher_test.rb @@ -96,6 +96,7 @@ class DispatcherTest < ActiveSupport::TestCase dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) dispatcher.expects(:interruptible_sleep).with(0.seconds).at_least(3) dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + dispatcher.expects(:handle_thread_error).never 3.times { AddToBufferJob.set(wait: 0.1).perform_later("I'm scheduled") } assert_equal 3, SolidQueue::ScheduledExecution.count @@ -112,8 +113,10 @@ class DispatcherTest < ActiveSupport::TestCase dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) dispatcher.expects(:interruptible_sleep).with(0.seconds).never dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + dispatcher.expects(:handle_thread_error).never + dispatcher.start - sleep 0.1 + wait_while_with_timeout(1.second) { !SolidQueue::ScheduledExecution.exists? } end private diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index 52b0d8e8..8db67912 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -176,6 +176,7 @@ class WorkerTest < ActiveSupport::TestCase @worker.expects(:interruptible_sleep).with(10.minutes).at_least_once @worker.expects(:interruptible_sleep).with(@worker.polling_interval).never + @worker.expects(:handle_thread_error).never @worker.start sleep 1.second @@ -186,6 +187,7 @@ class WorkerTest < ActiveSupport::TestCase @worker.expects(:interruptible_sleep).with(@worker.polling_interval).at_least_once @worker.expects(:interruptible_sleep).with(10.minutes).never + @worker.expects(:handle_thread_error).never @worker.start sleep 1.second