Add index, FKs, and method to clear finished jobs after a configured …

…time

The foreign keys with cascade delete are just a failsafe, to ensure we don't
fail to clear executions after deleting jobs.

app/models/solid_queue/job/clearable.rb

@@ -0,0 +1,13 @@
+module SolidQueue
+  class Job
+    module Clearable
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def clear_finished_in_batches(batch_size: 500, finished_before: SolidQueue.clear_finished_jobs_after.ago)
+          where(finished_at: ...finished_before).in_batches(of: batch_size).delete_all
+        end
+      end
+    end
+  end
+end

app/models/solid_queue/job/executable.rb

@@ -4,7 +4,7 @@ module Executable
       extend ActiveSupport::Concern
 
       included do
-        include ConcurrencyControls
+        include Clearable, ConcurrencyControls
 
         has_one :ready_execution, dependent: :destroy
         has_one :claimed_execution, dependent: :destroy

db/migrate/20231206150233_add_index_and_foreign_keys_for_clearing.rb

@@ -0,0 +1,9 @@
+class AddIndexAndForeignKeysForClearing < ActiveRecord::Migration[7.1]
+  def change
+    add_index :solid_queue_jobs, :finished_at
+
+    %w[ scheduled ready blocked claimed failed ].each do |execution_type|
+      add_foreign_key "solid_queue_#{execution_type}_executions", :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    end
+  end
+end

test/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2023_12_01_102514) do
+ActiveRecord::Schema[7.1].define(version: 2023_12_06_150233) do
   create_table "job_results", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.string "queue_name"
     t.string "status"
@@ -58,6 +58,7 @@
     t.string "concurrency_key"
     t.index ["active_job_id"], name: "index_solid_queue_jobs_on_job_id"
     t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+    t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
     t.index ["queue_name", "scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
   end
 
@@ -107,4 +108,9 @@
     t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
   end
 
+  add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
 end

test/integration/jobs_lifecycle_test.rb

@@ -64,6 +64,28 @@ class JobsLifecycleTest < ActiveSupport::TestCase
     assert_equal 0, SolidQueue::Job.count
   end
 
+  test "clear finished jobs after configured period" do
+    10.times { AddToBufferJob.perform_later(2) }
+    jobs = SolidQueue::Job.last(10)
+
+    assert_no_difference -> { SolidQueue::Job.count } do
+      SolidQueue::Job.clear_finished_in_batches
+    end
+
+    # Simulate that only 5 of these jobs finished
+    jobs.sample(5).each(&:finished!)
+
+    assert_no_difference -> { SolidQueue::Job.count } do
+      SolidQueue::Job.clear_finished_in_batches
+    end
+
+    travel_to 3.days.from_now
+
+    assert_difference -> { SolidQueue::Job.count }, -5 do
+      SolidQueue::Job.clear_finished_in_batches
+    end
+  end
+
   private
     def deleting_finished_jobs
       previous, SolidQueue.preserve_finished_jobs = SolidQueue.preserve_finished_jobs, false