From ca3a9e8d279f12e6b520cbc9f808afc0097306e9 Mon Sep 17 00:00:00 2001 From: Mitchell Henke Date: Tue, 18 Oct 2022 21:18:15 -0500 Subject: [PATCH 1/2] Add index to `good_jobs` to improve querying candidate jobs --- .../install/migrations/create_good_jobs.rb.erb | 2 ++ ...n_priority_created_at_when_unfinished.rb.erb | 17 +++++++++++++++++ ...bs_on_priority_created_at_when_unfinished.rb | 17 +++++++++++++++++ spec/test_app/db/schema.rb | 3 ++- 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb create mode 100644 spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb diff --git a/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb b/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb index 4e5dc3878..3319097a8 100644 --- a/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb +++ b/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb @@ -41,5 +41,7 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %> add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at + add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished end end diff --git a/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb b/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb new file mode 100644 index 000000000..d3fab1e6e --- /dev/null +++ b/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +class CreateIndexGoodJobsJobsOnPriorityCreatedAtWhenUnfinished < ActiveRecord::Migration<%= migration_version %> + disable_ddl_transaction! + + def change + reversible do |dir| + dir.up do + # Ensure this incremental update migration is idempotent + # with monolithic install migration. + return if connection.index_name_exists?(:good_jobs, :index_good_jobs_jobs_on_priority_created_at_when_unfinished) + end + end + + add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + end +end diff --git a/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb b/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb new file mode 100644 index 000000000..1bdacbe1b --- /dev/null +++ b/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +class CreateIndexGoodJobsJobsOnPriorityCreatedAtWhenUnfinished < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def change + reversible do |dir| + dir.up do + # Ensure this incremental update migration is idempotent + # with monolithic install migration. + return if connection.index_name_exists?(:good_jobs, :index_good_jobs_jobs_on_priority_created_at_when_unfinished) + end + end + + add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + end +end diff --git a/spec/test_app/db/schema.rb b/spec/test_app/db/schema.rb index 16a6ec07d..788be6b4a 100644 --- a/spec/test_app/db/schema.rb +++ b/spec/test_app/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_07_20_154438) do +ActiveRecord::Schema.define(version: 2022_10_19_021425) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" @@ -51,6 +51,7 @@ t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" + t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" end From 9a73ab55af909881678bb9c91d233cd45c99ab93 Mon Sep 17 00:00:00 2001 From: Ben Sheldon Date: Sat, 22 Oct 2022 08:56:10 -0700 Subject: [PATCH 2/2] Migration should create index concurrently --- .../templates/install/migrations/create_good_jobs.rb.erb | 2 +- ...d_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb | 5 +++-- ..._good_jobs_jobs_on_priority_created_at_when_unfinished.rb | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb b/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb index 3319097a8..87653b901 100644 --- a/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb +++ b/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb @@ -41,7 +41,7 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %> add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at - add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, + add_index :good_jobs, [:priority, :created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished end end diff --git a/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb b/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb index d3fab1e6e..6b84e23b7 100644 --- a/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb +++ b/lib/generators/good_job/templates/update/migrations/03_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb.erb @@ -11,7 +11,8 @@ class CreateIndexGoodJobsJobsOnPriorityCreatedAtWhenUnfinished < ActiveRecord::M end end - add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, - where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + add_index :good_jobs, [:priority, :created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished, + algorithm: :concurrently end end diff --git a/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb b/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb index 1bdacbe1b..af73e8255 100644 --- a/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb +++ b/spec/test_app/db/migrate/20221019021425_create_index_good_jobs_jobs_on_priority_created_at_when_unfinished.rb @@ -11,7 +11,8 @@ def change end end - add_index :good_jobs, [:priority, :created_at], order: {priority: "DESC NULLS LAST", created_at: :asc}, - where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished + add_index :good_jobs, [:priority, :created_at], order: { priority: "DESC NULLS LAST", created_at: :asc }, + where: "finished_at IS NULL", name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished, + algorithm: :concurrently end end