Skip to content

Commit

Permalink
perf: update delete in php to SQL + optimised query
Browse files Browse the repository at this point in the history
Locally improved from ~1.4s to ~0.42s = ~70% faster.

Also tweaked comments.
  • Loading branch information
keevan committed Aug 30, 2024
1 parent 65b1657 commit 9a371ee
Showing 1 changed file with 12 additions and 44 deletions.
56 changes: 12 additions & 44 deletions db/upgrade.php
Original file line number Diff line number Diff line change
Expand Up @@ -491,53 +491,21 @@ function xmldb_tool_excimer_upgrade($oldversion) {
}

if ($oldversion < 2024082301) {

// Change precision of name to 255 so it can be used as an index.
// First we need to drop a few edge cases that have a length of 256.
$DB->delete_records_select('tool_excimer_page_groups', $DB->sql_length('name') . ' > 255');

// Find all non-unique page groups and remove duplicates.
$sql = " SELECT pgroups.id,
LOWER(pgroups.name) name,
pgroups.month,
pgroups.fuzzycount
FROM {tool_excimer_page_groups} pgroups
JOIN (
SELECT LOWER(name) name,
month
FROM {tool_excimer_page_groups}
GROUP BY LOWER(name),
month
HAVING COUNT(*) > 1
) dupl ON LOWER(pgroups.name) = dupl.name AND pgroups.month = dupl.month
ORDER BY LOWER(pgroups.name),
pgroups.month,
pgroups.fuzzycount DESC
";
$duplicates = $DB->get_records_sql($sql);

if (!empty($duplicates)) {
$previouskey = '';
$removeids = [];

// Duplicates are ordered, so only keep the first occurence.
foreach ($duplicates as $row) {
$key = $row->name . '_' . $row->month;
if ($key === $previouskey) {
$removeids[] = $row->id;
} else {
$previouskey = $key;
}
}

// Remove the duplicate rows.
if (!empty($removeids)) {
$removeids = implode(',', $removeids);
$DB->delete_records_select('tool_excimer_page_groups', "id IN ($removeids)");
}
}

// Changing precision of field name on table tool_excimer_page_groups to (255).
// SQL to delete duplicates while keeping the first occurrence.
$sql = "DELETE FROM {tool_excimer_page_groups}
WHERE id NOT IN (
SELECT MIN(id)
FROM {tool_excimer_page_groups}
GROUP BY LOWER(name), month
)";

// Execute the delete query.
$DB->execute($sql);

// Change precision of name to 255 so it can be used as an index.
$table = new xmldb_table('tool_excimer_page_groups');
$field = new xmldb_field('name', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, 'id');

Expand Down

0 comments on commit 9a371ee

Please sign in to comment.