From 26ffa8eab36dc1e2b30873c302afea476df432a4 Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Fri, 6 Jan 2017 09:38:14 -0600 Subject: [PATCH 1/8] Process DB migration 5000 rows at a time instead of all at once. This should allow migrations to work for sites that have lower memory limits, a very large amount of data, or both. Returning all the rows of the stream_tmp table at once can take a lot of memory if there's a lot of data there, causing the migration process to fail. --- includes/db-updates.php | 46 +++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 87a0e55b0..87bf869b0 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -60,31 +60,37 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $plugin = wp_stream_get_instance(); $plugin->install->install( $current_version ); - $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp" ); + $starting_row = 0; + $rows_per_round = 5000; - foreach ( $stream_entries as $entry ) { - $context = $wpdb->get_row( - $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_context_tmp WHERE record_id = %s LIMIT 1", $entry->ID ) - ); + while ( ! empty( $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ) ) ) { + foreach ( $stream_entries as $entry ) { + $context = $wpdb->get_row( + $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_context_tmp WHERE record_id = %s LIMIT 1", + $entry->ID ) + ); + + $new_entry = array( + 'site_id' => $entry->site_id, + 'blog_id' => $entry->blog_id, + 'user_id' => $entry->author, + 'user_role' => $entry->author_role, + 'summary' => $entry->summary, + 'created' => $entry->created, + 'connector' => $context->connector, + 'context' => $context->context, + 'action' => $context->action, + 'ip' => $entry->ip, + ); - $new_entry = array( - 'site_id' => $entry->site_id, - 'blog_id' => $entry->blog_id, - 'user_id' => $entry->author, - 'user_role' => $entry->author_role, - 'summary' => $entry->summary, - 'created' => $entry->created, - 'connector' => $context->connector, - 'context' => $context->context, - 'action' => $context->action, - 'ip' => $entry->ip, - ); + if ( $entry->object_id && 0 !== $entry->object_id ) { + $new_entry['object_id'] = $entry->object_id; + } - if ( $entry->object_id && 0 !== $entry->object_id ) { - $new_entry['object_id'] = $entry->object_id; + $wpdb->insert( $wpdb->base_prefix . 'stream', $new_entry ); } - $wpdb->insert( $wpdb->base_prefix . 'stream', $new_entry ); + $starting_row += $rows_per_round; } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From c0b51e98c25e2c8747c1049d7d7ded3f5d237890 Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Mon, 9 Jan 2017 10:38:24 -0600 Subject: [PATCH 2/8] Split retrieval of stream entries and while loop to make it easier to read --- includes/db-updates.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 87bf869b0..77e5f36f1 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,7 +63,9 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - while ( ! empty( $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ) ) ) { + $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ); + + while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { $context = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_context_tmp WHERE record_id = %s LIMIT 1", @@ -91,6 +93,8 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { } $starting_row += $rows_per_round; + + $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ); } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From d9b7a6c42d76c028d10be034f2cf69d1ca7529af Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 10:55:13 -0600 Subject: [PATCH 3/8] Use $wpdb->prepare instead of passing raw variables to $wpdb->get_results --- includes/db-updates.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 77e5f36f1..3dbc837d0 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,7 +63,8 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ); + $prepared_query = $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round); + $stream_entries = $wpdb->get_results( $prepared_query ); while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { @@ -94,7 +95,8 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row += $rows_per_round; - $stream_entries = $wpdb->get_results( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT $starting_row, $rows_per_round" ); + $prepared_query = $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round); + $stream_entries = $wpdb->get_results( $prepared_query ); } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From 2fb8d3574065dda4f32beaf7277fa726fa61852a Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 10:56:20 -0600 Subject: [PATCH 4/8] Formatting updates for WP coding standards --- includes/db-updates.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 3dbc837d0..4e24aad2b 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,7 +63,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - $prepared_query = $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round); + $prepared_query = $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ); $stream_entries = $wpdb->get_results( $prepared_query ); while ( ! empty( $stream_entries ) ) { @@ -95,7 +95,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row += $rows_per_round; - $prepared_query = $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round); + $prepared_query = $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ); $stream_entries = $wpdb->get_results( $prepared_query ); } From d580de2445c510edb84c6d0ae64d2a183fbfab49 Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 11:05:02 -0600 Subject: [PATCH 5/8] Move $wpdb->prepare directly into $wpdb->get_results --- includes/db-updates.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 4e24aad2b..046f0c868 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,8 +63,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - $prepared_query = $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ); - $stream_entries = $wpdb->get_results( $prepared_query ); + $stream_entries = $wpdb->get_results( $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round) ); while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { @@ -95,8 +94,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row += $rows_per_round; - $prepared_query = $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ); - $stream_entries = $wpdb->get_results( $prepared_query ); + $stream_entries = $wpdb->get_results( $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round) ); } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From 775dfa4da51c3af4c132d2d5338d8a431b1beda8 Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 11:08:11 -0600 Subject: [PATCH 6/8] More formatting updates --- includes/db-updates.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 046f0c868..c30abe160 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,7 +63,8 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - $stream_entries = $wpdb->get_results( $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round) ); + $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", + $starting_row, $rows_per_round ) ); while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { @@ -94,7 +95,8 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row += $rows_per_round; - $stream_entries = $wpdb->get_results( $wpdb->prepare("SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round) ); + $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", + $starting_row, $rows_per_round ) ); } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From 1d7844d9234ee578575dda435b00901dd4e80d83 Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 11:10:51 -0600 Subject: [PATCH 7/8] Move function calls to single line --- includes/db-updates.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index c30abe160..093eab53f 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -63,8 +63,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row = 0; $rows_per_round = 5000; - $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", - $starting_row, $rows_per_round ) ); + $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ) ); while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { @@ -95,8 +94,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { $starting_row += $rows_per_round; - $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", - $starting_row, $rows_per_round ) ); + $stream_entries = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_tmp LIMIT %d, %d", $starting_row, $rows_per_round ) ); } $wpdb->query( "DROP TABLE {$wpdb->base_prefix}stream_tmp, {$wpdb->base_prefix}stream_context_tmp" ); From 2c5d7e25ef35bc6a02df0fe5af27be2d0347271a Mon Sep 17 00:00:00 2001 From: John Oleksowicz Date: Tue, 10 Jan 2017 11:12:45 -0600 Subject: [PATCH 8/8] Move another multiline function call to single line --- includes/db-updates.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/includes/db-updates.php b/includes/db-updates.php index 093eab53f..e156b690e 100644 --- a/includes/db-updates.php +++ b/includes/db-updates.php @@ -68,8 +68,7 @@ function wp_stream_update_auto_300( $db_version, $current_version ) { while ( ! empty( $stream_entries ) ) { foreach ( $stream_entries as $entry ) { $context = $wpdb->get_row( - $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_context_tmp WHERE record_id = %s LIMIT 1", - $entry->ID ) + $wpdb->prepare( "SELECT * FROM {$wpdb->base_prefix}stream_context_tmp WHERE record_id = %s LIMIT 1", $entry->ID ) ); $new_entry = array(