From e19ee93ee3ecc70d4811c498a0ddcdadb0aabfeb Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Thu, 14 Jan 2016 22:43:19 +0100 Subject: [PATCH] Stop using OFFSET in DB queries of phpBB3 importer http://use-the-index-luke.com/no-offset --- .../phpbb3/database/database_3_0.rb | 56 +++++++++---------- .../phpbb3/database/database_3_1.rb | 5 +- .../phpbb3/database/database_base.rb | 27 ++++++++- script/import_scripts/phpbb3/importer.rb | 15 +++-- 4 files changed, 65 insertions(+), 38 deletions(-) diff --git a/script/import_scripts/phpbb3/database/database_3_0.rb b/script/import_scripts/phpbb3/database/database_3_0.rb index b25395aba..4c750c2de 100644 --- a/script/import_scripts/phpbb3/database/database_3_0.rb +++ b/script/import_scripts/phpbb3/database/database_3_0.rb @@ -12,8 +12,8 @@ module ImportScripts::PhpBB3 SQL end - def fetch_users(offset) - query(<<-SQL) + def fetch_users(last_user_id) + query(<<-SQL, :user_id) SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip, u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason, u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar @@ -23,10 +23,9 @@ module ImportScripts::PhpBB3 u.user_id = b.ban_userid AND b.ban_exclude = 0 AND (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP()) ) - WHERE u.user_type != #{Constants::USER_TYPE_IGNORE} - ORDER BY u.user_id ASC + WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE} + ORDER BY u.user_id LIMIT #{@batch_size} - OFFSET #{offset} SQL end @@ -38,15 +37,16 @@ module ImportScripts::PhpBB3 SQL end - def fetch_anonymous_users(offset) - query(<<-SQL) + def fetch_anonymous_users(last_username) + last_username = escape(last_username) + + query(<<-SQL, :post_username) SELECT post_username, MIN(post_time) AS first_post_time FROM #{@table_prefix}_posts - WHERE post_username <> '' + WHERE post_username > '#{last_username}' GROUP BY post_username - ORDER BY post_username ASC + ORDER BY post_username LIMIT #{@batch_size} - OFFSET #{offset} SQL end @@ -60,7 +60,7 @@ module ImportScripts::PhpBB3 GROUP BY forum_id ) x ON (f.forum_id = x.forum_id) WHERE f.forum_type != #{Constants::FORUM_TYPE_LINK} - ORDER BY f.parent_id ASC, f.left_id ASC + ORDER BY f.parent_id, f.left_id SQL end @@ -71,17 +71,17 @@ module ImportScripts::PhpBB3 SQL end - def fetch_posts(offset) - query(<<-SQL) + def fetch_posts(last_post_id) + query(<<-SQL, :post_id) SELECT p.post_id, p.topic_id, t.forum_id, t.topic_title, t.topic_first_post_id, p.poster_id, p.post_text, p.post_time, p.post_username, t.topic_status, t.topic_type, t.poll_title, CASE WHEN t.poll_length > 0 THEN t.poll_start + t.poll_length ELSE NULL END AS poll_end, t.poll_max_options, p.post_attachment FROM #{@table_prefix}_posts p JOIN #{@table_prefix}_topics t ON (p.topic_id = t.topic_id) - ORDER BY p.post_id ASC + WHERE p.post_id > #{last_post_id} + ORDER BY p.post_id LIMIT #{@batch_size} - OFFSET #{offset} SQL end @@ -140,7 +140,7 @@ module ImportScripts::PhpBB3 SELECT physical_filename, real_filename FROM #{@table_prefix}_attachments WHERE topic_id = #{topic_id} AND post_msg_id = #{post_id} - ORDER BY filetime DESC, post_msg_id ASC + ORDER BY filetime DESC, post_msg_id SQL end @@ -158,9 +158,9 @@ module ImportScripts::PhpBB3 end end - def fetch_messages(use_fixed_messages, offset) + def fetch_messages(use_fixed_messages, last_msg_id) if use_fixed_messages - query(<<-SQL) + query(<<-SQL, :msg_id) SELECT m.msg_id, i.root_msg_id, m.author_id, m.message_time, m.message_subject, m.message_text, IFNULL(a.attachment_count, 0) AS attachment_count FROM #{@table_prefix}_privmsgs m @@ -171,12 +171,12 @@ module ImportScripts::PhpBB3 WHERE topic_id = 0 GROUP BY post_msg_id ) a ON (m.msg_id = a.post_msg_id) - ORDER BY i.root_msg_id ASC, m.msg_id ASC + WHERE m.msg_id > #{last_msg_id} + ORDER BY i.root_msg_id, m.msg_id LIMIT #{@batch_size} - OFFSET #{offset} SQL else - query(<<-SQL) + query(<<-SQL, :msg_id) SELECT m.msg_id, m.root_level AS root_msg_id, m.author_id, m.message_time, m.message_subject, m.message_text, IFNULL(a.attachment_count, 0) AS attachment_count FROM #{@table_prefix}_privmsgs m @@ -186,9 +186,9 @@ module ImportScripts::PhpBB3 WHERE topic_id = 0 GROUP BY post_msg_id ) a ON (m.msg_id = a.post_msg_id) - ORDER BY m.root_level ASC, m.msg_id ASC + WHERE m.msg_id > #{last_msg_id} + ORDER BY m.root_level, m.msg_id LIMIT #{@batch_size} - OFFSET #{offset} SQL end end @@ -229,14 +229,14 @@ module ImportScripts::PhpBB3 SQL end - def fetch_bookmarks(offset) - query(<<-SQL) + def fetch_bookmarks(last_user_id, last_topic_id) + query(<<-SQL, :user_id, :topic_first_post_id) SELECT b.user_id, t.topic_first_post_id FROM #{@table_prefix}_bookmarks b JOIN #{@table_prefix}_topics t ON (b.topic_id = t.topic_id) - ORDER BY b.user_id ASC, b.topic_id ASC + WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id} + ORDER BY b.user_id, b.topic_id LIMIT #{@batch_size} - OFFSET #{offset} SQL end @@ -322,7 +322,7 @@ module ImportScripts::PhpBB3 AND b.normalized_subject = i.normalized_subject AND a.msg_id <> m.msg_id AND a.message_time < m.message_time - ORDER BY a.message_time ASC + ORDER BY a.message_time LIMIT 1 ), 0) ELSE i.root_msg_id END AS root_msg_id FROM #{@table_prefix}_privmsgs m diff --git a/script/import_scripts/phpbb3/database/database_3_1.rb b/script/import_scripts/phpbb3/database/database_3_1.rb index 2dfc5054a..c9368edf3 100644 --- a/script/import_scripts/phpbb3/database/database_3_1.rb +++ b/script/import_scripts/phpbb3/database/database_3_1.rb @@ -3,7 +3,7 @@ require_relative '../support/constants' module ImportScripts::PhpBB3 class Database_3_1 < Database_3_0 - def fetch_users(offset) + def fetch_users(last_user_id) query(<<-SQL) SELECT u.user_id, u.user_email, u.username, CASE WHEN u.user_password LIKE '$2y$%' @@ -20,10 +20,9 @@ module ImportScripts::PhpBB3 u.user_id = b.ban_userid AND b.ban_exclude = 0 AND (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP()) ) - WHERE u.user_type != #{Constants::USER_TYPE_IGNORE} + WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE} ORDER BY u.user_id ASC LIMIT #{@batch_size} - OFFSET #{offset} SQL end end diff --git a/script/import_scripts/phpbb3/database/database_base.rb b/script/import_scripts/phpbb3/database/database_base.rb index a63cecf4c..fa8cb3d45 100644 --- a/script/import_scripts/phpbb3/database/database_base.rb +++ b/script/import_scripts/phpbb3/database/database_base.rb @@ -12,13 +12,36 @@ module ImportScripts::PhpBB3 protected # Executes a database query. - def query(sql) - @database_client.query(sql, cache_rows: true, symbolize_keys: true) + def query(sql, *last_columns) + rows = @database_client.query(sql, cache_rows: true, symbolize_keys: true) + return rows if last_columns.length == 0 + + result = [rows] + last_row = find_last_row(rows) + + last_columns.each { |column| result.push(last_row ? last_row[column] : nil) } + result end # Executes a database query and returns the value of the 'count' column. def count(sql) query(sql).first[:count] end + + def escape(value) + @database_client.escape(value) + end + + private + + def find_last_row(rows) + last_index = rows.size - 1 + + rows.each_with_index do |row, index| + return row if index == last_index + end + + nil + end end end diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb index d0e39eea1..b0c46c870 100644 --- a/script/import_scripts/phpbb3/importer.rb +++ b/script/import_scripts/phpbb3/importer.rb @@ -57,9 +57,10 @@ module ImportScripts::PhpBB3 puts '', 'creating users' total_count = @database.count_users importer = @importers.user_importer + last_user_id = 0 batches do |offset| - rows = @database.fetch_users(offset) + rows, last_user_id = @database.fetch_users(last_user_id) break if rows.size < 1 next if all_records_exist?(:users, importer.map_users_to_import_ids(rows)) @@ -74,9 +75,10 @@ module ImportScripts::PhpBB3 puts '', 'creating anonymous users' total_count = @database.count_anonymous_users importer = @importers.user_importer + last_username = '' batches do |offset| - rows = @database.fetch_anonymous_users(offset) + rows, last_username = @database.fetch_anonymous_users(last_username) break if rows.size < 1 next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows)) @@ -101,9 +103,10 @@ module ImportScripts::PhpBB3 puts '', 'creating topics and posts' total_count = @database.count_posts importer = @importers.post_importer + last_post_id = 0 batches do |offset| - rows = @database.fetch_posts(offset) + rows, last_post_id = @database.fetch_posts(last_post_id) break if rows.size < 1 next if all_records_exist?(:posts, importer.map_to_import_ids(rows)) @@ -123,9 +126,10 @@ module ImportScripts::PhpBB3 puts '', 'creating private messages' total_count = @database.count_messages(@settings.fix_private_messages) importer = @importers.message_importer + last_msg_id = 0 batches do |offset| - rows = @database.fetch_messages(@settings.fix_private_messages, offset) + rows, last_msg_id = @database.fetch_messages(@settings.fix_private_messages, last_msg_id) break if rows.size < 1 next if all_records_exist?(:posts, importer.map_to_import_ids(rows)) @@ -140,9 +144,10 @@ module ImportScripts::PhpBB3 puts '', 'creating bookmarks' total_count = @database.count_bookmarks importer = @importers.bookmark_importer + last_user_id = last_topic_id = 0 batches do |offset| - rows = @database.fetch_bookmarks(offset) + rows, last_user_id, last_topic_id = @database.fetch_bookmarks(last_user_id, last_topic_id) break if rows.size < 1 create_bookmarks(rows, total: total_count, offset: offset) do |row|