Merge pull request #4194 from gschlager/phpbb3-importer-no-offset

Stop using OFFSET in DB queries of phpBB3 importer

script/import_scripts/phpbb3/database/database_3_0.rb

@@ -12,8 +12,8 @@ module ImportScripts::PhpBB3
       SQL
     end
 
-    def fetch_users(offset)
-      query(<<-SQL)
+    def fetch_users(last_user_id)
+      query(<<-SQL, :user_id)
         SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip,
           u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
           u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar
@@ -23,10 +23,9 @@ module ImportScripts::PhpBB3
             u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
             (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
           )
-        WHERE u.user_type != #{Constants::USER_TYPE_IGNORE}
-        ORDER BY u.user_id ASC
+        WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE}
+        ORDER BY u.user_id
         LIMIT #{@batch_size}
-        OFFSET #{offset}
       SQL
     end
 
@@ -38,15 +37,16 @@ module ImportScripts::PhpBB3
       SQL
     end
 
-    def fetch_anonymous_users(offset)
-      query(<<-SQL)
+    def fetch_anonymous_users(last_username)
+      last_username = escape(last_username)
+
+      query(<<-SQL, :post_username)
         SELECT post_username, MIN(post_time) AS first_post_time
         FROM #{@table_prefix}_posts
-        WHERE post_username <> ''
+        WHERE post_username > '#{last_username}'
         GROUP BY post_username
-        ORDER BY post_username ASC
+        ORDER BY post_username
         LIMIT #{@batch_size}
-        OFFSET #{offset}
       SQL
     end
 
@@ -60,7 +60,7 @@ module ImportScripts::PhpBB3
             GROUP BY forum_id
           ) x ON (f.forum_id = x.forum_id)
         WHERE f.forum_type != #{Constants::FORUM_TYPE_LINK}
-        ORDER BY f.parent_id ASC, f.left_id ASC
+        ORDER BY f.parent_id, f.left_id
       SQL
     end
 
@@ -71,17 +71,17 @@ module ImportScripts::PhpBB3
       SQL
     end
 
-    def fetch_posts(offset)
-      query(<<-SQL)
+    def fetch_posts(last_post_id)
+      query(<<-SQL, :post_id)
         SELECT p.post_id, p.topic_id, t.forum_id, t.topic_title, t.topic_first_post_id, p.poster_id,
           p.post_text, p.post_time, p.post_username, t.topic_status, t.topic_type, t.poll_title,
           CASE WHEN t.poll_length > 0 THEN t.poll_start + t.poll_length ELSE NULL END AS poll_end,
           t.poll_max_options, p.post_attachment
         FROM #{@table_prefix}_posts p
           JOIN #{@table_prefix}_topics t ON (p.topic_id = t.topic_id)
-        ORDER BY p.post_id ASC
+        WHERE p.post_id > #{last_post_id}
+        ORDER BY p.post_id
         LIMIT #{@batch_size}
-        OFFSET #{offset}
       SQL
     end
 
@@ -140,7 +140,7 @@ module ImportScripts::PhpBB3
         SELECT physical_filename, real_filename
         FROM #{@table_prefix}_attachments
         WHERE topic_id = #{topic_id} AND post_msg_id = #{post_id}
-        ORDER BY filetime DESC, post_msg_id ASC
+        ORDER BY filetime DESC, post_msg_id
       SQL
     end
 
@@ -158,9 +158,9 @@ module ImportScripts::PhpBB3
       end
     end
 
-    def fetch_messages(use_fixed_messages, offset)
+    def fetch_messages(use_fixed_messages, last_msg_id)
       if use_fixed_messages
-        query(<<-SQL)
+        query(<<-SQL, :msg_id)
           SELECT m.msg_id, i.root_msg_id, m.author_id, m.message_time, m.message_subject, m.message_text,
             IFNULL(a.attachment_count, 0) AS attachment_count
           FROM #{@table_prefix}_privmsgs m
@@ -171,12 +171,12 @@ module ImportScripts::PhpBB3
               WHERE topic_id = 0
               GROUP BY post_msg_id
             ) a ON (m.msg_id = a.post_msg_id)
-          ORDER BY i.root_msg_id ASC, m.msg_id ASC
+          WHERE m.msg_id > #{last_msg_id}
+          ORDER BY i.root_msg_id, m.msg_id
           LIMIT #{@batch_size}
-          OFFSET #{offset}
         SQL
       else
-        query(<<-SQL)
+        query(<<-SQL, :msg_id)
           SELECT m.msg_id, m.root_level AS root_msg_id, m.author_id, m.message_time, m.message_subject,
             m.message_text, IFNULL(a.attachment_count, 0) AS attachment_count
           FROM #{@table_prefix}_privmsgs m
@@ -186,9 +186,9 @@ module ImportScripts::PhpBB3
               WHERE topic_id = 0
               GROUP BY post_msg_id
             ) a ON (m.msg_id = a.post_msg_id)
-          ORDER BY m.root_level ASC, m.msg_id ASC
+          WHERE m.msg_id > #{last_msg_id}
+          ORDER BY m.root_level, m.msg_id
           LIMIT #{@batch_size}
-          OFFSET #{offset}
         SQL
       end
     end
@@ -229,14 +229,14 @@ module ImportScripts::PhpBB3
       SQL
     end
 
-    def fetch_bookmarks(offset)
-      query(<<-SQL)
+    def fetch_bookmarks(last_user_id, last_topic_id)
+      query(<<-SQL, :user_id, :topic_first_post_id)
         SELECT b.user_id, t.topic_first_post_id
         FROM #{@table_prefix}_bookmarks b
           JOIN #{@table_prefix}_topics t ON (b.topic_id = t.topic_id)
-        ORDER BY b.user_id ASC, b.topic_id ASC
+        WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id}
+        ORDER BY b.user_id, b.topic_id
         LIMIT #{@batch_size}
-        OFFSET #{offset}
       SQL
     end
 
@@ -322,7 +322,7 @@ module ImportScripts::PhpBB3
               AND b.normalized_subject = i.normalized_subject
               AND a.msg_id <> m.msg_id
               AND a.message_time < m.message_time
-            ORDER BY a.message_time ASC
+            ORDER BY a.message_time
             LIMIT 1
           ), 0) ELSE i.root_msg_id END AS root_msg_id
         FROM #{@table_prefix}_privmsgs m

script/import_scripts/phpbb3/database/database_3_1.rb

@@ -3,7 +3,7 @@ require_relative '../support/constants'
 
 module ImportScripts::PhpBB3
   class Database_3_1 < Database_3_0
-    def fetch_users(offset)
+    def fetch_users(last_user_id)
       query(<<-SQL)
         SELECT u.user_id, u.user_email, u.username,
           CASE WHEN u.user_password LIKE '$2y$%'
@@ -20,10 +20,9 @@ module ImportScripts::PhpBB3
             u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
             (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
           )
-        WHERE u.user_type != #{Constants::USER_TYPE_IGNORE}
+        WHERE u.user_id > #{last_user_id} AND u.user_type != #{Constants::USER_TYPE_IGNORE}
         ORDER BY u.user_id ASC
         LIMIT #{@batch_size}
-        OFFSET #{offset}
       SQL
     end
   end

script/import_scripts/phpbb3/database/database_base.rb

@@ -12,13 +12,36 @@ module ImportScripts::PhpBB3
     protected
 
     # Executes a database query.
-    def query(sql)
-      @database_client.query(sql, cache_rows: true, symbolize_keys: true)
+    def query(sql, *last_columns)
+      rows = @database_client.query(sql, cache_rows: true, symbolize_keys: true)
+      return rows if last_columns.length == 0
+
+      result = [rows]
+      last_row = find_last_row(rows)
+
+      last_columns.each { |column| result.push(last_row ? last_row[column] : nil) }
+      result
     end
 
     # Executes a database query and returns the value of the 'count' column.
     def count(sql)
       query(sql).first[:count]
     end
+
+    def escape(value)
+      @database_client.escape(value)
+    end
+
+    private
+
+    def find_last_row(rows)
+      last_index = rows.size - 1
+
+      rows.each_with_index do |row, index|
+        return row if index == last_index
+      end
+
+      nil
+    end
   end
 end

script/import_scripts/phpbb3/importer.rb

@@ -57,9 +57,10 @@ module ImportScripts::PhpBB3
       puts '', 'creating users'
       total_count = @database.count_users
       importer = @importers.user_importer
+      last_user_id = 0
 
       batches do |offset|
-        rows = @database.fetch_users(offset)
+        rows, last_user_id = @database.fetch_users(last_user_id)
         break if rows.size < 1
 
         next if all_records_exist?(:users, importer.map_users_to_import_ids(rows))
@@ -74,9 +75,10 @@ module ImportScripts::PhpBB3
       puts '', 'creating anonymous users'
       total_count = @database.count_anonymous_users
       importer = @importers.user_importer
+      last_username = ''
 
       batches do |offset|
-        rows = @database.fetch_anonymous_users(offset)
+        rows, last_username = @database.fetch_anonymous_users(last_username)
         break if rows.size < 1
 
         next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows))
@@ -101,9 +103,10 @@ module ImportScripts::PhpBB3
       puts '', 'creating topics and posts'
       total_count = @database.count_posts
       importer = @importers.post_importer
+      last_post_id = 0
 
       batches do |offset|
-        rows = @database.fetch_posts(offset)
+        rows, last_post_id = @database.fetch_posts(last_post_id)
         break if rows.size < 1
 
         next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
@@ -123,9 +126,10 @@ module ImportScripts::PhpBB3
       puts '', 'creating private messages'
       total_count = @database.count_messages(@settings.fix_private_messages)
       importer = @importers.message_importer
+      last_msg_id = 0
 
       batches do |offset|
-        rows = @database.fetch_messages(@settings.fix_private_messages, offset)
+        rows, last_msg_id = @database.fetch_messages(@settings.fix_private_messages, last_msg_id)
         break if rows.size < 1
 
         next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
@@ -140,9 +144,10 @@ module ImportScripts::PhpBB3
       puts '', 'creating bookmarks'
       total_count = @database.count_bookmarks
       importer = @importers.bookmark_importer
+      last_user_id = last_topic_id = 0
 
       batches do |offset|
-        rows = @database.fetch_bookmarks(offset)
+        rows, last_user_id, last_topic_id = @database.fetch_bookmarks(last_user_id, last_topic_id)
         break if rows.size < 1
 
         create_bookmarks(rows, total: total_count, offset: offset) do |row|