From c145e747b62f81922c253d1363c7edb3c66130ca Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Mon, 11 Jul 2016 23:59:15 +0200 Subject: [PATCH] A few small changes to the phpBB3 importer (#4321) * Reconnect to phpBB3 database on connection loss * Map geek smiley to :nerd: emoji in phpBB3 importer * Import PMs to yourself from phpBB3 * Allow empty table prefix in phpBB3 importer --- .../phpbb3/database/database.rb | 5 +- .../phpbb3/database/database_3_0.rb | 88 +++++++++---------- .../phpbb3/database/database_3_1.rb | 8 +- .../phpbb3/importers/message_importer.rb | 5 +- script/import_scripts/phpbb3/settings.yml | 4 +- .../phpbb3/support/smiley_processor.rb | 3 +- 6 files changed, 57 insertions(+), 56 deletions(-) diff --git a/script/import_scripts/phpbb3/database/database.rb b/script/import_scripts/phpbb3/database/database.rb index b63d03563..fa8835234 100644 --- a/script/import_scripts/phpbb3/database/database.rb +++ b/script/import_scripts/phpbb3/database/database.rb @@ -37,7 +37,8 @@ module ImportScripts::PhpBB3 port: @database_settings.port, username: @database_settings.username, password: @database_settings.password, - database: @database_settings.schema + database: @database_settings.schema, + reconnect: true ) end @@ -46,7 +47,7 @@ module ImportScripts::PhpBB3 @database_client.query(<<-SQL, cache_rows: false, symbolize_keys: true).first[:config_value] SELECT config_value - FROM #{table_prefix}_config + FROM #{table_prefix}config WHERE config_name = 'version' SQL end diff --git a/script/import_scripts/phpbb3/database/database_3_0.rb b/script/import_scripts/phpbb3/database/database_3_0.rb index 94ec4ca50..c596b3733 100644 --- a/script/import_scripts/phpbb3/database/database_3_0.rb +++ b/script/import_scripts/phpbb3/database/database_3_0.rb @@ -6,8 +6,8 @@ module ImportScripts::PhpBB3 def count_users count(<<-SQL) SELECT COUNT(*) AS count - FROM #{@table_prefix}_users u - JOIN #{@table_prefix}_groups g ON g.group_id = u.group_id + FROM #{@table_prefix}users u + JOIN #{@table_prefix}groups g ON g.group_id = u.group_id WHERE u.user_type != #{Constants::USER_TYPE_IGNORE} SQL end @@ -17,9 +17,9 @@ module ImportScripts::PhpBB3 SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip, u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason, u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar - FROM #{@table_prefix}_users u - JOIN #{@table_prefix}_groups g ON (g.group_id = u.group_id) - LEFT OUTER JOIN #{@table_prefix}_banlist b ON ( + FROM #{@table_prefix}users u + JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id) + LEFT OUTER JOIN #{@table_prefix}banlist b ON ( u.user_id = b.ban_userid AND b.ban_exclude = 0 AND (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP()) ) @@ -32,7 +32,7 @@ module ImportScripts::PhpBB3 def count_anonymous_users count(<<-SQL) SELECT COUNT(DISTINCT post_username) AS count - FROM #{@table_prefix}_posts + FROM #{@table_prefix}posts WHERE post_username <> '' SQL end @@ -42,7 +42,7 @@ module ImportScripts::PhpBB3 query(<<-SQL, :post_username) SELECT post_username, MIN(post_time) AS first_post_time - FROM #{@table_prefix}_posts + FROM #{@table_prefix}posts WHERE post_username > '#{last_username}' GROUP BY post_username ORDER BY post_username @@ -53,10 +53,10 @@ module ImportScripts::PhpBB3 def fetch_categories query(<<-SQL) SELECT f.forum_id, f.parent_id, f.forum_name, f.forum_desc, x.first_post_time - FROM #{@table_prefix}_forums f + FROM #{@table_prefix}forums f LEFT OUTER JOIN ( SELECT MIN(topic_time) AS first_post_time, forum_id - FROM #{@table_prefix}_topics + FROM #{@table_prefix}topics GROUP BY forum_id ) x ON (f.forum_id = x.forum_id) WHERE f.forum_type != #{Constants::FORUM_TYPE_LINK} @@ -67,7 +67,7 @@ module ImportScripts::PhpBB3 def count_posts count(<<-SQL) SELECT COUNT(*) AS count - FROM #{@table_prefix}_posts + FROM #{@table_prefix}posts SQL end @@ -77,8 +77,8 @@ module ImportScripts::PhpBB3 p.post_text, p.post_time, p.post_username, t.topic_status, t.topic_type, t.poll_title, CASE WHEN t.poll_length > 0 THEN t.poll_start + t.poll_length ELSE NULL END AS poll_end, t.poll_max_options, p.post_attachment - FROM #{@table_prefix}_posts p - JOIN #{@table_prefix}_topics t ON (p.topic_id = t.topic_id) + FROM #{@table_prefix}posts p + JOIN #{@table_prefix}topics t ON (p.topic_id = t.topic_id) WHERE p.post_id > #{last_post_id} ORDER BY p.post_id LIMIT #{@batch_size} @@ -88,7 +88,7 @@ module ImportScripts::PhpBB3 def get_first_post_id(topic_id) query(<<-SQL).try(:first).try(:[], :topic_first_post_id) SELECT topic_first_post_id - FROM #{@table_prefix}_topics + FROM #{@table_prefix}topics WHERE topic_id = #{topic_id} SQL end @@ -98,12 +98,12 @@ module ImportScripts::PhpBB3 SELECT o.poll_option_id, o.poll_option_text, o.poll_option_total AS total_votes, o.poll_option_total - ( SELECT COUNT(DISTINCT v.vote_user_id) - FROM #{@table_prefix}_poll_votes v - JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id) - JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id) + FROM #{@table_prefix}poll_votes v + JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id) + JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id) WHERE v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id ) AS anonymous_votes - FROM #{@table_prefix}_poll_options o + FROM #{@table_prefix}poll_options o WHERE o.topic_id = #{topic_id} ORDER BY o.poll_option_id SQL @@ -113,10 +113,10 @@ module ImportScripts::PhpBB3 # this query ignores invalid votes that belong to non-existent users or topics query(<<-SQL) SELECT u.user_id, v.poll_option_id - FROM #{@table_prefix}_poll_votes v - JOIN #{@table_prefix}_poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id) - JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id) - JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id) + FROM #{@table_prefix}poll_votes v + JOIN #{@table_prefix}poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id) + JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id) + JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id) WHERE v.topic_id = #{topic_id} SQL end @@ -127,19 +127,19 @@ module ImportScripts::PhpBB3 SELECT MAX(x.total_voters) AS total_voters, MAX(x.total_voters) - ( SELECT COUNT(DISTINCT v.vote_user_id) - FROM #{@table_prefix}_poll_votes v - JOIN #{@table_prefix}_poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id) - JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id) - JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id) + FROM #{@table_prefix}poll_votes v + JOIN #{@table_prefix}poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id) + JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id) + JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id) WHERE v.topic_id = #{topic_id} ) AS anonymous_voters FROM ( SELECT COUNT(DISTINCT vote_user_id) AS total_voters - FROM #{@table_prefix}_poll_votes + FROM #{@table_prefix}poll_votes WHERE topic_id = #{topic_id} UNION SELECT MAX(poll_option_total) AS total_voters - FROM #{@table_prefix}_poll_options + FROM #{@table_prefix}poll_options WHERE topic_id = #{topic_id} ) x SQL @@ -148,14 +148,14 @@ module ImportScripts::PhpBB3 def get_max_attachment_size query(<<-SQL).first[:filesize] SELECT IFNULL(MAX(filesize), 0) AS filesize - FROM #{@table_prefix}_attachments + FROM #{@table_prefix}attachments SQL end def fetch_attachments(topic_id, post_id) query(<<-SQL) SELECT physical_filename, real_filename - FROM #{@table_prefix}_attachments + FROM #{@table_prefix}attachments WHERE topic_id = #{topic_id} AND post_msg_id = #{post_id} ORDER BY filetime DESC, post_msg_id SQL @@ -164,10 +164,10 @@ module ImportScripts::PhpBB3 def count_messages count(<<-SQL) SELECT COUNT(*) AS count - FROM #{@table_prefix}_privmsgs m + FROM #{@table_prefix}privmsgs m WHERE NOT EXISTS ( -- ignore duplicate messages SELECT 1 - FROM #{@table_prefix}_privmsgs x + FROM #{@table_prefix}privmsgs x WHERE x.msg_id < m.msg_id AND x.root_level = m.root_level AND x.author_id = m.author_id AND x.to_address = m.to_address AND x.message_time = m.message_time ) @@ -179,15 +179,15 @@ module ImportScripts::PhpBB3 SELECT m.msg_id, m.root_level AS root_msg_id, m.author_id, m.message_time, m.message_subject, m.message_text, m.to_address, r.author_id AS root_author_id, r.to_address AS root_to_address, ( SELECT COUNT(*) - FROM #{@table_prefix}_attachments a + FROM #{@table_prefix}attachments a WHERE a.topic_id = 0 AND m.msg_id = a.post_msg_id ) AS attachment_count - FROM #{@table_prefix}_privmsgs m - LEFT OUTER JOIN #{@table_prefix}_privmsgs r ON (m.root_level = r.msg_id) + FROM #{@table_prefix}privmsgs m + LEFT OUTER JOIN #{@table_prefix}privmsgs r ON (m.root_level = r.msg_id) WHERE m.msg_id > #{last_msg_id} AND NOT EXISTS ( -- ignore duplicate messages SELECT 1 - FROM #{@table_prefix}_privmsgs x + FROM #{@table_prefix}privmsgs x WHERE x.msg_id < m.msg_id AND x.root_level = m.root_level AND x.author_id = m.author_id AND x.to_address = m.to_address AND x.message_time = m.message_time ) @@ -199,15 +199,15 @@ module ImportScripts::PhpBB3 def count_bookmarks count(<<-SQL) SELECT COUNT(*) AS count - FROM #{@table_prefix}_bookmarks + FROM #{@table_prefix}bookmarks SQL end def fetch_bookmarks(last_user_id, last_topic_id) query(<<-SQL, :user_id, :topic_first_post_id) SELECT b.user_id, t.topic_first_post_id - FROM #{@table_prefix}_bookmarks b - JOIN #{@table_prefix}_topics t ON (b.topic_id = t.topic_id) + FROM #{@table_prefix}bookmarks b + JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id) WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id} ORDER BY b.user_id, b.topic_id LIMIT #{@batch_size} @@ -217,12 +217,12 @@ module ImportScripts::PhpBB3 def get_config_values query(<<-SQL).first SELECT - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'version') AS phpbb_version, - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_gallery_path') AS avatar_gallery_path, - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_path') AS avatar_path, - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_salt') AS avatar_salt, - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'smilies_path') AS smilies_path, - (SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'upload_path') AS attachment_path + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'version') AS phpbb_version, + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_gallery_path') AS avatar_gallery_path, + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_path') AS avatar_path, + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_salt') AS avatar_salt, + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'smilies_path') AS smilies_path, + (SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'upload_path') AS attachment_path SQL end end diff --git a/script/import_scripts/phpbb3/database/database_3_1.rb b/script/import_scripts/phpbb3/database/database_3_1.rb index 2d18880ab..0bd264856 100644 --- a/script/import_scripts/phpbb3/database/database_3_1.rb +++ b/script/import_scripts/phpbb3/database/database_3_1.rb @@ -13,10 +13,10 @@ module ImportScripts::PhpBB3 u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason, u.user_posts, f.pf_phpbb_website AS user_website, f.pf_phpbb_location AS user_from, u.user_birthday, u.user_avatar_type, u.user_avatar - FROM #{@table_prefix}_users u - LEFT OUTER JOIN #{@table_prefix}_profile_fields_data f ON (u.user_id = f.user_id) - JOIN #{@table_prefix}_groups g ON (g.group_id = u.group_id) - LEFT OUTER JOIN #{@table_prefix}_banlist b ON ( + FROM #{@table_prefix}users u + LEFT OUTER JOIN #{@table_prefix}profile_fields_data f ON (u.user_id = f.user_id) + JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id) + LEFT OUTER JOIN #{@table_prefix}banlist b ON ( u.user_id = b.ban_userid AND b.ban_exclude = 0 AND (b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP()) ) diff --git a/script/import_scripts/phpbb3/importers/message_importer.rb b/script/import_scripts/phpbb3/importers/message_importer.rb index c164806b6..1289122cb 100644 --- a/script/import_scripts/phpbb3/importers/message_importer.rb +++ b/script/import_scripts/phpbb3/importers/message_importer.rb @@ -56,7 +56,7 @@ module ImportScripts::PhpBB3 mapped[:target_usernames] = get_recipient_usernames(row) mapped[:custom_fields] = {import_user_ids: current_user_ids.join(',')} - if mapped[:target_usernames].empty? # pm with yourself? + if mapped[:target_usernames].empty? puts "Private message without recipients. Skipping #{row[:msg_id]}: #{row[:message_subject][0..40]}" return nil end @@ -80,11 +80,10 @@ module ImportScripts::PhpBB3 end def get_recipient_usernames(row) - author_id = row[:author_id].to_s import_user_ids = get_recipient_user_ids(row[:to_address]) import_user_ids.map! do |import_user_id| - import_user_id.to_s == author_id ? nil : @lookup.find_user_by_import_id(import_user_id).try(:username) + @lookup.find_user_by_import_id(import_user_id).try(:username) end.compact end diff --git a/script/import_scripts/phpbb3/settings.yml b/script/import_scripts/phpbb3/settings.yml index 8377860e8..eff7ac3a6 100644 --- a/script/import_scripts/phpbb3/settings.yml +++ b/script/import_scripts/phpbb3/settings.yml @@ -1,13 +1,13 @@ # This is an example settings file for the phpBB3 importer. database: - type: MySQL # currently only MySQL is supported - more to come soon + type: MySQL # currently only MySQL is supported host: localhost port: 3306 username: root password: schema: phpbb - table_prefix: phpbb # Usually all table names start with phpbb. Change this, if your forum is using a different prefix. + table_prefix: phpbb_ # Usually all table names start with phpbb_. Change this, if your forum is using a different prefix. batch_size: 1000 # Don't change this unless you know what you're doing. The default (1000) should work just fine. import: diff --git a/script/import_scripts/phpbb3/support/smiley_processor.rb b/script/import_scripts/phpbb3/support/smiley_processor.rb index 342cadfaa..9b4d643d0 100644 --- a/script/import_scripts/phpbb3/support/smiley_processor.rb +++ b/script/import_scripts/phpbb3/support/smiley_processor.rb @@ -47,7 +47,8 @@ module ImportScripts::PhpBB3 [':?:'] => ':question:', [':idea:'] => ':bulb:', [':arrow:'] => ':arrow_right:', - [':|', ':-|'] => ':neutral_face:' + [':|', ':-|'] => ':neutral_face:', + [':geek:'] => ':nerd:' }.each do |smilies, emoji| smilies.each { |smiley| @smiley_map[smiley] = emoji } end