codeninjasllc/discourse
1
0
Fork 0
mirror of https://github.com/codeninjasllc/discourse.git synced 2024-11-27 09:36:19 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

A few small changes to the phpBB3 importer (#4321)

Browse source 
* Reconnect to phpBB3 database on connection loss

* Map geek smiley to :nerd: emoji in phpBB3 importer

* Import PMs to yourself from phpBB3

* Allow empty table prefix in phpBB3 importer
This commit is contained in:
Gerhard Schlager 2016-07-11 23:59:15 +02:00 committed by Régis Hanol
parent 4d65370797
commit c145e747b6
6 changed files with 57 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
script/import_scripts/phpbb3/database/database.rb
View file

@ -37,7 +37,8 @@ module ImportScripts::PhpBB3
port: @database_settings.port,
username: @database_settings.username,
password: @database_settings.password,
database: @database_settings.schema
database: @database_settings.schema,
reconnect: true
)
end
@ -46,7 +47,7 @@ module ImportScripts::PhpBB3
@database_client.query(<<-SQL, cache_rows: false, symbolize_keys: true).first[:config_value]
SELECT config_value
FROM #{table_prefix}_config
FROM #{table_prefix}config
WHERE config_name = 'version'
SQL
end

88
script/import_scripts/phpbb3/database/database_3_0.rb
View file

@ -6,8 +6,8 @@ module ImportScripts::PhpBB3
def count_users
count(<<-SQL)
SELECT COUNT(*) AS count
FROM #{@table_prefix}_users u
JOIN #{@table_prefix}_groups g ON g.group_id = u.group_id
FROM #{@table_prefix}users u
JOIN #{@table_prefix}groups g ON g.group_id = u.group_id
WHERE u.user_type != #{Constants::USER_TYPE_IGNORE}
SQL
end
@ -17,9 +17,9 @@ module ImportScripts::PhpBB3
SELECT u.user_id, u.user_email, u.username, u.user_password, u.user_regdate, u.user_lastvisit, u.user_ip,
u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
u.user_posts, u.user_website, u.user_from, u.user_birthday, u.user_avatar_type, u.user_avatar
FROM #{@table_prefix}_users u
JOIN #{@table_prefix}_groups g ON (g.group_id = u.group_id)
LEFT OUTER JOIN #{@table_prefix}_banlist b ON (
FROM #{@table_prefix}users u
JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id)
LEFT OUTER JOIN #{@table_prefix}banlist b ON (
u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
(b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
)
@ -32,7 +32,7 @@ module ImportScripts::PhpBB3
def count_anonymous_users
count(<<-SQL)
SELECT COUNT(DISTINCT post_username) AS count
FROM #{@table_prefix}_posts
FROM #{@table_prefix}posts
WHERE post_username <> ''
SQL
end
@ -42,7 +42,7 @@ module ImportScripts::PhpBB3
query(<<-SQL, :post_username)
SELECT post_username, MIN(post_time) AS first_post_time
FROM #{@table_prefix}_posts
FROM #{@table_prefix}posts
WHERE post_username > '#{last_username}'
GROUP BY post_username
ORDER BY post_username
@ -53,10 +53,10 @@ module ImportScripts::PhpBB3
def fetch_categories
query(<<-SQL)
SELECT f.forum_id, f.parent_id, f.forum_name, f.forum_desc, x.first_post_time
FROM #{@table_prefix}_forums f
FROM #{@table_prefix}forums f
LEFT OUTER JOIN (
SELECT MIN(topic_time) AS first_post_time, forum_id
FROM #{@table_prefix}_topics
FROM #{@table_prefix}topics
GROUP BY forum_id
) x ON (f.forum_id = x.forum_id)
WHERE f.forum_type != #{Constants::FORUM_TYPE_LINK}
@ -67,7 +67,7 @@ module ImportScripts::PhpBB3
def count_posts
count(<<-SQL)
SELECT COUNT(*) AS count
FROM #{@table_prefix}_posts
FROM #{@table_prefix}posts
SQL
end
@ -77,8 +77,8 @@ module ImportScripts::PhpBB3
p.post_text, p.post_time, p.post_username, t.topic_status, t.topic_type, t.poll_title,
CASE WHEN t.poll_length > 0 THEN t.poll_start + t.poll_length ELSE NULL END AS poll_end,
t.poll_max_options, p.post_attachment
FROM #{@table_prefix}_posts p
JOIN #{@table_prefix}_topics t ON (p.topic_id = t.topic_id)
FROM #{@table_prefix}posts p
JOIN #{@table_prefix}topics t ON (p.topic_id = t.topic_id)
WHERE p.post_id > #{last_post_id}
ORDER BY p.post_id
LIMIT #{@batch_size}
@ -88,7 +88,7 @@ module ImportScripts::PhpBB3
def get_first_post_id(topic_id)
query(<<-SQL).try(:first).try(:[], :topic_first_post_id)
SELECT topic_first_post_id
FROM #{@table_prefix}_topics
FROM #{@table_prefix}topics
WHERE topic_id = #{topic_id}
SQL
end
@ -98,12 +98,12 @@ module ImportScripts::PhpBB3
SELECT o.poll_option_id, o.poll_option_text, o.poll_option_total AS total_votes,
o.poll_option_total - (
SELECT COUNT(DISTINCT v.vote_user_id)
FROM #{@table_prefix}_poll_votes v
JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id)
FROM #{@table_prefix}poll_votes v
JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id)
WHERE v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id
) AS anonymous_votes
FROM #{@table_prefix}_poll_options o
FROM #{@table_prefix}poll_options o
WHERE o.topic_id = #{topic_id}
ORDER BY o.poll_option_id
SQL
@ -113,10 +113,10 @@ module ImportScripts::PhpBB3
# this query ignores invalid votes that belong to non-existent users or topics
query(<<-SQL)
SELECT u.user_id, v.poll_option_id
FROM #{@table_prefix}_poll_votes v
JOIN #{@table_prefix}_poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id)
JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id)
FROM #{@table_prefix}poll_votes v
JOIN #{@table_prefix}poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id)
JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id)
WHERE v.topic_id = #{topic_id}
SQL
end
@ -127,19 +127,19 @@ module ImportScripts::PhpBB3
SELECT MAX(x.total_voters) AS total_voters,
MAX(x.total_voters) - (
SELECT COUNT(DISTINCT v.vote_user_id)
FROM #{@table_prefix}_poll_votes v
JOIN #{@table_prefix}_poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id)
JOIN #{@table_prefix}_users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}_topics t ON (v.topic_id = t.topic_id)
FROM #{@table_prefix}poll_votes v
JOIN #{@table_prefix}poll_options o ON (v.poll_option_id = o.poll_option_id AND v.topic_id = o.topic_id)
JOIN #{@table_prefix}users u ON (v.vote_user_id = u.user_id)
JOIN #{@table_prefix}topics t ON (v.topic_id = t.topic_id)
WHERE v.topic_id = #{topic_id}
) AS anonymous_voters
FROM (
SELECT COUNT(DISTINCT vote_user_id) AS total_voters
FROM #{@table_prefix}_poll_votes
FROM #{@table_prefix}poll_votes
WHERE topic_id = #{topic_id}
UNION
SELECT MAX(poll_option_total) AS total_voters
FROM #{@table_prefix}_poll_options
FROM #{@table_prefix}poll_options
WHERE topic_id = #{topic_id}
) x
SQL
@ -148,14 +148,14 @@ module ImportScripts::PhpBB3
def get_max_attachment_size
query(<<-SQL).first[:filesize]
SELECT IFNULL(MAX(filesize), 0) AS filesize
FROM #{@table_prefix}_attachments
FROM #{@table_prefix}attachments
SQL
end
def fetch_attachments(topic_id, post_id)
query(<<-SQL)
SELECT physical_filename, real_filename
FROM #{@table_prefix}_attachments
FROM #{@table_prefix}attachments
WHERE topic_id = #{topic_id} AND post_msg_id = #{post_id}
ORDER BY filetime DESC, post_msg_id
SQL
@ -164,10 +164,10 @@ module ImportScripts::PhpBB3
def count_messages
count(<<-SQL)
SELECT COUNT(*) AS count
FROM #{@table_prefix}_privmsgs m
FROM #{@table_prefix}privmsgs m
WHERE NOT EXISTS ( -- ignore duplicate messages
SELECT 1
FROM #{@table_prefix}_privmsgs x
FROM #{@table_prefix}privmsgs x
WHERE x.msg_id < m.msg_id AND x.root_level = m.root_level AND x.author_id = m.author_id
AND x.to_address = m.to_address AND x.message_time = m.message_time
)
@ -179,15 +179,15 @@ module ImportScripts::PhpBB3
SELECT m.msg_id, m.root_level AS root_msg_id, m.author_id, m.message_time, m.message_subject,
m.message_text, m.to_address, r.author_id AS root_author_id, r.to_address AS root_to_address, (
SELECT COUNT(*)
FROM #{@table_prefix}_attachments a
FROM #{@table_prefix}attachments a
WHERE a.topic_id = 0 AND m.msg_id = a.post_msg_id
) AS attachment_count
FROM #{@table_prefix}_privmsgs m
LEFT OUTER JOIN #{@table_prefix}_privmsgs r ON (m.root_level = r.msg_id)
FROM #{@table_prefix}privmsgs m
LEFT OUTER JOIN #{@table_prefix}privmsgs r ON (m.root_level = r.msg_id)
WHERE m.msg_id > #{last_msg_id}
AND NOT EXISTS ( -- ignore duplicate messages
SELECT 1
FROM #{@table_prefix}_privmsgs x
FROM #{@table_prefix}privmsgs x
WHERE x.msg_id < m.msg_id AND x.root_level = m.root_level AND x.author_id = m.author_id
AND x.to_address = m.to_address AND x.message_time = m.message_time
)
@ -199,15 +199,15 @@ module ImportScripts::PhpBB3
def count_bookmarks
count(<<-SQL)
SELECT COUNT(*) AS count
FROM #{@table_prefix}_bookmarks
FROM #{@table_prefix}bookmarks
SQL
end
def fetch_bookmarks(last_user_id, last_topic_id)
query(<<-SQL, :user_id, :topic_first_post_id)
SELECT b.user_id, t.topic_first_post_id
FROM #{@table_prefix}_bookmarks b
JOIN #{@table_prefix}_topics t ON (b.topic_id = t.topic_id)
FROM #{@table_prefix}bookmarks b
JOIN #{@table_prefix}topics t ON (b.topic_id = t.topic_id)
WHERE b.user_id > #{last_user_id} AND b.topic_id > #{last_topic_id}
ORDER BY b.user_id, b.topic_id
LIMIT #{@batch_size}
@ -217,12 +217,12 @@ module ImportScripts::PhpBB3
def get_config_values
query(<<-SQL).first
SELECT
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'version') AS phpbb_version,
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_gallery_path') AS avatar_gallery_path,
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_path') AS avatar_path,
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'avatar_salt') AS avatar_salt,
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'smilies_path') AS smilies_path,
(SELECT config_value FROM #{@table_prefix}_config WHERE config_name = 'upload_path') AS attachment_path
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'version') AS phpbb_version,
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_gallery_path') AS avatar_gallery_path,
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_path') AS avatar_path,
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'avatar_salt') AS avatar_salt,
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'smilies_path') AS smilies_path,
(SELECT config_value FROM #{@table_prefix}config WHERE config_name = 'upload_path') AS attachment_path
SQL
end
end

8
script/import_scripts/phpbb3/database/database_3_1.rb
View file

@ -13,10 +13,10 @@ module ImportScripts::PhpBB3
u.user_type, u.user_inactive_reason, g.group_name, b.ban_start, b.ban_end, b.ban_reason,
u.user_posts, f.pf_phpbb_website AS user_website, f.pf_phpbb_location AS user_from,
u.user_birthday, u.user_avatar_type, u.user_avatar
FROM #{@table_prefix}_users u
LEFT OUTER JOIN #{@table_prefix}_profile_fields_data f ON (u.user_id = f.user_id)
JOIN #{@table_prefix}_groups g ON (g.group_id = u.group_id)
LEFT OUTER JOIN #{@table_prefix}_banlist b ON (
FROM #{@table_prefix}users u
LEFT OUTER JOIN #{@table_prefix}profile_fields_data f ON (u.user_id = f.user_id)
JOIN #{@table_prefix}groups g ON (g.group_id = u.group_id)
LEFT OUTER JOIN #{@table_prefix}banlist b ON (
u.user_id = b.ban_userid AND b.ban_exclude = 0 AND
(b.ban_end = 0 OR b.ban_end >= UNIX_TIMESTAMP())
)

5
script/import_scripts/phpbb3/importers/message_importer.rb
View file

@ -56,7 +56,7 @@ module ImportScripts::PhpBB3
mapped[:target_usernames] = get_recipient_usernames(row)
mapped[:custom_fields] = {import_user_ids: current_user_ids.join(',')}
if mapped[:target_usernames].empty? # pm with yourself?
if mapped[:target_usernames].empty?
puts "Private message without recipients. Skipping #{row[:msg_id]}: #{row[:message_subject][0..40]}"
return nil
end
@ -80,11 +80,10 @@ module ImportScripts::PhpBB3
end
def get_recipient_usernames(row)
author_id = row[:author_id].to_s
import_user_ids = get_recipient_user_ids(row[:to_address])
import_user_ids.map! do |import_user_id|
import_user_id.to_s == author_id ? nil : @lookup.find_user_by_import_id(import_user_id).try(:username)
@lookup.find_user_by_import_id(import_user_id).try(:username)
end.compact
end

4
script/import_scripts/phpbb3/settings.yml
View file

@ -1,13 +1,13 @@
# This is an example settings file for the phpBB3 importer.
database:
type: MySQL # currently only MySQL is supported - more to come soon
type: MySQL # currently only MySQL is supported
host: localhost
port: 3306
username: root
password:
schema: phpbb
table_prefix: phpbb # Usually all table names start with phpbb. Change this, if your forum is using a different prefix.
table_prefix: phpbb_ # Usually all table names start with phpbb_. Change this, if your forum is using a different prefix.
batch_size: 1000 # Don't change this unless you know what you're doing. The default (1000) should work just fine.
import:

3
script/import_scripts/phpbb3/support/smiley_processor.rb
View file

@ -47,7 +47,8 @@ module ImportScripts::PhpBB3
[':?:'] => ':question:',
[':idea:'] => ':bulb:',
[':arrow:'] => ':arrow_right:',
[':|', ':-|'] => ':neutral_face:'
[':|', ':-|'] => ':neutral_face:',
[':geek:'] => ':nerd:'
}.each do |smilies, emoji|
smilies.each { |smiley| @smiley_map[smiley] = emoji }
end