diff --git a/script/import_scripts/askbot.rb b/script/import_scripts/askbot.rb
index c6af92b4e..ec0cfc7cb 100644
--- a/script/import_scripts/askbot.rb
+++ b/script/import_scripts/askbot.rb
@@ -73,7 +73,7 @@ class ImportScripts::MyAskBot < ImportScripts::Base
LIMIT #{BATCH_SIZE}
OFFSET #{offset}
SQL
- )
+ )
break if tags.ntuples() < 1
tags.each do |tag|
tid = tag["thread_id"].to_i
@@ -110,6 +110,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base
break if users.ntuples() < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(users, total: total_count, offset: offset) do |user|
{
id: user["id"],
@@ -153,6 +155,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base
break if posts.ntuples() < 1
+ next if all_records_exist? :posts, posts.map {|p| p["id"].to_i}
+
create_posts(posts, total: post_count, offset: offset) do |post|
pid = post["id"]
tid = post["thread_id"].to_i
@@ -206,6 +210,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base
break if posts.ntuples() < 1
+ next if all_records_exist? :posts, posts.map {|p| p["id"].to_i}
+
create_posts(posts, total: post_count, offset: offset) do |post|
tid = post["thread_id"].to_i
next unless thread = @thread_parents[tid]
@@ -267,4 +273,4 @@ class ImportScripts::MyAskBot < ImportScripts::Base
end
end
-ImportScripts::MyAskBot.new.perform
+ImportScripts::MyAskBot.new.perform
diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb
index 039e4de83..57bbc24e9 100644
--- a/script/import_scripts/base.rb
+++ b/script/import_scripts/base.rb
@@ -194,6 +194,23 @@ class ImportScripts::Base
g.tap(&:save)
end
+ def all_records_exist?(type, import_ids)
+ return false if import_ids.empty?
+
+ existing = "#{type.to_s.classify}CustomField".constantize.where(name: 'import_id')
+
+ if Fixnum === import_ids.first
+ existing = existing.where('cast(value as int) in (?)', import_ids)
+ else
+ existing = existing.where('value in (?)', import_ids)
+ end
+
+ if existing.count == import_ids.length
+ # puts "Skipping #{import_ids.length} already imported #{type}"
+ true
+ end
+ end
+
# Iterate through a list of user records to be imported.
# Takes a collection, and yields to the block for each element.
# Block should return a hash with the attributes for the User model.
diff --git a/script/import_scripts/bbpress.rb b/script/import_scripts/bbpress.rb
index 066c9c23d..aba552d11 100644
--- a/script/import_scripts/bbpress.rb
+++ b/script/import_scripts/bbpress.rb
@@ -85,6 +85,8 @@ class ImportScripts::Bbpress < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| p["id"].to_i}
+
create_posts(results, total: total_count, offset: offset) do |post|
skip = false
mapped = {}
diff --git a/script/import_scripts/discuz_x.rb b/script/import_scripts/discuz_x.rb
index 68f173806..f252ddb86 100644
--- a/script/import_scripts/discuz_x.rb
+++ b/script/import_scripts/discuz_x.rb
@@ -98,6 +98,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(results, total: total_count, offset: offset) do |user|
{ id: user['id'],
email: user['email'],
@@ -205,6 +207,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| p["id"].to_i}
+
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
mapped = {}
@@ -281,6 +285,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|m| "pm:#{m['id']}"}
+
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
mapped = {}
diff --git a/script/import_scripts/drupal.rb b/script/import_scripts/drupal.rb
index 82266d77c..a15933b10 100644
--- a/script/import_scripts/drupal.rb
+++ b/script/import_scripts/drupal.rb
@@ -121,6 +121,8 @@ class ImportScripts::Drupal < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "nid:#{p['nid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
{
id: "nid:#{row['nid']}",
@@ -167,6 +169,8 @@ class ImportScripts::Drupal < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}")
if topic_mapping && topic_id = topic_mapping[:topic_id]
diff --git a/script/import_scripts/drupal_qa.rb b/script/import_scripts/drupal_qa.rb
index 9ce0b556e..9899f73ad 100644
--- a/script/import_scripts/drupal_qa.rb
+++ b/script/import_scripts/drupal_qa.rb
@@ -56,6 +56,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "nid:#{p['nid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
{
id: "nid:#{row['nid']}",
@@ -100,6 +102,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}")
if topic_mapping && topic_id = topic_mapping[:topic_id]
@@ -151,6 +155,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}")
if topic_mapping && topic_id = topic_mapping[:topic_id]
@@ -201,6 +207,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"}
+
create_posts(results, total: total_count, offset: offset) do |row|
topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}")
if topic_mapping && topic_id = topic_mapping[:topic_id]
diff --git a/script/import_scripts/kunena.rb b/script/import_scripts/kunena.rb
index 8e1f6275b..3cb52e7b3 100644
--- a/script/import_scripts/kunena.rb
+++ b/script/import_scripts/kunena.rb
@@ -109,6 +109,8 @@ class ImportScripts::Kunena < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| p['id'].to_i}
+
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
mapped = {}
diff --git a/script/import_scripts/kunena3.rb b/script/import_scripts/kunena3.rb
index 56103861f..49b3f24e9 100644
--- a/script/import_scripts/kunena3.rb
+++ b/script/import_scripts/kunena3.rb
@@ -109,6 +109,8 @@ class ImportScripts::Kunena < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| p['id'].to_i}
+
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
mapped = {}
diff --git a/script/import_scripts/lithium.rb b/script/import_scripts/lithium.rb
index 4d7a84366..795f04f36 100644
--- a/script/import_scripts/lithium.rb
+++ b/script/import_scripts/lithium.rb
@@ -101,6 +101,8 @@ class ImportScripts::Lithium < ImportScripts::Base
break if users.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(users, total: user_count, offset: offset) do |user|
{
@@ -274,9 +276,10 @@ class ImportScripts::Lithium < ImportScripts::Base
OFFSET #{offset}
SQL
-
break if topics.size < 1
+ next if all_records_exist? :posts, topics.map {|topic| "#{topic["node_id"]} #{topic["id"]}"}
+
create_posts(topics, total: topic_count, offset: offset) do |topic|
category_id = category_id_from_imported_category_id(topic["node_id"])
@@ -322,6 +325,8 @@ class ImportScripts::Lithium < ImportScripts::Base
break if posts.size < 1
+ next if all_records_exist? :posts, posts.map {|post| "#{post["node_id"]} #{post["root_id"]} #{post["id"]}"}
+
create_posts(posts, total: post_count, offset: offset) do |post|
raw = post["raw"]
next unless topic = topic_lookup_from_imported_post_id("#{post["node_id"]} #{post["root_id"]}")
@@ -593,6 +598,8 @@ class ImportScripts::Lithium < ImportScripts::Base
break if topics.size < 1
+ next if all_records_exist? :posts, topics.map {|topic| "pm_#{topic["note_id"]}"}
+
create_posts(topics, total: topic_count, offset: offset) do |topic|
user_id = user_id_from_imported_user_id(topic["sender_user_id"]) || Discourse::SYSTEM_USER_ID
diff --git a/script/import_scripts/mbox.rb b/script/import_scripts/mbox.rb
index 78549ae90..d47b5553f 100755
--- a/script/import_scripts/mbox.rb
+++ b/script/import_scripts/mbox.rb
@@ -69,6 +69,7 @@ class ImportScripts::Mbox < ImportScripts::Base
batches(BATCH_SIZE) do |offset|
users = user_keys[offset..offset+BATCH_SIZE-1]
break if users.nil?
+ next if all_records_exist? :users, users
create_users(users, total: total_count, offset: offset) do |email|
{
@@ -99,6 +100,8 @@ class ImportScripts::Mbox < ImportScripts::Base
topics = all_topics[offset..offset+BATCH_SIZE-1]
break if topics.nil?
+ next if all_records_exist? :posts, topics.map {|t| t['id'].to_i}
+
create_posts(topics, total: topic_count, offset: offset) do |t|
raw_email = File.read(t['file'])
receiver = Email::Receiver.new(raw_email, skip_sanity_check: true)
@@ -136,6 +139,8 @@ class ImportScripts::Mbox < ImportScripts::Base
posts = replies[offset..offset+BATCH_SIZE-1]
break if posts.nil?
+ next if all_records_exist? :posts, posts.map {|p| p['id'].to_i}
+
create_posts(posts, total: post_count, offset: offset) do |p|
parent_id = p['topic']
id = p['id']
diff --git a/script/import_scripts/mybb.rb b/script/import_scripts/mybb.rb
index c2337d56a..b76e1e0d9 100644
--- a/script/import_scripts/mybb.rb
+++ b/script/import_scripts/mybb.rb
@@ -48,6 +48,8 @@ class ImportScripts::MyBB < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(results, total: total_count, offset: offset) do |user|
{ id: user['id'],
email: user['email'],
@@ -100,6 +102,8 @@ class ImportScripts::MyBB < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|m| m['id'].to_i}
+
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
mapped = {}
diff --git a/script/import_scripts/nabble.rb b/script/import_scripts/nabble.rb
index 607772da4..78e484975 100644
--- a/script/import_scripts/nabble.rb
+++ b/script/import_scripts/nabble.rb
@@ -40,6 +40,8 @@ class ImportScripts::Nabble < ImportScripts::Base
break if users.ntuples() < 1
+ next if all_records_exist? :users, users.map {|u| u["user_id"].to_i}
+
create_users(users, total: total_count, offset: offset) do |user|
{
id: user["user_id"],
@@ -80,6 +82,8 @@ class ImportScripts::Nabble < ImportScripts::Base
break if topics.ntuples() < 1
+ next if all_records_exist? :posts, topics.map {|t| t['node_id'].to_i}
+
create_posts(topics, total: topic_count, offset: offset) do |t|
raw = body_from(t)
next unless raw
@@ -122,6 +126,8 @@ class ImportScripts::Nabble < ImportScripts::Base
break if posts.ntuples() < 1
+ next if all_records_exist? :posts, posts.map {|p| p['node_id'].to_i}
+
create_posts(posts, total: post_count, offset: offset) do |p|
parent_id = p['parent_id']
id = p['node_id']
diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb
index 28c2ed6fa..11592b40d 100644
--- a/script/import_scripts/phpbb3/importer.rb
+++ b/script/import_scripts/phpbb3/importer.rb
@@ -56,6 +56,8 @@ module ImportScripts::PhpBB3
rows = @database.fetch_users(offset)
break if rows.size < 1
+ next if all_records_exist? :users, importer.map_to_import_ids(rows)
+
create_users(rows, total: total_count, offset: offset) do |row|
importer.map_user(row)
end
diff --git a/script/import_scripts/phpbb3/importers/user_importer.rb b/script/import_scripts/phpbb3/importers/user_importer.rb
index be0321d48..aeef5ec86 100644
--- a/script/import_scripts/phpbb3/importers/user_importer.rb
+++ b/script/import_scripts/phpbb3/importers/user_importer.rb
@@ -9,6 +9,10 @@ module ImportScripts::PhpBB3
@settings = settings
end
+ def map_to_import_ids(array)
+ array.map {|u| u[:user_id]}
+ end
+
def map_user(row)
is_active_user = row[:user_inactive_reason] != Constants::INACTIVE_REGISTER
diff --git a/script/import_scripts/punbb.rb b/script/import_scripts/punbb.rb
index 08ecc8831..a5d54372f 100644
--- a/script/import_scripts/punbb.rb
+++ b/script/import_scripts/punbb.rb
@@ -43,6 +43,8 @@ class ImportScripts::PunBB < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(results, total: total_count, offset: offset) do |user|
{ id: user['id'],
email: user['email'],
@@ -118,6 +120,7 @@ class ImportScripts::PunBB < ImportScripts::Base
").to_a
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|m| m['id'].to_i}
create_posts(results, total: total_count, offset: offset) do |m|
skip = false
diff --git a/script/import_scripts/sfn.rb b/script/import_scripts/sfn.rb
index f8a14f00d..14e54196c 100644
--- a/script/import_scripts/sfn.rb
+++ b/script/import_scripts/sfn.rb
@@ -77,6 +77,7 @@ class ImportScripts::Sfn < ImportScripts::Base
SQL
break if users.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
create_users(users, total: user_count, offset: offset) do |user|
external_user = @external_users[user["id"]]
@@ -231,6 +232,7 @@ class ImportScripts::Sfn < ImportScripts::Base
SQL
break if topics.size < 1
+ next if all_records_exist? :posts, topics.map {|t| t['id'].to_i}
create_posts(topics, total: topic_count, offset: offset) do |topic|
next unless category_id = CATEGORY_MAPPING[topic["category_id"]]
@@ -282,6 +284,8 @@ class ImportScripts::Sfn < ImportScripts::Base
break if posts.size < 1
+ next if all_records_exist? :posts, posts.map {|p| p['id'].to_i}
+
create_posts(posts, total: posts_count, offset: offset) do |post|
next unless parent = topic_lookup_from_imported_post_id(post["topic_id"])
diff --git a/script/import_scripts/tnation.rb b/script/import_scripts/tnation.rb
index b8563997e..438327c7b 100644
--- a/script/import_scripts/tnation.rb
+++ b/script/import_scripts/tnation.rb
@@ -173,6 +173,8 @@ class ImportScripts::Tnation < ImportScripts::Base
break if users.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
user_bios = {}
user_avatars = {}
user_properties = {}
@@ -317,6 +319,7 @@ class ImportScripts::Tnation < ImportScripts::Base
posts = posts.to_a
break if posts.size < 1
+ next if all_records_exist? :posts, posts.map {|p| p['id'].to_i}
# load images
forum_images = {}
diff --git a/script/import_scripts/vanilla_mysql.rb b/script/import_scripts/vanilla_mysql.rb
index 841bf1478..e472ce525 100644
--- a/script/import_scripts/vanilla_mysql.rb
+++ b/script/import_scripts/vanilla_mysql.rb
@@ -42,6 +42,8 @@ class ImportScripts::VanillaSQL < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :users, users.map {|u| u['UserID'].to_i}
+
create_users(results, total: total_count, offset: offset) do |user|
next if user['Email'].blank?
next if user['Name'].blank?
@@ -92,6 +94,7 @@ class ImportScripts::VanillaSQL < ImportScripts::Base
OFFSET #{offset};")
break if discussions.size < 1
+ next if all_records_exist? :posts, discussions.map {|t| "discussion#" + t['DiscussionID'].to_s}
create_posts(discussions, total: total_count, offset: offset) do |discussion|
{
@@ -121,6 +124,7 @@ class ImportScripts::VanillaSQL < ImportScripts::Base
OFFSET #{offset};")
break if comments.size < 1
+ next if all_records_exist? :posts, comments.map {|comment| "comment#" + comment['CommentID'].to_s}
create_posts(comments, total: total_count, offset: offset) do |comment|
next unless t = topic_lookup_from_imported_post_id("discussion#" + comment['DiscussionID'].to_s)
diff --git a/script/import_scripts/vbulletin.rb b/script/import_scripts/vbulletin.rb
index 6d3ac4415..22e5883db 100644
--- a/script/import_scripts/vbulletin.rb
+++ b/script/import_scripts/vbulletin.rb
@@ -71,6 +71,8 @@ class ImportScripts::VBulletin < ImportScripts::Base
break if users.size < 1
+ next if all_records_exist? :users, users.map {|u| u["userid"].to_i}
+
create_users(users, total: user_count, offset: offset) do |user|
username = @htmlentities.decode(user["username"]).strip
@@ -208,6 +210,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
SQL
break if topics.size < 1
+ next if all_records_exist? :posts, topics.map {|t| "thread-#{topic["threadid"]}" }
create_posts(topics, total: topic_count, offset: offset) do |topic|
raw = preprocess_post_raw(topic["raw"]) rescue nil
@@ -249,6 +252,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
SQL
break if posts.size < 1
+ next if all_records_exist? :posts, posts.map {|p| p["postid"] }
create_posts(posts, total: post_count, offset: offset) do |post|
raw = preprocess_post_raw(post["raw"]) rescue nil
diff --git a/script/import_scripts/xenforo.rb b/script/import_scripts/xenforo.rb
index 536c03ec1..41f7cd158 100644
--- a/script/import_scripts/xenforo.rb
+++ b/script/import_scripts/xenforo.rb
@@ -42,6 +42,8 @@ class ImportScripts::XenForo < ImportScripts::Base
break if results.size < 1
+ next if all_records_exist? :users, users.map {|u| u["id"].to_i}
+
create_users(results, total: total_count, offset: offset) do |user|
next if user['username'].blank?
{ id: user['id'],
@@ -98,6 +100,7 @@ class ImportScripts::XenForo < ImportScripts::Base
").to_a
break if results.size < 1
+ next if all_records_exist? :posts, results.map {|p| p['id'] }
create_posts(results, total: total_count, offset: offset) do |m|
skip = false