update bbcode dialect and fix vBulletin importer

app/assets/javascripts/discourse/dialects/bbcode_dialect.js

@@ -1,3 +1,5 @@
+Discourse.BBCode = {};
+
 /**
   Create a simple BBCode tag handler
 
@@ -13,7 +15,6 @@
     @param {Boolean} [opts.wordBoundary] If true, the match must be on a word boundary
     @param {Boolean} [opts.spaceBoundary] If true, the match must be on a sppace boundary
 **/
-Discourse.BBCode = {};
 
 Discourse.BBCode.register = function(codeName, args, emitter) {
 
@@ -64,7 +65,7 @@ Discourse.BBCode.register = function(codeName, args, emitter) {
   });
 };
 
-function replaceBBCode(tag, emitter, opts) {
+Discourse.BBCode.replaceBBCode = function (tag, emitter, opts) {
   opts = opts || {};
   opts = _.merge(opts, { start: "[" + tag + "]", stop: "[/" + tag + "]", emitter: emitter });
   Discourse.Dialect.inlineBetween(opts);
@@ -72,7 +73,7 @@ function replaceBBCode(tag, emitter, opts) {
   tag = tag.toUpperCase();
   opts = _.merge(opts, { start: "[" + tag + "]", stop: "[/" + tag + "]", emitter: emitter });
   Discourse.Dialect.inlineBetween(opts);
-}
+};
 
 /**
   Shortcut to call replaceBBCode with `rawContents` as true.
@@ -81,9 +82,9 @@ function replaceBBCode(tag, emitter, opts) {
   @param {tag} tag the tag we want to match
   @param {function} emitter the function that creates JsonML for the tag
 **/
-function rawBBCode(tag, emitter) {
+Discourse.BBCode.rawBBCode = function (tag, emitter) {
-  replaceBBCode(tag, emitter, { rawContents: true });
+  Discourse.BBCode.replaceBBCode(tag, emitter, { rawContents: true });
-}
+};
 
 /**
   Creates a BBCode handler that accepts parameters. Passes them to the emitter.
@@ -92,10 +93,8 @@ function rawBBCode(tag, emitter) {
   @param {tag} tag the tag we want to match
   @param {function} emitter the function that creates JsonML for the tag
 **/
-function replaceBBCodeParamsRaw(tag, emitter) {
+Discourse.BBCode.replaceBBCodeParamsRaw = function (tag, emitter) {
-  Discourse.Dialect.inlineBetween({
+  var opts = {
-    start: "[" + tag + "=",
-    stop: "[/" + tag + "]",
     rawContents: true,
     emitter: function(contents) {
       var regexp = /^([^\]]+)\]([\S\s]*)$/,
@@ -103,8 +102,13 @@ function replaceBBCodeParamsRaw(tag, emitter) {
 
       if (m) { return emitter.call(this, m[1], m[2]); }
     }
-  });
+  };
-}
+
+  Discourse.Dialect.inlineBetween(_.merge(opts, { start: "[" + tag + "=", stop: "[/" + tag + "]" }));
+
+  tag = tag.toUpperCase();
+  Discourse.Dialect.inlineBetween(_.merge(opts, { start: "[" + tag + "=", stop: "[/" + tag + "]" }));
+};
 
 /**
   Filters an array of JSON-ML nodes, removing nodes that represent empty lines ("\n").
@@ -112,28 +116,28 @@ function replaceBBCodeParamsRaw(tag, emitter) {
   @method removeEmptyLines
   @param {Array} [contents] Array of JSON-ML nodes
 **/
-function removeEmptyLines(contents) {
+Discourse.BBCode.removeEmptyLines = function (contents) {
   var result = [];
   for (var i=0; i < contents.length; i++) {
     if (contents[i] !== "\n") { result.push(contents[i]); }
   }
   return result;
-}
+};
 
-replaceBBCode('b', function(contents) { return ['span', {'class': 'bbcode-b'}].concat(contents); });
+Discourse.BBCode.replaceBBCode('b', function(contents) { return ['span', {'class': 'bbcode-b'}].concat(contents); });
-replaceBBCode('i', function(contents) { return ['span', {'class': 'bbcode-i'}].concat(contents); });
+Discourse.BBCode.replaceBBCode('i', function(contents) { return ['span', {'class': 'bbcode-i'}].concat(contents); });
-replaceBBCode('u', function(contents) { return ['span', {'class': 'bbcode-u'}].concat(contents); });
+Discourse.BBCode.replaceBBCode('u', function(contents) { return ['span', {'class': 'bbcode-u'}].concat(contents); });
-replaceBBCode('s', function(contents) { return ['span', {'class': 'bbcode-s'}].concat(contents); });
+Discourse.BBCode.replaceBBCode('s', function(contents) { return ['span', {'class': 'bbcode-s'}].concat(contents); });
 Discourse.Markdown.whiteListTag('span', 'class', /^bbcode-[bius]$/);
 
-replaceBBCode('ul', function(contents) { return ['ul'].concat(removeEmptyLines(contents)); });
+Discourse.BBCode.replaceBBCode('ul', function(contents) { return ['ul'].concat(Discourse.BBCode.removeEmptyLines(contents)); });
-replaceBBCode('ol', function(contents) { return ['ol'].concat(removeEmptyLines(contents)); });
+Discourse.BBCode.replaceBBCode('ol', function(contents) { return ['ol'].concat(Discourse.BBCode.removeEmptyLines(contents)); });
-replaceBBCode('li', function(contents) { return ['li'].concat(removeEmptyLines(contents)); });
+Discourse.BBCode.replaceBBCode('li', function(contents) { return ['li'].concat(Discourse.BBCode.removeEmptyLines(contents)); });
 
-rawBBCode('img', function(contents) { return ['img', {href: contents}]; });
+Discourse.BBCode.rawBBCode('img', function(contents) { return ['img', {href: contents}]; });
-rawBBCode('email', function(contents) { return ['a', {href: "mailto:" + contents, 'data-bbcode': true}, contents]; });
+Discourse.BBCode.rawBBCode('email', function(contents) { return ['a', {href: "mailto:" + contents, 'data-bbcode': true}, contents]; });
-rawBBCode('url', function(contents) { return ['a', {href: contents, 'data-bbcode': true}, contents]; });
+Discourse.BBCode.rawBBCode('url', function(contents) { return ['a', {href: contents, 'data-bbcode': true}, contents]; });
-rawBBCode('spoiler', function(contents) {
+Discourse.BBCode.rawBBCode('spoiler', function(contents) {
   if (/<img/i.test(contents)) {
     return ['div', { 'class': 'spoiler' }, contents];
   } else {
@@ -141,12 +145,12 @@ rawBBCode('spoiler', function(contents) {
   }
 });
 
-replaceBBCodeParamsRaw("url", function(param, contents) {
+Discourse.BBCode.replaceBBCodeParamsRaw("url", function(param, contents) {
-  return ['a', {href: param, 'data-bbcode': true}, contents];
+  return ['a', {href: param, 'data-bbcode': true}].concat(contents);
 });
 
-replaceBBCodeParamsRaw("email", function(param, contents) {
+Discourse.BBCode.replaceBBCodeParamsRaw("email", function(param, contents) {
-  return ['a', {href: "mailto:" + param, 'data-bbcode': true}, contents];
+  return ['a', {href: "mailto:" + param, 'data-bbcode': true}].concat(contents);
 });
 
 Discourse.BBCode.register('size', function(contents, params) {
@@ -165,4 +169,3 @@ Discourse.Dialect.replaceBlock({
     return ['p', ['pre', ['code', {'class': Discourse.SiteSettings.default_code_lang}, inner]]];
   }
 });
-

lib/user_name_suggester.rb

@@ -6,8 +6,6 @@ module UserNameSuggester
     find_available_username_based_on(name)
   end
 
-  private
-
   def self.parse_name_from_email(name)
     if name =~ User::EMAIL
       # When 'walter@white.com' take 'walter'
@@ -19,7 +17,7 @@ module UserNameSuggester
   end
 
   def self.find_available_username_based_on(name)
-    name = rightsize_username(sanitize_username!(name))
+    name = fix_username(name)
     i = 1
     attempt = name
     until User.username_available?(attempt)
@@ -31,11 +29,15 @@ module UserNameSuggester
     attempt
   end
 
-  def self.sanitize_username!(name)
+  def self.fix_username(name)
+    rightsize_username(sanitize_username(name))
+  end
+
+  def self.sanitize_username(name)
     name = ActiveSupport::Inflector.transliterate(name)
-    name.gsub!(/^[^[:alnum:]]+|\W+$/, "")
+    name = name.gsub(/^[^[:alnum:]]+|\W+$/, "")
-    name.gsub!(/\W+/, "_")
+               .gsub(/\W+/, "_")
-    name.gsub!(/^\_+/, '')
+               .gsub(/^\_+/, '')
     name
   end
 

script/import_scripts/base.rb

@@ -235,9 +235,11 @@ class ImportScripts::Base
 
   def create_user(opts, import_id)
     opts.delete(:id)
+    merge = opts.delete(:merge)
     post_create_action = opts.delete(:post_create_action)
+
     existing = User.where(email: opts[:email].downcase, username: opts[:username]).first
-    return existing if existing && existing.custom_fields["import_id"].to_i == import_id.to_i
+    return existing if existing && (merge || existing.custom_fields["import_id"].to_i == import_id.to_i)
 
     bio_raw = opts.delete(:bio_raw)
     website = opts.delete(:website)
@@ -434,12 +436,12 @@ class ImportScripts::Base
   end
 
   def update_bumped_at
-    puts "updating bumped_at on topics"
+    puts "", "updating bumped_at on topics"
     Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})")
   end
 
   def update_feature_topic_users
-    puts "updating featured topic users"
+    puts "", "updating featured topic users"
 
     total_count = Topic.count
     progress_count = 0
@@ -452,7 +454,7 @@ class ImportScripts::Base
   end
 
   def update_category_featured_topics
-    puts "updating featured topics in categories"
+    puts "", "updating featured topics in categories"
 
     total_count = Category.count
     progress_count = 0
@@ -465,7 +467,7 @@ class ImportScripts::Base
   end
 
   def update_topic_count_replies
-    puts "updating user topic reply counts"
+    puts "", "updating user topic reply counts"
 
     total_count = User.real.count
     progress_count = 0

script/import_scripts/vbulletin.rb

@@ -243,6 +243,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_groups
       puts "", "Importing groups..."
 
+      # sort the groups
+      @groups.sort_by! { |group| group[:usergroupid].to_i }
+
       create_groups(@groups) do |group|
         {
           id: group[:usergroupid],
@@ -255,10 +258,13 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_users
       puts "", "Importing users..."
 
+      # sort the users
+      @users.sort_by! { |user| user[:userid].to_i }
+
       @old_username_to_new_usernames = {}
 
       create_users(@users) do |user|
-        @old_username_to_new_usernames[user[:username]] = UserNameSuggester.suggest(user[:username])
+        @old_username_to_new_usernames[user[:username]] = UserNameSuggester.fix_username(user[:username])
 
         {
           id: user[:userid],
@@ -267,6 +273,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
           website: user[:homepage],
           title: user[:usertitle],
           primary_group_id: group_id_from_imported_group_id(user[:usergroupid]),
+          merge: true,
         }
       end
 
@@ -300,6 +307,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_categories
       puts "", "Importing categories..."
 
+      # sort categories
+      @categories.sort_by! { |category| category[:forumid].to_i }
+
       create_categories(@categories) do |category|
         {
           id: category[:forumid],
@@ -343,22 +353,18 @@ class ImportScripts::VBulletin < ImportScripts::Base
       raw = raw.gsub(/\[mention\](.+?)\[\/mention\]/i) do
         old_username = $1
         if @old_username_to_new_usernames.has_key?(old_username)
-          username = @old_username_to_new_usernames[old_username]
+          old_username = @old_username_to_new_usernames[old_username]
-          "@#{username}"
-        else
-          $&
         end
+        "@#{old_username}"
       end
 
-      # [MENTION=<user_id>]...[/MENTION]
+      # [MENTION=<user_id>]<username>[/MENTION]
-      raw = raw.gsub(/\[mention=(\d+)\].+?\[\/mention\]/i) do
+      raw = raw.gsub(/\[mention=(\d+)\](.+?)\[\/mention\]/i) do
-        user_id = $1
+        user_id, old_username = $1, $2
         if user = @users.select { |u| u[:userid] == user_id }.first
-          username = @old_username_to_new_usernames[user[:username]] || user[:username]
+          old_username = @old_username_to_new_usernames[user[:username]] || user[:username]
-          "@#{username}"
-        else
-          $&
         end
+        "@#{old_username}"
       end
 
       # [QUOTE]...[/QUOTE]
@@ -368,11 +374,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
       raw = raw.gsub(/\[quote=([^;\]]+)\](.+?)\[\/quote\]/im) do
         old_username, quote = $1, $2
         if @old_username_to_new_usernames.has_key?(old_username)
-          username = @old_username_to_new_usernames[old_username]
+          old_username = @old_username_to_new_usernames[old_username]
-          "\n[quote=\"#{username}\"]\n#{quote}\n[/quote]\n"
-        else
-          $&
         end
+        "\n[quote=\"#{old_username}\"]\n#{quote}\n[/quote]\n"
       end
 
       # [HTML]...[/HTML]
@@ -399,12 +403,24 @@ class ImportScripts::VBulletin < ImportScripts::Base
       # [MP3]<url>[/MP3]
       raw = raw.gsub(/\[MP3\](.+?)\[\/MP3\]/i) { "\n#{$1}\n" }
 
+      # replace all chevrons with HTML entities
+      raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub("<", "\u2603") + "`" }
+               .gsub("<", "&lt;")
+               .gsub("\u2603", "<")
+
+      raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub(">", "\u2603") + "`" }
+               .gsub(">", "&gt;")
+               .gsub("\u2603", ">")
+
       raw
     end
 
     def import_topics
       puts "", "Importing topics..."
 
+      # sort the topics
+      @topics.sort_by! { |topic| topic[:threadid].to_i }
+
       create_posts(@topics) do |topic|
         next unless post = @posts.select { |p| p[:postid] == topic[:firstpostid] }.first
 
@@ -441,8 +457,11 @@ class ImportScripts::VBulletin < ImportScripts::Base
       first_post_ids = Set.new(@topics.map { |t| t[:firstpostid] })
       @posts.reject! { |post| first_post_ids.include?(post[:postid]) }
 
+      # sort the posts
+      @posts.sort_by! { |post| post[:postid].to_i }
+
       create_posts(@posts) do |post|
-        t = topic_lookup_from_imported_post_id("thread#" + post[:threadid])
+        next unless t = topic_lookup_from_imported_post_id("thread#" + post[:threadid])
 
         p = {
           id: post[:postid],
@@ -487,13 +506,17 @@ class ImportScripts::VBulletin < ImportScripts::Base
       # [QUOTE=<username>;<post_id>]...[/QUOTE]
       raw = raw.gsub(/\[quote=([^;]+);(\d+)\](.+?)\[\/quote\]/im) do
         old_username, post_id, quote = $1, $2, $3
-        if @old_username_to_new_usernames.has_key?(old_username) && topic_lookup = topic_lookup_from_imported_post_id(post_id)
+
+        if @old_username_to_new_usernames.has_key?(old_username)
+          old_username = @old_username_to_new_usernames[old_username]
+        end
+
+        if topic_lookup = topic_lookup_from_imported_post_id(post_id)
           post_number = topic_lookup[:post_number]
           topic_id    = topic_lookup[:topic_id]
-          username    = @old_username_to_new_usernames[old_username]
+          "\n[quote=\"#{old_username},post:#{post_number},topic:#{topic_id}\"]\n#{quote}\n[/quote]\n"
-          "\n[quote=\"#{username},post:#{post_number},topic:#{topic_id}\"]\n#{quote}\n[/quote]\n"
         else
-          $&
+          "\n[quote=\"#{old_username}\"]\n#{quote}\n[/quote]\n"
         end
       end