From 700c005c26473a6963ba9273b7ebd9622b2c591f Mon Sep 17 00:00:00 2001
From: Arpit Jalan <arpit@techapj.com>
Date: Mon, 2 Nov 2015 11:01:08 +0530
Subject: [PATCH] PERF: optimize export user list CSV queries

---
 app/jobs/regular/export_csv_file.rb | 103 ++++++++++++++++------------
 spec/jobs/export_csv_file_spec.rb   |   2 +-
 2 files changed, 60 insertions(+), 45 deletions(-)

diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rb
index d3d2d3da8..236c3a4bd 100644
--- a/app/jobs/regular/export_csv_file.rb
+++ b/app/jobs/regular/export_csv_file.rb
@@ -51,14 +51,66 @@ module Jobs
     end
 
     def user_list_export
-      query = ::AdminUserIndexQuery.new
-      user_data = query.find_users_query.to_a
-      user_data.map do |user|
-        group_names = get_group_names(user).join(';')
-        user_array = get_user_list_fields(user)
-        user_array.push(group_names) if group_names != ''
-        user_array
+      user_array = []
+      user_field_ids = UserField.pluck(:id)
+
+      if SiteSetting.enable_sso
+        # SSO enabled
+        User.includes(:user_stat, :single_sign_on_record, :groups).find_each do |user|
+          user_info_string = "#{user.id},#{user.name},#{user.username},#{user.email},#{user.title},#{user.created_at},#{user.last_seen_at},#{user.last_posted_at},#{user.last_emailed_at},#{user.trust_level},#{user.approved},#{user.suspended_at},#{user.suspended_till},#{user.blocked},#{user.active},#{user.admin},#{user.moderator},#{user.ip_address},#{user.user_stat.topics_entered},#{user.user_stat.posts_read_count},#{user.user_stat.time_read},#{user.user_stat.topic_count},#{user.user_stat.post_count},#{user.user_stat.likes_given},#{user.user_stat.likes_received}"
+
+          # sso
+          if user.single_sign_on_record
+            user_info_string << ",#{user.single_sign_on_record.external_id},#{user.single_sign_on_record.external_email},#{user.single_sign_on_record.external_username},#{user.single_sign_on_record.external_name},#{user.single_sign_on_record.external_avatar_url}"
+          else
+            user_info_string << ",nil,nil,nil,nil,nil"
+          end
+
+          # custom fields
+          if user_field_ids.present?
+            user.user_fields.each do |custom_field|
+              user_info_string << ",#{custom_field[1]}"
+            end
+          end
+
+          # group names
+          group_names = ""
+          user.groups.each do |group|
+            group_names << "#{group.name};"
+          end
+          user_info_string << ",#{group_names[0..-2]}" unless group_names.blank?
+          group_names = nil
+
+          user_array.push(user_info_string.split(","))
+          user_info_string = nil
+        end
+      else
+        # SSO disabled
+        User.includes(:user_stat, :groups).find_each do |user|
+          user_info_string = "#{user.id},#{user.name},#{user.username},#{user.email},#{user.title},#{user.created_at},#{user.last_seen_at},#{user.last_posted_at},#{user.last_emailed_at},#{user.trust_level},#{user.approved},#{user.suspended_at},#{user.suspended_till},#{user.blocked},#{user.active},#{user.admin},#{user.moderator},#{user.ip_address},#{user.user_stat.topics_entered},#{user.user_stat.posts_read_count},#{user.user_stat.time_read},#{user.user_stat.topic_count},#{user.user_stat.post_count},#{user.user_stat.likes_given},#{user.user_stat.likes_received}"
+
+          # custom fields
+          if user_field_ids.present?
+            user.user_fields.each do |custom_field|
+              user_info_string << ",#{custom_field[1]}"
+            end
+          end
+
+          # group names
+          group_names = ""
+          user.groups.each do |group|
+            group_names << "#{group.name};"
+          end
+          user_info_string << ",#{group_names[0..-2]}" unless group_names.blank?
+          group_names = nil
+
+          user_array.push(user_info_string.split(","))
+          user_info_string = nil
+        end
       end
+
+      user_field_ids = nil
+      user_array
     end
 
     def staff_action_export
@@ -129,15 +181,6 @@ module Jobs
 
     private
 
-      def get_group_names(user)
-        group_names = []
-        groups = user.groups
-        groups.each do |group|
-          group_names.push(group.name)
-        end
-        return group_names
-      end
-
       def get_user_archive_fields(user_archive)
         user_archive_array = []
         topic_data = user_archive.topic
@@ -170,34 +213,6 @@ module Jobs
         user_archive_array
       end
 
-      def get_user_list_fields(user)
-        user_array = []
-
-        HEADER_ATTRS_FOR['user_list'].each do |attr|
-          user_array.push(user.attributes[attr])
-        end
-
-        HEADER_ATTRS_FOR['user_stats'].each do |stat|
-          user_array.push(user.user_stat.attributes[stat])
-        end
-
-        if SiteSetting.enable_sso
-          sso = user.single_sign_on_record
-          HEADER_ATTRS_FOR['user_sso'].each do |stat|
-            field = sso.attributes[stat] if sso
-            user_array.push(field)
-          end
-        end
-
-        if user.user_fields.present?
-          user.user_fields.each do |custom_field|
-            user_array.push(custom_field[1])
-          end
-        end
-
-        user_array
-      end
-
       def get_staff_action_fields(staff_action)
         staff_action_array = []
 
diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rb
index 80e211c9e..ebef5e599 100644
--- a/spec/jobs/export_csv_file_spec.rb
+++ b/spec/jobs/export_csv_file_spec.rb
@@ -25,7 +25,7 @@ describe Jobs::ExportCsvFile do
     user = Fabricate(:user)
     user.create_single_sign_on_record(external_id: "123", last_payload: "xxx", external_email: 'test@test.com')
 
-    user = to_hash(user_list_export.find{|u| u[0] == user.id})
+    user = to_hash(user_list_export.find{|u| u[0].to_i == user.id})
 
     expect(user["external_id"]).to eq("123")
     expect(user["external_email"]).to eq("test@test.com")