From d0736a06b6308f8535db6df25fd8e680dbe7beff Mon Sep 17 00:00:00 2001
From: Arpit Jalan <arpit@techapj.com>
Date: Sat, 9 Aug 2014 15:58:57 +0530
Subject: [PATCH 1/2] FEATURE: export user list
---
.../javascripts/admin/models/export_csv.js | 26 +++++++
.../admin/routes/admin_users_list_routes.js | 8 +-
.../admin/templates/users_list.js.handlebars | 3 +
.../admin/export_csv_controller.rb | 20 +++++
app/jobs/regular/export_csv_file.rb | 75 +++++++++++++++++++
app/models/export_csv.rb | 16 ++++
config/locales/client.en.yml | 7 ++
config/locales/server.en.yml | 11 +++
config/routes.rb | 9 +++
.../admin/export_csv_controller_spec.rb | 35 +++++++++
spec/jobs/export_csv_file_spec.rb | 13 ++++
11 files changed, 222 insertions(+), 1 deletion(-)
create mode 100644 app/assets/javascripts/admin/models/export_csv.js
create mode 100644 app/controllers/admin/export_csv_controller.rb
create mode 100644 app/jobs/regular/export_csv_file.rb
create mode 100644 app/models/export_csv.rb
create mode 100644 spec/controllers/admin/export_csv_controller_spec.rb
create mode 100644 spec/jobs/export_csv_file_spec.rb
diff --git a/app/assets/javascripts/admin/models/export_csv.js b/app/assets/javascripts/admin/models/export_csv.js
new file mode 100644
index 000000000..3afe84a1c
--- /dev/null
+++ b/app/assets/javascripts/admin/models/export_csv.js
@@ -0,0 +1,26 @@
+/**
+ Data model for representing an export
+
+ @class ExportCsv
+ @extends Discourse.Model
+ @namespace Discourse
+ @module Discourse
+**/
+Discourse.ExportCsv = Discourse.Model.extend({});
+
+Discourse.ExportCsv.reopenClass({
+ /**
+ Exports user list
+
+ @method export_user_list
+ **/
+ exportUserList: function() {
+ return Discourse.ajax("/admin/export_csv/users.json").then(function(result) {
+ if (result.success) {
+ bootbox.alert(I18n.t("admin.export_csv.success"));
+ } else {
+ bootbox.alert(I18n.t("admin.export_csv.failed"));
+ }
+ });
+ }
+});
diff --git a/app/assets/javascripts/admin/routes/admin_users_list_routes.js b/app/assets/javascripts/admin/routes/admin_users_list_routes.js
index 86b180d39..bc1516c18 100644
--- a/app/assets/javascripts/admin/routes/admin_users_list_routes.js
+++ b/app/assets/javascripts/admin/routes/admin_users_list_routes.js
@@ -9,6 +9,12 @@
Discourse.AdminUsersListRoute = Discourse.Route.extend({
renderTemplate: function() {
this.render('admin/templates/users_list', {into: 'admin/templates/admin'});
+ },
+
+ actions: {
+ exportUsers: function() {
+ Discourse.ExportCsv.exportUserList();
+ }
}
});
@@ -57,7 +63,7 @@ Discourse.AdminUsersListNewRoute = Discourse.Route.extend({
/**
Handles the route that lists pending users.
- @class AdminUsersListNewRoute
+ @class AdminUsersListPendingRoute
@extends Discourse.Route
@namespace Discourse
@module Discourse
diff --git a/app/assets/javascripts/admin/templates/users_list.js.handlebars b/app/assets/javascripts/admin/templates/users_list.js.handlebars
index 61cacc69f..131036732 100644
--- a/app/assets/javascripts/admin/templates/users_list.js.handlebars
+++ b/app/assets/javascripts/admin/templates/users_list.js.handlebars
@@ -15,6 +15,9 @@
<div class='username controls'>
{{text-field value=username placeholderKey="search_hint"}}
</div>
+ <div class="pull-right">
+ <button {{action exportUsers}} class="btn" title="{{i18n admin.export_csv.users.title}}"><i class="fa fa-download"></i>{{i18n admin.export_csv.users.text}}</button>
+ </div>
</div>
<div class="admin-container">
diff --git a/app/controllers/admin/export_csv_controller.rb b/app/controllers/admin/export_csv_controller.rb
new file mode 100644
index 000000000..f0e574105
--- /dev/null
+++ b/app/controllers/admin/export_csv_controller.rb
@@ -0,0 +1,20 @@
+class Admin::ExportCsvController < Admin::AdminController
+
+ skip_before_filter :check_xhr, only: [:download]
+
+ def export_user_list
+ # export csv file in a background thread
+ Jobs.enqueue(:export_csv_file, entity: 'user', user_id: current_user.id)
+ render json: success_json
+ end
+
+ def download
+ filename = params.fetch(:id)
+ if export_csv_path = ExportCsv.get_download_path(filename)
+ send_file export_csv_path
+ else
+ render nothing: true, status: 404
+ end
+ end
+
+end
diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rb
new file mode 100644
index 000000000..6e4a0b797
--- /dev/null
+++ b/app/jobs/regular/export_csv_file.rb
@@ -0,0 +1,75 @@
+require 'csv'
+require_dependency 'system_message'
+
+module Jobs
+
+ class ExportCsvFile < Jobs::Base
+ sidekiq_options retry: false
+ attr_accessor :current_user
+
+ def initialize
+ @file_name = ""
+ end
+
+ def execute(args)
+ entity = args[:entity]
+ @current_user = User.find_by(id: args[:user_id])
+
+ raise Discourse::InvalidParameters.new(:entity) if entity.blank?
+
+ case entity
+ when 'user'
+ query = ::AdminUserIndexQuery.new
+ user_data = query.find_users_query.to_a
+
+ data = Hash.new do |hash, key|
+ hash[key] = {}
+ end
+
+ user_data.each do |user|
+ id = user['id']
+ email = user['email']
+ data[id] = email
+ end
+ data = data.to_a
+ end
+
+ if data && data.length > 0
+ set_file_path
+ write_csv_file(data)
+ end
+
+ notify_user
+ end
+
+ private
+
+ def set_file_path
+ @file_name = "export_#{SecureRandom.hex(4)}.csv"
+ # ensure directory exists
+ dir = File.dirname("#{ExportCsv.base_directory}/#{@file_name}")
+ FileUtils.mkdir_p(dir) unless Dir.exists?(dir)
+ end
+
+ def write_csv_file(data)
+ # write to CSV file
+ CSV.open(File.expand_path("#{ExportCsv.base_directory}/#{@file_name}", __FILE__), "w") do |csv|
+ data.each do |value|
+ csv << [value[1]]
+ end
+ end
+ end
+
+ def notify_user
+ if @current_user
+ if @file_name != "" && File.exists?("#{ExportCsv.base_directory}/#{@file_name}")
+ SystemMessage.create_from_system_user(@current_user, :csv_export_succeeded, download_link: "#{Discourse.base_url}/admin/export_csv/#{@file_name}/download", file_name: @file_name)
+ else
+ SystemMessage.create_from_system_user(@current_user, :csv_export_failed)
+ end
+ end
+ end
+
+ end
+
+end
diff --git a/app/models/export_csv.rb b/app/models/export_csv.rb
new file mode 100644
index 000000000..214d650de
--- /dev/null
+++ b/app/models/export_csv.rb
@@ -0,0 +1,16 @@
+class ExportCsv
+
+ def self.get_download_path(filename)
+ path = File.join(ExportCsv.base_directory, filename)
+ if File.exists?(path)
+ return path
+ else
+ nil
+ end
+ end
+
+ def self.base_directory
+ File.join(Rails.root, "public", "uploads", "csv_exports", RailsMultisite::ConnectionManagement.current_db)
+ end
+
+end
diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml
index 8f30a7276..d25b0bbd4 100644
--- a/config/locales/client.en.yml
+++ b/config/locales/client.en.yml
@@ -1603,6 +1603,13 @@ en:
title: "Rollback the database to previous working state"
confirm: "Are your sure you want to rollback the database to the previous working state?"
+ export_csv:
+ users:
+ text: "Export Users"
+ title: "Export user list in a CSV file."
+ success: "Export has been initiated, you will be notified shortly with progress."
+ failed: "Export failed. Please check the logs."
+
customize:
title: "Customize"
long_title: "Site Customizations"
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 4acc4355f..512a8a276 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -1387,6 +1387,17 @@ en:
%{logs}
```
+ csv_export_succeeded:
+ subject_template: "Data Export completed successfully"
+ text_body_template: |
+ The data export was successful.
+
+ Download CSV file: <a class="attachment" href="%{download_link}">%{file_name}</a>
+
+ csv_export_failed:
+ subject_template: "Export failed"
+ text_body_template: "The export has failed. Please check the logs."
+
email_reject_trust_level:
subject_template: "Email issue -- Insufficient Trust Level"
text_body_template: |
diff --git a/config/routes.rb b/config/routes.rb
index 89782fe4c..add9d1604 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -145,6 +145,15 @@ Discourse::Application.routes.draw do
end
end
+ resources :export_csv, constraints: AdminConstraint.new do
+ member do
+ get "download" => "export_csv#download", constraints: { id: /[^\/]+/ }
+ end
+ collection do
+ get "users" => "export_csv#export_user_list"
+ end
+ end
+
resources :badges, constraints: AdminConstraint.new do
collection do
get "types" => "badges#badge_types"
diff --git a/spec/controllers/admin/export_csv_controller_spec.rb b/spec/controllers/admin/export_csv_controller_spec.rb
new file mode 100644
index 000000000..90863df2b
--- /dev/null
+++ b/spec/controllers/admin/export_csv_controller_spec.rb
@@ -0,0 +1,35 @@
+require "spec_helper"
+
+describe Admin::ExportCsvController do
+
+ it "is a subclass of AdminController" do
+ (Admin::ExportCsvController < Admin::AdminController).should be_true
+ end
+
+ let(:export_filename) { "export_b6a2bc87.csv" }
+
+ context "while logged in as an admin" do
+
+ before { @admin = log_in(:admin) }
+
+ describe ".download" do
+
+ it "uses send_file to transmit the export file" do
+ controller.stubs(:render)
+ export = ExportCsv.new()
+ ExportCsv.expects(:get_download_path).with(export_filename).returns(export)
+ subject.expects(:send_file).with(export)
+ get :download, id: export_filename
+ end
+
+ it "returns 404 when the export file does not exist" do
+ ExportCsv.expects(:get_download_path).returns(nil)
+ get :download, id: export_filename
+ response.should be_not_found
+ end
+
+ end
+
+ end
+
+end
diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rb
new file mode 100644
index 000000000..269fb12b2
--- /dev/null
+++ b/spec/jobs/export_csv_file_spec.rb
@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Jobs::ExportCsvFile do
+
+ context '.execute' do
+
+ it 'raises an error when the entity is missing' do
+ lambda { Jobs::ExportCsvFile.new.execute(user_id: "1") }.should raise_error(Discourse::InvalidParameters)
+ end
+
+ end
+end
+
From b0f44dcdf36225eb8687e7e9f907fdca88f4a199 Mon Sep 17 00:00:00 2001
From: Arpit Jalan <arpit@techapj.com>
Date: Fri, 15 Aug 2014 01:47:52 +0530
Subject: [PATCH 2/2] clean up exports weekly
---
app/jobs/scheduled/weekly.rb | 1 +
app/models/export_csv.rb | 9 +++++++++
2 files changed, 10 insertions(+)
diff --git a/app/jobs/scheduled/weekly.rb b/app/jobs/scheduled/weekly.rb
index 156cfd74e..9a079efb0 100644
--- a/app/jobs/scheduled/weekly.rb
+++ b/app/jobs/scheduled/weekly.rb
@@ -11,6 +11,7 @@ module Jobs
Post.calculate_avg_time
Topic.calculate_avg_time
ScoreCalculator.new.calculate
+ ExportCsv.remove_old_exports # delete exported CSV files older than 2 days
end
end
end
diff --git a/app/models/export_csv.rb b/app/models/export_csv.rb
index 214d650de..ffebf6c1e 100644
--- a/app/models/export_csv.rb
+++ b/app/models/export_csv.rb
@@ -9,6 +9,15 @@ class ExportCsv
end
end
+ def self.remove_old_exports
+ dir = Dir.new(ExportCsv.base_directory)
+ dir.each do |file|
+ if (File.mtime(File.join(ExportCsv.base_directory, file)) < 2.days.ago)
+ File.delete(File.join(ExportCsv.base_directory, file))
+ end
+ end
+ end
+
def self.base_directory
File.join(Rails.root, "public", "uploads", "csv_exports", RailsMultisite::ConnectionManagement.current_db)
end