From 727184641e608bc882c7c3a19b4dddbac5141a66 Mon Sep 17 00:00:00 2001 From: Arpit Jalan Date: Wed, 28 May 2014 01:44:37 +0530 Subject: [PATCH] FEATURE: Bulk Invite --- .../discourse/controllers/user-invited.js.es6 | 13 +++- .../discourse/routes/user_invited_route.js | 10 ++- .../templates/user/invited.js.handlebars | 11 ++- app/controllers/admin/backups_controller.rb | 21 ++--- app/controllers/invites_controller.rb | 46 ++++++++++- app/jobs/regular/backup_chunks_merger.rb | 24 +----- app/jobs/regular/bulk_invite.rb | 78 +++++++++++++++++++ app/models/invite.rb | 8 ++ app/services/handle_chunk_upload.rb | 65 ++++++++++++++++ config/locales/client.en.yml | 5 ++ config/locales/server.en.yml | 18 +++++ config/routes.rb | 11 ++- lib/guardian.rb | 4 + script/setup_dev | 2 - spec/controllers/invites_controller_spec.rb | 59 ++++++++++++++ spec/fixtures/csv/discourse.csv | 1 + 16 files changed, 330 insertions(+), 46 deletions(-) create mode 100644 app/jobs/regular/bulk_invite.rb create mode 100644 app/services/handle_chunk_upload.rb create mode 100644 spec/fixtures/csv/discourse.csv diff --git a/app/assets/javascripts/discourse/controllers/user-invited.js.es6 b/app/assets/javascripts/discourse/controllers/user-invited.js.es6 index 6e63475a3..ea1a43da7 100644 --- a/app/assets/javascripts/discourse/controllers/user-invited.js.es6 +++ b/app/assets/javascripts/discourse/controllers/user-invited.js.es6 @@ -14,6 +14,8 @@ export default Ember.ObjectController.extend({ this.set('searchTerm', ''); }, + uploadText: function() { return I18n.t("user.invited.bulk_invite.text"); }.property(), + /** Observe the search term box with a debouncer and change the results. @@ -42,6 +44,15 @@ export default Ember.ObjectController.extend({ return Discourse.User.currentProp('can_invite_to_forum'); }.property(), + /** + Can the currently logged in user bulk invite users to the site (only Admin is allowed to perform this operation) + + @property canBulkInvite + **/ + canBulkInvite: function() { + return Discourse.User.currentProp('admin'); + }.property(), + /** Should the search filter input box be displayed? @@ -75,5 +86,3 @@ export default Ember.ObjectController.extend({ } }); - - diff --git a/app/assets/javascripts/discourse/routes/user_invited_route.js b/app/assets/javascripts/discourse/routes/user_invited_route.js index 98afbdaad..797f4e5a0 100644 --- a/app/assets/javascripts/discourse/routes/user_invited_route.js +++ b/app/assets/javascripts/discourse/routes/user_invited_route.js @@ -34,7 +34,15 @@ Discourse.UserInvitedRoute = Discourse.Route.extend({ showInvite: function() { Discourse.Route.showModal(this, 'invite', Discourse.User.current()); this.controllerFor('invite').reset(); + }, + + uploadSuccess: function(filename) { + bootbox.alert(I18n.t("user.invited.bulk_invite.success", { filename: filename })); + }, + + uploadError: function(filename, message) { + bootbox.alert(I18n.t("user.invited.bulk_invite.error", { filename: filename, message: message })); } } -}); \ No newline at end of file +}); diff --git a/app/assets/javascripts/discourse/templates/user/invited.js.handlebars b/app/assets/javascripts/discourse/templates/user/invited.js.handlebars index 96b77f17c..23bd1e460 100644 --- a/app/assets/javascripts/discourse/templates/user/invited.js.handlebars +++ b/app/assets/javascripts/discourse/templates/user/invited.js.handlebars @@ -2,9 +2,14 @@

{{i18n user.invited.title}}

- {{#if canInviteToForum}} - - {{/if}} +
+ {{#if canInviteToForum}} + + {{/if}} + {{#if canBulkInvite}} + {{resumable-upload target="/invites/upload" success="uploadSuccess" error="uploadError" uploadText=uploadText}} + {{/if}} +
{{#if showSearch}}
diff --git a/app/controllers/admin/backups_controller.rb b/app/controllers/admin/backups_controller.rb index 94881c12b..453cbf2ba 100644 --- a/app/controllers/admin/backups_controller.rb +++ b/app/controllers/admin/backups_controller.rb @@ -2,7 +2,7 @@ require_dependency "backup_restore" class Admin::BackupsController < Admin::AdminController - skip_before_filter :check_xhr, only: [:index, :show, :logs, :check_chunk, :upload_chunk] + skip_before_filter :check_xhr, only: [:index, :show, :logs, :check_backup_chunk, :upload_backup_chunk] def index respond_to do |format| @@ -87,7 +87,7 @@ class Admin::BackupsController < Admin::AdminController render nothing: true end - def check_chunk + def check_backup_chunk identifier = params.fetch(:resumableIdentifier) filename = params.fetch(:resumableFilename) chunk_number = params.fetch(:resumableChunkNumber) @@ -95,15 +95,13 @@ class Admin::BackupsController < Admin::AdminController # path to chunk file chunk = Backup.chunk_path(identifier, filename, chunk_number) - # check whether the chunk has already been uploaded - has_chunk_been_uploaded = File.exists?(chunk) && File.size(chunk) == current_chunk_size - # 200 = exists, 404 = not uploaded yet - status = has_chunk_been_uploaded ? 200 : 404 + # check chunk upload status + status = HandleChunkUpload.check_chunk(chunk, current_chunk_size: current_chunk_size) render nothing: true, status: status end - def upload_chunk + def upload_backup_chunk filename = params.fetch(:resumableFilename) total_size = params.fetch(:resumableTotalSize).to_i @@ -118,15 +116,10 @@ class Admin::BackupsController < Admin::AdminController # path to chunk file chunk = Backup.chunk_path(identifier, filename, chunk_number) - dir = File.dirname(chunk) - - # ensure directory exists - FileUtils.mkdir_p(dir) unless Dir.exists?(dir) - # save chunk to the directory - File.open(chunk, "wb") { |f| f.write(file.tempfile.read) } + # upload chunk + HandleChunkUpload.upload_chunk(chunk, file: file) uploaded_file_size = chunk_number * chunk_size - # when all chunks are uploaded if uploaded_file_size + current_chunk_size >= total_size # merge all the chunks in a background thread diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb index 78623595b..6067ed74c 100644 --- a/app/controllers/invites_controller.rb +++ b/app/controllers/invites_controller.rb @@ -3,7 +3,7 @@ class InvitesController < ApplicationController skip_before_filter :check_xhr skip_before_filter :redirect_to_login_if_required - before_filter :ensure_logged_in, only: [:destroy, :create] + before_filter :ensure_logged_in, only: [:destroy, :create, :check_csv_chunk, :upload_csv_chunk] def show invite = Invite.find_by(invite_key: params[:id]) @@ -51,4 +51,48 @@ class InvitesController < ApplicationController render nothing: true end + def check_csv_chunk + guardian.ensure_can_bulk_invite_to_forum!(current_user) + + filename = params.fetch(:resumableFilename) + identifier = params.fetch(:resumableIdentifier) + chunk_number = params.fetch(:resumableChunkNumber) + current_chunk_size = params.fetch(:resumableCurrentChunkSize).to_i + + # path to chunk file + chunk = Invite.chunk_path(identifier, filename, chunk_number) + # check chunk upload status + status = HandleChunkUpload.check_chunk(chunk, current_chunk_size: current_chunk_size) + + render nothing: true, status: status + end + + def upload_csv_chunk + guardian.ensure_can_bulk_invite_to_forum!(current_user) + + filename = params.fetch(:resumableFilename) + return render status: 415, text: I18n.t("bulk_invite.file_should_be_csv") unless filename.to_s.end_with?(".csv") + + file = params.fetch(:file) + identifier = params.fetch(:resumableIdentifier) + chunk_number = params.fetch(:resumableChunkNumber).to_i + chunk_size = params.fetch(:resumableChunkSize).to_i + total_size = params.fetch(:resumableTotalSize).to_i + current_chunk_size = params.fetch(:resumableCurrentChunkSize).to_i + + # path to chunk file + chunk = Invite.chunk_path(identifier, filename, chunk_number) + # upload chunk + HandleChunkUpload.upload_chunk(chunk, file: file) + + uploaded_file_size = chunk_number * chunk_size + # when all chunks are uploaded + if uploaded_file_size + current_chunk_size >= total_size + # handle bulk_invite processing in a background thread + Jobs.enqueue(:bulk_invite, filename: filename, identifier: identifier, chunks: chunk_number, current_user_id: current_user.id) + end + + render nothing: true + end + end diff --git a/app/jobs/regular/backup_chunks_merger.rb b/app/jobs/regular/backup_chunks_merger.rb index 7cbd37427..97846fd47 100644 --- a/app/jobs/regular/backup_chunks_merger.rb +++ b/app/jobs/regular/backup_chunks_merger.rb @@ -14,27 +14,11 @@ module Jobs backup_path = "#{Backup.base_directory}/#{filename}" tmp_backup_path = "#{backup_path}.tmp" - - # delete destination files - File.delete(backup_path) rescue nil - File.delete(tmp_backup_path) rescue nil - - # merge all the chunks - File.open(tmp_backup_path, "a") do |backup| - (1..chunks).each do |chunk_number| - # path to chunk - chunk_path = Backup.chunk_path(identifier, filename, chunk_number) - # add chunk to backup - backup << File.open(chunk_path).read - end - end - - # rename tmp backup to final backup name - FileUtils.mv(tmp_backup_path, backup_path, force: true) - - # remove tmp directory + # path to tmp directory tmp_directory = File.dirname(Backup.chunk_path(identifier, filename, 0)) - FileUtils.rm_rf(tmp_directory) rescue nil + + # merge all chunks + HandleChunkUpload.merge_chunks(chunks, upload_path: backup_path, tmp_upload_path: tmp_backup_path, model: Backup, identifier: identifier, filename: filename, tmp_directory: tmp_directory) end end diff --git a/app/jobs/regular/bulk_invite.rb b/app/jobs/regular/bulk_invite.rb new file mode 100644 index 000000000..ed2ed63d1 --- /dev/null +++ b/app/jobs/regular/bulk_invite.rb @@ -0,0 +1,78 @@ +require 'csv' +require_dependency 'system_message' + +module Jobs + + class BulkInvite < Jobs::Base + sidekiq_options retry: false + + def initialize + @logs = [] + @sent = 0 + @failed = 0 + end + + def execute(args) + filename = args[:filename] + identifier = args[:identifier] + chunks = args[:chunks].to_i + current_user = User.find_by(id: args[:current_user_id]) + + raise Discourse::InvalidParameters.new(:filename) if filename.blank? + raise Discourse::InvalidParameters.new(:identifier) if identifier.blank? + raise Discourse::InvalidParameters.new(:chunks) if chunks <= 0 + + csv_path = "#{Invite.base_directory}/#{filename}" + tmp_csv_path = "#{csv_path}.tmp" + # path to tmp directory + tmp_directory = File.dirname(Invite.chunk_path(identifier, filename, 0)) + + # merge all chunks + HandleChunkUpload.merge_chunks(chunks, upload_path: csv_path, tmp_upload_path: tmp_csv_path, model: Invite, identifier: identifier, filename: filename, tmp_directory: tmp_directory) + + # read csv file, and send out invitations + CSV.foreach(csv_path) do |csv_info| + if !csv_info[0].nil? + if validate_email(csv_info[0]) + Invite.invite_by_email(csv_info[0], current_user, topic=nil) + @sent += 1 + else + log "Invalid email '#{csv_info[0]}' at line number '#{$INPUT_LINE_NUMBER}'" + @failed += 1 + end + end + end + + # send notification to user regarding progress + notify_user(current_user) + + # since emails have already been sent out, delete the uploaded csv file + FileUtils.rm_rf(csv_path) rescue nil + end + + def validate_email(email) + /\A[^@\s]+@([^@\s]+\.)+[^@\s]+\z/.match(email) + end + + def log(message) + puts(message) rescue nil + save_log(message) + end + + def save_log(message) + @logs << "[#{Time.now}] #{message}" + end + + def notify_user(current_user) + if current_user + if (@sent > 0 && @failed == 0) + SystemMessage.create(current_user, :bulk_invite_succeeded, sent: @sent) + else + SystemMessage.create(current_user, :bulk_invite_failed, sent: @sent, failed: @failed, logs: @logs.join("\n")) + end + end + end + + end + +end diff --git a/app/models/invite.rb b/app/models/invite.rb index 83a001c10..bbdf5edbd 100644 --- a/app/models/invite.rb +++ b/app/models/invite.rb @@ -129,6 +129,14 @@ class Invite < ActiveRecord::Base end i end + + def self.base_directory + File.join(Rails.root, "public", "csv", RailsMultisite::ConnectionManagement.current_db) + end + + def self.chunk_path(identifier, filename, chunk_number) + File.join(Invite.base_directory, "tmp", identifier, "#{filename}.part#{chunk_number}") + end end # == Schema Information diff --git a/app/services/handle_chunk_upload.rb b/app/services/handle_chunk_upload.rb new file mode 100644 index 000000000..c71a656da --- /dev/null +++ b/app/services/handle_chunk_upload.rb @@ -0,0 +1,65 @@ +class HandleChunkUpload + + def initialize(chunk, params={}) + @chunk = chunk + @params = params + end + + def self.check_chunk(chunk, params) + HandleChunkUpload.new(chunk, params).check_chunk + end + + def self.upload_chunk(chunk, params) + HandleChunkUpload.new(chunk, params).upload_chunk + end + + def self.merge_chunks(chunk, params) + HandleChunkUpload.new(chunk, params).merge_chunks + end + + def check_chunk + # check whether the chunk has already been uploaded + has_chunk_been_uploaded = File.exists?(@chunk) && File.size(@chunk) == @params[:current_chunk_size] + # 200 = exists, 404 = not uploaded yet + status = has_chunk_been_uploaded ? 200 : 404 + end + + def upload_chunk + # path to chunk file + dir = File.dirname(@chunk) + # ensure directory exists + FileUtils.mkdir_p(dir) unless Dir.exists?(dir) + # save chunk to the directory + File.open(@chunk, "wb") { |f| f.write(@params[:file].tempfile.read) } + end + + def merge_chunks + upload_path = @params[:upload_path] + tmp_upload_path = @params[:tmp_upload_path] + model = @params[:model] + identifier = @params[:identifier] + filename = @params[:filename] + tmp_directory = @params[:tmp_directory] + + # delete destination files + File.delete(upload_path) rescue nil + File.delete(tmp_upload_path) rescue nil + + # merge all the chunks + File.open(tmp_upload_path, "a") do |file| + (1..@chunk).each do |chunk_number| + # path to chunk + chunk_path = model.chunk_path(identifier, filename, chunk_number) + # add chunk to file + file << File.open(chunk_path).read + end + end + + # rename tmp file to final file name + FileUtils.mv(tmp_upload_path, upload_path, force: true) + + # remove tmp directory + FileUtils.rm_rf(tmp_directory) rescue nil + end + +end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 3c855f799..3ed52adeb 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -414,6 +414,11 @@ en: days_visited: "Days Visited" account_age_days: "Account age in days" create: "Send an Invite" + bulk_invite: + text: "Batch Invite from File" + uploading: "UPLOADING" + success: "File has been uploaded successfully, you will be notified shortly with progress." + error: "There has been an error while uploading '{{filename}}': {{message}}" password: title: "Password" diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index d7f69a0d6..927410b8f 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -39,6 +39,9 @@ en: embed: load_from_remote: "There was an error loading that post." + bulk_invite: + file_should_be_csv: "The uploaded file should be of csv format." + backup: operation_already_running: "An operation is currently running. Can't start a new job right now." backup_file_should_be_tar_gz: "The backup file should be a .tar.gz archive." @@ -1249,6 +1252,21 @@ en: %{logs} ``` + bulk_invite_succeeded: + subject_template: "Bulk Invite processed successfully!" + text_body_template: "The bulk invite has been processed, %{sent} invites sent." + + bulk_invite_failed: + subject_template: "Bulk Invite processed with some errors!" + text_body_template: | + The bulk invite has been processed, %{sent} invites sent and %{failed} invites failed. + + Here's the log: + + ``` + %{logs} + ``` + email_error_notification: subject_template: "Error parsing email" text_body_template: | diff --git a/config/routes.rb b/config/routes.rb index 880c561fe..7710d2a76 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -132,8 +132,8 @@ Discourse::Application.routes.draw do get "cancel" => "backups#cancel" get "rollback" => "backups#rollback" put "readonly" => "backups#readonly" - get "upload" => "backups#check_chunk" - post "upload" => "backups#upload_chunk" + get "upload" => "backups#check_backup_chunk" + post "upload" => "backups#upload_backup_chunk" end end @@ -360,7 +360,12 @@ Discourse::Application.routes.draw do get "/posts/:id/expand-embed" => "posts#expand_embed" get "raw/:topic_id(/:post_number)" => "posts#markdown" - resources :invites + resources :invites do + collection do + get "upload" => "invites#check_csv_chunk" + post "upload" => "invites#upload_csv_chunk" + end + end delete "invites" => "invites#destroy" get "onebox" => "onebox#show" diff --git a/lib/guardian.rb b/lib/guardian.rb index f19a18d93..bec29dc3f 100644 --- a/lib/guardian.rb +++ b/lib/guardian.rb @@ -204,6 +204,10 @@ class Guardian ( group_ids.blank? || is_admin? ) end + def can_bulk_invite_to_forum?(user) + user.admin? + end + def can_see_private_messages?(user_id) is_admin? || (authenticated? && @user.id == user_id) end diff --git a/script/setup_dev b/script/setup_dev index 133d76e5b..fc71f7f12 100755 --- a/script/setup_dev +++ b/script/setup_dev @@ -56,5 +56,3 @@ if User.count == 0 puts puts "To get started run: bundle exec thin start" end - - diff --git a/spec/controllers/invites_controller_spec.rb b/spec/controllers/invites_controller_spec.rb index c29b42698..436a0dc27 100644 --- a/spec/controllers/invites_controller_spec.rb +++ b/spec/controllers/invites_controller_spec.rb @@ -148,6 +148,65 @@ describe InvitesController do end + end + + context '.check_csv_chunk' do + it 'requires you to be logged in' do + lambda { + post :check_csv_chunk + }.should raise_error(Discourse::NotLoggedIn) + end + + context 'while logged in' do + let(:resumableChunkNumber) { 1 } + let(:resumableCurrentChunkSize) { 46 } + let(:resumableIdentifier) { '46-discoursecsv' } + let(:resumableFilename) { 'discourse.csv' } + + it "fails if you can't bulk invite to the forum" do + log_in + post :check_csv_chunk, resumableChunkNumber: resumableChunkNumber, resumableCurrentChunkSize: resumableCurrentChunkSize.to_i, resumableIdentifier: resumableIdentifier, resumableFilename: resumableFilename + response.should_not be_success + end + + end + + end + + context '.upload_csv_chunk' do + it 'requires you to be logged in' do + lambda { + post :upload_csv_chunk + }.should raise_error(Discourse::NotLoggedIn) + end + + context 'while logged in' do + let(:csv_file) { File.new("#{Rails.root}/spec/fixtures/csv/discourse.csv") } + let(:file) do + ActionDispatch::Http::UploadedFile.new({ filename: 'discourse.csv', tempfile: csv_file }) + end + let(:resumableChunkNumber) { 1 } + let(:resumableChunkSize) { 1048576 } + let(:resumableCurrentChunkSize) { 46 } + let(:resumableTotalSize) { 46 } + let(:resumableType) { 'text/csv' } + let(:resumableIdentifier) { '46-discoursecsv' } + let(:resumableFilename) { 'discourse.csv' } + let(:resumableRelativePath) { 'discourse.csv' } + + it "fails if you can't bulk invite to the forum" do + log_in + post :upload_csv_chunk, file: file, resumableChunkNumber: resumableChunkNumber.to_i, resumableChunkSize: resumableChunkSize.to_i, resumableCurrentChunkSize: resumableCurrentChunkSize.to_i, resumableTotalSize: resumableTotalSize.to_i, resumableType: resumableType, resumableIdentifier: resumableIdentifier, resumableFilename: resumableFilename + response.should_not be_success + end + + it "allows admins to bulk invite" do + log_in(:admin) + post :upload_csv_chunk, file: file, resumableChunkNumber: resumableChunkNumber.to_i, resumableChunkSize: resumableChunkSize.to_i, resumableCurrentChunkSize: resumableCurrentChunkSize.to_i, resumableTotalSize: resumableTotalSize.to_i, resumableType: resumableType, resumableIdentifier: resumableIdentifier, resumableFilename: resumableFilename + response.should be_success + end + + end end diff --git a/spec/fixtures/csv/discourse.csv b/spec/fixtures/csv/discourse.csv new file mode 100644 index 000000000..4bcc943c1 --- /dev/null +++ b/spec/fixtures/csv/discourse.csv @@ -0,0 +1 @@ +jeff@gmail.com sam@yahoo.com robin@outlook.com neil@aol.com regis@live.com \ No newline at end of file