From 652cc3efba074a00359a993b0bac191422bf76e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Mon, 29 Sep 2014 18:31:53 +0200 Subject: [PATCH] FEATURE: new rake task to clean up uploads & thumbnails --- app/controllers/user_avatars_controller.rb | 1 - lib/file_store/base_store.rb | 3 - lib/file_store/local_store.rb | 9 --- lib/file_store/s3_store.rb | 5 -- lib/letter_avatar.rb | 1 - lib/tasks/uploads.rake | 75 +++++++++++++++++++ .../components/file_store/local_store_spec.rb | 10 +-- spec/components/file_store/s3_store_spec.rb | 11 --- 8 files changed, 76 insertions(+), 39 deletions(-) diff --git a/app/controllers/user_avatars_controller.rb b/app/controllers/user_avatars_controller.rb index 1b1b034e1..23b53f36f 100644 --- a/app/controllers/user_avatars_controller.rb +++ b/app/controllers/user_avatars_controller.rb @@ -84,7 +84,6 @@ class UserAvatarsController < ApplicationController end end - # this protects us from a DoS def render_dot expires_in 10.minutes, public: true diff --git a/lib/file_store/base_store.rb b/lib/file_store/base_store.rb index 0769a52e6..a89e8d604 100644 --- a/lib/file_store/base_store.rb +++ b/lib/file_store/base_store.rb @@ -8,9 +8,6 @@ module FileStore def store_optimized_image(file, optimized_image) end - def store_avatar(file, avatar, size) - end - def remove_upload(upload) end diff --git a/lib/file_store/local_store.rb b/lib/file_store/local_store.rb index 7ebd95689..2986d8212 100644 --- a/lib/file_store/local_store.rb +++ b/lib/file_store/local_store.rb @@ -14,11 +14,6 @@ module FileStore store_file(file, path) end - def store_avatar(file, avatar, size) - path = get_path_for_avatar(file, avatar, size) - store_file(file, path) - end - def remove_upload(upload) remove_file(upload.url) end @@ -80,10 +75,6 @@ module FileStore "#{relative_base_url}/_optimized/#{optimized_image.sha1[0..2]}/#{optimized_image.sha1[3..5]}/#{filename}" end - def get_path_for_avatar(file, avatar, size) - relative_avatar_template(avatar).gsub("{size}", size.to_s) - end - def relative_avatar_template(avatar) File.join( relative_base_url, diff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rb index 3ac67dafe..5d24f0054 100644 --- a/lib/file_store/s3_store.rb +++ b/lib/file_store/s3_store.rb @@ -20,11 +20,6 @@ module FileStore store_file(file, path) end - def store_avatar(file, avatar, size) - path = get_path_for_avatar(file, avatar, size) - store_file(file, path) - end - def remove_upload(upload) remove_file(upload.url) end diff --git a/lib/letter_avatar.rb b/lib/letter_avatar.rb index d7da8541e..ecb031727 100644 --- a/lib/letter_avatar.rb +++ b/lib/letter_avatar.rb @@ -23,7 +23,6 @@ class LetterAvatar end end - def cache_path "public/uploads/letter_avatars/#{VERSION}" end diff --git a/lib/tasks/uploads.rake b/lib/tasks/uploads.rake index 63de5ae7b..e24e7b03a 100644 --- a/lib/tasks/uploads.rake +++ b/lib/tasks/uploads.rake @@ -77,3 +77,78 @@ task "uploads:migrate_from_s3" => :environment do puts end + +task "uploads:clean_up" => :environment do + + RailsMultisite::ConnectionManagement.each_connection do |db| + puts "Cleaning up uploads and thumbnails for '#{db}'..." + + if Discourse.store.external? + puts "This task only works for internal storages." + next + end + + public_directory = "#{Rails.root}/public" + + ## + ## DATABASE vs FILE SYSTEM + ## + + # uploads & avatars + Upload.order(:id).find_each do |upload| + path = "#{public_directory}#{upload.url}" + if !File.exists?(path) + upload.destroy rescue nil + putc "#" + else + putc "." + end + end + + # optimized images + OptimizedImage.order(:id).find_each do |optimized_image| + path = "#{public_directory}#{optimized_image.url}" + if !File.exists?(path) + optimized_image.destroy rescue nil + putc "#" + else + putc "." + end + end + + ## + ## FILE SYSTEM vs DATABASE + ## + + uploads_directory = "#{public_directory}/uploads/#{db}" + + # avatars (no avatar should be stored in that old directory) + FileUtils.rm_rf("#{uploads_directory}/avatars") rescue nil + + # uploads + Dir.glob("#{uploads_directory}/*/*.*").each do |f| + url = "/uploads/#{db}/" << f.split("/uploads/#{db}/")[1] + if !Upload.where(url: url).exists? + FileUtils.rm(f) rescue nil + putc "#" + else + putc "." + end + end + + # optimized images + Dir.glob("#{uploads_directory}/_optimized/*/*/*.*").each do |f| + url = "/uploads/#{db}/_optimized/" << f.split("/uploads/#{db}/_optimized/")[1] + if !OptimizedImage.where(url: url).exists? + FileUtils.rm(f) rescue nil + putc "#" + else + putc "." + end + end + + puts + + end + +end diff --git a/spec/components/file_store/local_store_spec.rb b/spec/components/file_store/local_store_spec.rb index c8cbb53bb..097976dcf 100644 --- a/spec/components/file_store/local_store_spec.rb +++ b/spec/components/file_store/local_store_spec.rb @@ -9,6 +9,7 @@ describe FileStore::LocalStore do let(:uploaded_file) { file_from_fixtures("logo.png") } let(:optimized_image) { build(:optimized_image) } + let(:avatar) { build(:upload) } describe ".store_upload" do @@ -31,15 +32,6 @@ describe FileStore::LocalStore do end - describe ".store_avatar" do - - it "returns a relative url" do - store.expects(:copy_file) - store.store_avatar({}, upload, 100).should == "/uploads/default/avatars/e9d/71f/5ee7c92d6d/100.png" - end - - end - describe ".remove_upload" do it "does not delete non uploaded" do diff --git a/spec/components/file_store/s3_store_spec.rb b/spec/components/file_store/s3_store_spec.rb index c7a233aea..ad0b2fecc 100644 --- a/spec/components/file_store/s3_store_spec.rb +++ b/spec/components/file_store/s3_store_spec.rb @@ -13,7 +13,6 @@ describe FileStore::S3Store do let(:optimized_image_file) { file_from_fixtures("logo.png") } let(:avatar) { build(:upload) } - let(:avatar_file) { file_from_fixtures("logo-dev.png") } before(:each) do SiteSetting.stubs(:s3_upload_bucket).returns("S3_Upload_Bucket") @@ -42,16 +41,6 @@ describe FileStore::S3Store do end - describe ".store_avatar" do - - it "returns an absolute schemaless url" do - avatar.stubs(:id).returns(42) - s3_helper.expects(:upload) - store.store_avatar(avatar_file, avatar, 100).should == "//s3_upload_bucket.s3.amazonaws.com/avatars/e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98/100.png" - end - - end - describe ".remove_upload" do it "calls remove_file with the url" do