add a tombstone for extra safety

app/jobs/scheduled/clean_up_uploads.rb

@@ -9,7 +9,7 @@ module Jobs
       uploads_used_in_posts = PostUpload.uniq.pluck(:upload_id)
       uploads_used_as_avatars = User.uniq.where('uploaded_avatar_id IS NOT NULL').pluck(:uploaded_avatar_id)
 
-      grace_period = [SiteSetting.uploads_grace_period_in_hours, 1].max
+      grace_period = [SiteSetting.clean_orphan_uploads_grace_period_hours, 1].max
 
       Upload.where("created_at < ?", grace_period.hour.ago)
             .where("id NOT IN (?)", uploads_used_in_posts + uploads_used_as_avatars)

app/jobs/scheduled/purge_deleted_uploads.rb

@@ -0,0 +1,13 @@
+module Jobs
+
+  class PurgeDeletedUploads < Jobs::Scheduled
+    recurrence { daily }
+
+    def execute(args)
+      grace_period = SiteSetting.purge_deleted_uploads_grace_period_days
+      Discourse.store.purge_tombstone(grace_period)
+    end
+
+  end
+
+end

config/locales/server.en.yml

@@ -633,7 +633,8 @@ en:
     suggested_topics: "Number of suggested topics shown at the bottom of a topic"
 
     clean_up_uploads: "Remove orphaned uploads to prevent illegal hosting. WARNING: you might want to make a backup of your /uploads directory before enabled this setting."
-    uploads_grace_period_in_hours: "Grace period (in hours) before an orphan upload is removed."
+    clean_orphan_uploads_grace_period_hours: "Grace period (in hours) before an orphan upload is removed."
+    purge_deleted_uploads_grace_period_days: "Grace period (in days) before a deleted upload is erased."
     enable_s3_uploads: "Place uploads on Amazon S3"
     s3_upload_bucket: "The Amazon S3 bucket name that files will be uploaded into. WARNING: must be lowercase (cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html)"
     s3_access_key_id: "The Amazon S3 access key id that will be used to upload images"

config/locales/server.fr.yml

@@ -557,7 +557,7 @@ fr:
     max_topics_per_day: "Quantité maximale de discussions que vous pouvez créer en un jour"
     max_private_messages_per_day: "Quantité maximale de messages privés que vous pouvez envoyer en un jour"
     suggested_topics: "Nombre de discussions suggérées à la fin d'une discussion"
-    uploads_grace_period_in_hours: "La période de grâce (en heures) avant qu'un téléchargement orphelin soit retiré."
+    clean_orphan_uploads_grace_period_hours: "La période de grâce (en heures) avant qu'un téléchargement orphelin soit retiré."
     enable_s3_uploads: "S'il faut ou non uploader sur Amazon S3"
     s3_upload_bucket: "Le bucket name Amazon S3 qui contiendra les fichiers téléchargés. ATTENTION : doit être en minuscule (cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html)"
     s3_access_key_id: "L' access key Amazon S3 qui sera utilisée pour uploader les images"

config/locales/server.ko.yml

@@ -623,7 +623,7 @@ ko:
     suggested_topics: "The number of suggested topics shown at the bottom of a topic"
 
     clean_up_uploads: "Remove orphaned uploads to prevent illegal hosting. WARNING: you might want to make a backup of your /uploads directory before enabled this setting."
-    uploads_grace_period_in_hours: "Grace period (in hours) before an orphan upload is removed."
+    clean_orphan_uploads_grace_period_hours: "Grace period (in hours) before an orphan upload is removed."
     enable_s3_uploads: "Place uploads on Amazon S3"
     s3_upload_bucket: "The Amazon S3 bucket name that files will be uploaded into"
     s3_access_key_id: "이미지 업로드를 위한 아마존 S3 ACCESS KEY"

config/locales/server.nl.yml

@@ -635,7 +635,7 @@ nl:
     suggested_topics: Het aantal aanbevolen topics dat is weergegeven aan de onderkant van een topic
 
     clean_up_uploads: "Verwijder weesbestanden om illegale hosting te voorkomen. LET OP: maak een backup van je /uploads directory voordat je deze instelling activeert."
-    uploads_grace_period_in_hours: Na hoeveel uur een weesbestand verwijderd wordt.
+    clean_orphan_uploads_grace_period_hours: Na hoeveel uur een weesbestand verwijderd wordt.
     enable_s3_uploads: Of we uploads op Amazon S3 willen zetten of niet
     s3_upload_bucket: "De 'bucket' waarin we onze uploads naar Amazon S3 willen zetten"
     s3_access_key_id: "De Amazon S3 access key id dat wordt gebruikt om afbeeldingen te uploaden"

config/locales/server.pt_BR.yml

@@ -615,7 +615,7 @@ pt_BR:
     suggested_topics: "Quantidade de tópicos sugeridos que você verá na parte debaixo dos tópicos"
 
     clean_up_uploads: "Remove envios(uploads) orfãos para previnir hospedagem ilegal. AVISO: Você pode querer fazer um backup do seu diretório /uploads antes de habilitar essa configuração."
-    uploads_grace_period_in_hours: "Carência (em horas), antes que um envio(upload) órfão é removido."
+    clean_orphan_uploads_grace_period_hours: "Carência (em horas), antes que um envio(upload) órfão é removido."
     enable_s3_uploads: "Enviar arquivos recebidos para o s3"
     s3_upload_bucket: "Bucket do s3 para qual enviar"
     s3_access_key_id: "Access key da Amazon S3 que será usada para envio de imagens"

config/locales/server.ru.yml

@@ -601,7 +601,7 @@ ru:
     max_private_messages_per_day: 'Максимальное количество личных сообщений, которое пользователь может послать в день'
     suggested_topics: 'Количество рекомендованных тем, отображаемых внизу текущей темы'
     clean_up_uploads: 'Удалить неиспользуемые загрузки для предотвращения хранения нелегального контента. ВНИМАНИЕ: рекомендуется сделать резервную копию директории /uploads перед включением данной настройки.'
-    uploads_grace_period_in_hours: 'Период (в часах) после которого неопубликованные вложения удаляются.'
+    clean_orphan_uploads_grace_period_hours: 'Период (в часах) после которого неопубликованные вложения удаляются.'
     enable_s3_uploads: 'Размещать загруженные файлы на Amazon S3'
     s3_upload_bucket: 'Наименование Amazon S3 bucket в который будут загружаться файлы. ВНИМАНИЕ: имя должно быть в нижнем регистре (см. http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html)'
     s3_access_key_id: 'Amazon S3 access key для загрузки и хранения изображений'

config/site_settings.yml

@@ -233,7 +233,8 @@ files:
     default: 500
   create_thumbnails: true
   clean_up_uploads: false
-  uploads_grace_period_in_hours: 1
+  clean_orphan_uploads_grace_period_hours: 1
+  purge_deleted_uploads_grace_period_days: 30
   enable_s3_uploads: false
   s3_access_key_id: ''
   s3_secret_access_key: ''

lib/file_store/base_store.rb

@@ -41,6 +41,9 @@ module FileStore
     def absolute_avatar_template(avatar)
     end
 
+    def purge_tombstone(grace_period)
+    end
+
   end
 
 end

lib/file_store/local_store.rb

@@ -55,6 +55,10 @@ module FileStore
       avatar_template(avatar, absolute_base_url)
     end
 
+    def purge_tombstone(grace_period)
+      `find #{tombstone_dir} -mtime +#{grace_period} -type f -delete`
+    end
+
     private
 
     def get_path_for_upload(file, upload)
@@ -105,14 +109,16 @@ module FileStore
     def copy_file(file, path)
       FileUtils.mkdir_p(Pathname.new(path).dirname)
       # move the file to the right location
-      # not using cause mv, cause permissions are no good on move
+      # not using mv, cause permissions are no good on move
-      File.open(path, "wb") do |f|
+      File.open(path, "wb") { |f| f.write(file.read) }
-        f.write(file.read)
-      end
     end
 
     def remove_file(url)
-      File.delete("#{public_dir}#{url}") if is_relative?(url)
+      return unless is_relative?(url)
+      path = public_dir + url
+      tombstone = public_dir + url.gsub("/uploads/", "/tombstone/")
+      FileUtils.mkdir_p(Pathname.new(tombstone).dirname)
+      FileUtils.move(path, tombstone)
     rescue Errno::ENOENT
       # don't care if the file isn't there
     end
@@ -134,6 +140,10 @@ module FileStore
       "#{Rails.root}/public"
     end
 
+    def tombstone_dir
+      public_dir + relative_base_url.gsub("/uploads/", "/tombstone/")
+    end
+
   end
 
 end

lib/file_store/s3_store.rb

@@ -62,6 +62,10 @@ module FileStore
       "#{absolute_base_url}/avatars/#{avatar.sha1}/{size}#{avatar.extension}"
     end
 
+    def purge_tombstone(grace_period)
+      update_tombstone_lifecycle(grace_period)
+    end
+
     private
 
     def get_path_for_upload(file, upload)
@@ -99,16 +103,6 @@ module FileStore
       raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank?
     end
 
-    def get_or_create_directory(bucket)
-      check_missing_site_settings
-
-      fog = Fog::Storage.new(s3_options)
-
-      directory = fog.directories.get(bucket)
-      directory = fog.directories.create(key: bucket) unless directory
-      directory
-    end
-
     def s3_options
       options = {
         provider: 'AWS',
@@ -119,6 +113,18 @@ module FileStore
       options
     end
 
+    def fog_with_options
+      check_missing_site_settings
+      Fog::Storage.new(s3_options)
+    end
+
+    def get_or_create_directory(bucket)
+      fog = fog_with_options
+      directory = fog.directories.get(bucket)
+      directory = fog.directories.create(key: bucket) unless directory
+      directory
+    end
+
     def upload(file, unique_filename, filename=nil, content_type=nil)
       args = {
         key: unique_filename,
@@ -132,13 +138,32 @@ module FileStore
     end
 
     def remove(unique_filename)
-      check_missing_site_settings
+      fog = fog_with_options
-
+      # copy the file in tombstone
-      fog = Fog::Storage.new(s3_options)
+      fog.copy_object(unique_filename, s3_bucket, tombstone_prefix + unique_filename, s3_bucket)
-
+      # delete the file
       fog.delete_object(s3_bucket, unique_filename)
     end
 
+    def update_tombstone_lifecycle(grace_period)
+      # cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
+      fog_with_options.put_bucket_lifecycle(s3_bucket, lifecycle(grace_period))
+    end
+
+    def lifecycle(grace_period)
+      {
+        "Rules" => [{
+          "Prefix" => tombstone_prefix,
+          "Enabled" => true,
+          "Expiration" => { "Days" => grace_period }
+        }]
+      }
+    end
+
+    def tombstone_prefix
+      "tombstone/"
+    end
+
   end
 
 end

script/user_simulator.rb

@@ -49,7 +49,6 @@ puts "Simulating activity for user id #{user.id}: #{user.name}"
 
 while true
   puts "Creating a random topic"
-
   category = Category.where(read_restricted: false).order('random()').first
   PostCreator.create(user, raw: sentence, title: sentence[0..50].strip, category:  category.name)
 

spec/components/file_store/local_store_spec.rb

@@ -48,14 +48,15 @@ describe FileStore::LocalStore do
   describe ".remove_upload" do
 
     it "does not delete non uploaded" do
-      File.expects(:delete).never
+      FileUtils.expects(:mkdir_p).never
       upload = Upload.new
       upload.stubs(:url).returns("/path/to/file")
       store.remove_upload(upload)
     end
 
-    it "deletes the file locally" do
+    it "moves the file to the tombstone" do
-      File.expects(:delete)
+      FileUtils.expects(:mkdir_p)
+      FileUtils.expects(:move)
       upload = Upload.new
       upload.stubs(:url).returns("/uploads/default/42/253dc8edf9d4ada1.png")
       store.remove_upload(upload)
@@ -65,11 +66,12 @@ describe FileStore::LocalStore do
 
   describe ".remove_optimized_image" do
 
-    it "deletes the file locally" do
+    it "moves the file to the tombstone" do
-      File.expects(:delete)
+      FileUtils.expects(:mkdir_p)
+      FileUtils.expects(:move)
       oi = OptimizedImage.new
       oi.stubs(:url).returns("/uploads/default/_optimized/42/253dc8edf9d4ada1.png")
-      store.remove_upload(upload)
+      store.remove_optimized_image(upload)
     end
 
   end

spec/components/file_store/s3_store_spec.rb

@@ -35,6 +35,8 @@ describe FileStore::S3Store do
     SiteSetting.stubs(:s3_access_key_id).returns("s3_access_key_id")
     SiteSetting.stubs(:s3_secret_access_key).returns("s3_secret_access_key")
     Fog.mock!
+    Fog::Mock.reset
+    Fog::Mock.delay = 0
   end
 
   after(:each) { Fog.unmock! }