storage engines refactor

lib/file_store/base_store.rb

@@ -3,33 +3,50 @@ module FileStore
   class BaseStore
 
     def store_upload(file, upload, content_type = nil)
+      path = get_path_for_upload(upload)
+      store_file(file, path)
     end
 
     def store_optimized_image(file, optimized_image)
+      path = get_path_for_optimized_image(optimized_image)
+      store_file(file, path)
+    end
+
+    def store_file(file, path, opts = {})
     end
 
     def remove_upload(upload)
+      remove_file(upload.url)
     end
 
     def remove_optimized_image(optimized_image)
+      remove_file(optimized_image.url)
+    end
+
+    def remove_file(url)
     end
 
     def has_been_uploaded?(url)
     end
 
+    def download_url(upload)
+    end
+
+    def cdn_url(url)
+      url
+    end
+
     def absolute_base_url
     end
 
     def relative_base_url
     end
 
-    def download_url(upload)
-    end
-
     def external?
     end
 
     def internal?
+      !external?
     end
 
     def path_for(upload)
@@ -38,16 +55,9 @@ module FileStore
     def download(upload)
     end
 
-    def avatar_template(avatar)
-    end
-
     def purge_tombstone(grace_period)
     end
 
-    def cdn_url(url)
-      url
-    end
-
     def get_path_for(type, id, sha, extension)
       depth = [0, Math.log(id / 1_000.0, 16).ceil].max
       tree = File.join(*sha[0, depth].split(""), "")

lib/file_store/local_store.rb

@@ -4,22 +4,19 @@ module FileStore
 
   class LocalStore < BaseStore
 
-    def store_upload(file, upload, content_type = nil)
-      path = get_path_for_upload(upload)
-      store_file(file, path)
+    def store_file(file, path)
+      copy_file(file, "#{public_dir}#{path}")
+      "#{Discourse.base_uri}#{path}"
     end
 
-    def store_optimized_image(file, optimized_image)
-      path = get_path_for_optimized_image(optimized_image)
-      store_file(file, path)
-    end
-
-    def remove_upload(upload)
-      remove_file(upload.url)
-    end
-
-    def remove_optimized_image(optimized_image)
-      remove_file(optimized_image.url)
+    def remove_file(url)
+      return unless is_relative?(url)
+      path = public_dir + url
+      tombstone = public_dir + url.gsub("/uploads/", "/tombstone/")
+      FileUtils.mkdir_p(Pathname.new(tombstone).dirname)
+      FileUtils.move(path, tombstone)
+    rescue Errno::ENOENT
+      # don't care if the file isn't there
     end
 
     def has_been_uploaded?(url)
@@ -34,19 +31,15 @@ module FileStore
       "/uploads/#{RailsMultisite::ConnectionManagement.current_db}"
     end
 
+    def external?
+      false
+    end
+
     def download_url(upload)
       return unless upload
       "#{relative_base_url}/#{upload.sha1}"
     end
 
-    def external?
-      !internal?
-    end
-
-    def internal?
-      true
-    end
-
     def path_for(upload)
       "#{public_dir}#{upload.url}"
     end
@@ -55,19 +48,10 @@ module FileStore
       `find #{tombstone_dir} -mtime +#{grace_period} -type f -delete`
     end
 
-    private
-
     def get_path_for(type, upload_id, sha, extension)
       "#{relative_base_url}/#{super(type, upload_id, sha, extension)}"
     end
 
-    def store_file(file, path)
-      # copy the file to the right location
-      copy_file(file, "#{public_dir}#{path}")
-      # url
-      "#{Discourse.base_uri}#{path}"
-    end
-
     def copy_file(file, path)
       FileUtils.mkdir_p(Pathname.new(path).dirname)
       # move the file to the right location
@@ -75,16 +59,6 @@ module FileStore
       File.open(path, "wb") { |f| f.write(file.read) }
     end
 
-    def remove_file(url)
-      return unless is_relative?(url)
-      path = public_dir + url
-      tombstone = public_dir + url.gsub("/uploads/", "/tombstone/")
-      FileUtils.mkdir_p(Pathname.new(tombstone).dirname)
-      FileUtils.move(path, tombstone)
-    rescue Errno::ENOENT
-      # don't care if the file isn't there
-    end
-
     def is_relative?(url)
       url.present? && url.start_with?(relative_base_url)
     end

lib/file_store/s3_store.rb

@@ -13,22 +13,37 @@ module FileStore
       @s3_helper = s3_helper || S3Helper.new(s3_bucket, TOMBSTONE_PREFIX)
     end
 
-    def store_upload(file, upload, content_type=nil)
+    def store_upload(file, upload, content_type = nil)
       path = get_path_for_upload(upload)
       store_file(file, path, filename: upload.original_filename, content_type: content_type, cache_locally: true)
     end
 
-    def store_optimized_image(file, optimized_image)
-      path = get_path_for_optimized_image(optimized_image)
-      store_file(file, path)
+    # options
+    #   - filename
+    #   - content_type
+    #   - cache_locally
+    def store_file(file, path, opts={})
+      filename      = opts[:filename].presence
+      content_type  = opts[:content_type].presence
+      # cache file locally when needed
+      cache_file(file, File.basename(path)) if opts[:cache_locally]
+      # stored uploaded are public by default
+      options = { acl: "public-read" }
+      # add a "content disposition" header for "attachments"
+      options[:content_disposition] = "attachment; filename=\"#{filename}\"" if filename && !FileHelper.is_image?(filename)
+      # add a "content type" header when provided
+      options[:content_type] = content_type if content_type
+      # if this fails, it will throw an exception
+      @s3_helper.upload(file, path, options)
+      # return the upload url
+      "#{absolute_base_url}/#{path}"
     end
 
-    def remove_upload(upload)
-      remove_file(upload.url)
-    end
-
-    def remove_optimized_image(optimized_image)
-      remove_file(optimized_image.url)
+    def remove_file(url)
+      return unless has_been_uploaded?(url)
+      filename = File.basename(url)
+      # copy the removed file to tombstone
+      @s3_helper.remove(filename, true)
     end
 
     def has_been_uploaded?(url)
@@ -51,10 +66,6 @@ module FileStore
       true
     end
 
-    def internal?
-      !external?
-    end
-
     def download(upload)
       return unless has_been_uploaded?(upload.url)
 
@@ -85,69 +96,47 @@ module FileStore
     end
 
     def cdn_url(url)
-      if SiteSetting.s3_cdn_url.present?
-        url.sub(absolute_base_url, SiteSetting.s3_cdn_url)
-      else
-        url
-      end
+      return url if SiteSetting.s3_cdn_url.blank?
+      url.sub(absolute_base_url, SiteSetting.s3_cdn_url)
     end
 
-    private
+    def cache_avatar(avatar, user_id)
+      source = avatar.url.sub(absolute_base_url + "/", "")
+      destination = avatar_template(avatar, user_id).sub(absolute_base_url + "/", "")
+      @s3_helper.copy(source, destination)
+    end
 
-      # options
-      #   - filename
-      #   - content_type
-      #   - cache_locally
-      def store_file(file, path, opts={})
-        filename      = opts[:filename].presence
-        content_type  = opts[:content_type].presence
-        # cache file locally when needed
-        cache_file(file, File.basename(path)) if opts[:cache_locally]
-        # stored uploaded are public by default
-        options = { acl: "public-read" }
-        # add a "content disposition" header for "attachments"
-        options[:content_disposition] = "attachment; filename=\"#{filename}\"" if filename && !FileHelper.is_image?(filename)
-        # add a "content type" header when provided
-        options[:content_type] = content_type if content_type
-        # if this fails, it will throw an exception
-        @s3_helper.upload(file, path, options)
-        # return the upload url
-        "#{absolute_base_url}/#{path}"
-      end
+    def avatar_template(avatar, user_id)
+      UserAvatar.external_avatar_url(user_id, avatar.upload_id, avatar.width)
+    end
 
-      def remove_file(url)
-        return unless has_been_uploaded?(url)
-        filename = File.basename(url)
-        # copy the removed file to tombstone
-        @s3_helper.remove(filename, true)
-      end
+    CACHE_DIR ||= "#{Rails.root}/tmp/s3_cache/"
+    CACHE_MAXIMUM_SIZE ||= 500
 
-      CACHE_DIR ||= "#{Rails.root}/tmp/s3_cache/"
-      CACHE_MAXIMUM_SIZE ||= 500
+    def get_cache_path_for(filename)
+      "#{CACHE_DIR}#{filename}"
+    end
 
-      def get_cache_path_for(filename)
-        "#{CACHE_DIR}#{filename}"
-      end
+    def get_from_cache(filename)
+      path = get_cache_path_for(filename)
+      File.open(path) if File.exists?(path)
+    end
 
-      def get_from_cache(filename)
-        path = get_cache_path_for(filename)
-        File.open(path) if File.exists?(path)
-      end
+    def cache_file(file, filename)
+      path = get_cache_path_for(filename)
+      dir = File.dirname(path)
+      FileUtils.mkdir_p(dir) unless Dir[dir].present?
+      FileUtils.cp(file.path, path)
+      # keep up to 500 files
+      `ls -tr #{CACHE_DIR} | head -n +#{CACHE_MAXIMUM_SIZE} | xargs rm -f`
+    end
 
-      def cache_file(file, filename)
-        path = get_cache_path_for(filename)
-        dir = File.dirname(path)
-        FileUtils.mkdir_p(dir) unless Dir[dir].present?
-        FileUtils.cp(file.path, path)
-        # keep up to 500 files
-        `ls -tr #{CACHE_DIR} | head -n +#{CACHE_MAXIMUM_SIZE} | xargs rm -f`
-      end
+    def s3_bucket
+      return @s3_bucket if @s3_bucket
+      raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
+      @s3_bucket = SiteSetting.s3_upload_bucket.downcase
+    end
 
-      def s3_bucket
-        return @s3_bucket if @s3_bucket
-        raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
-        @s3_bucket = SiteSetting.s3_upload_bucket.downcase
-      end
   end
 
 end

spec/components/file_store/local_store_spec.rb

@@ -10,8 +10,6 @@ describe FileStore::LocalStore do
 
   let(:optimized_image) { Fabricate(:optimized_image) }
 
-  let(:avatar) { Fabricate(:upload) }
-
   describe ".store_upload" do
 
     it "returns a relative url" do

spec/components/file_store/s3_store_spec.rb

@@ -13,8 +13,6 @@ describe FileStore::S3Store do
   let(:optimized_image) { Fabricate(:optimized_image) }
   let(:optimized_image_file) { file_from_fixtures("logo.png") }
 
-  let(:avatar) { Fabricate(:upload) }
-
   before(:each) do
     SiteSetting.stubs(:s3_upload_bucket).returns("S3_Upload_Bucket")
     SiteSetting.stubs(:s3_access_key_id).returns("s3_access_key_id")

spec/models/optimized_image_spec.rb

@@ -110,12 +110,8 @@ end
 
 class FakeInternalStore
 
-  def internal?
-    true
-  end
-
   def external?
-    !internal?
+    false
   end
 
   def path_for(upload)
@@ -134,10 +130,6 @@ class FakeExternalStore
     true
   end
 
-  def internal?
-    !external?
-  end
-
   def store_optimized_image(file, optimized_image)
     "/externally/stored/optimized/image#{optimized_image.extension}"
   end