From f079dd63ed58930e06f823505e8c34bda9024671 Mon Sep 17 00:00:00 2001
From: Sam <sam.saffron@gmail.com>
Date: Mon, 25 May 2015 17:57:06 +1000
Subject: [PATCH] PERF: remove "fog" dependency
---
Gemfile | 3 +-
Gemfile.lock | 89 +++++------------------------
lib/file_store/s3_store.rb | 2 +-
lib/s3_helper.rb | 111 +++++++++++++++----------------------
4 files changed, 61 insertions(+), 144 deletions(-)
diff --git a/Gemfile b/Gemfile
index b3e58c654..bab99a602 100644
--- a/Gemfile
+++ b/Gemfile
@@ -57,7 +57,8 @@ gem 'fast_xor'
# while we sort out https://github.com/sdsykes/fastimage/pull/46
gem 'fastimage_discourse', require: 'fastimage'
-gem 'fog', '1.26.0', require: false
+gem 'aws-sdk', require: false
+gem 'excon', require: false
gem 'unf', require: false
gem 'email_reply_parser'
diff --git a/Gemfile.lock b/Gemfile.lock
index 2b207db1d..fcd40076f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -6,7 +6,6 @@ PATH
GEM
remote: https://rubygems.org/
specs:
- CFPropertyList (2.2.8)
actionmailer (4.1.10)
actionpack (= 4.1.10)
actionview (= 4.1.10)
@@ -41,6 +40,14 @@ GEM
activerecord (>= 2.3.0)
rake (~> 10.4.2, >= 10.4.2)
arel (5.0.1.20140414130214)
+ aws-sdk (2.0.45)
+ aws-sdk-resources (= 2.0.45)
+ aws-sdk-core (2.0.45)
+ builder (~> 3.0)
+ jmespath (~> 1.0)
+ multi_json (~> 1.0)
+ aws-sdk-resources (2.0.45)
+ aws-sdk-core (= 2.0.45)
babel-source (4.6.6)
babel-transpiler (0.6.0)
babel-source (>= 4.0, < 5)
@@ -84,7 +91,7 @@ GEM
ember-source (1.11.3.1)
erubis (2.7.0)
eventmachine (1.0.7)
- excon (0.44.4)
+ excon (0.45.3)
execjs (2.5.2)
exifr (1.1.3)
fabrication (2.9.8)
@@ -101,79 +108,11 @@ GEM
fast_xs (0.8.0)
fastimage_discourse (1.6.6)
ffi (1.9.6)
- fission (0.5.0)
- CFPropertyList (~> 2.2)
flamegraph (0.1.0)
fast_stack
- fog (1.26.0)
- fog-atmos
- fog-brightbox (~> 0.4)
- fog-core (~> 1.27, >= 1.27.1)
- fog-ecloud
- fog-json
- fog-profitbricks
- fog-radosgw (>= 0.0.2)
- fog-sakuracloud (>= 0.0.4)
- fog-softlayer
- fog-storm_on_demand
- fog-terremark
- fog-vmfusion
- fog-voxel
- fog-xml (~> 0.1.1)
- ipaddress (~> 0.5)
- nokogiri (~> 1.5, >= 1.5.11)
- fog-atmos (0.1.0)
- fog-core
- fog-xml
- fog-brightbox (0.7.1)
- fog-core (~> 1.22)
- fog-json
- inflecto (~> 0.0.2)
- fog-core (1.27.2)
- builder
- excon (~> 0.38)
- formatador (~> 0.2)
- mime-types
- net-scp (~> 1.1)
- net-ssh (>= 2.1.3)
- fog-ecloud (0.0.2)
- fog-core
- fog-xml
- fog-json (1.0.0)
- multi_json (~> 1.0)
- fog-profitbricks (0.0.1)
- fog-core
- fog-xml
- nokogiri
- fog-radosgw (0.0.3)
- fog-core (>= 1.21.0)
- fog-json
- fog-xml (>= 0.0.1)
- fog-sakuracloud (0.1.1)
- fog-core
- fog-json
- fog-softlayer (0.3.26)
- fog-core
- fog-json
- fog-storm_on_demand (0.1.0)
- fog-core
- fog-json
- fog-terremark (0.0.3)
- fog-core
- fog-xml
- fog-vmfusion (0.0.1)
- fission
- fog-core
- fog-voxel (0.0.2)
- fog-core
- fog-xml
- fog-xml (0.1.1)
- fog-core
- nokogiri (~> 1.5, >= 1.5.11)
foreman (0.77.0)
dotenv (~> 1.0.2)
thor (~> 0.19.1)
- formatador (0.2.5)
fspath (2.0.6)
gctools (0.2.3)
given_core (3.5.4)
@@ -195,8 +134,8 @@ GEM
progress (~> 3.0.0)
image_size (1.1.5)
in_threads (1.2.2)
- inflecto (0.0.2)
- ipaddress (0.8.0)
+ jmespath (1.0.2)
+ multi_json (~> 1.0)
jquery-rails (3.1.2)
railties (>= 3.0, < 5.0)
thor (>= 0.14, < 2.0)
@@ -231,9 +170,6 @@ GEM
multi_xml (0.5.5)
multipart-post (2.0.0)
mustache (0.99.8)
- net-scp (1.2.1)
- net-ssh (>= 2.6.5)
- net-ssh (2.9.2)
netrc (0.10.3)
nokogiri (1.6.6.2)
mini_portile (~> 0.6.0)
@@ -459,6 +395,7 @@ DEPENDENCIES
actionpack-action_caching
active_model_serializers (~> 0.8.3)
annotate
+ aws-sdk
babel-transpiler
barber
better_errors
@@ -468,6 +405,7 @@ DEPENDENCIES
email_reply_parser
ember-rails
ember-source (= 1.11.3.1)
+ excon
fabrication (= 2.9.8)
fakeweb (~> 1.3.0)
fast_blank
@@ -475,7 +413,6 @@ DEPENDENCIES
fast_xs
fastimage_discourse
flamegraph
- fog (= 1.26.0)
foreman
gctools
handlebars-source (= 2.0.0)
diff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rb
index 5d24f0054..ea8a82575 100644
--- a/lib/file_store/s3_store.rb
+++ b/lib/file_store/s3_store.rb
@@ -80,7 +80,7 @@ module FileStore
def store_file(file, path, filename=nil, content_type=nil)
# stored uploaded are public by default
- options = { public: true }
+ options = { acl: 'public-read' }
# add a "content disposition" header for "attachments"
options[:content_disposition] = "attachment; filename=\"#{filename}\"" if filename && !FileHelper.is_image?(filename)
# add a "content type" header when provided (ie. for "attachments")
diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb
index d512b655b..29b1078c9 100644
--- a/lib/s3_helper.rb
+++ b/lib/s3_helper.rb
@@ -1,50 +1,76 @@
-require "fog"
+require "aws-sdk"
class S3Helper
- def initialize(s3_bucket, tombstone_prefix=nil, fog=nil)
+ def initialize(s3_bucket, tombstone_prefix=nil)
raise Discourse::InvalidParameters.new("s3_bucket") if s3_bucket.blank?
@s3_bucket = s3_bucket
@tombstone_prefix = tombstone_prefix
check_missing_site_settings
-
- @fog = fog || Fog::Storage.new(s3_options)
end
def upload(file, unique_filename, options={})
- args = {
- body: file,
- key: unique_filename,
- public: false,
- }
-
- args.merge!(options)
-
- directory = get_or_create_directory(@s3_bucket)
- directory.files.create(args)
+ obj = s3_bucket.object(unique_filename)
+ obj.upload_file(file, options)
end
def remove(unique_filename, copy_to_tombstone=false)
+ bucket = s3_bucket
+
# copy the file in tombstone
if copy_to_tombstone && @tombstone_prefix.present?
- @fog.copy_object(unique_filename, @s3_bucket, @tombstone_prefix + unique_filename, @s3_bucket)
+ bucket.object(@tombstone_prefix + unique_filename).copy_from(copy_source: "#{@s3_bucket}/#{unique_filename}")
end
# delete the file
- @fog.delete_object(@s3_bucket, unique_filename)
- rescue Excon::Errors::NotFound
- # if the file cannot be found, don't raise an error
+ bucket.object(unique_filename).delete
+ rescue Aws::S3::Errors::NoSuchKey
end
def update_tombstone_lifecycle(grace_period)
+
return if @tombstone_prefix.blank?
# cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
- @fog.put_bucket_lifecycle(@s3_bucket, lifecycle(grace_period))
+ s3_resource.client.put_bucket_lifecycle({
+ bucket: @s3_bucket,
+ lifecycle_configuration: {
+ rules: [
+ {
+ id: 'purge-tombstone',
+ status: 'Enabled',
+ expiration: {
+ days: grace_period
+ },
+ prefix: @tombstone_prefix
+ }
+ ]
+ }
+ })
end
private
+ def s3_resource
+ opts = {}
+
+ opts = {
+ access_key_id: SiteSetting.s3_access_key_id,
+ secret_access_key: SiteSetting.s3_secret_access_key
+ } unless SiteSetting.s3_use_iam_profile
+
+ opts[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank?
+
+ Aws::S3::Resource.new(opts)
+ end
+
+ def s3_bucket
+ bucket = s3_resource.bucket(@s3_bucket)
+ bucket.create unless bucket.exists?
+ bucket
+ end
+
+
def check_missing_site_settings
unless SiteSetting.s3_use_iam_profile
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
@@ -52,51 +78,4 @@ class S3Helper
end
end
- def s3_options
- options = { provider: 'AWS', scheme: SiteSetting.scheme }
-
- # cf. https://github.com/fog/fog/issues/2381
- options[:path_style] = dns_compatible?(@s3_bucket, SiteSetting.use_https?)
-
- options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank?
-
- if SiteSetting.s3_use_iam_profile
- options.merge!(use_iam_profile: true)
- else
- options.merge!(aws_access_key_id: SiteSetting.s3_access_key_id,
- aws_secret_access_key: SiteSetting.s3_secret_access_key)
- end
-
- options
- end
-
- def get_or_create_directory(bucket)
- directory = @fog.directories.get(bucket)
- directory = @fog.directories.create(key: bucket) unless directory
- directory
- end
-
- def lifecycle(grace_period)
- {
- "Rules" => [{
- "Prefix" => @tombstone_prefix,
- "Enabled" => true,
- "Expiration" => { "Days" => grace_period }
- }]
- }
- end
-
- # cf. https://github.com/aws/aws-sdk-core-ruby/blob/master/aws-sdk-core/lib/aws-sdk-core/plugins/s3_bucket_dns.rb#L65-L80
- def dns_compatible?(bucket_name, ssl)
- return false unless valid_subdomain?(bucket_name)
- bucket_name.match(/\./) && ssl ? false : true
- end
-
- def valid_subdomain?(bucket_name)
- bucket_name.size < 64 &&
- bucket_name =~ /^[a-z0-9][a-z0-9.-]+[a-z0-9]$/ &&
- bucket_name !~ /(\d+\.){3}\d+/ &&
- bucket_name !~ /[.-]{2}/
- end
-
end