codeninjasllc/discourse
1
0
Fork 0
mirror of https://github.com/codeninjasllc/discourse.git synced 2024-11-27 09:36:19 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

FEATURE: move migrate_to_new_scheme into a background job

Browse source 
- new hidden site setting 'migrate_to_new_scheme' (defaults to false)
- new rake tasks to toggle migration to new scheme
- FIX: migrate_to_new_scheme also works with CDN
- PERF: improve perf of the DbHelper.remap method
- REFACTOR: UrlHelper is now a class
This commit is contained in:
Régis Hanol 2015-06-12 12:02:36 +02:00
parent 74e825fff2
commit 189cb3ff12
19 changed files with 236 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/controllers/topics_controller.rb
View file

@ -5,8 +5,6 @@ require_dependency 'topics_bulk_action'
require_dependency 'discourse_event'
class TopicsController < ApplicationController
include UrlHelper
before_filter :ensure_logged_in, only: [:timings,
:destroy_timings,
:update,
@ -81,7 +79,7 @@ class TopicsController < ApplicationController
perform_show_response
canonical_url absolute_without_cdn("#{Discourse.base_uri}#{@topic_view.canonical_path}")
canonical_url UrlHelper.absolute_without_cdn("#{Discourse.base_uri}#{@topic_view.canonical_path}")
rescue Discourse::InvalidAccess => ex
if current_user

2
app/jobs/regular/pull_hotlinked_images.rb
View file

@ -4,8 +4,6 @@ require_dependency 'file_helper'
module Jobs
class PullHotlinkedImages < Jobs::Base
include UrlHelper
def initialize
# maximum size of the file in bytes
@max_size = SiteSetting.max_image_size_kb.kilobytes

48
app/jobs/scheduled/migrate_scheme.rb Normal file
View file

@ -0,0 +1,48 @@
module Jobs
class MigrateScheme < Jobs::Scheduled
every 10.minutes
sidekiq_options retry: false
MIGRATE_SCHEME_KEY ||= "migrate_to_new_scheme"
def execute(args)
begin
return unless SiteSetting.migrate_to_new_scheme
return if $redis.exists(MIGRATE_SCHEME_KEY)
# use a mutex to make sure this job is only run once
DistributedMutex.synchronize(MIGRATE_SCHEME_KEY) do
# clean up failed uploads
Upload.where("created_at < ?", 1.hour.ago)
.where("LENGTH(COALESCE(url, '')) = 0")
.destroy_all
# migrate uploads to new scheme
problems = Upload.migrate_to_new_scheme
problems.each do |hash|
upload_id = hash[:upload].id
Discourse.handle_job_exception(hash[:ex], error_context(args, "Migrating upload id #{upload_id}", upload_id: upload_id))
end
# clean up failed optimized images
OptimizedImage.where("LENGTH(COALESCE(url, '')) = 0").destroy_all
# Clean up orphan optimized images
OptimizedImage.where("upload_id NOT IN (SELECT id FROM uploads)").destroy_all
# migrate optimized_images to new scheme
problems = OptimizedImage.migrate_to_new_scheme
problems.each do |hash|
optimized_image_id = hash[:optimized_image].id
Discourse.handle_job_exception(hash[:ex], error_context(args, "Migrating optimized_image id #{optimized_image_id}", optimized_image_id: optimized_image_id))
end
end
rescue => e
puts e.message
puts e.backtrace.join("\n")
end
end
end
end

4
app/jobs/scheduled/periodical_updates.rb
View file

@ -8,7 +8,6 @@ module Jobs
every 15.minutes
def execute(args)
# Feature topics in categories
CategoryFeaturedTopic.feature_topics
@ -22,7 +21,8 @@ module Jobs
unless UserAvatar.where("last_gravatar_download_attempt IS NULL").limit(1).first
problems = Post.rebake_old(250)
problems.each do |hash|
Discourse.handle_job_exception(hash[:ex], error_context(args, "Rebaking post id #{hash[:post].id}", post_id: hash[:post].id))
post_id = hash[:post].id
Discourse.handle_job_exception(hash[:ex], error_context(args, "Rebaking post id #{post_id}", post_id: post_id))
end
end

3
app/models/backup.rb
View file

@ -1,5 +1,4 @@
class Backup
include UrlHelper
include ActiveModel::SerializerSupport
attr_reader :filename
@ -72,7 +71,7 @@ class Backup
def self.create_from_filename(filename)
Backup.new(filename).tap do |b|
b.path = File.join(Backup.base_directory, b.filename)
b.link = b.schemaless "#{Discourse.base_url}/admin/backups/#{b.filename}"
b.link = UrlHelper.schemaless "#{Discourse.base_url}/admin/backups/#{b.filename}"
b.size = File.size(b.path)
end
end

62
app/models/optimized_image.rb
View file

@ -1,4 +1,8 @@
require "digest/sha1"
require_dependency "file_helper"
require_dependency "url_helper"
require_dependency "db_helper"
require_dependency "file_store/local_store"
class OptimizedImage < ActiveRecord::Base
belongs_to :upload
@ -170,6 +174,64 @@ class OptimizedImage < ActiveRecord::Base
false
end
def self.migrate_to_new_scheme(limit=50)
problems = []
if SiteSetting.migrate_to_new_scheme
max_file_size_kb = SiteSetting.max_image_size_kb.kilobytes
local_store = FileStore::LocalStore.new
OptimizedImage.includes(:upload)
.where("url NOT LIKE '%/optimized/_X/%'")
.limit(limit)
.order(id: :desc)
.each do |optimized_image|
begin
# keep track of the url
previous_url = optimized_image.url.dup
# where is the file currently stored?
external = previous_url =~ /^\/\//
# download if external
if external
url = SiteSetting.scheme + ":" + previous_url
file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
next unless file
path = file.path
else
path = local_store.path_for(optimized_image)
next unless File.exists?(path)
file = File.open(path)
end
# compute SHA if missing
if optimized_image.sha1.blank?
optimized_image.sha1 = Digest::SHA1.file(path).hexdigest
end
# optimize if image
ImageOptim.new.optimize_image!(path)
# store to new location & update the filesize
File.open(path) do |f|
optimized_image.url = Discourse.store.store_optimized_image(f, optimized_image)
optimized_image.save
end
# remap the URLs
DbHelper.remap(UrlHelper.absolute(previous_url), optimized_image.url) unless external
DbHelper.remap(previous_url, optimized_image.url)
# remove the old file (when local)
unless external
FileUtils.rm(path, force: true) rescue nil
end
rescue => e
problems << { optimized_image: optimized_image, ex: e }
ensure
file.try(:unlink) rescue nil
file.try(:close) rescue nil
end
end
end
problems
end
end
# == Schema Information

9
app/models/topic_link_click.rb
View file

@ -2,10 +2,6 @@ require_dependency 'discourse'
require 'ipaddr'
require 'url_helper'
class TopicLinkClickHelper
include UrlHelper
end
class TopicLinkClick < ActiveRecord::Base
belongs_to :topic_link, counter_cache: :clicks
belongs_to :user
@ -20,7 +16,6 @@ class TopicLinkClick < ActiveRecord::Base
url = args[:url]
return nil if url.blank?
helper = TopicLinkClickHelper.new
uri = URI.parse(url) rescue nil
urls = Set.new
@ -28,9 +23,9 @@ class TopicLinkClick < ActiveRecord::Base
if url =~ /^http/
urls << url.sub(/^https/, 'http')
urls << url.sub(/^http:/, 'https:')
urls << helper.schemaless(url)
urls << UrlHelper.schemaless(url)
end
urls << helper.absolute_without_cdn(url)
urls << UrlHelper.absolute_without_cdn(url)
urls << uri.path if uri.try(:host) == Discourse.current_hostname
urls << url.sub(/\?.*$/, '') if url.include?('?')

62
app/models/upload.rb
View file

@ -1,7 +1,10 @@
require "digest/sha1"
require_dependency "image_sizer"
require_dependency "file_helper"
require_dependency "url_helper"
require_dependency "db_helper"
require_dependency "validators/upload_validator"
require_dependency "file_store/local_store"
class Upload < ActiveRecord::Base
belongs_to :user
@ -144,6 +147,65 @@ class Upload < ActiveRecord::Base
`convert #{path} -auto-orient #{path}`
end
def self.migrate_to_new_scheme(limit=50)
problems = []
if SiteSetting.migrate_to_new_scheme
max_file_size_kb = [SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb].max.kilobytes
local_store = FileStore::LocalStore.new
Upload.where("url NOT LIKE '%/original/_X/%'")
.limit(limit)
.order(id: :desc)
.each do |upload|
begin
# keep track of the url
previous_url = upload.url.dup
# where is the file currently stored?
external = previous_url =~ /^\/\//
# download if external
if external
url = SiteSetting.scheme + ":" + previous_url
file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
next unless file
path = file.path
else
path = local_store.path_for(upload)
next unless File.exists?(path)
end
# compute SHA if missing
if upload.sha1.blank?
upload.sha1 = Digest::SHA1.file(path).hexdigest
end
# optimize if image
if FileHelper.is_image?(File.basename(path))
ImageOptim.new.optimize_image!(path)
end
# store to new location & update the filesize
File.open(path) do |f|
upload.url = Discourse.store.store_upload(f, upload)
upload.filesize = f.size
upload.save
end
# remap the URLs
DbHelper.remap(UrlHelper.absolute(previous_url), upload.url) unless external
DbHelper.remap(previous_url, upload.url)
# remove the old file (when local)
unless external
FileUtils.rm(path, force: true) rescue nil
end
rescue => e
problems << { upload: upload, ex: e }
ensure
file.try(:unlink) rescue nil
file.try(:close) rescue nil
end
end
end
problems
end
end
# == Schema Information

3
app/models/user.rb
View file

@ -12,7 +12,6 @@ require_dependency 'promotion'
class User < ActiveRecord::Base
include Roleable
include UrlHelper
include HasCustomFields
has_many :posts
@ -427,7 +426,7 @@ class User < ActiveRecord::Base
end
def avatar_template_url
schemaless absolute avatar_template
UrlHelper.schemaless UrlHelper.absolute avatar_template
end
def self.avatar_template(username,uploaded_avatar_id)

4
app/serializers/post_wordpress_serializer.rb
View file

@ -2,11 +2,9 @@
class PostWordpressSerializer < BasicPostSerializer
attributes :post_number
include UrlHelper
def avatar_template
if object.user
absolute object.user.avatar_template
UrlHelper.absolute object.user.avatar_template
else
nil
end

6
app/serializers/user_wordpress_serializer.rb
View file

@ -1,12 +1,10 @@
class UserWordpressSerializer < BasicUserSerializer
include UrlHelper
def avatar_template
if Hash === object
absolute User.avatar_template(user[:username], user[:uploaded_avatar_id])
UrlHelper.absolute User.avatar_template(user[:username], user[:uploaded_avatar_id])
else
absolute object.avatar_template
UrlHelper.absolute object.avatar_template
end
end

3
config/site_settings.yml
View file

@ -727,6 +727,9 @@ developer:
verbose_localization:
default: false
client: true
migrate_to_new_scheme:
hidden: true
default: false
embedding:
embeddable_hosts:

5
lib/cooked_post_processor.rb
View file

@ -5,7 +5,6 @@ require_dependency 'url_helper'
class CookedPostProcessor
include ActionView::Helpers::NumberHelper
include UrlHelper
def initialize(post, opts={})
@dirty = false
@ -228,13 +227,13 @@ class CookedPostProcessor
%w{href data-download-href}.each do |selector|
@doc.css("a[#{selector}]").each do |a|
href = a["#{selector}"].to_s
a["#{selector}"] = schemaless absolute(href) if is_local(href)
a["#{selector}"] = UrlHelper.schemaless UrlHelper.absolute(href) if UrlHelper.is_local(href)
end
end
@doc.css("img[src]").each do |img|
src = img["src"].to_s
img["src"] = schemaless absolute(src) if is_local(src)
img["src"] = UrlHelper.schemaless UrlHelper.absolute(src) if UrlHelper.is_local(src)
end
end

23
lib/db_helper.rb Normal file
View file

@ -0,0 +1,23 @@
class DbHelper
REMAP_SQL ||= "
SELECT table_name, column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND is_updatable = 'YES'
AND (data_type LIKE 'char%' OR data_type LIKE 'text%')
ORDER BY table_name, column_name"
def self.remap(from, to)
connection = ActiveRecord::Base.connection.raw_connection
remappable_columns = connection.async_exec(REMAP_SQL).to_a
args = [from, to, "%#{from}%"]
remappable_columns.each do |rc|
table_name = rc["table_name"]
column_name = rc["column_name"]
connection.async_exec("UPDATE #{table_name} SET #{column_name} = REPLACE(#{column_name}, $1, $2) WHERE #{column_name} LIKE $3", args) rescue nil
end
end
end

4
lib/pretty_text.rb
View file

@ -7,8 +7,6 @@ require_dependency 'post'
module PrettyText
class Helpers
include UrlHelper
def t(key, opts)
key = "js." + key
unless opts
@ -40,7 +38,7 @@ module PrettyText
avatar_template = user.avatar_template
end
schemaless absolute avatar_template
UrlHelper.schemaless UrlHelper.absolute avatar_template
end
def is_username_valid(username)

173
lib/tasks/uploads.rake
View file

@ -96,6 +96,7 @@ end
task "uploads:migrate_to_s3" => :environment do
require "file_store/s3_store"
require "file_store/local_store"
require "db_helper"
ENV["RAILS_DB"] ? migrate_to_s3 : migrate_to_s3_all_sites
end
@ -150,7 +151,7 @@ def migrate_to_s3
end
# remap the URL
remap(from, to)
DbHelper.remap(from, to)
putc "."
end
@ -363,171 +364,15 @@ def regenerate_missing_optimized
end
################################################################################
# migrate_to_new_pattern #
# migrate_to_new_scheme #
################################################################################
task "uploads:migrate_to_new_pattern" => :environment do
require "file_helper"
require "file_store/local_store"
ENV["RAILS_DB"] ? migrate_to_new_pattern : migrate_to_new_pattern_all_sites
task "uploads:start_migration" => :environment do
SiteSetting.migrate_to_new_scheme = true
puts "Migration started!"
end
def migrate_to_new_pattern_all_sites
RailsMultisite::ConnectionManagement.each_connection { migrate_to_new_pattern }
end
def migrate_to_new_pattern
db = RailsMultisite::ConnectionManagement.current_db
puts "Migrating uploads to new pattern for '#{db}'..."
migrate_uploads_to_new_pattern
puts "Migrating optimized images to new pattern for '#{db}'..."
migrate_optimized_images_to_new_pattern
puts "Done!"
end
def migrate_uploads_to_new_pattern
puts "Moving uploads to new location..."
max_file_size_kb = [SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb].max.kilobytes
local_store = FileStore::LocalStore.new
Upload.where("LENGTH(COALESCE(url, '')) = 0").destroy_all
Upload.where("url NOT LIKE '%/original/_X/%'").find_each do |upload|
begin
successful = false
# keep track of the url
previous_url = upload.url.dup
# where is the file currently stored?
external = previous_url =~ /^\/\//
# download if external
if external
url = SiteSetting.scheme + ":" + previous_url
file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
next unless file
path = file.path
else
path = local_store.path_for(upload)
next unless File.exists?(path)
end
# compute SHA if missing
if upload.sha1.blank?
upload.sha1 = Digest::SHA1.file(path).hexdigest
end
# optimize if image
if FileHelper.is_image?(File.basename(path))
ImageOptim.new.optimize_image!(path)
end
# store to new location & update the filesize
File.open(path) do |f|
upload.url = Discourse.store.store_upload(f, upload)
upload.filesize = f.size
upload.save
end
# remap the URLs
remap(previous_url, upload.url)
# remove the old file (when local)
unless external
FileUtils.rm(path, force: true) rescue nil
end
# succesfully migrated
successful = true
rescue => e
puts e.message
puts e.backtrace.join("\n")
ensure
putc successful ? '.' : 'X'
file.try(:unlink) rescue nil
file.try(:close) rescue nil
end
end
puts
end
def migrate_optimized_images_to_new_pattern
max_file_size_kb = SiteSetting.max_image_size_kb.kilobytes
local_store = FileStore::LocalStore.new
OptimizedImage.where("LENGTH(COALESCE(url, '')) = 0").destroy_all
OptimizedImage.where("url NOT LIKE '%/original/_X/%'").find_each do |optimized_image|
begin
successful = false
# keep track of the url
previous_url = optimized_image.url.dup
# where is the file currently stored?
external = previous_url =~ /^\/\//
# download if external
if external
url = SiteSetting.scheme + ":" + previous_url
file = FileHelper.download(url, max_file_size_kb, "discourse", true) rescue nil
next unless file
path = file.path
else
path = local_store.path_for(optimized_image)
next unless File.exists?(path)
file = File.open(path)
end
# compute SHA if missing
if optimized_image.sha1.blank?
optimized_image.sha1 = Digest::SHA1.file(path).hexdigest
end
# optimize if image
ImageOptim.new.optimize_image!(path)
# store to new location & update the filesize
File.open(path) do |f|
optimized_image.url = Discourse.store.store_optimized_image(f, optimized_image)
optimized_image.save
end
# remap the URLs
remap(previous_url, optimized_image.url)
# remove the old file (when local)
unless external
FileUtils.rm(path, force: true) rescue nil
end
# succesfully migrated
successful = true
rescue => e
puts e.message
puts e.backtrace.join("\n")
ensure
putc successful ? '.' : 'X'
file.try(:unlink) rescue nil
file.try(:close) rescue nil
end
end
puts
end
REMAP_SQL ||= "
SELECT table_name, column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND is_updatable = 'YES'
AND (data_type LIKE 'char%' OR data_type LIKE 'text%')
ORDER BY table_name, column_name
"
def remap(from, to)
connection ||= ActiveRecord::Base.connection.raw_connection
remappable_columns ||= connection.async_exec(REMAP_SQL).to_a
remappable_columns.each do |rc|
table_name = rc["table_name"]
column_name = rc["column_name"]
begin
connection.async_exec("
UPDATE #{table_name}
SET #{column_name} = REPLACE(#{column_name}, $1, $2)
WHERE #{column_name} IS NOT NULL
AND #{column_name} <> REPLACE(#{column_name}, $1, $2)", [from, to])
rescue
end
end
task "uploads:stop_migration" => :environment do
SiteSetting.migrate_to_new_scheme = false
puts "Migration stoped!"
end

12
lib/url_helper.rb
View file

@ -1,6 +1,6 @@
module UrlHelper
class UrlHelper
def is_local(url)
def self.is_local(url)
url.present? && (
Discourse.store.has_been_uploaded?(url) ||
!!(url =~ /^\/assets\//) ||
@ -9,15 +9,15 @@ module UrlHelper
)
end
def absolute(url, cdn = Discourse.asset_host)
def self.absolute(url, cdn = Discourse.asset_host)
url =~ /^\/[^\/]/ ? (cdn || Discourse.base_url_no_prefix) + url : url
end
def absolute_without_cdn(url)
absolute(url, nil)
def self.absolute_without_cdn(url)
self.absolute(url, nil)
end
def schemaless(url)
def self.schemaless(url)
url.sub(/^https?:/, "")
end

4
script/discourse
View file

@ -152,10 +152,6 @@ WHERE table_schema='public' and (data_type like 'char%' or data_type like 'text%
require File.expand_path(File.dirname(__FILE__) + "/../config/environment")
end
def schemaless(url)
url.gsub(/^https?:/, "")
end
end
DiscourseCLI.start(ARGV)

26
spec/components/url_helper_spec.rb
View file

@ -3,33 +3,27 @@ require_dependency 'url_helper'
describe UrlHelper do
class DummyClass
include UrlHelper
end
let(:helper) { DummyClass.new }
describe "#is_local" do
it "is true when the file has been uploaded" do
store = stub
store.expects(:has_been_uploaded?).returns(true)
Discourse.stubs(:store).returns(store)
expect(helper.is_local("http://discuss.site.com/path/to/file.png")).to eq(true)
expect(UrlHelper.is_local("http://discuss.site.com/path/to/file.png")).to eq(true)
end
it "is true for relative assets" do
store = stub
store.expects(:has_been_uploaded?).returns(false)
Discourse.stubs(:store).returns(store)
expect(helper.is_local("/assets/javascripts/all.js")).to eq(true)
expect(UrlHelper.is_local("/assets/javascripts/all.js")).to eq(true)
end
it "is true for plugin assets" do
store = stub
store.expects(:has_been_uploaded?).returns(false)
Discourse.stubs(:store).returns(store)
expect(helper.is_local("/plugins/all.js")).to eq(true)
expect(UrlHelper.is_local("/plugins/all.js")).to eq(true)
end
end
@ -37,16 +31,16 @@ describe UrlHelper do
describe "#absolute" do
it "does not change non-relative url" do
expect(helper.absolute("http://www.discourse.org")).to eq("http://www.discourse.org")
expect(UrlHelper.absolute("http://www.discourse.org")).to eq("http://www.discourse.org")
end
it "changes a relative url to an absolute one using base url by default" do
expect(helper.absolute("/path/to/file")).to eq("http://test.localhost/path/to/file")
expect(UrlHelper.absolute("/path/to/file")).to eq("http://test.localhost/path/to/file")
end
it "changes a relative url to an absolute one using the cdn when enabled" do
Rails.configuration.action_controller.stubs(:asset_host).returns("http://my.cdn.com")
expect(helper.absolute("/path/to/file")).to eq("http://my.cdn.com/path/to/file")
expect(UrlHelper.absolute("/path/to/file")).to eq("http://my.cdn.com/path/to/file")
end
end
@ -55,7 +49,7 @@ describe UrlHelper do
it "changes a relative url to an absolute one using base url even when cdn is enabled" do
Rails.configuration.action_controller.stubs(:asset_host).returns("http://my.cdn.com")
expect(helper.absolute_without_cdn("/path/to/file")).to eq("http://test.localhost/path/to/file")
expect(UrlHelper.absolute_without_cdn("/path/to/file")).to eq("http://test.localhost/path/to/file")
end
end
@ -63,9 +57,9 @@ describe UrlHelper do
describe "#schemaless" do
it "removes http or https schemas only" do
expect(helper.schemaless("http://www.discourse.org")).to eq("//www.discourse.org")
expect(helper.schemaless("https://secure.discourse.org")).to eq("//secure.discourse.org")
expect(helper.schemaless("ftp://ftp.discourse.org")).to eq("ftp://ftp.discourse.org")
expect(UrlHelper.schemaless("http://www.discourse.org")).to eq("//www.discourse.org")
expect(UrlHelper.schemaless("https://secure.discourse.org")).to eq("//secure.discourse.org")
expect(UrlHelper.schemaless("ftp://ftp.discourse.org")).to eq("ftp://ftp.discourse.org")
end
end