diff --git a/config/database.yml b/config/database.yml index 19da362a0..fb6d923ae 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,13 +1,7 @@ development: prepared_statements: false - adapter: postgresql_fallback - host: 172.17.0.2 - port: 6432 + adapter: postgresql database: discourse_development - username: tgxworld - password: test - replica_host: 172.17.0.3 - replica_port: 6432 min_messages: warning pool: 5 timeout: 5000 diff --git a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb index 18f76d6ec..df5a23b9e 100644 --- a/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb +++ b/lib/active_record/connection_adapters/postgresql_fallback_adapter.rb @@ -14,7 +14,7 @@ class TaskObserver logger.info { "PG connection heartbeat: Master connection is not active.".freeze } else logger.error { "PG connection heartbeat failed with error: \"#{ex}\"" } - end + end end end @@ -28,15 +28,21 @@ end module ActiveRecord module ConnectionHandling def postgresql_fallback_connection(config) - master_connection = postgresql_connection(config) + begin + connection = postgresql_connection(config) + rescue PG::ConnectionBad => e + connection = postgresql_connection(config.dup.merge({ + "host" => config["replica_host"], "port" => config["replica_port"] + })) - replica_connection = postgresql_connection(config.dup.merge({ - host: config[:replica_host], port: config[:replica_port] - })) - verify_replica(replica_connection) + verify_replica(connection) - klass = ConnectionAdapters::PostgreSQLFallbackAdapter.proxy_pass(master_connection.class) - klass.new(master_connection, replica_connection, logger, config) + Discourse.enable_readonly_mode if !Discourse.readonly_mode? + + start_connection_heartbeart(connection, config) + end + + connection end private @@ -45,92 +51,24 @@ module ActiveRecord value = connection.raw_connection.exec("SELECT pg_is_in_recovery()").values[0][0] raise "Replica database server is not in recovery mode." if value == 'f' end - end - module ConnectionAdapters - class PostgreSQLFallbackAdapter < AbstractAdapter - ADAPTER_NAME = "PostgreSQLFallback".freeze - MAX_FAILURE = 5 - HEARTBEAT_INTERVAL = 5 + def interval + 5 + end - attr_reader :main_connection + def start_connection_heartbeart(existing_connection, config) + timer_task = Concurrent::TimerTask.new(execution_interval: interval) do |task| + connection = postgresql_connection(config) - def self.all_methods(klass) - methods = [] - - (klass.ancestors - AbstractAdapter.ancestors).each do |_klass| - %w(public protected private).map do |level| - methods << _klass.send("#{level}_instance_methods", false) - end - end - - methods.flatten.uniq.sort - end - - def self.proxy_pass(klass) - Class.new(self) do - (self.all_methods(klass) - self.all_methods(self)).each do |method| - self.class_eval <<-EOF - def #{method}(*args, &block) - proxy_method(:#{method}, *args, &block) - end - EOF - end + if connection.active? + existing_connection.disconnect! + Discourse.disable_readonly_mode if Discourse.readonly_mode? + task.shutdown end end - def initialize(master_connection, replica_connection, logger, config) - super(nil, logger, config) - - @master_connection = master_connection - @main_connection = @master_connection - @replica_connection = replica_connection - @failure_count = 0 - load! - end - - def proxy_method(method, *args, &block) - @main_connection.send(method, *args, &block) - rescue ActiveRecord::StatementInvalid => e - if e.message.include?("PG::UnableToSend") && @main_connection == @master_connection - @failure_count += 1 - - if @failure_count == MAX_FAILURE - Discourse.enable_readonly_mode if !Discourse.readonly_mode? - @main_connection = @replica_connection - load! - connection_heartbeart(@master_connection) - @failure_count = 0 - else - proxy_method(method, *args, &block) - end - end - - raise e - end - - private - - def load! - @visitor = @main_connection.visitor - @connection = @main_connection.raw_connection - end - - def connection_heartbeart(connection, interval = HEARTBEAT_INTERVAL) - timer_task = Concurrent::TimerTask.new(execution_interval: interval) do |task| - connection.reconnect! - - if connection.active? - @main_connection = connection - load! - Discourse.disable_readonly_mode if Discourse.readonly_mode? - task.shutdown - end - end - - timer_task.add_observer(TaskObserver.new) - timer_task.execute - end + timer_task.add_observer(TaskObserver.new) + timer_task.execute end end end diff --git a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb index e0195e7d8..66dd0707c 100644 --- a/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb +++ b/spec/components/active_record/connection_adapters/postgresql_fallback_adapter_spec.rb @@ -1,55 +1,67 @@ require 'rails_helper' require_dependency 'active_record/connection_adapters/postgresql_fallback_adapter' -describe ActiveRecord::ConnectionAdapters::PostgreSQLFallbackAdapter do - let(:master_connection) { ActiveRecord::Base.connection } - let(:replica_connection) { master_connection.dup } - let(:adapter) { described_class.new(master_connection, replica_connection, nil, nil) } - - before :each do - ActiveRecord::Base.clear_all_connections! +describe ActiveRecord::ConnectionHandling do + let(:config) do + ActiveRecord::Base.configurations["test"].merge({ + "adapter" => "postgresql_fallback", + "replica_host" => "localhost", + "replica_port" => "6432" + }) end - describe "proxy_method" do - context "when master connection is not active" do + after do + ActiveRecord::Base.clear_all_connections! + Discourse.disable_readonly_mode + end + + describe "#postgresql_fallback_connection" do + it 'should return a PostgreSQL adapter' do + expect(ActiveRecord::Base.postgresql_fallback_connection(config)) + .to be_an_instance_of(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) + end + + context 'when master server is down' do before do - replica_connection.stubs(:send) - master_connection.stubs(:send).raises(ActiveRecord::StatementInvalid.new('PG::UnableToSend')) - master_connection.stubs(:reconnect!) - master_connection.stubs(:active?).returns(false) + @replica_connection = mock('replica_connection') - @old_const = described_class::HEARTBEAT_INTERVAL - described_class.const_set("HEARTBEAT_INTERVAL", 0.1) + ActiveRecord::Base.expects(:postgresql_connection).with(config).raises(PG::ConnectionBad) + + ActiveRecord::Base.expects(:postgresql_connection).with(config.merge({ + "host" => "localhost", "port" => "6432" + })).returns(@replica_connection) + + ActiveRecord::Base.expects(:verify_replica).with(@replica_connection) + + @replica_connection.expects(:disconnect!) + + ActiveRecord::Base.stubs(:interval).returns(0.1) + + Concurrent::TimerTask.any_instance.expects(:shutdown) end - after do - Discourse.disable_readonly_mode - described_class.const_set("HEARTBEAT_INTERVAL", @old_const) - end + it 'should failover to a replica server' do + ActiveRecord::Base.postgresql_fallback_connection(config) - it "should set site to readonly mode and carry out failover and switch back procedures" do - expect(adapter.main_connection).to eq(master_connection) - adapter.proxy_method('some method') expect(Discourse.readonly_mode?).to eq(true) - expect(adapter.main_connection).to eq(replica_connection) - master_connection.stubs(:active?).returns(true) + ActiveRecord::Base.unstub(:postgresql_connection) sleep 0.15 expect(Discourse.readonly_mode?).to eq(false) - expect(adapter.main_connection).to eq(master_connection) + + expect(ActiveRecord::Base.connection) + .to be_an_instance_of(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) end end - it 'should raise errors not related to the database connection' do - master_connection.stubs(:send).raises(StandardError.new) - expect { adapter.proxy_method('some method') }.to raise_error(StandardError) - end + context 'when both master and replica server is down' do + it 'should raise the right error' do + ActiveRecord::Base.expects(:postgresql_connection).raises(PG::ConnectionBad).twice - it 'should proxy methods successfully' do - expect(adapter.proxy_method(:execute, 'SELECT 1').values[0][0]).to eq("1") - expect(adapter.proxy_method(:active?)).to eq(true) - expect(adapter.proxy_method(:raw_connection)).to eq(master_connection.raw_connection) + expect { ActiveRecord::Base.postgresql_fallback_connection(config) } + .to raise_error(PG::ConnectionBad) + end end end end