diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb
index 81468c3979..00b94b46ff 100644
--- a/lib/fluent/plugin/out_forward.rb
+++ b/lib/fluent/plugin/out_forward.rb
@@ -229,10 +229,11 @@ def configure(conf)
socket_cache: socket_cache,
)
+ @load_balancer = LoadBalancer.new(log)
service_discovery_configure(
:out_forward_service_discovery_watcher,
static_default_service_directive: 'server',
- load_balancer: LoadBalancer.new(log),
+ load_balancer: @load_balancer,
custom_build_method: method(:build_node),
)
@@ -472,7 +473,8 @@ def build_node(server)
def on_heartbeat_timer
need_rebuild = false
- service_discovery_services.each do |n|
+ services = service_discovery_services
+ services.each do |n|
begin
log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
n.usock = @usock if @usock
@@ -486,7 +488,7 @@ def on_heartbeat_timer
need_rebuild = n.tick || need_rebuild
end
- if need_rebuild
+ if need_rebuild || @load_balancer.availability_changed?(services)
service_discovery_rebalance
end
end
diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb
index 5f408dbf92..f226cfa872 100644
--- a/lib/fluent/plugin/out_forward/load_balancer.rb
+++ b/lib/fluent/plugin/out_forward/load_balancer.rb
@@ -26,6 +26,7 @@ def initialize(log)
@rand_seed = Random.new.seed
@rr = 0
@mutex = Mutex.new
+ @availability = {}
end
def select_healthy_node
@@ -55,14 +56,20 @@ def select_healthy_node
raise NoNodesAvailable, "no nodes are available"
end
+ def availability_changed?(nodes)
+ availability = @mutex.synchronize { @availability }
+ nodes.any? { |n| availability[n] != n.available? }
+ end
+
def rebuild_weight_array(nodes)
+ availability = nodes.to_h { |n| [n, n.available?] }
standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
n.standby?
}
lost_weight = 0
regular_nodes.each {|n|
- unless n.available?
+ unless availability[n]
lost_weight += n.weight
end
}
@@ -70,7 +77,7 @@ def rebuild_weight_array(nodes)
if lost_weight > 0
standby_nodes.each {|n|
- if n.available?
+ if availability[n]
regular_nodes << n
@log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
lost_weight -= n.weight
@@ -83,6 +90,7 @@ def rebuild_weight_array(nodes)
if regular_nodes.empty?
@log.warn('No nodes are available')
@mutex.synchronize do
+ @availability = availability
@weight_array = weight_array
end
return @weight_array
@@ -103,6 +111,7 @@ def rebuild_weight_array(nodes)
weight_array.sort_by! { r.rand }
@mutex.synchronize do
+ @availability = availability
@weight_array = weight_array
end
end
diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb
index cc7926af4a..41ed1e5685 100644
--- a/test/plugin/out_forward/test_load_balancer.rb
+++ b/test/plugin/out_forward/test_load_balancer.rb
@@ -103,4 +103,32 @@ class LoadBalancerTest < Test::Unit::TestCase
end
end
end
+
+ sub_test_case 'availability_changed?' do
+ test 'compares the nodes with their availability at the last rebuild' do
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+ available = true
+ n1 = flexmock('node', :'standby?' => false, weight: 1)
+ n1.should_receive(:available?).and_return { available }
+
+ lb.rebuild_weight_array([n1])
+ assert_false lb.availability_changed?([n1])
+
+ available = false
+ assert_true lb.availability_changed?([n1])
+
+ lb.rebuild_weight_array([n1])
+ assert_false lb.availability_changed?([n1])
+ end
+
+ test 'regards a node added after the last rebuild as a change' do
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+ n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+ n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+ lb.rebuild_weight_array([n1])
+
+ assert_true lb.availability_changed?([n1, n2])
+ end
+ end
end
diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb
index 1f4821df4e..7a98f65668 100644
--- a/test/plugin/test_out_forward.rb
+++ b/test/plugin/test_out_forward.rb
@@ -1495,4 +1495,79 @@ def create_ack_driver
instance.__send__(:ack_check, 0)
end
end
+
+ sub_test_case 'a node disabled outside the heartbeat' do
+ def create_standby_driver(primary_host: TARGET_HOST, conf: '')
+ @d = create_driver(%[
+ heartbeat_type transport
+ recover_wait 0
+ #{conf}
+
+ name primary
+ host #{primary_host}
+ port #{@target_port}
+
+
+ name standby
+ host #{TARGET_HOST}
+ port #{unused_port(protocol: :tcp)}
+ standby true
+
+ ])
+ @instance = @d.instance
+ @primary = @instance.nodes.find { |n| n.name == 'primary' }
+ @standby = @instance.nodes.find { |n| n.name == 'standby' }
+ @instance.nodes.each { |n| stub(n).send_heartbeat { false } }
+ end
+
+ def run_heartbeat_timer
+ @instance.__send__(:on_heartbeat_timer)
+ end
+
+ def selected_node
+ _, node = @instance.__send__(:service_discovery_select_service) { |n| n }
+ node
+ end
+
+ test 'the heartbeat timer switches to the standby node' do
+ create_standby_driver
+ @primary.disable!
+ run_heartbeat_timer
+
+ assert_equal @standby, selected_node
+ end
+
+ test 'rebuilds the weight array only once
+ while the availability stays the same' do
+ create_standby_driver
+ mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything).once
+
+ @primary.disable!
+ 2.times { run_heartbeat_timer }
+ end
+
+ test 'the heartbeat timer switches to the standby node again after the primary recovers' do
+ create_standby_driver
+ @primary.disable!
+ run_heartbeat_timer
+ stub(@primary).send_heartbeat { @primary.heartbeat }
+ run_heartbeat_timer
+ assert_equal @primary, selected_node
+
+ stub(@primary).send_heartbeat { false }
+ @primary.disable!
+ run_heartbeat_timer
+
+ assert_equal @standby, selected_node
+ end
+
+ test 'the heartbeat timer switches to the standby node for a primary disabled at startup' do
+ create_standby_driver(primary_host: 'unexisting.yaaaaaaaaaaaaaay.host.example.com', conf: 'ignore_network_errors_at_startup true')
+ assert_false @primary.available?
+
+ run_heartbeat_timer
+
+ assert_equal @standby, selected_node
+ end
+ end
end