diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 81468c3979..00b94b46ff 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -229,10 +229,11 @@ def configure(conf) socket_cache: socket_cache, ) + @load_balancer = LoadBalancer.new(log) service_discovery_configure( :out_forward_service_discovery_watcher, static_default_service_directive: 'server', - load_balancer: LoadBalancer.new(log), + load_balancer: @load_balancer, custom_build_method: method(:build_node), ) @@ -472,7 +473,8 @@ def build_node(server) def on_heartbeat_timer need_rebuild = false - service_discovery_services.each do |n| + services = service_discovery_services + services.each do |n| begin log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type n.usock = @usock if @usock @@ -486,7 +488,7 @@ def on_heartbeat_timer need_rebuild = n.tick || need_rebuild end - if need_rebuild + if need_rebuild || @load_balancer.availability_changed?(services) service_discovery_rebalance end end diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb index 5f408dbf92..f226cfa872 100644 --- a/lib/fluent/plugin/out_forward/load_balancer.rb +++ b/lib/fluent/plugin/out_forward/load_balancer.rb @@ -26,6 +26,7 @@ def initialize(log) @rand_seed = Random.new.seed @rr = 0 @mutex = Mutex.new + @availability = {} end def select_healthy_node @@ -55,14 +56,20 @@ def select_healthy_node raise NoNodesAvailable, "no nodes are available" end + def availability_changed?(nodes) + availability = @mutex.synchronize { @availability } + nodes.any? { |n| availability[n] != n.available? } + end + def rebuild_weight_array(nodes) + availability = nodes.to_h { |n| [n, n.available?] } standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n| n.standby? } lost_weight = 0 regular_nodes.each {|n| - unless n.available? + unless availability[n] lost_weight += n.weight end } @@ -70,7 +77,7 @@ def rebuild_weight_array(nodes) if lost_weight > 0 standby_nodes.each {|n| - if n.available? + if availability[n] regular_nodes << n @log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight lost_weight -= n.weight @@ -83,6 +90,7 @@ def rebuild_weight_array(nodes) if regular_nodes.empty? @log.warn('No nodes are available') @mutex.synchronize do + @availability = availability @weight_array = weight_array end return @weight_array @@ -103,6 +111,7 @@ def rebuild_weight_array(nodes) weight_array.sort_by! { r.rand } @mutex.synchronize do + @availability = availability @weight_array = weight_array end end diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb index cc7926af4a..41ed1e5685 100644 --- a/test/plugin/out_forward/test_load_balancer.rb +++ b/test/plugin/out_forward/test_load_balancer.rb @@ -103,4 +103,32 @@ class LoadBalancerTest < Test::Unit::TestCase end end end + + sub_test_case 'availability_changed?' do + test 'compares the nodes with their availability at the last rebuild' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + available = true + n1 = flexmock('node', :'standby?' => false, weight: 1) + n1.should_receive(:available?).and_return { available } + + lb.rebuild_weight_array([n1]) + assert_false lb.availability_changed?([n1]) + + available = false + assert_true lb.availability_changed?([n1]) + + lb.rebuild_weight_array([n1]) + assert_false lb.availability_changed?([n1]) + end + + test 'regards a node added after the last rebuild as a change' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + + lb.rebuild_weight_array([n1]) + + assert_true lb.availability_changed?([n1, n2]) + end + end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 1f4821df4e..7a98f65668 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1495,4 +1495,79 @@ def create_ack_driver instance.__send__(:ack_check, 0) end end + + sub_test_case 'a node disabled outside the heartbeat' do + def create_standby_driver(primary_host: TARGET_HOST, conf: '') + @d = create_driver(%[ + heartbeat_type transport + recover_wait 0 + #{conf} + + name primary + host #{primary_host} + port #{@target_port} + + + name standby + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + standby true + + ]) + @instance = @d.instance + @primary = @instance.nodes.find { |n| n.name == 'primary' } + @standby = @instance.nodes.find { |n| n.name == 'standby' } + @instance.nodes.each { |n| stub(n).send_heartbeat { false } } + end + + def run_heartbeat_timer + @instance.__send__(:on_heartbeat_timer) + end + + def selected_node + _, node = @instance.__send__(:service_discovery_select_service) { |n| n } + node + end + + test 'the heartbeat timer switches to the standby node' do + create_standby_driver + @primary.disable! + run_heartbeat_timer + + assert_equal @standby, selected_node + end + + test 'rebuilds the weight array only once + while the availability stays the same' do + create_standby_driver + mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything).once + + @primary.disable! + 2.times { run_heartbeat_timer } + end + + test 'the heartbeat timer switches to the standby node again after the primary recovers' do + create_standby_driver + @primary.disable! + run_heartbeat_timer + stub(@primary).send_heartbeat { @primary.heartbeat } + run_heartbeat_timer + assert_equal @primary, selected_node + + stub(@primary).send_heartbeat { false } + @primary.disable! + run_heartbeat_timer + + assert_equal @standby, selected_node + end + + test 'the heartbeat timer switches to the standby node for a primary disabled at startup' do + create_standby_driver(primary_host: 'unexisting.yaaaaaaaaaaaaaay.host.example.com', conf: 'ignore_network_errors_at_startup true') + assert_false @primary.available? + + run_heartbeat_timer + + assert_equal @standby, selected_node + end + end end