From 50afa48aa75bb75bc82b6228dcd31178f437d11b Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Tue, 10 Nov 2026 17:52:43 +0900 Subject: [PATCH] out_forward: use the standby node after a failed ACK or handshake (#5623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Which issue(s) this PR fixes**: Fixes #5620 **What this PR does / why we need it**: A failed ACK or a failed handshake disables a node outside the heartbeat, and the heartbeat timer did not rebuild the weight array for a node that was already disabled. So the standby node was never used, and every flush failed with `NoNodesAvailable` until the primary came back. This remembers the availability of each node when the weight array is built, and the heartbeat timer rebuilds it when the availability has changed. The same applies to a node disabled at startup by `ignore_network_errors_at_startup`, so the standby is now used there too. With `heartbeat_type udp`, a primary that is alive but fails one ACK is replaced by the standby until its next UDP heartbeat response arrives, and `using standby node` and `recovered forwarding server` are logged. Before this change, flushes failed with `NoNodesAvailable` during that time. **Docs Changes**: None. **Release Note**: out_forward: use the standby node after a failed ACK or handshake When a failed ACK (with `require_ack_response`) or a failed handshake (with ``) disabled the primary, the standby node was not used, and every flush failed with `NoNodesAvailable` until the primary came back. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Signed-off-by: Shizuo Fujita Co-authored-by: Claude Opus 5.5 Co-authored-by: Kentaro Hayashi Signed-off-by: github-actions[bot] --- lib/fluent/plugin/out_forward.rb | 8 +- .../plugin/out_forward/load_balancer.rb | 13 +++- test/plugin/out_forward/test_load_balancer.rb | 28 +++++++ test/plugin/test_out_forward.rb | 75 +++++++++++++++++++ 4 files changed, 119 insertions(+), 5 deletions(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 81468c3979..00b94b46ff 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -229,10 +229,11 @@ def configure(conf) socket_cache: socket_cache, ) + @load_balancer = LoadBalancer.new(log) service_discovery_configure( :out_forward_service_discovery_watcher, static_default_service_directive: 'server', - load_balancer: LoadBalancer.new(log), + load_balancer: @load_balancer, custom_build_method: method(:build_node), ) @@ -472,7 +473,8 @@ def build_node(server) def on_heartbeat_timer need_rebuild = false - service_discovery_services.each do |n| + services = service_discovery_services + services.each do |n| begin log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type n.usock = @usock if @usock @@ -486,7 +488,7 @@ def on_heartbeat_timer need_rebuild = n.tick || need_rebuild end - if need_rebuild + if need_rebuild || @load_balancer.availability_changed?(services) service_discovery_rebalance end end diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb index 5f408dbf92..f226cfa872 100644 --- a/lib/fluent/plugin/out_forward/load_balancer.rb +++ b/lib/fluent/plugin/out_forward/load_balancer.rb @@ -26,6 +26,7 @@ def initialize(log) @rand_seed = Random.new.seed @rr = 0 @mutex = Mutex.new + @availability = {} end def select_healthy_node @@ -55,14 +56,20 @@ def select_healthy_node raise NoNodesAvailable, "no nodes are available" end + def availability_changed?(nodes) + availability = @mutex.synchronize { @availability } + nodes.any? { |n| availability[n] != n.available? } + end + def rebuild_weight_array(nodes) + availability = nodes.to_h { |n| [n, n.available?] } standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n| n.standby? } lost_weight = 0 regular_nodes.each {|n| - unless n.available? + unless availability[n] lost_weight += n.weight end } @@ -70,7 +77,7 @@ def rebuild_weight_array(nodes) if lost_weight > 0 standby_nodes.each {|n| - if n.available? + if availability[n] regular_nodes << n @log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight lost_weight -= n.weight @@ -83,6 +90,7 @@ def rebuild_weight_array(nodes) if regular_nodes.empty? @log.warn('No nodes are available') @mutex.synchronize do + @availability = availability @weight_array = weight_array end return @weight_array @@ -103,6 +111,7 @@ def rebuild_weight_array(nodes) weight_array.sort_by! { r.rand } @mutex.synchronize do + @availability = availability @weight_array = weight_array end end diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb index cc7926af4a..41ed1e5685 100644 --- a/test/plugin/out_forward/test_load_balancer.rb +++ b/test/plugin/out_forward/test_load_balancer.rb @@ -103,4 +103,32 @@ class LoadBalancerTest < Test::Unit::TestCase end end end + + sub_test_case 'availability_changed?' do + test 'compares the nodes with their availability at the last rebuild' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + available = true + n1 = flexmock('node', :'standby?' => false, weight: 1) + n1.should_receive(:available?).and_return { available } + + lb.rebuild_weight_array([n1]) + assert_false lb.availability_changed?([n1]) + + available = false + assert_true lb.availability_changed?([n1]) + + lb.rebuild_weight_array([n1]) + assert_false lb.availability_changed?([n1]) + end + + test 'regards a node added after the last rebuild as a change' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + + lb.rebuild_weight_array([n1]) + + assert_true lb.availability_changed?([n1, n2]) + end + end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 1f4821df4e..7a98f65668 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1495,4 +1495,79 @@ def create_ack_driver instance.__send__(:ack_check, 0) end end + + sub_test_case 'a node disabled outside the heartbeat' do + def create_standby_driver(primary_host: TARGET_HOST, conf: '') + @d = create_driver(%[ + heartbeat_type transport + recover_wait 0 + #{conf} + + name primary + host #{primary_host} + port #{@target_port} + + + name standby + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + standby true + + ]) + @instance = @d.instance + @primary = @instance.nodes.find { |n| n.name == 'primary' } + @standby = @instance.nodes.find { |n| n.name == 'standby' } + @instance.nodes.each { |n| stub(n).send_heartbeat { false } } + end + + def run_heartbeat_timer + @instance.__send__(:on_heartbeat_timer) + end + + def selected_node + _, node = @instance.__send__(:service_discovery_select_service) { |n| n } + node + end + + test 'the heartbeat timer switches to the standby node' do + create_standby_driver + @primary.disable! + run_heartbeat_timer + + assert_equal @standby, selected_node + end + + test 'rebuilds the weight array only once + while the availability stays the same' do + create_standby_driver + mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything).once + + @primary.disable! + 2.times { run_heartbeat_timer } + end + + test 'the heartbeat timer switches to the standby node again after the primary recovers' do + create_standby_driver + @primary.disable! + run_heartbeat_timer + stub(@primary).send_heartbeat { @primary.heartbeat } + run_heartbeat_timer + assert_equal @primary, selected_node + + stub(@primary).send_heartbeat { false } + @primary.disable! + run_heartbeat_timer + + assert_equal @standby, selected_node + end + + test 'the heartbeat timer switches to the standby node for a primary disabled at startup' do + create_standby_driver(primary_host: 'unexisting.yaaaaaaaaaaaaaay.host.example.com', conf: 'ignore_network_errors_at_startup true') + assert_false @primary.available? + + run_heartbeat_timer + + assert_equal @standby, selected_node + end + end end