From 295f6c4a55a1f5ae3432e710ec9c9805709e5c68 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:07:38 +0600 Subject: [PATCH 1/7] out_forward: start retry after a failed ACK when heartbeat_type is none Fixes #5619 When require_ack_response is enabled, a failed ACK disables the node and rolls the chunk back with update_retry: false, so the next write can start the retry state via NoNodesAvailable. NoneHeartbeatNode stays available after disable!, so that path never runs and chunks are resent immediately, ignoring retry_timeout and . Start the retry on rollback when the node is still available after disable!. Leave update_retry false when disable! works, to avoid counting the same failure twice. Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- lib/fluent/plugin/out_forward.rb | 9 ++++- test/plugin/test_out_forward.rb | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 8298203706..45d297cb61 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -546,7 +546,14 @@ def ack_check(select_interval) # could later be read against a chunk sent on the reused socket (#4057). @connection_manager.discard(sock) node&.disable! - rollback_write(chunk_id, update_retry: false) if chunk_id + # rollback_write(..., update_retry: false) is used so a later write + # failure (after disable!) starts the retry state, instead of + # counting a retry here and again when NoNodesAvailable is raised. + # NoneHeartbeatNode#available? stays true after disable!, so that + # later write never fails this way and retry_timeout / secondary + # would never apply. Start the retry here in that case (#5619). + update_retry = node.nil? || node.available? + rollback_write(chunk_id, update_retry: update_retry) if chunk_id when AckHandler::Result::CHUNKID_UNMATCHED # The ack read from this socket belongs to a different chunk than the # one tracked for it: the keepalive connection's ack stream is out of diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index e5ea0f71fe..88fa3a7496 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1520,6 +1520,64 @@ def create_ack_driver end end + sub_test_case 'ack_check starts retry when a failed ACK leaves the node available (#5619)' do + def create_ack_driver(heartbeat_type) + create_driver(%[ + send_timeout 51 + heartbeat_type #{heartbeat_type} + require_ack_response true + + name test + host #{TARGET_HOST} + port #{@target_port} + + ]) + end + + def stub_failed_ack(instance, node) + sock = 'sock' + fake_ack_handler = Object.new + fake_ack_handler.instance_variable_set(:@yielded, false) + fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| + next if @yielded + @yielded = true + block.call('chunk-id', node, sock, Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED) + end + instance.instance_variable_set(:@ack_handler, fake_ack_handler) + sock + end + + test 'starts retry on rollback when heartbeat_type is none' do + @d = d = create_ack_driver('none') + instance = d.instance + node = instance.nodes.first + sock = stub_failed_ack(instance, node) + + cm = instance.instance_variable_get(:@connection_manager) + stub(cm).discard(sock) + + assert_true node.available? + mock(instance).rollback_write('chunk-id', update_retry: true).once + instance.__send__(:ack_check, 0) + assert_true node.available? + end + + test 'keeps update_retry false when disable! makes the node unavailable' do + @d = d = create_ack_driver('transport') + instance = d.instance + node = instance.nodes.first + sock = stub_failed_ack(instance, node) + + cm = instance.instance_variable_get(:@connection_manager) + stub(cm).discard(sock) + + assert_true node.available? + mock(instance).rollback_write('chunk-id', update_retry: false).once + instance.__send__(:ack_check, 0) + assert_false node.available? + end + end + sub_test_case 'a node disabled outside the heartbeat' do def create_standby_driver(primary_host: TARGET_HOST, conf: '') @d = create_driver(%[ From 9bb842363d994b1eca9b363989cff1bdb92b6eb0 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:34:07 +0600 Subject: [PATCH 2/7] out_forward: detach the node after a failed ACK instead of counting a retry Follow the direction suggested in the review of #5627: after a failed ACK (timeout or closed connection), keep the node out for recover_wait and roll the chunk back with update_retry: false, so its chunks go to another node instead of starting a retry for the whole output. - NoneHeartbeatNode is now unavailable while detached, and the weight array is rebuilt before selecting a node with heartbeat_type none, so a standby node is used while the active node is out and switched back afterwards. The rebuild also runs right after the detach. - For transport/udp, the hold is set before disable!, so a heartbeat or in-flight write samples cannot bring the node back before recover_wait. - The last available node is not detached (and nothing is detached with recover_wait 0). The chunk is rolled back and a retry is counted with an AckResponseError, so a transient failure is resent after retry_wait, retry_timeout/retry_max_times/ apply, and the retry or the discard is logged as usual. - A detach is logged as a warning with the chunk id. Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- lib/fluent/plugin/out_forward.rb | 98 ++++++++-- lib/fluent/plugin/out_forward/error.rb | 1 + test/plugin/test_out_forward.rb | 240 +++++++++++++++++++++---- 3 files changed, 292 insertions(+), 47 deletions(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 45d297cb61..321af4b81e 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -52,7 +52,8 @@ class ForwardOutput < Output config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport desc 'The interval of the heartbeat packer.' config_param :heartbeat_interval, :time, default: 1 - desc 'The wait time before accepting a server fault recovery.' + desc 'The wait time before accepting a server fault recovery. ' \ + 'A server whose ack response fails is also kept out for this long while another server can take its chunks.' config_param :recover_wait, :time, default: 10 desc 'The hard timeout used to detect server failure.' config_param :hard_timeout, :time, default: 60 @@ -379,6 +380,7 @@ def write(chunk) return if chunk.empty? tag = chunk.metadata.tag + rebalance_if_availability_changed service_discovery_select_service { |node| node.send_data(tag, chunk) } end @@ -389,6 +391,7 @@ def try_write(chunk) return end tag = chunk.metadata.tag + rebalance_if_availability_changed service_discovery_select_service { |node| node.send_data(tag, chunk) } last_ack if @require_ack_response && @suspend_flush end @@ -501,6 +504,63 @@ def on_heartbeat_timer end end + # heartbeat_type none has no heartbeat timer to rebuild the weight array, so + # a node detached after a failed ack (and the standby used in its place) + # would never be switched back. Check it before selecting a node instead. + def rebalance_if_availability_changed + return unless @heartbeat_type == :none + + if @load_balancer.availability_changed?(service_discovery_services) + service_discovery_rebalance + end + end + + # Whether another node can take the chunks of a node whose ack failed. + # With recover_wait 0 a detached node would be back at once, so there is + # no point in detaching it. + def failover_node_available?(failed_node) + return false if failed_node.nil? || @recover_wait <= 0 + + service_discovery_services.any? { |n| !n.equal?(failed_node) && n.weight > 0 && n.available? } + end + + def node_label(node) + node.name || "#{node.host}:#{node.port}" + end + + def on_ack_failed(chunk_id, node) + chunk_id_hex = chunk_id ? dump_unique_id_hex(chunk_id) : nil + + if failover_node_available?(node) + # Keep the node out for recover_wait so that its chunks go to the + # other nodes (a standby node is used if needed). No retry is counted: + # the chunk has not failed on every node, and the retry state belongs + # to the whole output. If no node is left, the next write raises + # NoNodesAvailable and the usual retry applies. + node.detach! + service_discovery_rebalance + log.warn "failed to receive the ack response, detached forwarding server '#{node_label(node)}' for #{@recover_wait}s. the chunk will be sent to another server", + host: node.host, port: node.port, chunk_id: chunk_id_hex + rollback_write(chunk_id, update_retry: false) if chunk_id + return + end + + # No other node can take the chunk. Keeping the last node out would make + # every retry fail with NoNodesAvailable until it comes back (and could use + # up retry_max_times on a single transient failure), so leave it in and + # count a retry instead: the chunk is resent after retry_wait, and + # retry_timeout, retry_max_times and apply. The error is + # passed so that the retry and a discard are logged as usual. + return unless chunk_id + + if rollback_write(chunk_id, update_retry: false) + error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'") + error.set_backtrace(caller(0, 1)) + primary = @as_secondary ? @primary_instance : self + primary.update_retry_state(chunk_id, @as_secondary, error) + end + end + def on_udp_heartbeat_response_recv(data, sock) sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host) if node = service_discovery_services.find { |n| n.sockaddr == sockaddr } @@ -545,15 +605,7 @@ def ack_check(select_interval) # instead of returning it to the reuse pool; otherwise a late ack # could later be read against a chunk sent on the reused socket (#4057). @connection_manager.discard(sock) - node&.disable! - # rollback_write(..., update_retry: false) is used so a later write - # failure (after disable!) starts the retry state, instead of - # counting a retry here and again when NoNodesAvailable is raised. - # NoneHeartbeatNode#available? stays true after disable!, so that - # later write never fails this way and retry_timeout / secondary - # would never apply. Start the retry here in that case (#5619). - update_retry = node.nil? || node.available? - rollback_write(chunk_id, update_retry: update_retry) if chunk_id + on_ack_failed(chunk_id, node) when AckHandler::Result::CHUNKID_UNMATCHED # The ack read from this socket belongs to a different chunk than the # one tracked for it: the keepalive connection's ack stream is out of @@ -592,6 +644,7 @@ def initialize(sender, server, failure:, connection_manager:, ack_handler:) @standby = server.standby @failure = failure @available = true + @detached_until = nil # @hostname is used for certificate verification & TLS SNI host_is_hostname = !(IPAddr.new(@host) rescue false) @@ -637,6 +690,21 @@ def disable! @available = false end + # Keep the node out for recover_wait after a failed ack. The hold is set + # before disabling so a heartbeat running in between cannot bring the + # node back at once, and samples added by in-flight writes cannot end it + # early. After the hold, recovery still needs fresh heartbeats. + def detach! + @detached_until = Fluent::Clock.now + @sender.recover_wait + disable! + @failure.clear + end + + def detached? + detached_until = @detached_until + !detached_until.nil? && Fluent::Clock.now < detached_until + end + def standby? @standby end @@ -830,7 +898,7 @@ def tick def heartbeat(detect=true) now = Time.now.to_f @failure.add(now) - if detect && !available? && @failure.sample_size > @sender.recover_sample_size + if detect && !available? && !detached? && @failure.sample_size > @sender.recover_sample_size @available = true @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port true @@ -858,8 +926,14 @@ def connect(host = nil, ack: false, &block) # Override Node to disable heartbeat class NoneHeartbeatNode < Node + # Without heartbeats, a node is only kept out for recover_wait after a + # failed ack (see ForwardOutput#on_ack_failed). def available? - true + !detached? + end + + def detach! + @detached_until = Fluent::Clock.now + @sender.recover_wait end def tick diff --git a/lib/fluent/plugin/out_forward/error.rb b/lib/fluent/plugin/out_forward/error.rb index 87de15d9b4..8ec3fd8159 100644 --- a/lib/fluent/plugin/out_forward/error.rb +++ b/lib/fluent/plugin/out_forward/error.rb @@ -21,6 +21,7 @@ class ForwardOutput < Output class Error < StandardError; end class NoNodesAvailable < Error; end class ConnectionClosedError < Error; end + class AckResponseError < Error; end class HandshakeError < Error; end class HeloError < HandshakeError; end class PingpongError < HandshakeError; end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 88fa3a7496..1a4c5fc413 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1520,61 +1520,231 @@ def create_ack_driver end end - sub_test_case 'ack_check starts retry when a failed ACK leaves the node available (#5619)' do - def create_ack_driver(heartbeat_type) - create_driver(%[ + sub_test_case 'a failed ack (#5619)' do + def create_ack_driver(heartbeat_type: 'none', recover_wait: 10, servers: [{name: 'test'}]) + @d = create_driver(%[ send_timeout 51 heartbeat_type #{heartbeat_type} + recover_wait #{recover_wait} require_ack_response true + #{servers.map { |sv| %[ - name test + name #{sv[:name]} host #{TARGET_HOST} - port #{@target_port} - + port #{sv[:port] || @target_port} + #{sv[:standby] ? 'standby true' : ''} + ] }.join} ]) + @instance = @d.instance + @instance.nodes.each { |n| stub(n).send_heartbeat { false } } + stub(@instance.instance_variable_get(:@connection_manager)).discard + @d end - def stub_failed_ack(instance, node) - sock = 'sock' + def node_named(name) + @instance.nodes.find { |n| n.name == name } + end + + def fail_ack(node, chunk_id = 'chunk-id') fake_ack_handler = Object.new - fake_ack_handler.instance_variable_set(:@yielded, false) fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| - next if @yielded - @yielded = true - block.call('chunk-id', node, sock, Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED) + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED) end - instance.instance_variable_set(:@ack_handler, fake_ack_handler) - sock + original = @instance.instance_variable_get(:@ack_handler) + @instance.instance_variable_set(:@ack_handler, fake_ack_handler) + @instance.__send__(:ack_check, 0) + ensure + @instance.instance_variable_set(:@ack_handler, original) end - test 'starts retry on rollback when heartbeat_type is none' do - @d = d = create_ack_driver('none') - instance = d.instance - node = instance.nodes.first - sock = stub_failed_ack(instance, node) + # the same as what #write does before selecting a node + def selected_node + @instance.__send__(:rebalance_if_availability_changed) + _, node = @instance.__send__(:service_discovery_select_service) { |n| n } + node + end - cm = instance.instance_variable_get(:@connection_manager) - stub(cm).discard(sock) + def two_servers(standby: false) + [{name: 'primary'}, {name: 'secondary', port: unused_port(protocol: :tcp), standby: standby}] + end - assert_true node.available? - mock(instance).rollback_write('chunk-id', update_retry: true).once - instance.__send__(:ack_check, 0) - assert_true node.available? + data('none' => 'none', 'transport' => 'transport') + test 'detaches the node and sends its chunks to another server without counting a retry' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type, servers: two_servers) + primary = node_named('primary') + secondary = node_named('secondary') + + mock(@instance).rollback_write('chunk-id', update_retry: false).once + mock(@instance).update_retry_state.never + fail_ack(primary) + + assert_false primary.available? + assert_true secondary.available? + 5.times { assert_equal secondary, selected_node } + assert{ @d.logs.any? { |log| log.include?("detached forwarding server 'primary' for 10.0s") } } end - test 'keeps update_retry false when disable! makes the node unavailable' do - @d = d = create_ack_driver('transport') - instance = d.instance - node = instance.nodes.first - sock = stub_failed_ack(instance, node) + data('none' => 'none', 'transport' => 'transport') + test 'uses the standby node while the active node is detached' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type, servers: two_servers(standby: true)) + primary = node_named('primary') + standby = node_named('secondary') + assert_equal primary, selected_node - cm = instance.instance_variable_get(:@connection_manager) - stub(cm).discard(sock) + mock(@instance).rollback_write('chunk-id', update_retry: false).once + mock(@instance).update_retry_state.never + fail_ack(primary) + + assert_false primary.available? + 5.times { assert_equal standby, selected_node } + end + + test 'heartbeat_type none switches back to the active node after recover_wait' do + create_ack_driver(heartbeat_type: 'none', servers: two_servers(standby: true)) + primary = node_named('primary') + standby = node_named('secondary') + stub(@instance).rollback_write + + now = Fluent::Clock.now + Fluent::Clock.freeze(now) do + fail_ack(primary) + assert_equal standby, selected_node + end + Fluent::Clock.freeze(now + 9) do + assert_equal standby, selected_node + end + Fluent::Clock.freeze(now + 10) do + assert_true primary.available? + assert_equal primary, selected_node + end + end + + test 'heartbeat_type transport does not bring a detached node back before recover_wait' do + create_ack_driver(heartbeat_type: 'transport', recover_wait: 2, servers: two_servers) + primary = node_named('primary') + stub(@instance).rollback_write + + now = Fluent::Clock.now + Fluent::Clock.freeze(now) do + fail_ack(primary) + # enough samples to recover, as a heartbeat or in-flight writes would add + (@instance.recover_sample_size.ceil + 2).times { primary.heartbeat(false) } + primary.heartbeat + assert_false primary.available? + end + Fluent::Clock.freeze(now + 2) do + primary.heartbeat + assert_true primary.available? + end + end + + data('none' => 'none', 'transport' => 'transport') + test 'keeps the last node and counts a retry with the error' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type) + node = @instance.nodes.first + + errors = [] + mock(@instance).rollback_write('chunk-id', update_retry: false).once { true } + mock(@instance).update_retry_state('chunk-id', false, anything).once { |_id, _sec, error| errors << error } + fail_ack(node) assert_true node.available? - mock(instance).rollback_write('chunk-id', update_retry: false).once - instance.__send__(:ack_check, 0) - assert_false node.available? + assert_equal [Fluent::Plugin::ForwardOutput::AckResponseError], errors.map(&:class) + assert_equal "failed to receive the ack response from forwarding server 'test'", errors.first.message + end + + test 'counts a retry when the other node is detached too' do + create_ack_driver(heartbeat_type: 'none', servers: two_servers) + primary = node_named('primary') + secondary = node_named('secondary') + stub(@instance).rollback_write { true } + mock(@instance).update_retry_state('chunk-2', false, anything).once + + fail_ack(primary, 'chunk-1') + fail_ack(secondary, 'chunk-2') + + assert_false primary.available? + assert_true secondary.available? + end + + test 'does not detach the node when recover_wait is 0' do + create_ack_driver(heartbeat_type: 'none', recover_wait: 0, servers: two_servers) + primary = node_named('primary') + stub(@instance).rollback_write { true } + mock(@instance).update_retry_state('chunk-id', false, anything).once + + fail_ack(primary) + + assert_true primary.available? + end + end + + sub_test_case 'a server that never sends the ack (#5619)' do + # Reproduces the scenarios in the review of #5627: one server accepts + # chunks but never acks, the other acks normally. No chunk may be lost + # even with retry_max_times 0. + def run_with_silent_server(standby:) + silent_port = unused_port(protocol: :all) + silent = create_target_input_driver(response_stub: ->(_option) { nil }, + conf: %[port #{silent_port}\nbind #{TARGET_HOST}]) + healthy = create_target_input_driver + + @d = d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1 + + name silent + host #{TARGET_HOST} + port #{silent_port} + + + name healthy + host #{TARGET_HOST} + port #{@target_port} + #{standby ? 'standby true' : ''} + + + flush_mode immediate + flush_thread_interval 0.1 + flush_thread_burst_interval 0.1 + retry_wait 1 + retry_max_times 0 + flush_at_shutdown false + + ]) + + time = event_time + tags = (1..8).map { |i| "test.#{i}" } + # The logs are taken when every chunk has arrived: a chunk still in + # flight to the healthy server can fail when the drivers shut down. + logs = nil + delivered = -> { healthy.events.map(&:first).uniq.sort_by { |t| t.split('.').last.to_i } } + done = -> { delivered.call.size >= tags.size && (logs ||= d.logs.dup) } + [healthy, d].each { |drv| drv.end_if(&done) } + + silent.instance_start + begin + healthy.run(timeout: 30) do + d.run(timeout: 25, wait_flush_completion: false, shutdown: false, flush: false) do + tags.each { |tag| d.feed(tag, time, {'tag' => tag}) } + end + end + ensure + silent.instance_shutdown + end + + assert_equal tags, delivered.call + assert{ logs.any? { |log| log.include?("detached forwarding server 'silent'") } } + assert{ logs.none? { |log| log.include?('Hit limit for retries') } } + end + + test 'delivers every chunk to the healthy server' do + run_with_silent_server(standby: false) + end + + test 'delivers every chunk to the healthy standby server' do + run_with_silent_server(standby: true) end end From 8a61903c12cd2a25d01c99540aebdb8905a8ddb7 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:22:35 +0600 Subject: [PATCH 3/7] out_forward: do not regard a received ack as missing in the ack reader The ack reader checked the timeout before reading, and a socket enqueued while it was waiting in IO.select was only selected in the next loop. With a select interval as long as ack_response_timeout, an ack that had already arrived could be regarded as missing. Expired acks were also reported only after a full select wait, so Output#try_rollback_write could roll the chunk back first on delayed_commit_timeout and count a retry without an error. Read an ack that is already in the socket buffer, and report expired acks without waiting for the select interval. --- lib/fluent/plugin/out_forward/ack_handler.rb | 21 +++++++- test/plugin/out_forward/test_ack_handler.rb | 53 +++++++++++++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/lib/fluent/plugin/out_forward/ack_handler.rb b/lib/fluent/plugin/out_forward/ack_handler.rb index 04948eafa3..b819df55dc 100644 --- a/lib/fluent/plugin/out_forward/ack_handler.rb +++ b/lib/fluent/plugin/out_forward/ack_handler.rb @@ -45,7 +45,7 @@ def collect_response(select_interval) new_list = [] @mutex.synchronize do @ack_waitings.each do |info| - if info.expired?(now) + if info.expired?(now) && !readable?(info.sock) # There are 2 types of cases when no response has been received from socket: # (1) the node does not support sending responses # (2) the node does support sending response but responses have not arrived for some reasons. @@ -59,8 +59,15 @@ def collect_response(select_interval) @ack_waitings = new_list end + # Report expired acks right away instead of after a full select + # wait. Otherwise a failed ack can be reported up to two select + # intervals after it expired, which reaches delayed_commit_timeout + # (ack_response_timeout + 2). Then Output#try_rollback_write rolls the + # chunk back first and counts a retry without an error, and with + # retry_max_times 0 the whole queue is cleared without a log. + timeout = results.empty? ? select_interval : 0 begin - readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval) + readable_sockets, _, _ = IO.select(sockets, nil, nil, timeout) rescue IOError @log.info "connection closed while waiting for readable sockets" readable_sockets = nil @@ -110,6 +117,16 @@ def enqueue(node, sock, cid) private + # A socket enqueued while this thread was waiting in IO.select is not in + # that select call, so its ack can already be in the socket buffer when the + # timeout is checked. Read it instead of regarding the node as unavailable. + def readable?(sock) + readable_sockets, _, _ = IO.select([sock], nil, nil, 0) + !readable_sockets.nil? + rescue IOError + false + end + def read_ack_from_sock(sock) begin raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length) diff --git a/test/plugin/out_forward/test_ack_handler.rb b/test/plugin/out_forward/test_ack_handler.rb index 309822deb9..5605162d0d 100644 --- a/test/plugin/out_forward/test_ack_handler.rb +++ b/test/plugin/out_forward/test_ack_handler.rb @@ -80,7 +80,6 @@ class AckHandlerTest < Test::Unit::TestCase r, w = IO.pipe begin - w.write(chunk_id) mock(r).recv(anything).never ack.enqueue(r) @@ -100,6 +99,58 @@ class AckHandlerTest < Test::Unit::TestCase end end + test 'reports an expired ack without waiting for the select interval' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + ack = ack_handler.create_ack(chunk_id, node) + + r, w = IO.pipe + begin + ack.enqueue(r) + + result = nil + started = Fluent::Clock.now + ack_handler.collect_response(5) do |_cid, _n, _s, ret| + result = ret + end + elapsed = Fluent::Clock.now - started + + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, result + assert_operator elapsed, :<, 1 + ensure + r.close rescue nil + w.close rescue nil + end + end + + test 'reads an ack that is already received when the timeout is checked' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + ack = ack_handler.create_ack(chunk_id, node) + + r, w = IO.pipe + begin + w.write(chunk_id) + mock(r).recv(anything) { |_| MessagePack.pack({ 'ack' => Base64.encode64(chunk_id) }) } # IO does not have recv + ack.enqueue(r) + + a1 = a4 = nil + ack_handler.collect_response(1) do |cid, _n, _s, ret| + a1 = cid; a4 = ret + end + + assert_equal chunk_id, a1 + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS, a4 + ensure + r.close rescue nil + w.close rescue nil + end + end + # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case. # If exclusive control for this case is implemented, this test may not be necessary. test 'raises no error when another thread closes a socket' do From 54cfc285ecefa1fd6e64d4c6dcce07cf5c1e7610 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:31:08 +0600 Subject: [PATCH 4/7] out_forward: drop stale rebalances and skip a retry already counted by delayed commit A rebuild of the weight array from the ack thread could publish after a newer service-discovery rebuild, putting removed nodes back into the array. Tag each rebuild with a generation taken when the node list is read, and ignore an older result. A late ack failure could also count a second retry after Output#try_rollback_write had already rolled the same attempt back on delayed_commit_timeout. Bind each ack to that write attempt and skip the extra retry. A later send of the same chunk still counts if its ack fails and delayed commit has not already handled that attempt. Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- lib/fluent/plugin/out_forward.rb | 77 ++++++++++- lib/fluent/plugin/out_forward/ack_handler.rb | 15 ++- .../plugin/out_forward/load_balancer.rb | 16 ++- .../service_discovery/manager.rb | 11 +- .../service_discovery/round_robin_balancer.rb | 2 +- test/plugin/out_forward/test_load_balancer.rb | 19 +++ test/plugin/test_out_forward.rb | 124 +++++++++++++++++- 7 files changed, 240 insertions(+), 24 deletions(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 321af4b81e..a12d4607d6 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -172,6 +172,9 @@ def initialize @suspend_flush = false @healthy_nodes_count_metrics = nil @registered_nodes_count_metrics = nil + @write_attempt_seq = 0 + @delayed_commit_rolled_back_attempts = {} + @delayed_commit_attempts_mutex = Mutex.new end def configure(conf) @@ -528,7 +531,57 @@ def node_label(node) node.name || "#{node.host}:#{node.port}" end - def on_ack_failed(chunk_id, node) + public + + # Bind a write to the DequeuedChunkInfo created at the start of try_write, + # so a late ack can be matched to that attempt after a later resend. + def write_attempt_id_for(chunk_id) + return nil unless chunk_id && @dequeued_chunks && @dequeued_chunks_mutex + + @dequeued_chunks_mutex.synchronize do + info = @dequeued_chunks.find { |i| i.chunk_id == chunk_id } + if info + unless info.instance_variable_defined?(:@write_attempt_id) + @write_attempt_seq += 1 + info.instance_variable_set(:@write_attempt_id, @write_attempt_seq) + end + info.instance_variable_get(:@write_attempt_id) + end + end + end + + def remember_delayed_commit_rollback(attempt_id) + return unless attempt_id + + @delayed_commit_attempts_mutex.synchronize do + @delayed_commit_rolled_back_attempts[attempt_id] = true + end + end + + def delayed_commit_already_counted?(attempt_id) + return false unless attempt_id + + @delayed_commit_attempts_mutex.synchronize do + @delayed_commit_rolled_back_attempts[attempt_id] + end + end + + # Record attempts that Output#try_rollback_write has already counted, so a + # late ack for the same send does not count a second retry. + def try_rollback_write + expired_attempt_ids = [] + if @dequeued_chunks && @dequeued_chunks_mutex + @dequeued_chunks_mutex.synchronize do + @dequeued_chunks.each do |info| + expired_attempt_ids << info.instance_variable_get(:@write_attempt_id) if info.expired? + end + end + end + super + expired_attempt_ids.each { |attempt_id| remember_delayed_commit_rollback(attempt_id) } + end + + def on_ack_failed(chunk_id, node, attempt_id = nil) chunk_id_hex = chunk_id ? dump_unique_id_hex(chunk_id) : nil if failover_node_available?(node) @@ -554,10 +607,15 @@ def on_ack_failed(chunk_id, node) return unless chunk_id if rollback_write(chunk_id, update_retry: false) - error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'") - error.set_backtrace(caller(0, 1)) - primary = @as_secondary ? @primary_instance : self - primary.update_retry_state(chunk_id, @as_secondary, error) + # rollback_write returns true when it took a chunk with this id back, + # which may now be a later send of the same chunk. Do not count a retry + # if delayed commit has already rolled this attempt back. + unless delayed_commit_already_counted?(attempt_id) + error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'") + error.set_backtrace(caller(0, 1)) + primary = @as_secondary ? @primary_instance : self + primary.update_retry_state(chunk_id, @as_secondary, error) + end end end @@ -594,7 +652,7 @@ def ack_reader end def ack_check(select_interval) - @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result| + @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result, attempt_id| case result when AckHandler::Result::SUCCESS @connection_manager.close(sock) @@ -605,7 +663,7 @@ def ack_check(select_interval) # instead of returning it to the reuse pool; otherwise a late ack # could later be read against a chunk sent on the reused socket (#4057). @connection_manager.discard(sock) - on_ack_failed(chunk_id, node) + on_ack_failed(chunk_id, node, attempt_id) when AckHandler::Result::CHUNKID_UNMATCHED # The ack read from this socket belongs to a different chunk than the # one tracked for it: the keepalive connection's ack stream is out of @@ -675,9 +733,14 @@ def initialize(sender, server, failure:, connection_manager:, ack_handler:) attr_accessor :usock attr_reader :state + attr_reader :sender attr_reader :sockaddr # used by on_udp_heartbeat_response_recv attr_reader :failure # for test + def write_attempt_id(chunk_id) + @sender.write_attempt_id_for(chunk_id) + end + def validate_host_resolution! resolved_host end diff --git a/lib/fluent/plugin/out_forward/ack_handler.rb b/lib/fluent/plugin/out_forward/ack_handler.rb index b819df55dc..179943b284 100644 --- a/lib/fluent/plugin/out_forward/ack_handler.rb +++ b/lib/fluent/plugin/out_forward/ack_handler.rb @@ -83,7 +83,7 @@ def collect_response(select_interval) if info.nil? yield nil, nil, nil, ret else - yield info.chunk_id, info.node, info.sock, ret + yield info.chunk_id, info.node, info.sock, ret, info.attempt_id end end rescue => e @@ -92,24 +92,25 @@ def collect_response(select_interval) end end - ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do + ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time, :attempt_id) do def expired?(now) expired_time < now end end - Ack = Struct.new(:chunk_id, :node, :handler) do + Ack = Struct.new(:chunk_id, :node, :handler, :attempt_id) do def enqueue(sock) - handler.enqueue(node, sock, chunk_id) + handler.enqueue(node, sock, chunk_id, attempt_id) end end def create_ack(chunk_id, node) - Ack.new(chunk_id, node, self) + attempt_id = node.respond_to?(:write_attempt_id) ? node.write_attempt_id(chunk_id) : nil + Ack.new(chunk_id, node, self, attempt_id) end - def enqueue(node, sock, cid) - info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout) + def enqueue(node, sock, cid, attempt_id = nil) + info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout, attempt_id) @mutex.synchronize do @ack_waitings << info end diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb index f226cfa872..c22cf807e1 100644 --- a/lib/fluent/plugin/out_forward/load_balancer.rb +++ b/lib/fluent/plugin/out_forward/load_balancer.rb @@ -27,6 +27,7 @@ def initialize(log) @rr = 0 @mutex = Mutex.new @availability = {} + @published_generation = 0 end def select_healthy_node @@ -61,7 +62,7 @@ def availability_changed?(nodes) nodes.any? { |n| availability[n] != n.available? } end - def rebuild_weight_array(nodes) + def rebuild_weight_array(nodes, generation = nil) availability = nodes.to_h { |n| [n, n.available?] } standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n| n.standby? @@ -89,10 +90,7 @@ def rebuild_weight_array(nodes) weight_array = [] if regular_nodes.empty? @log.warn('No nodes are available') - @mutex.synchronize do - @availability = availability - @weight_array = weight_array - end + publish_weight_array(weight_array, availability, generation) return @weight_array end @@ -110,7 +108,15 @@ def rebuild_weight_array(nodes) r = Random.new(@rand_seed) weight_array.sort_by! { r.rand } + publish_weight_array(weight_array, availability, generation) + end + + def publish_weight_array(weight_array, availability, generation) @mutex.synchronize do + if generation && @published_generation && generation < @published_generation + return @weight_array + end + @published_generation = generation if generation @availability = availability @weight_array = weight_array end diff --git a/lib/fluent/plugin_helper/service_discovery/manager.rb b/lib/fluent/plugin_helper/service_discovery/manager.rb index 66adec82fe..2da2acd85d 100644 --- a/lib/fluent/plugin_helper/service_discovery/manager.rb +++ b/lib/fluent/plugin_helper/service_discovery/manager.rb @@ -30,6 +30,8 @@ def initialize(log:, load_balancer: nil, custom_build_method: nil) @services = {} @queue = Queue.new @static_config = true + @rebalance_mutex = Mutex.new + @rebalance_generation = 0 end def configure(configs, parent: nil) @@ -101,7 +103,14 @@ def run_once end def rebalance - @load_balancer.rebalance(services) + # Capture a generation with the node list so an older rebuild, which + # may still be building after a later one has published, cannot + # replace the newer weight array. + generation, snapshot = @rebalance_mutex.synchronize do + @rebalance_generation += 1 + [@rebalance_generation, services] + end + @load_balancer.rebalance(snapshot, generation) end def select_service(&block) diff --git a/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb b/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb index 4a403881da..ffdc013b2a 100644 --- a/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb +++ b/lib/fluent/plugin_helper/service_discovery/round_robin_balancer.rb @@ -23,7 +23,7 @@ def initialize @mutex = Mutex.new end - def rebalance(services) + def rebalance(services, generation = nil) @mutex.synchronize do @services = services end diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb index 41ed1e5685..ba1c0584a8 100644 --- a/test/plugin/out_forward/test_load_balancer.rb +++ b/test/plugin/out_forward/test_load_balancer.rb @@ -131,4 +131,23 @@ class LoadBalancerTest < Test::Unit::TestCase assert_true lb.availability_changed?([n1, n2]) end end + + sub_test_case 'rebuild_weight_array generation' do + test 'an older rebuild does not replace a newer weight array' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1, host: 'n1', port: 1) + n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1, host: 'n2', port: 2) + n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1, host: 'n3', port: 3) + + lb.rebuild_weight_array([n1, n2, n3], 1) + # A later rebuild publishes only n3 + lb.rebuild_weight_array([n3], 2) + # The older rebuild resumes and must not bring n1/n2 back + lb.rebuild_weight_array([n1, n2, n3], 1) + + selected = 6.times.map { lb.select_healthy_node { |n| n }[1] }.uniq + assert_equal [n3], selected + end + end + end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 1a4c5fc413..2e26ad971f 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1545,10 +1545,10 @@ def node_named(name) @instance.nodes.find { |n| n.name == name } end - def fail_ack(node, chunk_id = 'chunk-id') + def fail_ack(node, chunk_id = 'chunk-id', attempt_id: nil) fake_ack_handler = Object.new fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| - block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED) + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, attempt_id) end original = @instance.instance_variable_get(:@ack_handler) @instance.instance_variable_set(:@ack_handler, fake_ack_handler) @@ -1677,6 +1677,124 @@ def two_servers(standby: false) assert_true primary.available? end + + test 'an older rebuild does not replace a newer weight array after service discovery' do + create_ack_driver(heartbeat_type: 'none', servers: [ + {name: 'failed'}, + {name: 'removed', port: unused_port(protocol: :tcp)}, + {name: 'retained', port: unused_port(protocol: :tcp)}, + ]) + failed = node_named('failed') + removed = node_named('removed') + retained = node_named('retained') + stub(@instance).rollback_write + + lb = @instance.instance_variable_get(:@load_balancer) + manager = @instance.discovery_manager + first_started = Queue.new + first_continue = Queue.new + first_nodes = nil + calls = 0 + orig = lb.method(:rebuild_weight_array) + # Manager#rebalance calls LoadBalancer#rebalance (an alias), so stub that + # entry point rather than rebuild_weight_array. + stub(lb).rebalance do |*args| + calls += 1 + if calls == 1 + first_nodes = args[0] + first_started.push(true) + first_continue.pop + end + orig.call(*args) + end + + t = Thread.new { fail_ack(failed) } + Timeout.timeout(5) { first_started.pop } + assert_equal ['failed', 'removed', 'retained'], first_nodes.map(&:name).sort + + services = manager.instance_variable_get(:@services) + services.delete(failed.discovery_id) + services.delete(removed.discovery_id) + manager.rebalance + + first_continue.push(true) + Timeout.timeout(5) { t.join } + + names = 8.times.map { selected_node.name }.uniq + assert_equal ['retained'], names + end + + test 'does not count a retry for a late ack after delayed commit already rolled the attempt back' do + create_ack_driver(heartbeat_type: 'none') + node = @instance.nodes.first + chunk_id = 'chunk-id' + info1 = Fluent::Plugin::Output::DequeuedChunkInfo.new(chunk_id, Time.now, 60) + @instance.dequeued_chunks = [info1] + @instance.dequeued_chunks_mutex = Mutex.new + attempt1 = @instance.write_attempt_id_for(chunk_id) + + info1.define_singleton_method(:expired?) { true } + stub(@instance.buffer).takeback_chunk(chunk_id) { true } + retries = 0 + stub(@instance).update_retry_state { retries += 1 } + @instance.try_rollback_write + assert_equal 1, retries + + info2 = Fluent::Plugin::Output::DequeuedChunkInfo.new(chunk_id, Time.now, 60) + @instance.dequeued_chunks = [info2] + attempt2 = @instance.write_attempt_id_for(chunk_id) + assert_not_equal attempt1, attempt2 + + stub(@instance).rollback_write(chunk_id, update_retry: false) { true } + fail_ack(node, chunk_id, attempt_id: attempt1) + assert_equal 1, retries + end + + test 'counts a retry when a later send of the same chunk fails on its own' do + create_ack_driver(heartbeat_type: 'none') + node = @instance.nodes.first + chunk_id = 'chunk-id' + info1 = Fluent::Plugin::Output::DequeuedChunkInfo.new(chunk_id, Time.now, 60) + @instance.dequeued_chunks = [info1] + @instance.dequeued_chunks_mutex = Mutex.new + attempt1 = @instance.write_attempt_id_for(chunk_id) + + info1.define_singleton_method(:expired?) { true } + stub(@instance.buffer).takeback_chunk(chunk_id) { true } + retries = 0 + stub(@instance).update_retry_state { retries += 1 } + @instance.try_rollback_write + assert_equal 1, retries + + info2 = Fluent::Plugin::Output::DequeuedChunkInfo.new(chunk_id, Time.now, 60) + @instance.dequeued_chunks = [info2] + attempt2 = @instance.write_attempt_id_for(chunk_id) + + stub(@instance).rollback_write(chunk_id, update_retry: false) { true } + fail_ack(node, chunk_id, attempt_id: attempt2) + assert_equal 2, retries + end + + test 'does not count a retry when delayed commit already rolled back the later attempt too' do + create_ack_driver(heartbeat_type: 'none') + node = @instance.nodes.first + chunk_id = 'chunk-id' + info2 = Fluent::Plugin::Output::DequeuedChunkInfo.new(chunk_id, Time.now, 60) + @instance.dequeued_chunks = [info2] + @instance.dequeued_chunks_mutex = Mutex.new + attempt2 = @instance.write_attempt_id_for(chunk_id) + + info2.define_singleton_method(:expired?) { true } + stub(@instance.buffer).takeback_chunk(chunk_id) { true } + retries = 0 + stub(@instance).update_retry_state { retries += 1 } + @instance.try_rollback_write + assert_equal 1, retries + + stub(@instance).rollback_write(chunk_id, update_retry: false) { true } + fail_ack(node, chunk_id, attempt_id: attempt2) + assert_equal 1, retries + end end sub_test_case 'a server that never sends the ack (#5619)' do @@ -1792,7 +1910,7 @@ def selected_node test 'rebuilds the weight array only once while the availability stays the same' do create_standby_driver - mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything).once + mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything, anything).once @primary.disable! 2.times { run_heartbeat_timer } From 6e6e1eb9d39091ff2e9febd6aa54e6b9351c7e67 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:17:08 +0600 Subject: [PATCH 5/7] out_forward: do not let an older weight array rebuild replace a newer one With heartbeat_type none, the ack thread (after a failed ack) and the flush threads rebuild the weight array as well as the event loop (heartbeat timer and service discovery). A rebuild that read the node list earlier could publish its array after a newer one, e.g. after service discovery had removed a server, and the removed server stayed selectable. Each rebuild now reads the node list and publishes its array while holding a rebuild lock in the load balancer, so the last published array is always built from the latest node list. Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- lib/fluent/plugin/out_forward.rb | 3 + .../plugin/out_forward/load_balancer.rb | 30 +++++++-- test/plugin/out_forward/test_load_balancer.rb | 39 +++++++++++ test/plugin/test_out_forward.rb | 67 +++++++++++++++++++ 4 files changed, 135 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 321af4b81e..c7190d2d52 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -239,6 +239,9 @@ def configure(conf) load_balancer: @load_balancer, custom_build_method: method(:build_node), ) + # Read the node list under the rebuild lock of the load balancer, so a + # rebuild never uses a list that is older than the last published one. + @load_balancer.nodes_provider = method(:service_discovery_services) service_discovery_services.each do |server| # it's only for test diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb index f226cfa872..7c9e27a30b 100644 --- a/lib/fluent/plugin/out_forward/load_balancer.rb +++ b/lib/fluent/plugin/out_forward/load_balancer.rb @@ -27,8 +27,21 @@ def initialize(log) @rr = 0 @mutex = Mutex.new @availability = {} + # The weight array is rebuilt from several threads: the event loop + # (heartbeat timer and service discovery), the ack thread after a + # failed ack and, with heartbeat_type none, the flush threads. Each + # rebuild reads the node list and publishes its array while holding + # this lock, so an older rebuild cannot publish its array after a + # newer one (#5627). + @rebuild_mutex = Mutex.new + @nodes_provider = nil end + # Returns the current node list. When it is set, every rebuild reads the + # node list from it under the rebuild lock instead of using the list the + # caller read before. + attr_writer :nodes_provider + def select_healthy_node error = nil @@ -61,7 +74,19 @@ def availability_changed?(nodes) nodes.any? { |n| availability[n] != n.available? } end - def rebuild_weight_array(nodes) + def rebuild_weight_array(nodes = nil) + @rebuild_mutex.synchronize do + nodes = @nodes_provider.call if @nodes_provider + rebuild_weight_array_unlocked(nodes) + end + end + + alias select_service select_healthy_node + alias rebalance rebuild_weight_array + + private + + def rebuild_weight_array_unlocked(nodes) availability = nodes.to_h { |n| [n, n.available?] } standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n| n.standby? @@ -115,9 +140,6 @@ def rebuild_weight_array(nodes) @weight_array = weight_array end end - - alias select_service select_healthy_node - alias rebalance rebuild_weight_array end end end diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb index 41ed1e5685..de17bf1a12 100644 --- a/test/plugin/out_forward/test_load_balancer.rb +++ b/test/plugin/out_forward/test_load_balancer.rb @@ -131,4 +131,43 @@ class LoadBalancerTest < Test::Unit::TestCase assert_true lb.availability_changed?([n1, n2]) end end + + sub_test_case 'nodes_provider' do + test 'reads the node list from the provider instead of the given list' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + old_node = flexmock('old', :'standby?' => false, :'available?' => true, weight: 1) + new_node = flexmock('new', :'standby?' => false, :'available?' => true, weight: 1) + lb.nodes_provider = -> { [new_node] } + + lb.rebuild_weight_array([old_node]) + + 3.times { assert_equal new_node, lb.select_healthy_node { |n| n }.last } + end + + test 'reads the node list while holding the rebuild lock' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + node = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + reading = Queue.new + resume = Queue.new + calls = 0 + lb.nodes_provider = -> { + calls += 1 + if calls == 1 + reading << true + resume.pop + end + [node] + } + + first = Thread.new { lb.rebuild_weight_array } + reading.pop + second = Thread.new { lb.rebuild_weight_array } + Timeout.timeout(10) { sleep 0.01 until second.status == 'sleep' || second.status == false } + # the second rebuild waits for the first one before reading the list + assert_equal 1, calls + resume << true + [first, second].each { |t| t.join(10) } + assert_equal 2, calls + end + end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 1a4c5fc413..24791500cb 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1679,6 +1679,73 @@ def two_servers(standby: false) end end + sub_test_case 'concurrent rebuilds of the weight array (#5627)' do + # With heartbeat_type none, the ack thread and the flush threads rebuild + # the weight array as well as the event loop (heartbeat timer and service + # discovery). An older rebuild must not publish its array after a newer one. + test 'a rebuild that read the old node list does not replace a newer one' do + @d = create_driver(%[ + heartbeat_type none + require_ack_response true + + name failed + host #{TARGET_HOST} + port #{@target_port} + + + name removed + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + + name retained + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + ]) + instance = @d.instance + manager = instance.discovery_manager + load_balancer = instance.instance_variable_get(:@load_balancer) + + # Pause the rebuild of the "ack thread" right after it has read the node list. + read_list = Queue.new + resume = Queue.new + paused_thread = nil + original_services = manager.method(:services) + manager.define_singleton_method(:services) do + list = original_services.call + if Thread.current.equal?(paused_thread) + read_list << true + resume.pop + end + list + end + + paused_thread = Thread.new { instance.__send__(:service_discovery_rebalance) } + read_list.pop + + # Service discovery removes 'failed' and 'removed' and rebuilds the array. + static_services = manager.instance_variable_get(:@discoveries).first.services + discovery = Thread.new do + static_services.select { |s| %w[failed removed].include?(s.name) }.each do |s| + manager.instance_variable_get(:@queue) << Fluent::Plugin::ServiceDiscovery.service_out_msg(s) + end + manager.run_once + end + # Without the rebuild lock this rebuild finishes now; with it, it waits + # for the paused rebuild. + Timeout.timeout(10) { sleep 0.01 until discovery.status == 'sleep' || discovery.status == false } + + resume << true + [paused_thread, discovery].each { |t| t.join(10) } + + assert_equal ['retained'], manager.services.map(&:name) + assert_equal ['retained'], load_balancer.instance_variable_get(:@weight_array).map(&:name).uniq + selected = 10.times.map { load_balancer.select_healthy_node { |n| n }.last.name } + assert_equal ['retained'], selected.uniq + end + end + sub_test_case 'a server that never sends the ack (#5619)' do # Reproduces the scenarios in the review of #5627: one server accepts # chunks but never acks, the other acks normally. No chunk may be lost From 0420c88ea478ee1594a75e2b7c6095a4addfca17 Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:17:19 +0600 Subject: [PATCH 6/7] out_forward: do not count a retry twice for a late ack failure A send of a chunk can take longer than delayed_commit_timeout, and the ack timeout only starts when the send is done. Then Output#try_rollback_write rolls the attempt back and counts a retry before the ack thread reports the failed ack. When the ack thread reported it while the same chunk was in flight again, rollback_write took back the new attempt and returned true, so a second retry was counted for the same failure, and with a small retry_max_times the chunk could be dropped. The ack now carries the delayed commit entry that Output#try_flush added for its send. A failed ack only rolls the chunk back and counts a retry if that entry is still pending; it is removed under the same lock that Output#try_rollback_write uses. If the delayed commit timeout has already rolled the attempt back, the late failure is logged and nothing else is done. A later send of the same chunk that fails on its own still counts. Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- lib/fluent/plugin/out_forward.rb | 73 ++++++++-- lib/fluent/plugin/out_forward/ack_handler.rb | 16 +-- test/plugin/out_forward/test_ack_handler.rb | 27 ++++ test/plugin/test_out_forward.rb | 141 ++++++++++++++++++- 4 files changed, 239 insertions(+), 18 deletions(-) diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index c7190d2d52..32db3f20fe 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -394,8 +394,9 @@ def try_write(chunk) return end tag = chunk.metadata.tag + attempt = delayed_commit_attempt(chunk.unique_id) rebalance_if_availability_changed - service_discovery_select_service { |node| node.send_data(tag, chunk) } + service_discovery_select_service { |node| node.send_data(tag, chunk, attempt: attempt) } last_ack if @require_ack_response && @suspend_flush end @@ -531,7 +532,50 @@ def node_label(node) node.name || "#{node.host}:#{node.port}" end - def on_ack_failed(chunk_id, node) + # The instance whose flush thread sent the chunk and keeps its delayed + # commit state (Output#try_flush and #try_rollback_write). + def delayed_commit_owner + @as_secondary ? @primary_instance : self + end + + # The delayed commit entry (Output::DequeuedChunkInfo) that Output#try_flush + # added for this send of the chunk. It identifies the send attempt: when + # the delayed commit timeout rolls the attempt back, the entry is removed, + # and the next send of the same chunk gets a new one. + def delayed_commit_attempt(chunk_id) + return nil unless @ack_handler + + owner = delayed_commit_owner + mutex = owner.dequeued_chunks_mutex + return nil unless mutex + + mutex.synchronize do + owner.dequeued_chunks.reverse_each.find { |info| info.chunk_id == chunk_id } + end + end + + # Rolls back the send attempt whose ack failed. + # Returns nil if the delayed commit timeout (or another error) has already + # rolled this attempt back. The chunk may be in flight again then, so it is + # neither taken back nor counted again. Otherwise returns the result of + # #rollback_write. + def rollback_failed_attempt(chunk_id, attempt) + if attempt + owner = delayed_commit_owner + # Remove the entry in the same lock that Output#try_rollback_write + # holds, so only one of them can roll this attempt back. + current = owner.dequeued_chunks_mutex.synchronize do + index = owner.dequeued_chunks.index { |info| info.equal?(attempt) } + owner.dequeued_chunks.delete_at(index) if index + !index.nil? + end + return nil unless current + end + + rollback_write(chunk_id, update_retry: false) + end + + def on_ack_failed(chunk_id, node, attempt = nil) chunk_id_hex = chunk_id ? dump_unique_id_hex(chunk_id) : nil if failover_node_available?(node) @@ -544,7 +588,9 @@ def on_ack_failed(chunk_id, node) service_discovery_rebalance log.warn "failed to receive the ack response, detached forwarding server '#{node_label(node)}' for #{@recover_wait}s. the chunk will be sent to another server", host: node.host, port: node.port, chunk_id: chunk_id_hex - rollback_write(chunk_id, update_retry: false) if chunk_id + if chunk_id && rollback_failed_attempt(chunk_id, attempt).nil? + log.debug "the send attempt of the chunk was already rolled back by the delayed commit timeout", chunk_id: chunk_id_hex + end return end @@ -556,7 +602,17 @@ def on_ack_failed(chunk_id, node) # passed so that the retry and a discard are logged as usual. return unless chunk_id - if rollback_write(chunk_id, update_retry: false) + rolled_back = rollback_failed_attempt(chunk_id, attempt) + if rolled_back.nil? + # Output#try_rollback_write has already rolled this attempt back and + # counted a retry for it ("timeout to commit"). Counting another one + # here could use up retry_max_times for a single failure. + log.warn "failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}' after the delayed commit timeout rolled the chunk back. no retry is counted again", + chunk_id: chunk_id_hex + return + end + + if rolled_back error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'") error.set_backtrace(caller(0, 1)) primary = @as_secondary ? @primary_instance : self @@ -597,7 +653,7 @@ def ack_reader end def ack_check(select_interval) - @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result| + @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result, attempt| case result when AckHandler::Result::SUCCESS @connection_manager.close(sock) @@ -608,7 +664,7 @@ def ack_check(select_interval) # instead of returning it to the reuse pool; otherwise a late ack # could later be read against a chunk sent on the reused socket (#4057). @connection_manager.discard(sock) - on_ack_failed(chunk_id, node) + on_ack_failed(chunk_id, node, attempt) when AckHandler::Result::CHUNKID_UNMATCHED # The ack read from this socket belongs to a different chunk than the # one tracked for it: the keepalive connection's ack stream is out of @@ -790,8 +846,9 @@ def send_data_actual(sock, tag, chunk) # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported end - def send_data(tag, chunk) - ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self) + # attempt identifies this send of the chunk (see ForwardOutput#delayed_commit_attempt) + def send_data(tag, chunk, attempt: nil) + ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self, attempt) connect(nil, ack: ack) do |sock, ri| ensure_established_connection(sock, ri) send_data_actual(sock, tag, chunk) diff --git a/lib/fluent/plugin/out_forward/ack_handler.rb b/lib/fluent/plugin/out_forward/ack_handler.rb index b819df55dc..c05afc95f3 100644 --- a/lib/fluent/plugin/out_forward/ack_handler.rb +++ b/lib/fluent/plugin/out_forward/ack_handler.rb @@ -83,7 +83,7 @@ def collect_response(select_interval) if info.nil? yield nil, nil, nil, ret else - yield info.chunk_id, info.node, info.sock, ret + yield info.chunk_id, info.node, info.sock, ret, info.attempt end end rescue => e @@ -92,24 +92,24 @@ def collect_response(select_interval) end end - ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do + ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time, :attempt) do def expired?(now) expired_time < now end end - Ack = Struct.new(:chunk_id, :node, :handler) do + Ack = Struct.new(:chunk_id, :node, :handler, :attempt) do def enqueue(sock) - handler.enqueue(node, sock, chunk_id) + handler.enqueue(node, sock, chunk_id, attempt) end end - def create_ack(chunk_id, node) - Ack.new(chunk_id, node, self) + def create_ack(chunk_id, node, attempt = nil) + Ack.new(chunk_id, node, self, attempt) end - def enqueue(node, sock, cid) - info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout) + def enqueue(node, sock, cid, attempt = nil) + info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout, attempt) @mutex.synchronize do @ack_waitings << info end diff --git a/test/plugin/out_forward/test_ack_handler.rb b/test/plugin/out_forward/test_ack_handler.rb index 5605162d0d..7e56581c4c 100644 --- a/test/plugin/out_forward/test_ack_handler.rb +++ b/test/plugin/out_forward/test_ack_handler.rb @@ -42,6 +42,33 @@ class AckHandlerTest < Test::Unit::TestCase end end + test 'returns the send attempt given to the ack' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + attempt = Object.new + ack = ack_handler.create_ack(chunk_id, node, attempt) + + r, w = IO.pipe + begin + w.write(chunk_id) + mock(r).recv(anything) { |_| nil } # the connection is closed: the ack failed + ack.enqueue(r) + + result = yielded_attempt = nil + ack_handler.collect_response(1) do |_cid, _n, _s, ret, att| + result = ret; yielded_attempt = att + end + + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, result + assert_same attempt, yielded_attempt + ensure + r.close rescue nil + w.close rescue nil + end + end + test 'returns nil if raise an error' do ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 24791500cb..2b929e892b 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -2,6 +2,7 @@ require 'fluent/test/driver/output' require 'fluent/plugin/out_forward' require 'flexmock/test_unit' +require 'timecop' require 'fluent/test/driver/input' require 'fluent/plugin/in_forward' @@ -1545,10 +1546,10 @@ def node_named(name) @instance.nodes.find { |n| n.name == name } end - def fail_ack(node, chunk_id = 'chunk-id') + def fail_ack(node, chunk_id = 'chunk-id', attempt = nil) fake_ack_handler = Object.new fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| - block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED) + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, attempt) end original = @instance.instance_variable_get(:@ack_handler) @instance.instance_variable_set(:@ack_handler, fake_ack_handler) @@ -1679,6 +1680,142 @@ def two_servers(standby: false) end end + sub_test_case 'a late ack failure after the delayed commit timeout (#5627)' do + # A send of a chunk can take longer than delayed_commit_timeout, and the + # ack timeout only starts when the send is done. Then Output#try_rollback_write + # rolls the attempt back and counts a retry before the ack thread reports + # the failed ack. By then the chunk may be in flight again. + setup do + @d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1 + + name test + host #{TARGET_HOST} + port #{@target_port} + + + flush_mode interval + flush_interval 3600 + flush_thread_interval 3600 + flush_thread_burst_interval 3600 + flush_at_shutdown false + + ]) + @instance = @d.instance + @node = @instance.nodes.first + @attempts = attempts = [] + # record the attempt of each send instead of sending the chunk + @node.define_singleton_method(:send_data) { |_tag, _chunk, attempt: nil| attempts << attempt; nil } + @retry_errors = retry_errors = [] + @instance.singleton_class.prepend(Module.new { + define_method(:update_retry_state) do |chunk_id, using_secondary, error = nil| + retry_errors << error + super(chunk_id, using_secondary, error) + end + }) + @d.instance_start + # keep the flush thread from rolling back chunks by itself + @instance.interrupt_flushes + stub(@instance.instance_variable_get(:@connection_manager)).discard + + @instance.emit_events('test', Fluent::OneEventStream.new(event_time, {'k' => 'v'})) + @instance.buffer.enqueue_all + @chunk_id = @instance.buffer.queue.first.unique_id + end + + # Returns the delayed commit entry that Output#try_flush added for the send. + def send_chunk + @instance.try_flush + assert_true @instance.buffer.dequeued.key?(@chunk_id) + dequeued_chunk_infos.last + end + + def expire_delayed_commit(attempt) + Timecop.travel(attempt.time + attempt.timeout + 1) do + @instance.try_rollback_write + end + assert_false @instance.buffer.dequeued.key?(@chunk_id) + end + + def fail_ack(attempt) + fake_ack_handler = Object.new + chunk_id, node = @chunk_id, @node + fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, attempt) + end + original = @instance.instance_variable_get(:@ack_handler) + @instance.instance_variable_set(:@ack_handler, fake_ack_handler) + @instance.__send__(:ack_check, 0) + ensure + @instance.instance_variable_set(:@ack_handler, original) + end + + def dequeued_chunk_infos + @instance.dequeued_chunks_mutex.synchronize { @instance.dequeued_chunks.dup } + end + + test 'passes the delayed commit entry of the send to the ack' do + attempt = send_chunk + assert_equal @chunk_id, attempt.chunk_id + assert_same attempt, @attempts.last + end + + test 'does not count a second retry for an attempt that was already rolled back' do + first = send_chunk + expire_delayed_commit(first) + assert_equal [nil], @retry_errors # "timeout to commit" + + second = send_chunk + fail_ack(first) + + # the late failure belongs to the first attempt: the chunk that is in + # flight again is not taken back, and no retry is counted for it + assert_equal [nil], @retry_errors + assert_true @instance.buffer.dequeued.key?(@chunk_id) + assert_equal [second], dequeued_chunk_infos + assert{ @d.logs.any? { |log| log.include?("after the delayed commit timeout rolled the chunk back. no retry is counted again") } } + end + + test 'counts a retry when a later send of the chunk fails on its own' do + first = send_chunk + expire_delayed_commit(first) + second = send_chunk + fail_ack(first) + + fail_ack(second) + + assert_equal [nil, Fluent::Plugin::ForwardOutput::AckResponseError], @retry_errors.map { |e| e&.class } + assert_false @instance.buffer.dequeued.key?(@chunk_id) + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + assert_equal [], dequeued_chunk_infos + end + + test 'does not count a retry when the delayed commit timeout rolled back the later send too' do + first = send_chunk + expire_delayed_commit(first) + second = send_chunk + expire_delayed_commit(second) + assert_equal [nil, nil], @retry_errors + + fail_ack(first) + fail_ack(second) + + assert_equal [nil, nil], @retry_errors + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + end + + test 'counts a retry for a failed ack before the delayed commit timeout' do + attempt = send_chunk + fail_ack(attempt) + + assert_equal [Fluent::Plugin::ForwardOutput::AckResponseError], @retry_errors.map(&:class) + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + assert_equal [], dequeued_chunk_infos + end + end + sub_test_case 'concurrent rebuilds of the weight array (#5627)' do # With heartbeat_type none, the ack thread and the flush threads rebuild # the weight array as well as the event loop (heartbeat timer and service From a2a3990190750532804a944278ccfa0932b8a2bb Mon Sep 17 00:00:00 2001 From: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:21:22 +0600 Subject: [PATCH 7/7] test_out_forward: use Array.new instead of times.map (RuboCop) Signed-off-by: SahinurDEV <68515168+SahinurDEV@users.noreply.github.com> --- test/plugin/test_out_forward.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 2b929e892b..804e84f782 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1878,7 +1878,7 @@ def dequeued_chunk_infos assert_equal ['retained'], manager.services.map(&:name) assert_equal ['retained'], load_balancer.instance_variable_get(:@weight_array).map(&:name).uniq - selected = 10.times.map { load_balancer.select_healthy_node { |n| n }.last.name } + selected = Array.new(10) { load_balancer.select_healthy_node { |n| n }.last.name } assert_equal ['retained'], selected.uniq end end