From 2200f8a679074b32f72dce15d5a298255365234a Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Fri, 13 Nov 2026 23:33:17 -0300 Subject: [PATCH 1/2] output: Pass update_retry as a keyword in the secondary rollback_write acts_as_secondary defines rollback_write to delegate to the primary with rollback_write(chunk_id, update_retry). Output#rollback_write has taken update_retry as a keyword argument since v0.14.25, so the call always raised ArgumentError. A delayed commit output used as , such as out_forward with require_ack_response, could not roll back a chunk after an ack timeout, and the chunk was not sent again. Fixes #5633 Signed-off-by: Lucas Barros --- lib/fluent/plugin/output.rb | 2 +- .../test_output_as_buffered_secondary.rb | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb index b733ec1698..8b971afd7a 100644 --- a/lib/fluent/plugin/output.rb +++ b/lib/fluent/plugin/output.rb @@ -253,7 +253,7 @@ def acts_as_secondary(primary) singleton_class.module_eval do define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) } - define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) } + define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry: update_retry) } end end diff --git a/test/plugin/test_output_as_buffered_secondary.rb b/test/plugin/test_output_as_buffered_secondary.rb index 15ff633f71..258144c7ce 100644 --- a/test/plugin/test_output_as_buffered_secondary.rb +++ b/test/plugin/test_output_as_buffered_secondary.rb @@ -455,6 +455,33 @@ def dummy_event_stream assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } end + data('update_retry false' => [{update_retry: false}, false], + 'default update_retry' => [{}, true]) + test 'secondary plugin can rollback a chunk through the primary' do |(kwargs, update_retry)| + priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.register(:prefer_delayed_commit){ false } + @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } + @i.register(:write){|chunk| raise "yay, your #write must fail" } + @i.secondary.register(:prefer_delayed_commit){ true } + @i.start + @i.after_start + + @i.interrupt_flushes + + @i.emit_events("test.tag.1", dummy_event_stream()) + @i.buffer.enqueue_all + chunk = @i.buffer.dequeue_chunk + + mock.proxy(@i).update_retry_state(chunk.unique_id, false).times(update_retry ? 1 : 0) + assert_true @i.secondary.rollback_write(chunk.unique_id, **kwargs) + + assert_nil @i.buffer.dequeued[chunk.unique_id] + assert_equal [chunk.unique_id], @i.buffer.queue.map(&:unique_id) + end + test 'secondary plugin can do delayed commit even if primary does not do it' do written = [] chunks = [] From 2bd73f2d73ca8a5d7db222e0ffe62a38ffed1771 Mon Sep 17 00:00:00 2001 From: Lucas Barros Date: Mon, 16 Nov 2026 00:10:57 -0300 Subject: [PATCH 2/2] output: Update the retry state on a rollback from the secondary A secondary runs while the primary is retrying. With out_forward as the secondary, an ack timeout rolls the chunk back with update_retry: false, so retry_timeout and retry_max_times were never reached when the secondary never acked, and Fluentd stayed on the secondary even after the primary recovered. A rollback through the secondary now always updates the retry state of the primary. An ack failure on the primary still does not. Signed-off-by: Lucas Barros --- lib/fluent/plugin/output.rb | 5 +- test/plugin/test_out_forward.rb | 66 +++++++++++++++++++ .../test_output_as_buffered_secondary.rb | 16 ++--- 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb index 8b971afd7a..e26cae2d7c 100644 --- a/lib/fluent/plugin/output.rb +++ b/lib/fluent/plugin/output.rb @@ -253,7 +253,10 @@ def acts_as_secondary(primary) singleton_class.module_eval do define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) } - define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry: update_retry) } + # A secondary runs while the primary is already retrying, and only the retry limit + # ends that phase if the secondary never succeeds. So a rollback from the secondary + # always updates the retry state, even when the plugin passes update_retry: false. + define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry: true) } end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index e5ea0f71fe..555675ca12 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1594,4 +1594,70 @@ def selected_node assert_equal @standby, selected_node end end + + sub_test_case 'as secondary with a destination that never acks' do + setup do + @silent_server = TCPServer.new(TARGET_HOST, 0) + @silent_port = @silent_server.addr[1] + @silent_connections = 0 + @silent_thread = Thread.new do + loop do + sock = @silent_server.accept + @silent_connections += 1 + Thread.new(sock) { |s| s.read rescue nil } + end + rescue IOError, Errno::EBADF + end + end + + teardown do + @silent_server.close rescue nil + @silent_thread.join(1) + end + + data('retry_max_times' => {'retry_type' => 'exponential_backoff', 'retry_max_times' => '2'}, + 'retry_timeout' => {'retry_type' => 'periodic', 'retry_timeout' => '4s'}) + test 'the retry limit ends the retry after ack timeouts of the secondary' do |retry_conf| + @d = d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1s + + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + + @type memory + flush_mode immediate + flush_at_shutdown false + retry_wait 1s + retry_randomize false + retry_secondary_threshold 0.25 + #{retry_conf.map { |k, v| "#{k} #{v}" }.join("\n")} + + + @type forward + heartbeat_type none + require_ack_response true + ack_response_timeout 1s + + host #{TARGET_HOST} + port #{@silent_port} + + + ]) + buffer = d.instance.buffer + limit_reached = lambda do + @silent_connections > 0 && + d.instance.retry.nil? && + buffer.stage.empty? && buffer.queue.empty? && buffer.dequeued.empty? + end + d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false, force_flush_retry: true) do + d.feed(event_time, {"a" => 1}) + waiting(15) { sleep 0.1 until limit_reached.call } + end + + assert_true limit_reached.call + end + end end diff --git a/test/plugin/test_output_as_buffered_secondary.rb b/test/plugin/test_output_as_buffered_secondary.rb index 258144c7ce..cdc0a701dd 100644 --- a/test/plugin/test_output_as_buffered_secondary.rb +++ b/test/plugin/test_output_as_buffered_secondary.rb @@ -455,27 +455,21 @@ def dummy_event_stream assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } end - data('update_retry false' => [{update_retry: false}, false], - 'default update_retry' => [{}, true]) - test 'secondary plugin can rollback a chunk through the primary' do |(kwargs, update_retry)| - priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'retry_randomize' => false}) + data('update_retry false' => {update_retry: false}, + 'default update_retry' => {}) + test 'secondary plugin can rollback a chunk through the primary and update the retry state' do |kwargs| + priconf = config_element('buffer','tag',{'flush_thread_count' => 0, 'queued_chunks_limit_size' => 1}) secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) @i.configure(config_element('ROOT','',{},[priconf,secconf])) @i.register(:prefer_buffered_processing){ true } - @i.register(:prefer_delayed_commit){ false } - @i.register(:format){|tag,time,record| [tag,time.to_i,record].to_json + "\n" } - @i.register(:write){|chunk| raise "yay, your #write must fail" } - @i.secondary.register(:prefer_delayed_commit){ true } @i.start @i.after_start - @i.interrupt_flushes - @i.emit_events("test.tag.1", dummy_event_stream()) @i.buffer.enqueue_all chunk = @i.buffer.dequeue_chunk - mock.proxy(@i).update_retry_state(chunk.unique_id, false).times(update_retry ? 1 : 0) + mock.proxy(@i).update_retry_state(chunk.unique_id, false).once assert_true @i.secondary.rollback_write(chunk.unique_id, **kwargs) assert_nil @i.buffer.dequeued[chunk.unique_id]