diff --git a/lib/fluent/plugin/output.rb b/lib/fluent/plugin/output.rb index b733ec1698..e26cae2d7c 100644 --- a/lib/fluent/plugin/output.rb +++ b/lib/fluent/plugin/output.rb @@ -253,7 +253,10 @@ def acts_as_secondary(primary) singleton_class.module_eval do define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) } - define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) } + # A secondary runs while the primary is already retrying, and only the retry limit + # ends that phase if the secondary never succeeds. So a rollback from the secondary + # always updates the retry state, even when the plugin passes update_retry: false. + define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry: true) } end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index e5ea0f71fe..555675ca12 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -1594,4 +1594,70 @@ def selected_node assert_equal @standby, selected_node end end + + sub_test_case 'as secondary with a destination that never acks' do + setup do + @silent_server = TCPServer.new(TARGET_HOST, 0) + @silent_port = @silent_server.addr[1] + @silent_connections = 0 + @silent_thread = Thread.new do + loop do + sock = @silent_server.accept + @silent_connections += 1 + Thread.new(sock) { |s| s.read rescue nil } + end + rescue IOError, Errno::EBADF + end + end + + teardown do + @silent_server.close rescue nil + @silent_thread.join(1) + end + + data('retry_max_times' => {'retry_type' => 'exponential_backoff', 'retry_max_times' => '2'}, + 'retry_timeout' => {'retry_type' => 'periodic', 'retry_timeout' => '4s'}) + test 'the retry limit ends the retry after ack timeouts of the secondary' do |retry_conf| + @d = d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1s + + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + + @type memory + flush_mode immediate + flush_at_shutdown false + retry_wait 1s + retry_randomize false + retry_secondary_threshold 0.25 + #{retry_conf.map { |k, v| "#{k} #{v}" }.join("\n")} + + + @type forward + heartbeat_type none + require_ack_response true + ack_response_timeout 1s + + host #{TARGET_HOST} + port #{@silent_port} + + + ]) + buffer = d.instance.buffer + limit_reached = lambda do + @silent_connections > 0 && + d.instance.retry.nil? && + buffer.stage.empty? && buffer.queue.empty? && buffer.dequeued.empty? + end + d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false, force_flush_retry: true) do + d.feed(event_time, {"a" => 1}) + waiting(15) { sleep 0.1 until limit_reached.call } + end + + assert_true limit_reached.call + end + end end diff --git a/test/plugin/test_output_as_buffered_secondary.rb b/test/plugin/test_output_as_buffered_secondary.rb index 15ff633f71..cdc0a701dd 100644 --- a/test/plugin/test_output_as_buffered_secondary.rb +++ b/test/plugin/test_output_as_buffered_secondary.rb @@ -455,6 +455,27 @@ def dummy_event_stream assert{ logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } } end + data('update_retry false' => {update_retry: false}, + 'default update_retry' => {}) + test 'secondary plugin can rollback a chunk through the primary and update the retry state' do |kwargs| + priconf = config_element('buffer','tag',{'flush_thread_count' => 0, 'queued_chunks_limit_size' => 1}) + secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'}) + @i.configure(config_element('ROOT','',{},[priconf,secconf])) + @i.register(:prefer_buffered_processing){ true } + @i.start + @i.after_start + + @i.emit_events("test.tag.1", dummy_event_stream()) + @i.buffer.enqueue_all + chunk = @i.buffer.dequeue_chunk + + mock.proxy(@i).update_retry_state(chunk.unique_id, false).once + assert_true @i.secondary.rollback_write(chunk.unique_id, **kwargs) + + assert_nil @i.buffer.dequeued[chunk.unique_id] + assert_equal [chunk.unique_id], @i.buffer.queue.map(&:unique_id) + end + test 'secondary plugin can do delayed commit even if primary does not do it' do written = [] chunks = []