diff --git a/lib/fluent/plugin/out_forward.rb b/lib/fluent/plugin/out_forward.rb index 8298203706..32db3f20fe 100644 --- a/lib/fluent/plugin/out_forward.rb +++ b/lib/fluent/plugin/out_forward.rb @@ -52,7 +52,8 @@ class ForwardOutput < Output config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport desc 'The interval of the heartbeat packer.' config_param :heartbeat_interval, :time, default: 1 - desc 'The wait time before accepting a server fault recovery.' + desc 'The wait time before accepting a server fault recovery. ' \ + 'A server whose ack response fails is also kept out for this long while another server can take its chunks.' config_param :recover_wait, :time, default: 10 desc 'The hard timeout used to detect server failure.' config_param :hard_timeout, :time, default: 60 @@ -238,6 +239,9 @@ def configure(conf) load_balancer: @load_balancer, custom_build_method: method(:build_node), ) + # Read the node list under the rebuild lock of the load balancer, so a + # rebuild never uses a list that is older than the last published one. + @load_balancer.nodes_provider = method(:service_discovery_services) service_discovery_services.each do |server| # it's only for test @@ -379,6 +383,7 @@ def write(chunk) return if chunk.empty? tag = chunk.metadata.tag + rebalance_if_availability_changed service_discovery_select_service { |node| node.send_data(tag, chunk) } end @@ -389,7 +394,9 @@ def try_write(chunk) return end tag = chunk.metadata.tag - service_discovery_select_service { |node| node.send_data(tag, chunk) } + attempt = delayed_commit_attempt(chunk.unique_id) + rebalance_if_availability_changed + service_discovery_select_service { |node| node.send_data(tag, chunk, attempt: attempt) } last_ack if @require_ack_response && @suspend_flush end @@ -501,6 +508,118 @@ def on_heartbeat_timer end end + # heartbeat_type none has no heartbeat timer to rebuild the weight array, so + # a node detached after a failed ack (and the standby used in its place) + # would never be switched back. Check it before selecting a node instead. + def rebalance_if_availability_changed + return unless @heartbeat_type == :none + + if @load_balancer.availability_changed?(service_discovery_services) + service_discovery_rebalance + end + end + + # Whether another node can take the chunks of a node whose ack failed. + # With recover_wait 0 a detached node would be back at once, so there is + # no point in detaching it. + def failover_node_available?(failed_node) + return false if failed_node.nil? || @recover_wait <= 0 + + service_discovery_services.any? { |n| !n.equal?(failed_node) && n.weight > 0 && n.available? } + end + + def node_label(node) + node.name || "#{node.host}:#{node.port}" + end + + # The instance whose flush thread sent the chunk and keeps its delayed + # commit state (Output#try_flush and #try_rollback_write). + def delayed_commit_owner + @as_secondary ? @primary_instance : self + end + + # The delayed commit entry (Output::DequeuedChunkInfo) that Output#try_flush + # added for this send of the chunk. It identifies the send attempt: when + # the delayed commit timeout rolls the attempt back, the entry is removed, + # and the next send of the same chunk gets a new one. + def delayed_commit_attempt(chunk_id) + return nil unless @ack_handler + + owner = delayed_commit_owner + mutex = owner.dequeued_chunks_mutex + return nil unless mutex + + mutex.synchronize do + owner.dequeued_chunks.reverse_each.find { |info| info.chunk_id == chunk_id } + end + end + + # Rolls back the send attempt whose ack failed. + # Returns nil if the delayed commit timeout (or another error) has already + # rolled this attempt back. The chunk may be in flight again then, so it is + # neither taken back nor counted again. Otherwise returns the result of + # #rollback_write. + def rollback_failed_attempt(chunk_id, attempt) + if attempt + owner = delayed_commit_owner + # Remove the entry in the same lock that Output#try_rollback_write + # holds, so only one of them can roll this attempt back. + current = owner.dequeued_chunks_mutex.synchronize do + index = owner.dequeued_chunks.index { |info| info.equal?(attempt) } + owner.dequeued_chunks.delete_at(index) if index + !index.nil? + end + return nil unless current + end + + rollback_write(chunk_id, update_retry: false) + end + + def on_ack_failed(chunk_id, node, attempt = nil) + chunk_id_hex = chunk_id ? dump_unique_id_hex(chunk_id) : nil + + if failover_node_available?(node) + # Keep the node out for recover_wait so that its chunks go to the + # other nodes (a standby node is used if needed). No retry is counted: + # the chunk has not failed on every node, and the retry state belongs + # to the whole output. If no node is left, the next write raises + # NoNodesAvailable and the usual retry applies. + node.detach! + service_discovery_rebalance + log.warn "failed to receive the ack response, detached forwarding server '#{node_label(node)}' for #{@recover_wait}s. the chunk will be sent to another server", + host: node.host, port: node.port, chunk_id: chunk_id_hex + if chunk_id && rollback_failed_attempt(chunk_id, attempt).nil? + log.debug "the send attempt of the chunk was already rolled back by the delayed commit timeout", chunk_id: chunk_id_hex + end + return + end + + # No other node can take the chunk. Keeping the last node out would make + # every retry fail with NoNodesAvailable until it comes back (and could use + # up retry_max_times on a single transient failure), so leave it in and + # count a retry instead: the chunk is resent after retry_wait, and + # retry_timeout, retry_max_times and apply. The error is + # passed so that the retry and a discard are logged as usual. + return unless chunk_id + + rolled_back = rollback_failed_attempt(chunk_id, attempt) + if rolled_back.nil? + # Output#try_rollback_write has already rolled this attempt back and + # counted a retry for it ("timeout to commit"). Counting another one + # here could use up retry_max_times for a single failure. + log.warn "failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}' after the delayed commit timeout rolled the chunk back. no retry is counted again", + chunk_id: chunk_id_hex + return + end + + if rolled_back + error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'") + error.set_backtrace(caller(0, 1)) + primary = @as_secondary ? @primary_instance : self + primary.update_retry_state(chunk_id, @as_secondary, error) + end + end + def on_udp_heartbeat_response_recv(data, sock) sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host) if node = service_discovery_services.find { |n| n.sockaddr == sockaddr } @@ -534,7 +653,7 @@ def ack_reader end def ack_check(select_interval) - @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result| + @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result, attempt| case result when AckHandler::Result::SUCCESS @connection_manager.close(sock) @@ -545,8 +664,7 @@ def ack_check(select_interval) # instead of returning it to the reuse pool; otherwise a late ack # could later be read against a chunk sent on the reused socket (#4057). @connection_manager.discard(sock) - node&.disable! - rollback_write(chunk_id, update_retry: false) if chunk_id + on_ack_failed(chunk_id, node, attempt) when AckHandler::Result::CHUNKID_UNMATCHED # The ack read from this socket belongs to a different chunk than the # one tracked for it: the keepalive connection's ack stream is out of @@ -585,6 +703,7 @@ def initialize(sender, server, failure:, connection_manager:, ack_handler:) @standby = server.standby @failure = failure @available = true + @detached_until = nil # @hostname is used for certificate verification & TLS SNI host_is_hostname = !(IPAddr.new(@host) rescue false) @@ -630,6 +749,21 @@ def disable! @available = false end + # Keep the node out for recover_wait after a failed ack. The hold is set + # before disabling so a heartbeat running in between cannot bring the + # node back at once, and samples added by in-flight writes cannot end it + # early. After the hold, recovery still needs fresh heartbeats. + def detach! + @detached_until = Fluent::Clock.now + @sender.recover_wait + disable! + @failure.clear + end + + def detached? + detached_until = @detached_until + !detached_until.nil? && Fluent::Clock.now < detached_until + end + def standby? @standby end @@ -712,8 +846,9 @@ def send_data_actual(sock, tag, chunk) # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported end - def send_data(tag, chunk) - ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self) + # attempt identifies this send of the chunk (see ForwardOutput#delayed_commit_attempt) + def send_data(tag, chunk, attempt: nil) + ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self, attempt) connect(nil, ack: ack) do |sock, ri| ensure_established_connection(sock, ri) send_data_actual(sock, tag, chunk) @@ -823,7 +958,7 @@ def tick def heartbeat(detect=true) now = Time.now.to_f @failure.add(now) - if detect && !available? && @failure.sample_size > @sender.recover_sample_size + if detect && !available? && !detached? && @failure.sample_size > @sender.recover_sample_size @available = true @log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port true @@ -851,8 +986,14 @@ def connect(host = nil, ack: false, &block) # Override Node to disable heartbeat class NoneHeartbeatNode < Node + # Without heartbeats, a node is only kept out for recover_wait after a + # failed ack (see ForwardOutput#on_ack_failed). def available? - true + !detached? + end + + def detach! + @detached_until = Fluent::Clock.now + @sender.recover_wait end def tick diff --git a/lib/fluent/plugin/out_forward/ack_handler.rb b/lib/fluent/plugin/out_forward/ack_handler.rb index 04948eafa3..c05afc95f3 100644 --- a/lib/fluent/plugin/out_forward/ack_handler.rb +++ b/lib/fluent/plugin/out_forward/ack_handler.rb @@ -45,7 +45,7 @@ def collect_response(select_interval) new_list = [] @mutex.synchronize do @ack_waitings.each do |info| - if info.expired?(now) + if info.expired?(now) && !readable?(info.sock) # There are 2 types of cases when no response has been received from socket: # (1) the node does not support sending responses # (2) the node does support sending response but responses have not arrived for some reasons. @@ -59,8 +59,15 @@ def collect_response(select_interval) @ack_waitings = new_list end + # Report expired acks right away instead of after a full select + # wait. Otherwise a failed ack can be reported up to two select + # intervals after it expired, which reaches delayed_commit_timeout + # (ack_response_timeout + 2). Then Output#try_rollback_write rolls the + # chunk back first and counts a retry without an error, and with + # retry_max_times 0 the whole queue is cleared without a log. + timeout = results.empty? ? select_interval : 0 begin - readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval) + readable_sockets, _, _ = IO.select(sockets, nil, nil, timeout) rescue IOError @log.info "connection closed while waiting for readable sockets" readable_sockets = nil @@ -76,7 +83,7 @@ def collect_response(select_interval) if info.nil? yield nil, nil, nil, ret else - yield info.chunk_id, info.node, info.sock, ret + yield info.chunk_id, info.node, info.sock, ret, info.attempt end end rescue => e @@ -85,24 +92,24 @@ def collect_response(select_interval) end end - ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do + ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time, :attempt) do def expired?(now) expired_time < now end end - Ack = Struct.new(:chunk_id, :node, :handler) do + Ack = Struct.new(:chunk_id, :node, :handler, :attempt) do def enqueue(sock) - handler.enqueue(node, sock, chunk_id) + handler.enqueue(node, sock, chunk_id, attempt) end end - def create_ack(chunk_id, node) - Ack.new(chunk_id, node, self) + def create_ack(chunk_id, node, attempt = nil) + Ack.new(chunk_id, node, self, attempt) end - def enqueue(node, sock, cid) - info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout) + def enqueue(node, sock, cid, attempt = nil) + info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout, attempt) @mutex.synchronize do @ack_waitings << info end @@ -110,6 +117,16 @@ def enqueue(node, sock, cid) private + # A socket enqueued while this thread was waiting in IO.select is not in + # that select call, so its ack can already be in the socket buffer when the + # timeout is checked. Read it instead of regarding the node as unavailable. + def readable?(sock) + readable_sockets, _, _ = IO.select([sock], nil, nil, 0) + !readable_sockets.nil? + rescue IOError + false + end + def read_ack_from_sock(sock) begin raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length) diff --git a/lib/fluent/plugin/out_forward/error.rb b/lib/fluent/plugin/out_forward/error.rb index 87de15d9b4..8ec3fd8159 100644 --- a/lib/fluent/plugin/out_forward/error.rb +++ b/lib/fluent/plugin/out_forward/error.rb @@ -21,6 +21,7 @@ class ForwardOutput < Output class Error < StandardError; end class NoNodesAvailable < Error; end class ConnectionClosedError < Error; end + class AckResponseError < Error; end class HandshakeError < Error; end class HeloError < HandshakeError; end class PingpongError < HandshakeError; end diff --git a/lib/fluent/plugin/out_forward/load_balancer.rb b/lib/fluent/plugin/out_forward/load_balancer.rb index f226cfa872..7c9e27a30b 100644 --- a/lib/fluent/plugin/out_forward/load_balancer.rb +++ b/lib/fluent/plugin/out_forward/load_balancer.rb @@ -27,8 +27,21 @@ def initialize(log) @rr = 0 @mutex = Mutex.new @availability = {} + # The weight array is rebuilt from several threads: the event loop + # (heartbeat timer and service discovery), the ack thread after a + # failed ack and, with heartbeat_type none, the flush threads. Each + # rebuild reads the node list and publishes its array while holding + # this lock, so an older rebuild cannot publish its array after a + # newer one (#5627). + @rebuild_mutex = Mutex.new + @nodes_provider = nil end + # Returns the current node list. When it is set, every rebuild reads the + # node list from it under the rebuild lock instead of using the list the + # caller read before. + attr_writer :nodes_provider + def select_healthy_node error = nil @@ -61,7 +74,19 @@ def availability_changed?(nodes) nodes.any? { |n| availability[n] != n.available? } end - def rebuild_weight_array(nodes) + def rebuild_weight_array(nodes = nil) + @rebuild_mutex.synchronize do + nodes = @nodes_provider.call if @nodes_provider + rebuild_weight_array_unlocked(nodes) + end + end + + alias select_service select_healthy_node + alias rebalance rebuild_weight_array + + private + + def rebuild_weight_array_unlocked(nodes) availability = nodes.to_h { |n| [n, n.available?] } standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n| n.standby? @@ -115,9 +140,6 @@ def rebuild_weight_array(nodes) @weight_array = weight_array end end - - alias select_service select_healthy_node - alias rebalance rebuild_weight_array end end end diff --git a/test/plugin/out_forward/test_ack_handler.rb b/test/plugin/out_forward/test_ack_handler.rb index 309822deb9..7e56581c4c 100644 --- a/test/plugin/out_forward/test_ack_handler.rb +++ b/test/plugin/out_forward/test_ack_handler.rb @@ -42,6 +42,33 @@ class AckHandlerTest < Test::Unit::TestCase end end + test 'returns the send attempt given to the ack' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + attempt = Object.new + ack = ack_handler.create_ack(chunk_id, node, attempt) + + r, w = IO.pipe + begin + w.write(chunk_id) + mock(r).recv(anything) { |_| nil } # the connection is closed: the ack failed + ack.enqueue(r) + + result = yielded_attempt = nil + ack_handler.collect_response(1) do |_cid, _n, _s, ret, att| + result = ret; yielded_attempt = att + end + + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, result + assert_same attempt, yielded_attempt + ensure + r.close rescue nil + w.close rescue nil + end + end + test 'returns nil if raise an error' do ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100) @@ -80,7 +107,6 @@ class AckHandlerTest < Test::Unit::TestCase r, w = IO.pipe begin - w.write(chunk_id) mock(r).recv(anything).never ack.enqueue(r) @@ -100,6 +126,58 @@ class AckHandlerTest < Test::Unit::TestCase end end + test 'reports an expired ack without waiting for the select interval' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + ack = ack_handler.create_ack(chunk_id, node) + + r, w = IO.pipe + begin + ack.enqueue(r) + + result = nil + started = Fluent::Clock.now + ack_handler.collect_response(5) do |_cid, _n, _s, ret| + result = ret + end + elapsed = Fluent::Clock.now - started + + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, result + assert_operator elapsed, :<, 1 + ensure + r.close rescue nil + w.close rescue nil + end + end + + test 'reads an ack that is already received when the timeout is checked' do + ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100) + + node = flexmock('node', host: '127.0.0.1', port: '1000') # for log + chunk_id = 'chunk_id 111' + ack = ack_handler.create_ack(chunk_id, node) + + r, w = IO.pipe + begin + w.write(chunk_id) + mock(r).recv(anything) { |_| MessagePack.pack({ 'ack' => Base64.encode64(chunk_id) }) } # IO does not have recv + ack.enqueue(r) + + a1 = a4 = nil + ack_handler.collect_response(1) do |cid, _n, _s, ret| + a1 = cid; a4 = ret + end + + assert_equal chunk_id, a1 + assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS, a4 + ensure + r.close rescue nil + w.close rescue nil + end + end + # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case. # If exclusive control for this case is implemented, this test may not be necessary. test 'raises no error when another thread closes a socket' do diff --git a/test/plugin/out_forward/test_load_balancer.rb b/test/plugin/out_forward/test_load_balancer.rb index 41ed1e5685..de17bf1a12 100644 --- a/test/plugin/out_forward/test_load_balancer.rb +++ b/test/plugin/out_forward/test_load_balancer.rb @@ -131,4 +131,43 @@ class LoadBalancerTest < Test::Unit::TestCase assert_true lb.availability_changed?([n1, n2]) end end + + sub_test_case 'nodes_provider' do + test 'reads the node list from the provider instead of the given list' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + old_node = flexmock('old', :'standby?' => false, :'available?' => true, weight: 1) + new_node = flexmock('new', :'standby?' => false, :'available?' => true, weight: 1) + lb.nodes_provider = -> { [new_node] } + + lb.rebuild_weight_array([old_node]) + + 3.times { assert_equal new_node, lb.select_healthy_node { |n| n }.last } + end + + test 'reads the node list while holding the rebuild lock' do + lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log) + node = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1) + reading = Queue.new + resume = Queue.new + calls = 0 + lb.nodes_provider = -> { + calls += 1 + if calls == 1 + reading << true + resume.pop + end + [node] + } + + first = Thread.new { lb.rebuild_weight_array } + reading.pop + second = Thread.new { lb.rebuild_weight_array } + Timeout.timeout(10) { sleep 0.01 until second.status == 'sleep' || second.status == false } + # the second rebuild waits for the first one before reading the list + assert_equal 1, calls + resume << true + [first, second].each { |t| t.join(10) } + assert_equal 2, calls + end + end end diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index e5ea0f71fe..804e84f782 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -2,6 +2,7 @@ require 'fluent/test/driver/output' require 'fluent/plugin/out_forward' require 'flexmock/test_unit' +require 'timecop' require 'fluent/test/driver/input' require 'fluent/plugin/in_forward' @@ -1520,6 +1521,437 @@ def create_ack_driver end end + sub_test_case 'a failed ack (#5619)' do + def create_ack_driver(heartbeat_type: 'none', recover_wait: 10, servers: [{name: 'test'}]) + @d = create_driver(%[ + send_timeout 51 + heartbeat_type #{heartbeat_type} + recover_wait #{recover_wait} + require_ack_response true + #{servers.map { |sv| %[ + + name #{sv[:name]} + host #{TARGET_HOST} + port #{sv[:port] || @target_port} + #{sv[:standby] ? 'standby true' : ''} + ] }.join} + ]) + @instance = @d.instance + @instance.nodes.each { |n| stub(n).send_heartbeat { false } } + stub(@instance.instance_variable_get(:@connection_manager)).discard + @d + end + + def node_named(name) + @instance.nodes.find { |n| n.name == name } + end + + def fail_ack(node, chunk_id = 'chunk-id', attempt = nil) + fake_ack_handler = Object.new + fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, attempt) + end + original = @instance.instance_variable_get(:@ack_handler) + @instance.instance_variable_set(:@ack_handler, fake_ack_handler) + @instance.__send__(:ack_check, 0) + ensure + @instance.instance_variable_set(:@ack_handler, original) + end + + # the same as what #write does before selecting a node + def selected_node + @instance.__send__(:rebalance_if_availability_changed) + _, node = @instance.__send__(:service_discovery_select_service) { |n| n } + node + end + + def two_servers(standby: false) + [{name: 'primary'}, {name: 'secondary', port: unused_port(protocol: :tcp), standby: standby}] + end + + data('none' => 'none', 'transport' => 'transport') + test 'detaches the node and sends its chunks to another server without counting a retry' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type, servers: two_servers) + primary = node_named('primary') + secondary = node_named('secondary') + + mock(@instance).rollback_write('chunk-id', update_retry: false).once + mock(@instance).update_retry_state.never + fail_ack(primary) + + assert_false primary.available? + assert_true secondary.available? + 5.times { assert_equal secondary, selected_node } + assert{ @d.logs.any? { |log| log.include?("detached forwarding server 'primary' for 10.0s") } } + end + + data('none' => 'none', 'transport' => 'transport') + test 'uses the standby node while the active node is detached' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type, servers: two_servers(standby: true)) + primary = node_named('primary') + standby = node_named('secondary') + assert_equal primary, selected_node + + mock(@instance).rollback_write('chunk-id', update_retry: false).once + mock(@instance).update_retry_state.never + fail_ack(primary) + + assert_false primary.available? + 5.times { assert_equal standby, selected_node } + end + + test 'heartbeat_type none switches back to the active node after recover_wait' do + create_ack_driver(heartbeat_type: 'none', servers: two_servers(standby: true)) + primary = node_named('primary') + standby = node_named('secondary') + stub(@instance).rollback_write + + now = Fluent::Clock.now + Fluent::Clock.freeze(now) do + fail_ack(primary) + assert_equal standby, selected_node + end + Fluent::Clock.freeze(now + 9) do + assert_equal standby, selected_node + end + Fluent::Clock.freeze(now + 10) do + assert_true primary.available? + assert_equal primary, selected_node + end + end + + test 'heartbeat_type transport does not bring a detached node back before recover_wait' do + create_ack_driver(heartbeat_type: 'transport', recover_wait: 2, servers: two_servers) + primary = node_named('primary') + stub(@instance).rollback_write + + now = Fluent::Clock.now + Fluent::Clock.freeze(now) do + fail_ack(primary) + # enough samples to recover, as a heartbeat or in-flight writes would add + (@instance.recover_sample_size.ceil + 2).times { primary.heartbeat(false) } + primary.heartbeat + assert_false primary.available? + end + Fluent::Clock.freeze(now + 2) do + primary.heartbeat + assert_true primary.available? + end + end + + data('none' => 'none', 'transport' => 'transport') + test 'keeps the last node and counts a retry with the error' do |heartbeat_type| + create_ack_driver(heartbeat_type: heartbeat_type) + node = @instance.nodes.first + + errors = [] + mock(@instance).rollback_write('chunk-id', update_retry: false).once { true } + mock(@instance).update_retry_state('chunk-id', false, anything).once { |_id, _sec, error| errors << error } + fail_ack(node) + + assert_true node.available? + assert_equal [Fluent::Plugin::ForwardOutput::AckResponseError], errors.map(&:class) + assert_equal "failed to receive the ack response from forwarding server 'test'", errors.first.message + end + + test 'counts a retry when the other node is detached too' do + create_ack_driver(heartbeat_type: 'none', servers: two_servers) + primary = node_named('primary') + secondary = node_named('secondary') + stub(@instance).rollback_write { true } + mock(@instance).update_retry_state('chunk-2', false, anything).once + + fail_ack(primary, 'chunk-1') + fail_ack(secondary, 'chunk-2') + + assert_false primary.available? + assert_true secondary.available? + end + + test 'does not detach the node when recover_wait is 0' do + create_ack_driver(heartbeat_type: 'none', recover_wait: 0, servers: two_servers) + primary = node_named('primary') + stub(@instance).rollback_write { true } + mock(@instance).update_retry_state('chunk-id', false, anything).once + + fail_ack(primary) + + assert_true primary.available? + end + end + + sub_test_case 'a late ack failure after the delayed commit timeout (#5627)' do + # A send of a chunk can take longer than delayed_commit_timeout, and the + # ack timeout only starts when the send is done. Then Output#try_rollback_write + # rolls the attempt back and counts a retry before the ack thread reports + # the failed ack. By then the chunk may be in flight again. + setup do + @d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1 + + name test + host #{TARGET_HOST} + port #{@target_port} + + + flush_mode interval + flush_interval 3600 + flush_thread_interval 3600 + flush_thread_burst_interval 3600 + flush_at_shutdown false + + ]) + @instance = @d.instance + @node = @instance.nodes.first + @attempts = attempts = [] + # record the attempt of each send instead of sending the chunk + @node.define_singleton_method(:send_data) { |_tag, _chunk, attempt: nil| attempts << attempt; nil } + @retry_errors = retry_errors = [] + @instance.singleton_class.prepend(Module.new { + define_method(:update_retry_state) do |chunk_id, using_secondary, error = nil| + retry_errors << error + super(chunk_id, using_secondary, error) + end + }) + @d.instance_start + # keep the flush thread from rolling back chunks by itself + @instance.interrupt_flushes + stub(@instance.instance_variable_get(:@connection_manager)).discard + + @instance.emit_events('test', Fluent::OneEventStream.new(event_time, {'k' => 'v'})) + @instance.buffer.enqueue_all + @chunk_id = @instance.buffer.queue.first.unique_id + end + + # Returns the delayed commit entry that Output#try_flush added for the send. + def send_chunk + @instance.try_flush + assert_true @instance.buffer.dequeued.key?(@chunk_id) + dequeued_chunk_infos.last + end + + def expire_delayed_commit(attempt) + Timecop.travel(attempt.time + attempt.timeout + 1) do + @instance.try_rollback_write + end + assert_false @instance.buffer.dequeued.key?(@chunk_id) + end + + def fail_ack(attempt) + fake_ack_handler = Object.new + chunk_id, node = @chunk_id, @node + fake_ack_handler.define_singleton_method(:collect_response) do |_interval, &block| + block.call(chunk_id, node, 'sock', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, attempt) + end + original = @instance.instance_variable_get(:@ack_handler) + @instance.instance_variable_set(:@ack_handler, fake_ack_handler) + @instance.__send__(:ack_check, 0) + ensure + @instance.instance_variable_set(:@ack_handler, original) + end + + def dequeued_chunk_infos + @instance.dequeued_chunks_mutex.synchronize { @instance.dequeued_chunks.dup } + end + + test 'passes the delayed commit entry of the send to the ack' do + attempt = send_chunk + assert_equal @chunk_id, attempt.chunk_id + assert_same attempt, @attempts.last + end + + test 'does not count a second retry for an attempt that was already rolled back' do + first = send_chunk + expire_delayed_commit(first) + assert_equal [nil], @retry_errors # "timeout to commit" + + second = send_chunk + fail_ack(first) + + # the late failure belongs to the first attempt: the chunk that is in + # flight again is not taken back, and no retry is counted for it + assert_equal [nil], @retry_errors + assert_true @instance.buffer.dequeued.key?(@chunk_id) + assert_equal [second], dequeued_chunk_infos + assert{ @d.logs.any? { |log| log.include?("after the delayed commit timeout rolled the chunk back. no retry is counted again") } } + end + + test 'counts a retry when a later send of the chunk fails on its own' do + first = send_chunk + expire_delayed_commit(first) + second = send_chunk + fail_ack(first) + + fail_ack(second) + + assert_equal [nil, Fluent::Plugin::ForwardOutput::AckResponseError], @retry_errors.map { |e| e&.class } + assert_false @instance.buffer.dequeued.key?(@chunk_id) + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + assert_equal [], dequeued_chunk_infos + end + + test 'does not count a retry when the delayed commit timeout rolled back the later send too' do + first = send_chunk + expire_delayed_commit(first) + second = send_chunk + expire_delayed_commit(second) + assert_equal [nil, nil], @retry_errors + + fail_ack(first) + fail_ack(second) + + assert_equal [nil, nil], @retry_errors + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + end + + test 'counts a retry for a failed ack before the delayed commit timeout' do + attempt = send_chunk + fail_ack(attempt) + + assert_equal [Fluent::Plugin::ForwardOutput::AckResponseError], @retry_errors.map(&:class) + assert_equal [@chunk_id], @instance.buffer.queue.map(&:unique_id) + assert_equal [], dequeued_chunk_infos + end + end + + sub_test_case 'concurrent rebuilds of the weight array (#5627)' do + # With heartbeat_type none, the ack thread and the flush threads rebuild + # the weight array as well as the event loop (heartbeat timer and service + # discovery). An older rebuild must not publish its array after a newer one. + test 'a rebuild that read the old node list does not replace a newer one' do + @d = create_driver(%[ + heartbeat_type none + require_ack_response true + + name failed + host #{TARGET_HOST} + port #{@target_port} + + + name removed + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + + name retained + host #{TARGET_HOST} + port #{unused_port(protocol: :tcp)} + + ]) + instance = @d.instance + manager = instance.discovery_manager + load_balancer = instance.instance_variable_get(:@load_balancer) + + # Pause the rebuild of the "ack thread" right after it has read the node list. + read_list = Queue.new + resume = Queue.new + paused_thread = nil + original_services = manager.method(:services) + manager.define_singleton_method(:services) do + list = original_services.call + if Thread.current.equal?(paused_thread) + read_list << true + resume.pop + end + list + end + + paused_thread = Thread.new { instance.__send__(:service_discovery_rebalance) } + read_list.pop + + # Service discovery removes 'failed' and 'removed' and rebuilds the array. + static_services = manager.instance_variable_get(:@discoveries).first.services + discovery = Thread.new do + static_services.select { |s| %w[failed removed].include?(s.name) }.each do |s| + manager.instance_variable_get(:@queue) << Fluent::Plugin::ServiceDiscovery.service_out_msg(s) + end + manager.run_once + end + # Without the rebuild lock this rebuild finishes now; with it, it waits + # for the paused rebuild. + Timeout.timeout(10) { sleep 0.01 until discovery.status == 'sleep' || discovery.status == false } + + resume << true + [paused_thread, discovery].each { |t| t.join(10) } + + assert_equal ['retained'], manager.services.map(&:name) + assert_equal ['retained'], load_balancer.instance_variable_get(:@weight_array).map(&:name).uniq + selected = Array.new(10) { load_balancer.select_healthy_node { |n| n }.last.name } + assert_equal ['retained'], selected.uniq + end + end + + sub_test_case 'a server that never sends the ack (#5619)' do + # Reproduces the scenarios in the review of #5627: one server accepts + # chunks but never acks, the other acks normally. No chunk may be lost + # even with retry_max_times 0. + def run_with_silent_server(standby:) + silent_port = unused_port(protocol: :all) + silent = create_target_input_driver(response_stub: ->(_option) { nil }, + conf: %[port #{silent_port}\nbind #{TARGET_HOST}]) + healthy = create_target_input_driver + + @d = d = create_driver(%[ + heartbeat_type none + require_ack_response true + ack_response_timeout 1 + + name silent + host #{TARGET_HOST} + port #{silent_port} + + + name healthy + host #{TARGET_HOST} + port #{@target_port} + #{standby ? 'standby true' : ''} + + + flush_mode immediate + flush_thread_interval 0.1 + flush_thread_burst_interval 0.1 + retry_wait 1 + retry_max_times 0 + flush_at_shutdown false + + ]) + + time = event_time + tags = (1..8).map { |i| "test.#{i}" } + # The logs are taken when every chunk has arrived: a chunk still in + # flight to the healthy server can fail when the drivers shut down. + logs = nil + delivered = -> { healthy.events.map(&:first).uniq.sort_by { |t| t.split('.').last.to_i } } + done = -> { delivered.call.size >= tags.size && (logs ||= d.logs.dup) } + [healthy, d].each { |drv| drv.end_if(&done) } + + silent.instance_start + begin + healthy.run(timeout: 30) do + d.run(timeout: 25, wait_flush_completion: false, shutdown: false, flush: false) do + tags.each { |tag| d.feed(tag, time, {'tag' => tag}) } + end + end + ensure + silent.instance_shutdown + end + + assert_equal tags, delivered.call + assert{ logs.any? { |log| log.include?("detached forwarding server 'silent'") } } + assert{ logs.none? { |log| log.include?('Hit limit for retries') } } + end + + test 'delivers every chunk to the healthy server' do + run_with_silent_server(standby: false) + end + + test 'delivers every chunk to the healthy standby server' do + run_with_silent_server(standby: true) + end + end + sub_test_case 'a node disabled outside the heartbeat' do def create_standby_driver(primary_host: TARGET_HOST, conf: '') @d = create_driver(%[