Skip to content
Open
159 changes: 150 additions & 9 deletions lib/fluent/plugin/out_forward.rb
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ class ForwardOutput < Output
config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport
desc 'The interval of the heartbeat packer.'
config_param :heartbeat_interval, :time, default: 1
desc 'The wait time before accepting a server fault recovery.'
desc 'The wait time before accepting a server fault recovery. ' \
'A server whose ack response fails is also kept out for this long while another server can take its chunks.'
config_param :recover_wait, :time, default: 10
desc 'The hard timeout used to detect server failure.'
config_param :hard_timeout, :time, default: 60
Expand Down Expand Up @@ -238,6 +239,9 @@ def configure(conf)
load_balancer: @load_balancer,
custom_build_method: method(:build_node),
)
# Read the node list under the rebuild lock of the load balancer, so a
# rebuild never uses a list that is older than the last published one.
@load_balancer.nodes_provider = method(:service_discovery_services)

service_discovery_services.each do |server|
# it's only for test
Expand Down Expand Up @@ -379,6 +383,7 @@ def write(chunk)
return if chunk.empty?
tag = chunk.metadata.tag

rebalance_if_availability_changed
service_discovery_select_service { |node| node.send_data(tag, chunk) }
end

Expand All @@ -389,7 +394,9 @@ def try_write(chunk)
return
end
tag = chunk.metadata.tag
service_discovery_select_service { |node| node.send_data(tag, chunk) }
attempt = delayed_commit_attempt(chunk.unique_id)
rebalance_if_availability_changed
service_discovery_select_service { |node| node.send_data(tag, chunk, attempt: attempt) }
last_ack if @require_ack_response && @suspend_flush
end

Expand Down Expand Up @@ -501,6 +508,118 @@ def on_heartbeat_timer
end
end

# heartbeat_type none has no heartbeat timer to rebuild the weight array, so
# a node detached after a failed ack (and the standby used in its place)
# would never be switched back. Check it before selecting a node instead.
def rebalance_if_availability_changed
return unless @heartbeat_type == :none

if @load_balancer.availability_changed?(service_discovery_services)
service_discovery_rebalance
end
end

# Whether another node can take the chunks of a node whose ack failed.
# With recover_wait 0 a detached node would be back at once, so there is
# no point in detaching it.
def failover_node_available?(failed_node)
return false if failed_node.nil? || @recover_wait <= 0

service_discovery_services.any? { |n| !n.equal?(failed_node) && n.weight > 0 && n.available? }
end

def node_label(node)
node.name || "#{node.host}:#{node.port}"
end

# The instance whose flush thread sent the chunk and keeps its delayed
# commit state (Output#try_flush and #try_rollback_write).
def delayed_commit_owner
@as_secondary ? @primary_instance : self
end

# The delayed commit entry (Output::DequeuedChunkInfo) that Output#try_flush
# added for this send of the chunk. It identifies the send attempt: when
# the delayed commit timeout rolls the attempt back, the entry is removed,
# and the next send of the same chunk gets a new one.
def delayed_commit_attempt(chunk_id)
return nil unless @ack_handler

owner = delayed_commit_owner
mutex = owner.dequeued_chunks_mutex
return nil unless mutex

mutex.synchronize do
owner.dequeued_chunks.reverse_each.find { |info| info.chunk_id == chunk_id }
end
end

# Rolls back the send attempt whose ack failed.
# Returns nil if the delayed commit timeout (or another error) has already
# rolled this attempt back. The chunk may be in flight again then, so it is
# neither taken back nor counted again. Otherwise returns the result of
# #rollback_write.
def rollback_failed_attempt(chunk_id, attempt)
if attempt
owner = delayed_commit_owner
# Remove the entry in the same lock that Output#try_rollback_write
# holds, so only one of them can roll this attempt back.
current = owner.dequeued_chunks_mutex.synchronize do
index = owner.dequeued_chunks.index { |info| info.equal?(attempt) }
owner.dequeued_chunks.delete_at(index) if index
!index.nil?
end
return nil unless current
end

rollback_write(chunk_id, update_retry: false)
end

def on_ack_failed(chunk_id, node, attempt = nil)
chunk_id_hex = chunk_id ? dump_unique_id_hex(chunk_id) : nil

if failover_node_available?(node)
# Keep the node out for recover_wait so that its chunks go to the
# other nodes (a standby node is used if needed). No retry is counted:
# the chunk has not failed on every node, and the retry state belongs
# to the whole output. If no node is left, the next write raises
# NoNodesAvailable and the usual retry applies.
node.detach!
service_discovery_rebalance

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MUST: This rebuilds the weight array from the ack thread. With heartbeat_type none, rebalance_if_availability_changed also rebuilds it from the flush threads. On master, after start, only the event loop rebuilds it (the heartbeat timer and service discovery), so two rebuilds never overlap. Now a rebuild that read the node list earlier can publish its array after a newer one.

I reproduced this with dynamic service discovery and three servers (failed, removed, retained):

  1. The ack for failed fails, and this service_discovery_rebalance reads the current services. I paused the ack thread there.
  2. Service discovery removes failed and removed, and publishes a weight array with only retained.
  3. The ack thread resumes and publishes its array, built from the old list.

After that, master sends only to retained. With this PR, removed is still selected. availability_changed? only checks the current services, so the check before a write does not notice it, and the stale array stays until another rebuild. A lock around the assignment of @weight_array alone would not help, because the node list is read before it.

Please make sure that an older rebuild cannot replace a newer one, and add a test that forces this order. The standby case (delivers every chunk to the healthy standby server) should keep passing.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for reproducing this with a forced order. Fixed in 6e6e1eb.

LoadBalancer#rebuild_weight_array now holds a rebuild lock while it reads the node list, builds the array, and publishes it. As you pointed out, locking only the assignment wouldn't help, so out_forward gives the load balancer a nodes_provider (service_discovery_services). Every rebuild reads the list under the lock and ignores the list the caller read earlier. That covers the ack thread, the flush threads, the heartbeat timer, and the service discovery rebuild through Manager#rebalance. A rebuild that read the old list has to publish before the next rebuild can take the lock. Any later rebuild, such as the one service discovery runs after removing failed and removed, then reads the new list and publishes last. The service discovery helper and its rebalance(services) interface are unchanged.

Tests:

  • concurrent rebuilds of the weight array (#5627) / a rebuild that read the old node list does not replace a newer one in test_out_forward.rb follows your steps. It uses three servers (failed, removed, retained) and pauses the ack-thread rebuild right after it has read the node list. Meanwhile, another thread sends SERVICE_OUT for failed and removed and calls Manager#run_once. The test then resumes the paused rebuild and checks that the weight array and the selected nodes contain only retained. It fails on 8a61903 (removed is still in the array) and passes now.
  • Two unit tests in test_load_balancer.rb check that the provider is used and that the list is read under the lock.
  • delivers every chunk to the healthy standby server and the other never sends the ack and a failed ack tests still pass.

log.warn "failed to receive the ack response, detached forwarding server '#{node_label(node)}' for #{@recover_wait}s. the chunk will be sent to another server",
host: node.host, port: node.port, chunk_id: chunk_id_hex
if chunk_id && rollback_failed_attempt(chunk_id, attempt).nil?
log.debug "the send attempt of the chunk was already rolled back by the delayed commit timeout", chunk_id: chunk_id_hex
end
return
end

# No other node can take the chunk. Keeping the last node out would make
# every retry fail with NoNodesAvailable until it comes back (and could use
# up retry_max_times on a single transient failure), so leave it in and
# count a retry instead: the chunk is resent after retry_wait, and
# retry_timeout, retry_max_times and <secondary> apply. The error is
# passed so that the retry and a discard are logged as usual.
return unless chunk_id

rolled_back = rollback_failed_attempt(chunk_id, attempt)
if rolled_back.nil?
# Output#try_rollback_write has already rolled this attempt back and
# counted a retry for it ("timeout to commit"). Counting another one
# here could use up retry_max_times for a single failure.
log.warn "failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}' after the delayed commit timeout rolled the chunk back. no retry is counted again",
chunk_id: chunk_id_hex
return
end

if rolled_back
error = AckResponseError.new("failed to receive the ack response from forwarding server '#{node ? node_label(node) : 'unknown'}'")
error.set_backtrace(caller(0, 1))
primary = @as_secondary ? @primary_instance : self
primary.update_retry_state(chunk_id, @as_secondary, error)
end
end

def on_udp_heartbeat_response_recv(data, sock)
sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
Expand Down Expand Up @@ -534,7 +653,7 @@ def ack_reader
end

def ack_check(select_interval)
@ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
@ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result, attempt|
case result
when AckHandler::Result::SUCCESS
@connection_manager.close(sock)
Expand All @@ -545,8 +664,7 @@ def ack_check(select_interval)
# instead of returning it to the reuse pool; otherwise a late ack
# could later be read against a chunk sent on the reused socket (#4057).
@connection_manager.discard(sock)
node&.disable!
rollback_write(chunk_id, update_retry: false) if chunk_id
on_ack_failed(chunk_id, node, attempt)
when AckHandler::Result::CHUNKID_UNMATCHED
# The ack read from this socket belongs to a different chunk than the
# one tracked for it: the keepalive connection's ack stream is out of
Expand Down Expand Up @@ -585,6 +703,7 @@ def initialize(sender, server, failure:, connection_manager:, ack_handler:)
@standby = server.standby
@failure = failure
@available = true
@detached_until = nil

# @hostname is used for certificate verification & TLS SNI
host_is_hostname = !(IPAddr.new(@host) rescue false)
Expand Down Expand Up @@ -630,6 +749,21 @@ def disable!
@available = false
end

# Keep the node out for recover_wait after a failed ack. The hold is set
# before disabling so a heartbeat running in between cannot bring the
# node back at once, and samples added by in-flight writes cannot end it
# early. After the hold, recovery still needs fresh heartbeats.
def detach!
@detached_until = Fluent::Clock.now + @sender.recover_wait
disable!
@failure.clear
end

def detached?
detached_until = @detached_until
!detached_until.nil? && Fluent::Clock.now < detached_until
end

def standby?
@standby
end
Expand Down Expand Up @@ -712,8 +846,9 @@ def send_data_actual(sock, tag, chunk)
# TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported
end

def send_data(tag, chunk)
ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self)
# attempt identifies this send of the chunk (see ForwardOutput#delayed_commit_attempt)
def send_data(tag, chunk, attempt: nil)
ack = @ack_handler && @ack_handler.create_ack(chunk.unique_id, self, attempt)
connect(nil, ack: ack) do |sock, ri|
ensure_established_connection(sock, ri)
send_data_actual(sock, tag, chunk)
Expand Down Expand Up @@ -823,7 +958,7 @@ def tick
def heartbeat(detect=true)
now = Time.now.to_f
@failure.add(now)
if detect && !available? && @failure.sample_size > @sender.recover_sample_size
if detect && !available? && !detached? && @failure.sample_size > @sender.recover_sample_size
@available = true
@log.warn "recovered forwarding server '#{@name}'", host: @host, port: @port
true
Expand Down Expand Up @@ -851,8 +986,14 @@ def connect(host = nil, ack: false, &block)

# Override Node to disable heartbeat
class NoneHeartbeatNode < Node
# Without heartbeats, a node is only kept out for recover_wait after a
# failed ack (see ForwardOutput#on_ack_failed).
def available?
true
!detached?
end

def detach!
@detached_until = Fluent::Clock.now + @sender.recover_wait
end

def tick
Expand Down
37 changes: 27 additions & 10 deletions lib/fluent/plugin/out_forward/ack_handler.rb
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def collect_response(select_interval)
new_list = []
@mutex.synchronize do
@ack_waitings.each do |info|
if info.expired?(now)
if info.expired?(now) && !readable?(info.sock)
# There are 2 types of cases when no response has been received from socket:
# (1) the node does not support sending responses
# (2) the node does support sending response but responses have not arrived for some reasons.
Expand All @@ -59,8 +59,15 @@ def collect_response(select_interval)
@ack_waitings = new_list
end

# Report expired acks right away instead of after a full select
# wait. Otherwise a failed ack can be reported up to two select
# intervals after it expired, which reaches delayed_commit_timeout
# (ack_response_timeout + 2). Then Output#try_rollback_write rolls the
# chunk back first and counts a retry without an error, and with
# retry_max_times 0 the whole queue is cleared without a log.
timeout = results.empty? ? select_interval : 0
begin
readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
readable_sockets, _, _ = IO.select(sockets, nil, nil, timeout)
rescue IOError
@log.info "connection closed while waiting for readable sockets"
readable_sockets = nil
Expand All @@ -76,7 +83,7 @@ def collect_response(select_interval)
if info.nil?
yield nil, nil, nil, ret
else
yield info.chunk_id, info.node, info.sock, ret
yield info.chunk_id, info.node, info.sock, ret, info.attempt
end
end
rescue => e
Expand All @@ -85,31 +92,41 @@ def collect_response(select_interval)
end
end

ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do
ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time, :attempt) do
def expired?(now)
expired_time < now
end
end

Ack = Struct.new(:chunk_id, :node, :handler) do
Ack = Struct.new(:chunk_id, :node, :handler, :attempt) do
def enqueue(sock)
handler.enqueue(node, sock, chunk_id)
handler.enqueue(node, sock, chunk_id, attempt)
end
end

def create_ack(chunk_id, node)
Ack.new(chunk_id, node, self)
def create_ack(chunk_id, node, attempt = nil)
Ack.new(chunk_id, node, self, attempt)
end

def enqueue(node, sock, cid)
info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout)
def enqueue(node, sock, cid, attempt = nil)
info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout, attempt)
@mutex.synchronize do
@ack_waitings << info
end
end

private

# A socket enqueued while this thread was waiting in IO.select is not in
# that select call, so its ack can already be in the socket buffer when the
# timeout is checked. Read it instead of regarding the node as unavailable.
def readable?(sock)
readable_sockets, _, _ = IO.select([sock], nil, nil, 0)
!readable_sockets.nil?
rescue IOError
false
end

def read_ack_from_sock(sock)
begin
raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
Expand Down
1 change: 1 addition & 0 deletions lib/fluent/plugin/out_forward/error.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class ForwardOutput < Output
class Error < StandardError; end
class NoNodesAvailable < Error; end
class ConnectionClosedError < Error; end
class AckResponseError < Error; end
class HandshakeError < Error; end
class HeloError < HandshakeError; end
class PingpongError < HandshakeError; end
Expand Down
30 changes: 26 additions & 4 deletions lib/fluent/plugin/out_forward/load_balancer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,21 @@ def initialize(log)
@rr = 0
@mutex = Mutex.new
@availability = {}
# The weight array is rebuilt from several threads: the event loop
# (heartbeat timer and service discovery), the ack thread after a
# failed ack and, with heartbeat_type none, the flush threads. Each
# rebuild reads the node list and publishes its array while holding
# this lock, so an older rebuild cannot publish its array after a
# newer one (#5627).
@rebuild_mutex = Mutex.new
@nodes_provider = nil
end

# Returns the current node list. When it is set, every rebuild reads the
# node list from it under the rebuild lock instead of using the list the
# caller read before.
attr_writer :nodes_provider

def select_healthy_node
error = nil

Expand Down Expand Up @@ -61,7 +74,19 @@ def availability_changed?(nodes)
nodes.any? { |n| availability[n] != n.available? }
end

def rebuild_weight_array(nodes)
def rebuild_weight_array(nodes = nil)
@rebuild_mutex.synchronize do
nodes = @nodes_provider.call if @nodes_provider
rebuild_weight_array_unlocked(nodes)
end
end

alias select_service select_healthy_node
alias rebalance rebuild_weight_array

private

def rebuild_weight_array_unlocked(nodes)
availability = nodes.to_h { |n| [n, n.available?] }
standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
n.standby?
Expand Down Expand Up @@ -115,9 +140,6 @@ def rebuild_weight_array(nodes)
@weight_array = weight_array
end
end

alias select_service select_healthy_node
alias rebalance rebuild_weight_array
end
end
end
Loading