Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions lib/fluent/plugin/out_forward.rb
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,11 @@ def configure(conf)
socket_cache: socket_cache,
)

@load_balancer = LoadBalancer.new(log)
service_discovery_configure(
:out_forward_service_discovery_watcher,
static_default_service_directive: 'server',
load_balancer: LoadBalancer.new(log),
load_balancer: @load_balancer,
custom_build_method: method(:build_node),
)

Expand Down Expand Up @@ -472,7 +473,8 @@ def build_node(server)

def on_heartbeat_timer
need_rebuild = false
service_discovery_services.each do |n|
services = service_discovery_services
services.each do |n|
begin
log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
n.usock = @usock if @usock
Expand All @@ -486,7 +488,7 @@ def on_heartbeat_timer
need_rebuild = n.tick || need_rebuild
end

if need_rebuild
if need_rebuild || @load_balancer.availability_changed?(services)
service_discovery_rebalance
end
end
Expand Down
13 changes: 11 additions & 2 deletions lib/fluent/plugin/out_forward/load_balancer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ def initialize(log)
@rand_seed = Random.new.seed
@rr = 0
@mutex = Mutex.new
@availability = {}
end

def select_healthy_node
Expand Down Expand Up @@ -55,22 +56,28 @@ def select_healthy_node
raise NoNodesAvailable, "no nodes are available"
end

def availability_changed?(nodes)
availability = @mutex.synchronize { @availability }
nodes.any? { |n| availability[n] != n.available? }
end

def rebuild_weight_array(nodes)
availability = nodes.to_h { |n| [n, n.available?] }
standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
n.standby?
}

lost_weight = 0
regular_nodes.each {|n|
unless n.available?
unless availability[n]
lost_weight += n.weight
end
}
@log.debug("rebuilding weight array", lost_weight: lost_weight)

if lost_weight > 0
standby_nodes.each {|n|
if n.available?
if availability[n]
regular_nodes << n
@log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
lost_weight -= n.weight
Expand All @@ -83,6 +90,7 @@ def rebuild_weight_array(nodes)
if regular_nodes.empty?
@log.warn('No nodes are available')
@mutex.synchronize do
@availability = availability
@weight_array = weight_array
end
return @weight_array
Expand All @@ -103,6 +111,7 @@ def rebuild_weight_array(nodes)
weight_array.sort_by! { r.rand }

@mutex.synchronize do
@availability = availability
@weight_array = weight_array
end
end
Expand Down
28 changes: 28 additions & 0 deletions test/plugin/out_forward/test_load_balancer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,32 @@ class LoadBalancerTest < Test::Unit::TestCase
end
end
end

sub_test_case 'availability_changed?' do
test 'compares the nodes with their availability at the last rebuild' do
lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
available = true
n1 = flexmock('node', :'standby?' => false, weight: 1)
n1.should_receive(:available?).and_return { available }

lb.rebuild_weight_array([n1])
assert_false lb.availability_changed?([n1])

available = false
assert_true lb.availability_changed?([n1])

lb.rebuild_weight_array([n1])
assert_false lb.availability_changed?([n1])
end

test 'regards a node added after the last rebuild as a change' do
lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)

lb.rebuild_weight_array([n1])

assert_true lb.availability_changed?([n1, n2])
end
end
end
75 changes: 75 additions & 0 deletions test/plugin/test_out_forward.rb
Original file line number Diff line number Diff line change
Expand Up @@ -1495,4 +1495,79 @@ def create_ack_driver
instance.__send__(:ack_check, 0)
end
end

sub_test_case 'a node disabled outside the heartbeat' do
def create_standby_driver(primary_host: TARGET_HOST, conf: '')
@d = create_driver(%[
heartbeat_type transport
recover_wait 0
#{conf}
<server>
name primary
host #{primary_host}
port #{@target_port}
</server>
<server>
name standby
host #{TARGET_HOST}
port #{unused_port(protocol: :tcp)}
standby true
</server>
])
@instance = @d.instance
@primary = @instance.nodes.find { |n| n.name == 'primary' }
@standby = @instance.nodes.find { |n| n.name == 'standby' }
@instance.nodes.each { |n| stub(n).send_heartbeat { false } }
end

def run_heartbeat_timer
@instance.__send__(:on_heartbeat_timer)
end

def selected_node
_, node = @instance.__send__(:service_discovery_select_service) { |n| n }
node
end

test 'the heartbeat timer switches to the standby node' do
create_standby_driver
@primary.disable!
run_heartbeat_timer

assert_equal @standby, selected_node
end

test 'rebuilds the weight array only once
while the availability stays the same' do
create_standby_driver
mock.proxy(@instance.instance_variable_get(:@load_balancer)).rebalance(anything).once

@primary.disable!
2.times { run_heartbeat_timer }
end

test 'the heartbeat timer switches to the standby node again after the primary recovers' do
create_standby_driver
@primary.disable!
run_heartbeat_timer
stub(@primary).send_heartbeat { @primary.heartbeat }
run_heartbeat_timer
assert_equal @primary, selected_node

stub(@primary).send_heartbeat { false }
@primary.disable!
run_heartbeat_timer

assert_equal @standby, selected_node
end

test 'the heartbeat timer switches to the standby node for a primary disabled at startup' do
create_standby_driver(primary_host: 'unexisting.yaaaaaaaaaaaaaay.host.example.com', conf: 'ignore_network_errors_at_startup true')
assert_false @primary.available?

run_heartbeat_timer

assert_equal @standby, selected_node
end
end
end
Loading