diff --git a/lib/fluent/event_router.rb b/lib/fluent/event_router.rb index bfc192ff0a..83f8e087b9 100644 --- a/lib/fluent/event_router.rb +++ b/lib/fluent/event_router.rb @@ -83,6 +83,10 @@ def suppress_missing_match! # called by Agent to add new match pattern and collector def add_rule(pattern, collector) @match_rules << Rule.new(pattern, collector) + # A tag may already be cached as unmatched (or as an earlier rule) + # if an event arrived before this rule was added, e.g. during a + # graceful reload. Drop the cache so the new rule can match. + @match_cache.clear end def add_metric_callbacks(caller_plugin_id, callback) @@ -158,6 +162,11 @@ def get(key) @keys << key collector end + + def clear + @map.clear + @keys.clear + end end private diff --git a/lib/fluent/plugin_helper/event_emitter.rb b/lib/fluent/plugin_helper/event_emitter.rb index 23bc15ce53..58d265ece0 100644 --- a/lib/fluent/plugin_helper/event_emitter.rb +++ b/lib/fluent/plugin_helper/event_emitter.rb @@ -61,10 +61,15 @@ def event_emitter_cancel_source_only def event_emitter_router(label_name) if label_name - if label_name == "@ROOT" - Engine.root_agent.event_router + owner = event_emitter_owner_agent + current = Engine.root_agent + if owner.nil? || owner.equal?(current) || owner.shutdown_started? + LabelRouter.find(current, label_name) else - Engine.root_agent.find_label(label_name).event_router + # During a graceful reload, this plugin belongs to the old + # configuration while Engine.root_agent is the new one. + # See LabelRouter. + LabelRouter.new(owner, label_name).tap(&:current) end elsif self.respond_to?(:as_secondary) && self.as_secondary if @primary_instance.has_router? @@ -84,6 +89,7 @@ def initialize @_event_emitter_used_actually = false @_event_emitter_lazy_init = false @_event_emitter_force_source_only_router = false + @_event_emitter_owner_agent = nil @router = nil end @@ -107,6 +113,82 @@ def terminate @router = nil super end + + private + + # The RootAgent of the configuration this plugin belongs to, or nil if + # it is not known (e.g. a plugin created without an agent in tests). + # The agents answer this themselves (Label#root_agent, RootAgent#root_agent). + # It does not change after configure, so it is cached once known. + def event_emitter_owner_agent + return @_event_emitter_owner_agent if @_event_emitter_owner_agent + router = self.context_router + handler = router.respond_to?(:emit_error_handler) ? router.emit_error_handler : nil + @_event_emitter_owner_agent = handler.respond_to?(:root_agent) ? handler.root_agent : nil + end + + # The router that event_emitter_router(label_name) returns to a plugin + # of the old configuration during a graceful reload. Outside a reload, + # event_emitter_router returns the label's EventRouter itself. + # + # Engine.root_agent becomes the new configuration before its + # sections are added, so a label looked up there can drop the event and + # leave the tag cached as unmatched (#5594). While the old configuration + # is still running, events go to its own label instead, like @label and + # relabel. + # + # Once the old configuration has started to shut down, its labels may + # already have dropped their routers, so events go to the label of + # Engine.root_agent, as before. This is decided on each emit, because a + # plugin may take the router before the old configuration starts to shut + # down and emit after that. + class LabelRouter + def self.find(agent, label_name) + if label_name == "@ROOT" + agent.event_router + else + agent.find_label(label_name).event_router + end + end + + attr_reader :label_name + + def initialize(owner_agent, label_name) + @owner_agent = owner_agent + @label_name = label_name + end + + def current + owner = @owner_agent + agent = (owner && !owner.shutdown_started?) ? owner : Engine.root_agent + LabelRouter.find(agent, @label_name) + end + + def emit(tag, time, record) + current.emit(tag, time, record) + end + + def emit_array(tag, array) + current.emit_array(tag, array) + end + + def emit_stream(tag, es) + current.emit_stream(tag, es) + end + + def respond_to_missing?(name, include_private = false) + current.respond_to?(name, include_private) || super + end + + def method_missing(name, *args, &block) + router = current + if router.respond_to?(name) + router.__send__(name, *args, &block) + else + super + end + end + end end end end diff --git a/lib/fluent/root_agent.rb b/lib/fluent/root_agent.rb index a12e9b928e..49c9870617 100644 --- a/lib/fluent/root_agent.rb +++ b/lib/fluent/root_agent.rb @@ -87,6 +87,7 @@ def initialize(log:, system_config: SystemConfig.new, start_in_parallel: false) @source_only_mode = SourceOnlyMode.new(system_config.with_source_only, start_in_parallel) @source_only_buffer_agent = nil @enable_input_metrics = system_config.enable_input_metrics + @shutdown_started = false suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil? end @@ -94,6 +95,17 @@ def initialize(log:, system_config: SystemConfig.new, start_in_parallel: false) attr_reader :inputs attr_reader :labels + # Labels answer the RootAgent they belong to; a RootAgent is its own. + def root_agent + self + end + + # true once #shutdown of the whole configuration has begun, e.g. for the + # old configuration during a graceful reload. + def shutdown_started? + @shutdown_started + end + def source_only_router raise "[BUG] 'RootAgent#source_only_router' should not be called when 'with_source_only' is false" unless @source_only_mode.enabled? @source_only_buffer_agent.event_router @@ -324,6 +336,7 @@ def cancel_source_only! end def shutdown(kind_or_agent_list: nil) + @shutdown_started = true unless kind_or_agent_list # Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins # These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible # if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others. diff --git a/test/plugin_helper/test_event_emitter.rb b/test/plugin_helper/test_event_emitter.rb index 26580f0087..d96fd65304 100644 --- a/test/plugin_helper/test_event_emitter.rb +++ b/test/plugin_helper/test_event_emitter.rb @@ -1,6 +1,7 @@ require_relative '../helper' require 'fluent/plugin_helper/event_emitter' require 'fluent/plugin/base' +require 'fluent/event_router' require 'flexmock/test_unit' class EventEmitterTest < Test::Unit::TestCase @@ -106,4 +107,87 @@ class Dummy < Fluent::Plugin::TestBase [router_when_source_only, router_after_canceled] ) end + + sub_test_case 'event_emitter_router during a graceful reload' do + class RecordingRouter + attr_reader :events + def initialize; @events = []; end + def emit_stream(tag, es); @events << tag; end + def emit(tag, time, record); @events << tag; end + end + + setup do + require 'fluent/root_agent' + @old_agent = Fluent::RootAgent.new(log: $log) + @new_agent = Fluent::RootAgent.new(log: $log) + @routers = {} + [[:old, @old_agent], [:new, @new_agent]].each do |name, agent| + agent.add_label('@L') + @routers[name] = RecordingRouter.new + @routers[:"#{name}_root"] = RecordingRouter.new + label_router = @routers[name] + root_router = @routers[:"#{name}_root"] + stub(agent.find_label('@L')).event_router { label_router } + stub(agent).event_router { root_router } + end + stub(Fluent::Engine).root_agent { @new_agent } + end + + def plugin_in(router) + d = Dummy.new + d.context_router = router + d + end + + def old_root_plugin + plugin_in(Fluent::EventRouter.new(nil, @old_agent)) + end + + def old_label_plugin + plugin_in(Fluent::EventRouter.new(nil, @old_agent.find_label('@L'))) + end + + test 'RootAgent#root_agent and Label#root_agent answer the owning configuration' do + assert_same @old_agent, @old_agent.root_agent + assert_same @old_agent, @old_agent.find_label('@L').root_agent + end + + data('plugin in of the old configuration' => :old_root_plugin, + 'plugin in