diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb index 8eaf007936..bb6e6a9564 100644 --- a/lib/fluent/plugin/in_tcp.rb +++ b/lib/fluent/plugin/in_tcp.rb @@ -141,7 +141,7 @@ def start next end - @parser.parse(msg) do |time, record| + parse_message(msg) do |time, record| unless time && record log.on_warn { warn_pattern_not_matched(msg) } next @@ -192,7 +192,7 @@ def start next end - @parser.parse(msg) do |time, record| + parse_message(msg) do |time, record| unless time && record log.on_warn { warn_pattern_not_matched(msg) } next @@ -221,6 +221,20 @@ def start private + # A parser can raise ParserError on a line it can't handle (e.g. the json + # parser with a time that doesn't match time_format). Drop only that line + # and keep the connection so that the other lines are not lost. + def parse_message(msg, &block) + @parser.parse(msg, &block) + rescue Fluent::Plugin::Parser::ParserError => e + log.on_warn { warn_parse_error(msg, e) } + end + + def warn_parse_error(msg, error) + head = msg.length > MAX_LOGGED_MESSAGE_LENGTH ? msg[...MAX_LOGGED_MESSAGE_LENGTH] : msg + log.warn "failed to parse message", message: head, error: error + end + def warn_pattern_not_matched(msg) if msg.length > MAX_LOGGED_MESSAGE_LENGTH log.warn "pattern not matched", message: msg[...MAX_LOGGED_MESSAGE_LENGTH], size: msg.bytesize diff --git a/test/plugin/test_in_tcp.rb b/test/plugin/test_in_tcp.rb index 933073a340..b91aac7dbd 100755 --- a/test/plugin/test_in_tcp.rb +++ b/test/plugin/test_in_tcp.rb @@ -395,6 +395,31 @@ def create_tcp_socket(host, port, &block) end end + sub_test_case "parser error" do + data("batch_emit", { extract: "" }, keep: true) + data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true) + test "drop only the line that the parser fails on and keep the connection" do |data| + d = create_driver(base_config + %! + + @type json + time_key time + time_format %Y-%m-%dT%H:%M:%S + + #{data[:extract]} + !) + d.run(expect_records: 3, timeout: 5) do + create_tcp_socket('127.0.0.1', @port) do |sock| + sock.write(%Q({"k":"1"}\n{"time":"bad","k":"2"}\n{"k":"3"}\n)) + sleep 0.5 + sock.write(%Q({"k":"4"}\n)) + end + end + + assert_equal(["1", "3", "4"], d.events.map { |_tag, _time, record| record["k"] }) + assert_equal(1, d.logs.count { |log| log.include?("[warn]: failed to parse message") }) + end + end + sub_test_case "pattern not matched" do data("batch_emit", { extract: "" }, keep: true) data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true)