From d50d583ff1ce5be16f2974754551d63a9c80f1f9 Mon Sep 17 00:00:00 2001 From: shreeraj Date: Sun, 15 Nov 2026 14:09:59 +0530 Subject: [PATCH] in_tcp: drop only the line that the parser fails on When @parser.parse raised an error for a line (for example ParserError from the json parser when the time field does not match time_format), the error was not rescued in in_tcp. It went up to the server helper, which logged "unexpected error on reading data" and closed the connection. With batch emit the lines that were already parsed in the same read were not emitted, and with single emit the lines after the bad line were lost. One bad line from a client dropped its valid lines. Rescue ParserError per line, log a warning and go on with the next line, as is done for a line that doesn't match the pattern. Fixes #5629 Signed-off-by: shreeraj --- lib/fluent/plugin/in_tcp.rb | 18 ++++++++++++++++-- test/plugin/test_in_tcp.rb | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb index 8eaf007936..bb6e6a9564 100644 --- a/lib/fluent/plugin/in_tcp.rb +++ b/lib/fluent/plugin/in_tcp.rb @@ -141,7 +141,7 @@ def start next end - @parser.parse(msg) do |time, record| + parse_message(msg) do |time, record| unless time && record log.on_warn { warn_pattern_not_matched(msg) } next @@ -192,7 +192,7 @@ def start next end - @parser.parse(msg) do |time, record| + parse_message(msg) do |time, record| unless time && record log.on_warn { warn_pattern_not_matched(msg) } next @@ -221,6 +221,20 @@ def start private + # A parser can raise ParserError on a line it can't handle (e.g. the json + # parser with a time that doesn't match time_format). Drop only that line + # and keep the connection so that the other lines are not lost. + def parse_message(msg, &block) + @parser.parse(msg, &block) + rescue Fluent::Plugin::Parser::ParserError => e + log.on_warn { warn_parse_error(msg, e) } + end + + def warn_parse_error(msg, error) + head = msg.length > MAX_LOGGED_MESSAGE_LENGTH ? msg[...MAX_LOGGED_MESSAGE_LENGTH] : msg + log.warn "failed to parse message", message: head, error: error + end + def warn_pattern_not_matched(msg) if msg.length > MAX_LOGGED_MESSAGE_LENGTH log.warn "pattern not matched", message: msg[...MAX_LOGGED_MESSAGE_LENGTH], size: msg.bytesize diff --git a/test/plugin/test_in_tcp.rb b/test/plugin/test_in_tcp.rb index 933073a340..b91aac7dbd 100755 --- a/test/plugin/test_in_tcp.rb +++ b/test/plugin/test_in_tcp.rb @@ -395,6 +395,31 @@ def create_tcp_socket(host, port, &block) end end + sub_test_case "parser error" do + data("batch_emit", { extract: "" }, keep: true) + data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true) + test "drop only the line that the parser fails on and keep the connection" do |data| + d = create_driver(base_config + %! + + @type json + time_key time + time_format %Y-%m-%dT%H:%M:%S + + #{data[:extract]} + !) + d.run(expect_records: 3, timeout: 5) do + create_tcp_socket('127.0.0.1', @port) do |sock| + sock.write(%Q({"k":"1"}\n{"time":"bad","k":"2"}\n{"k":"3"}\n)) + sleep 0.5 + sock.write(%Q({"k":"4"}\n)) + end + end + + assert_equal(["1", "3", "4"], d.events.map { |_tag, _time, record| record["k"] }) + assert_equal(1, d.logs.count { |log| log.include?("[warn]: failed to parse message") }) + end + end + sub_test_case "pattern not matched" do data("batch_emit", { extract: "" }, keep: true) data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true)