diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb
index 8eaf007936..bb6e6a9564 100644
--- a/lib/fluent/plugin/in_tcp.rb
+++ b/lib/fluent/plugin/in_tcp.rb
@@ -141,7 +141,7 @@ def start
next
end
- @parser.parse(msg) do |time, record|
+ parse_message(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
next
@@ -192,7 +192,7 @@ def start
next
end
- @parser.parse(msg) do |time, record|
+ parse_message(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
next
@@ -221,6 +221,20 @@ def start
private
+ # A parser can raise ParserError on a line it can't handle (e.g. the json
+ # parser with a time that doesn't match time_format). Drop only that line
+ # and keep the connection so that the other lines are not lost.
+ def parse_message(msg, &block)
+ @parser.parse(msg, &block)
+ rescue Fluent::Plugin::Parser::ParserError => e
+ log.on_warn { warn_parse_error(msg, e) }
+ end
+
+ def warn_parse_error(msg, error)
+ head = msg.length > MAX_LOGGED_MESSAGE_LENGTH ? msg[...MAX_LOGGED_MESSAGE_LENGTH] : msg
+ log.warn "failed to parse message", message: head, error: error
+ end
+
def warn_pattern_not_matched(msg)
if msg.length > MAX_LOGGED_MESSAGE_LENGTH
log.warn "pattern not matched", message: msg[...MAX_LOGGED_MESSAGE_LENGTH], size: msg.bytesize
diff --git a/test/plugin/test_in_tcp.rb b/test/plugin/test_in_tcp.rb
index 933073a340..b91aac7dbd 100755
--- a/test/plugin/test_in_tcp.rb
+++ b/test/plugin/test_in_tcp.rb
@@ -395,6 +395,31 @@ def create_tcp_socket(host, port, &block)
end
end
+ sub_test_case "parser error" do
+ data("batch_emit", { extract: "" }, keep: true)
+ data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true)
+ test "drop only the line that the parser fails on and keep the connection" do |data|
+ d = create_driver(base_config + %!
+
+ @type json
+ time_key time
+ time_format %Y-%m-%dT%H:%M:%S
+
+ #{data[:extract]}
+ !)
+ d.run(expect_records: 3, timeout: 5) do
+ create_tcp_socket('127.0.0.1', @port) do |sock|
+ sock.write(%Q({"k":"1"}\n{"time":"bad","k":"2"}\n{"k":"3"}\n))
+ sleep 0.5
+ sock.write(%Q({"k":"4"}\n))
+ end
+ end
+
+ assert_equal(["1", "3", "4"], d.events.map { |_tag, _time, record| record["k"] })
+ assert_equal(1, d.logs.count { |log| log.include?("[warn]: failed to parse message") })
+ end
+ end
+
sub_test_case "pattern not matched" do
data("batch_emit", { extract: "" }, keep: true)
data("single_emit", { extract: "\ntag_key tag\n\n" }, keep: true)