From 72776855aa6a6d4492313519a94517fa52d7d253 Mon Sep 17 00:00:00 2001 From: Raed Date: Wed, 18 Nov 2026 03:24:28 +0530 Subject: [PATCH] in_tcp: keep the message_length_limit discard state per connection discard_till_next_delimiter was a local variable of start, so every connection shared it. When one connection exceeded message_length_limit and had its buffer cleared, the flag stayed set, and the next delimiter from any connection, whichever came first, was dropped as if it were the tail of the long message. The rest of the long message was then emitted as a record, because the flag was already reset when its own delimiter came. Move the flag into the connection block, as in_syslog already does, so each connection discards only its own data until its own next delimiter. The read loop is unchanged apart from that scope. Fixes #5647 Signed-off-by: Raed Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/fluent/plugin/in_tcp.rb | 165 +++++++++++++++++++----------------- test/plugin/test_in_tcp.rb | 29 +++++++ 2 files changed, 115 insertions(+), 79 deletions(-) diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb index 8eaf007936..b18a2935ff 100644 --- a/lib/fluent/plugin/in_tcp.rb +++ b/lib/fluent/plugin/in_tcp.rb @@ -115,105 +115,112 @@ def start # The received data is binary (ASCII-8BIT), so match the delimiter as bytes. delimiter = @delimiter.b del_size = delimiter.bytesize - discard_till_next_delimiter = false if @_extract_enabled && @_extract_tag_key - server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn| - unless check_client(conn) - conn.close - next - end - - buf = conn.buffer - search_pos = [buf.length - del_size + 1, 0].max - buf << data - pos = 0 - while i = buf.index(delimiter, search_pos) - msg = buf[pos...i] - pos = search_pos = i + del_size - - if discard_till_next_delimiter - discard_till_next_delimiter = false + server_create_connection(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn| + # Each connection discards only its own data until its own next delimiter. + discard_till_next_delimiter = false + conn.data do |data| + unless check_client(conn) + conn.close next end - if !@message_length_limit.nil? && @message_length_limit < msg.bytesize - log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] - next - end + buf = conn.buffer + search_pos = [buf.length - del_size + 1, 0].max + buf << data + pos = 0 + while i = buf.index(delimiter, search_pos) + msg = buf[pos...i] + pos = search_pos = i + del_size + + if discard_till_next_delimiter + discard_till_next_delimiter = false + next + end - @parser.parse(msg) do |time, record| - unless time && record - log.on_warn { warn_pattern_not_matched(msg) } + if !@message_length_limit.nil? && @message_length_limit < msg.bytesize + log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] next end - tag = extract_tag_from_record(record) - tag ||= @tag - time ||= extract_time_from_record(record) || Fluent::EventTime.now - record[@source_address_key] = conn.remote_addr if @source_address_key - record[@source_hostname_key] = conn.remote_host if @source_hostname_key - router.emit(tag, time, record) + @parser.parse(msg) do |time, record| + unless time && record + log.on_warn { warn_pattern_not_matched(msg) } + next + end + + tag = extract_tag_from_record(record) + tag ||= @tag + time ||= extract_time_from_record(record) || Fluent::EventTime.now + record[@source_address_key] = conn.remote_addr if @source_address_key + record[@source_hostname_key] = conn.remote_host if @source_hostname_key + router.emit(tag, time, record) + end + end + buf.slice!(0, pos) if pos > 0 + # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. + # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. + if !@message_length_limit.nil? && @message_length_limit < buf.bytesize + log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] + buf.clear + # We should discard the subsequent data until the next delimiter comes. + discard_till_next_delimiter = true + next end - end - buf.slice!(0, pos) if pos > 0 - # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. - # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. - if !@message_length_limit.nil? && @message_length_limit < buf.bytesize - log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] - buf.clear - # We should discard the subsequent data until the next delimiter comes. - discard_till_next_delimiter = true - next end end else - server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn| - unless check_client(conn) - conn.close - next - end - - buf = conn.buffer - search_pos = [buf.length - del_size + 1, 0].max - buf << data - pos = 0 - es = Fluent::MultiEventStream.new - while i = buf.index(delimiter, search_pos) - msg = buf[pos...i] - pos = search_pos = i + del_size - - if discard_till_next_delimiter - discard_till_next_delimiter = false + server_create_connection(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn| + # Each connection discards only its own data until its own next delimiter. + discard_till_next_delimiter = false + conn.data do |data| + unless check_client(conn) + conn.close next end - if !@message_length_limit.nil? && @message_length_limit < msg.bytesize - log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] - next - end + buf = conn.buffer + search_pos = [buf.length - del_size + 1, 0].max + buf << data + pos = 0 + es = Fluent::MultiEventStream.new + while i = buf.index(delimiter, search_pos) + msg = buf[pos...i] + pos = search_pos = i + del_size + + if discard_till_next_delimiter + discard_till_next_delimiter = false + next + end - @parser.parse(msg) do |time, record| - unless time && record - log.on_warn { warn_pattern_not_matched(msg) } + if !@message_length_limit.nil? && @message_length_limit < msg.bytesize + log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] next end - time ||= extract_time_from_record(record) || Fluent::EventTime.now - record[@source_address_key] = conn.remote_addr if @source_address_key - record[@source_hostname_key] = conn.remote_host if @source_hostname_key - es.add(time, record) + @parser.parse(msg) do |time, record| + unless time && record + log.on_warn { warn_pattern_not_matched(msg) } + next + end + + time ||= extract_time_from_record(record) || Fluent::EventTime.now + record[@source_address_key] = conn.remote_addr if @source_address_key + record[@source_hostname_key] = conn.remote_host if @source_hostname_key + es.add(time, record) + end + end + router.emit_stream(@tag, es) unless es.empty? + buf.slice!(0, pos) if pos > 0 + # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. + # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. + if !@message_length_limit.nil? && @message_length_limit < buf.bytesize + log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] + buf.clear + # We should discard the subsequent data until the next delimiter comes. + discard_till_next_delimiter = true + next end - end - router.emit_stream(@tag, es) unless es.empty? - buf.slice!(0, pos) if pos > 0 - # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. - # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. - if !@message_length_limit.nil? && @message_length_limit < buf.bytesize - log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] - buf.clear - # We should discard the subsequent data until the next delimiter comes. - discard_till_next_delimiter = true - next end end end diff --git a/test/plugin/test_in_tcp.rb b/test/plugin/test_in_tcp.rb index 933073a340..1801940715 100755 --- a/test/plugin/test_in_tcp.rb +++ b/test/plugin/test_in_tcp.rb @@ -354,6 +354,35 @@ def create_tcp_socket(host, port, &block) assert_equal expected_records, actual_records end + test "discard only the data of the connection that exceeded the limit" do |data| + message_length_limit = 10 + d = create_driver(base_config + %! + message_length_limit #{message_length_limit} + + @type none + + #{data[:extract]} + !) + d.run(expect_records: 2, timeout: 10) do + create_tcp_socket('127.0.0.1', @port) do |long_sock| + create_tcp_socket('127.0.0.1', @port) do |other_sock| + long_sock.send("x" * (message_length_limit + 1), 0) + sleep 1 # To make the server read the data and clear the buffer of long_sock here. + other_sock.send("hello\n", 0) + long_sock.send("tail\n", 0) # The end of the long message of long_sock, which must be discarded. + other_sock.send("world\n", 0) + sleep 1 # To make the server emit the records before the sockets are closed. + end + end + end + + actual_records = d.events.collect do |event| + event[2]["message"] + end + + assert_equal(["hello", "world"], actual_records) + end + test "clear buffer and discard the subsequent data until the next delimiter" do |data| message_length_limit = 12 d = create_driver(base_config + %!