diff --git a/lib/fluent/plugin/in_tcp.rb b/lib/fluent/plugin/in_tcp.rb index 8eaf007936..b18a2935ff 100644 --- a/lib/fluent/plugin/in_tcp.rb +++ b/lib/fluent/plugin/in_tcp.rb @@ -115,105 +115,112 @@ def start # The received data is binary (ASCII-8BIT), so match the delimiter as bytes. delimiter = @delimiter.b del_size = delimiter.bytesize - discard_till_next_delimiter = false if @_extract_enabled && @_extract_tag_key - server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn| - unless check_client(conn) - conn.close - next - end - - buf = conn.buffer - search_pos = [buf.length - del_size + 1, 0].max - buf << data - pos = 0 - while i = buf.index(delimiter, search_pos) - msg = buf[pos...i] - pos = search_pos = i + del_size - - if discard_till_next_delimiter - discard_till_next_delimiter = false + server_create_connection(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn| + # Each connection discards only its own data until its own next delimiter. + discard_till_next_delimiter = false + conn.data do |data| + unless check_client(conn) + conn.close next end - if !@message_length_limit.nil? && @message_length_limit < msg.bytesize - log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] - next - end + buf = conn.buffer + search_pos = [buf.length - del_size + 1, 0].max + buf << data + pos = 0 + while i = buf.index(delimiter, search_pos) + msg = buf[pos...i] + pos = search_pos = i + del_size + + if discard_till_next_delimiter + discard_till_next_delimiter = false + next + end - @parser.parse(msg) do |time, record| - unless time && record - log.on_warn { warn_pattern_not_matched(msg) } + if !@message_length_limit.nil? && @message_length_limit < msg.bytesize + log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] next end - tag = extract_tag_from_record(record) - tag ||= @tag - time ||= extract_time_from_record(record) || Fluent::EventTime.now - record[@source_address_key] = conn.remote_addr if @source_address_key - record[@source_hostname_key] = conn.remote_host if @source_hostname_key - router.emit(tag, time, record) + @parser.parse(msg) do |time, record| + unless time && record + log.on_warn { warn_pattern_not_matched(msg) } + next + end + + tag = extract_tag_from_record(record) + tag ||= @tag + time ||= extract_time_from_record(record) || Fluent::EventTime.now + record[@source_address_key] = conn.remote_addr if @source_address_key + record[@source_hostname_key] = conn.remote_host if @source_hostname_key + router.emit(tag, time, record) + end + end + buf.slice!(0, pos) if pos > 0 + # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. + # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. + if !@message_length_limit.nil? && @message_length_limit < buf.bytesize + log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] + buf.clear + # We should discard the subsequent data until the next delimiter comes. + discard_till_next_delimiter = true + next end - end - buf.slice!(0, pos) if pos > 0 - # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. - # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. - if !@message_length_limit.nil? && @message_length_limit < buf.bytesize - log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] - buf.clear - # We should discard the subsequent data until the next delimiter comes. - discard_till_next_delimiter = true - next end end else - server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn| - unless check_client(conn) - conn.close - next - end - - buf = conn.buffer - search_pos = [buf.length - del_size + 1, 0].max - buf << data - pos = 0 - es = Fluent::MultiEventStream.new - while i = buf.index(delimiter, search_pos) - msg = buf[pos...i] - pos = search_pos = i + del_size - - if discard_till_next_delimiter - discard_till_next_delimiter = false + server_create_connection(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn| + # Each connection discards only its own data until its own next delimiter. + discard_till_next_delimiter = false + conn.data do |data| + unless check_client(conn) + conn.close next end - if !@message_length_limit.nil? && @message_length_limit < msg.bytesize - log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] - next - end + buf = conn.buffer + search_pos = [buf.length - del_size + 1, 0].max + buf << data + pos = 0 + es = Fluent::MultiEventStream.new + while i = buf.index(delimiter, search_pos) + msg = buf[pos...i] + pos = search_pos = i + del_size + + if discard_till_next_delimiter + discard_till_next_delimiter = false + next + end - @parser.parse(msg) do |time, record| - unless time && record - log.on_warn { warn_pattern_not_matched(msg) } + if !@message_length_limit.nil? && @message_length_limit < msg.bytesize + log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32] next end - time ||= extract_time_from_record(record) || Fluent::EventTime.now - record[@source_address_key] = conn.remote_addr if @source_address_key - record[@source_hostname_key] = conn.remote_host if @source_hostname_key - es.add(time, record) + @parser.parse(msg) do |time, record| + unless time && record + log.on_warn { warn_pattern_not_matched(msg) } + next + end + + time ||= extract_time_from_record(record) || Fluent::EventTime.now + record[@source_address_key] = conn.remote_addr if @source_address_key + record[@source_hostname_key] = conn.remote_host if @source_hostname_key + es.add(time, record) + end + end + router.emit_stream(@tag, es) unless es.empty? + buf.slice!(0, pos) if pos > 0 + # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. + # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. + if !@message_length_limit.nil? && @message_length_limit < buf.bytesize + log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] + buf.clear + # We should discard the subsequent data until the next delimiter comes. + discard_till_next_delimiter = true + next end - end - router.emit_stream(@tag, es) unless es.empty? - buf.slice!(0, pos) if pos > 0 - # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit. - # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes. - if !@message_length_limit.nil? && @message_length_limit < buf.bytesize - log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32] - buf.clear - # We should discard the subsequent data until the next delimiter comes. - discard_till_next_delimiter = true - next end end end diff --git a/test/plugin/test_in_tcp.rb b/test/plugin/test_in_tcp.rb index 933073a340..1801940715 100755 --- a/test/plugin/test_in_tcp.rb +++ b/test/plugin/test_in_tcp.rb @@ -354,6 +354,35 @@ def create_tcp_socket(host, port, &block) assert_equal expected_records, actual_records end + test "discard only the data of the connection that exceeded the limit" do |data| + message_length_limit = 10 + d = create_driver(base_config + %! + message_length_limit #{message_length_limit} + + @type none + + #{data[:extract]} + !) + d.run(expect_records: 2, timeout: 10) do + create_tcp_socket('127.0.0.1', @port) do |long_sock| + create_tcp_socket('127.0.0.1', @port) do |other_sock| + long_sock.send("x" * (message_length_limit + 1), 0) + sleep 1 # To make the server read the data and clear the buffer of long_sock here. + other_sock.send("hello\n", 0) + long_sock.send("tail\n", 0) # The end of the long message of long_sock, which must be discarded. + other_sock.send("world\n", 0) + sleep 1 # To make the server emit the records before the sockets are closed. + end + end + end + + actual_records = d.events.collect do |event| + event[2]["message"] + end + + assert_equal(["hello", "world"], actual_records) + end + test "clear buffer and discard the subsequent data until the next delimiter" do |data| message_length_limit = 12 d = create_driver(base_config + %!