Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 86 additions & 79 deletions lib/fluent/plugin/in_tcp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -115,105 +115,112 @@ def start
# The received data is binary (ASCII-8BIT), so match the delimiter as bytes.
delimiter = @delimiter.b
del_size = delimiter.bytesize
discard_till_next_delimiter = false
if @_extract_enabled && @_extract_tag_key
server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
unless check_client(conn)
conn.close
next
end

buf = conn.buffer
search_pos = [buf.length - del_size + 1, 0].max
buf << data
pos = 0
while i = buf.index(delimiter, search_pos)
msg = buf[pos...i]
pos = search_pos = i + del_size

if discard_till_next_delimiter
discard_till_next_delimiter = false
server_create_connection(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn|
# Each connection discards only its own data until its own next delimiter.
discard_till_next_delimiter = false
conn.data do |data|
unless check_client(conn)
conn.close
next
end

if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
next
end
buf = conn.buffer
search_pos = [buf.length - del_size + 1, 0].max
buf << data
pos = 0
while i = buf.index(delimiter, search_pos)
msg = buf[pos...i]
pos = search_pos = i + del_size

if discard_till_next_delimiter
discard_till_next_delimiter = false
next
end

@parser.parse(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
next
end

tag = extract_tag_from_record(record)
tag ||= @tag
time ||= extract_time_from_record(record) || Fluent::EventTime.now
record[@source_address_key] = conn.remote_addr if @source_address_key
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
router.emit(tag, time, record)
@parser.parse(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
next
end

tag = extract_tag_from_record(record)
tag ||= @tag
time ||= extract_time_from_record(record) || Fluent::EventTime.now
record[@source_address_key] = conn.remote_addr if @source_address_key
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
router.emit(tag, time, record)
end
end
buf.slice!(0, pos) if pos > 0
# If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
# So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
buf.clear
# We should discard the subsequent data until the next delimiter comes.
discard_till_next_delimiter = true
next
end
end
buf.slice!(0, pos) if pos > 0
# If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
# So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
buf.clear
# We should discard the subsequent data until the next delimiter comes.
discard_till_next_delimiter = true
next
end
end
else
server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
unless check_client(conn)
conn.close
next
end

buf = conn.buffer
search_pos = [buf.length - del_size + 1, 0].max
buf << data
pos = 0
es = Fluent::MultiEventStream.new
while i = buf.index(delimiter, search_pos)
msg = buf[pos...i]
pos = search_pos = i + del_size

if discard_till_next_delimiter
discard_till_next_delimiter = false
server_create_connection(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |conn|
# Each connection discards only its own data until its own next delimiter.
discard_till_next_delimiter = false
conn.data do |data|
unless check_client(conn)
conn.close
next
end

if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
next
end
buf = conn.buffer
search_pos = [buf.length - del_size + 1, 0].max
buf << data
pos = 0
es = Fluent::MultiEventStream.new
while i = buf.index(delimiter, search_pos)
msg = buf[pos...i]
pos = search_pos = i + del_size

if discard_till_next_delimiter
discard_till_next_delimiter = false
next
end

@parser.parse(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
next
end

time ||= extract_time_from_record(record) || Fluent::EventTime.now
record[@source_address_key] = conn.remote_addr if @source_address_key
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
es.add(time, record)
@parser.parse(msg) do |time, record|
unless time && record
log.on_warn { warn_pattern_not_matched(msg) }
next
end

time ||= extract_time_from_record(record) || Fluent::EventTime.now
record[@source_address_key] = conn.remote_addr if @source_address_key
record[@source_hostname_key] = conn.remote_host if @source_hostname_key
es.add(time, record)
end
end
router.emit_stream(@tag, es) unless es.empty?
buf.slice!(0, pos) if pos > 0
# If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
# So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
buf.clear
# We should discard the subsequent data until the next delimiter comes.
discard_till_next_delimiter = true
next
end
end
router.emit_stream(@tag, es) unless es.empty?
buf.slice!(0, pos) if pos > 0
# If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
# So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
buf.clear
# We should discard the subsequent data until the next delimiter comes.
discard_till_next_delimiter = true
next
end
end
end
Expand Down
29 changes: 29 additions & 0 deletions test/plugin/test_in_tcp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,35 @@ def create_tcp_socket(host, port, &block)
assert_equal expected_records, actual_records
end

test "discard only the data of the connection that exceeded the limit" do |data|
message_length_limit = 10
d = create_driver(base_config + %!
message_length_limit #{message_length_limit}
<parse>
@type none
</parse>
#{data[:extract]}
!)
d.run(expect_records: 2, timeout: 10) do
create_tcp_socket('127.0.0.1', @port) do |long_sock|
create_tcp_socket('127.0.0.1', @port) do |other_sock|
long_sock.send("x" * (message_length_limit + 1), 0)
sleep 1 # To make the server read the data and clear the buffer of long_sock here.
other_sock.send("hello\n", 0)
long_sock.send("tail\n", 0) # The end of the long message of long_sock, which must be discarded.
other_sock.send("world\n", 0)
sleep 1 # To make the server emit the records before the sockets are closed.
end
end
end

actual_records = d.events.collect do |event|
event[2]["message"]
end

assert_equal(["hello", "world"], actual_records)
end

test "clear buffer and discard the subsequent data until the next delimiter" do |data|
message_length_limit = 12
d = create_driver(base_config + %!
Expand Down