Calculate batch sizes in bytes (#44)

* fix: count max payload size in bytes

* Changelog and bump version

* Verify payload always under max size

* Review comments

* Use bytesize in tests

Author: dyladan

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.7.1
+
+- Fix an error where max payload size was calculated using character count instead of bytes
+
 ## 0.7.0
 
 - Add new development dependency `rackup` for logstash 8.x compatibility

lib/logstash/outputs/dynatrace.rb

@@ -122,11 +122,11 @@ def initialize(max_batch_size)
 
         def offer(serialized_event)
           # 2 square brackets, the length of all previously serialized strings, commas, and the current event size
-          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
+          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.bytesize
           return false if batch_size_bytes > @max_batch_size
 
           @serialized_events.push(serialized_event)
-          @batch_events_size += serialized_event.length
+          @batch_events_size += serialized_event.bytesize
           true
         end
 
@@ -178,8 +178,8 @@ def send_events(events)
 
         events.each do |event|
           serialized_event = LogStash::Json.dump(event.to_hash)
-          if serialized_event.length > @max_payload_size
-            log_params = { size: serialized_event.length }
+          if serialized_event.bytesize > @max_payload_size
+            log_params = { size: serialized_event.bytesize }
             log_params[:body] = serialized_event if @debug_include_body
             log_warning('Event larger than max_payload_size dropped', log_params)
             next

spec/outputs/dynatrace_spec.rb

@@ -232,17 +232,38 @@
     end
   end
 
-  context 'max_payload_size 2MB' do
-    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
+  context 'max_payload_size 500' do
+    let(:max_payload_size) { 500 }
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => max_payload_size } }
     subject { LogStash::Outputs::Dynatrace.new(config) }
 
     before do
       allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
-      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
     end
 
-    it 'should split the chunk into multiple requests' do
-      expect(subject).to have_received(:send_event).exactly(2).times
+    it 'should send 2 400B messages in multiple requests' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 400 }) })
+      expect(subject).to have_received(:send_event).exactly(2).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should send 2 100B messages in a single request' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 100 }) })
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should drop messages larger than max_payload_size' do
+      subject.multi_receive([
+        LogStash::Event.new({ 'event' => '🤣' * 400 }),
+        LogStash::Event.new({ 'event' => 'n' * 600 }),
+        LogStash::Event.new({ 'event' => 'n' * 400 }),
+      ])
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
     end
   end
 

version.yaml

@@ -1 +1 @@
-logstash-output-dynatrace: '0.7.0'
+logstash-output-dynatrace: '0.7.1'