Skip to content

Commit 33a374a

Browse files
committed
Keep the trailing zero with an allocator; copy the allocator
release wrote the zero after the size, then shrank the buffer through the allocator with used == size: an allocator that keeps only the used bytes, as its contract allows, lost the zero (a 2,500-byte JSON string: 7.5 kB reserved, 2,503 bytes written). We shrink first, then write the zero. The writers kept a pointer to the allocator argument: a temporary dangled at the next grow and in the destructor. The buffer keeps a copy. Tests with an allocator that keeps only the used bytes, and one that goes away before the writers.
1 parent 1a437a1 commit 33a374a

4 files changed

Lines changed: 98 additions & 15 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,10 @@ struct allocator {
233233
};
234234
```
235235

236-
`release` shrinks the buffer (through `reallocate`) when much of it is unused.
236+
The writers keep a copy of the allocator (the functions and the context must
237+
stay valid, not the struct). `release` shrinks the buffer (through
238+
`reallocate`) when much of it is unused; the buffer it gives up holds a zero
239+
byte after its `*size` bytes, with your allocator too.
237240

238241
## The C interface
239242

‎include/simdjson_msgpack.h‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ struct status {
2424
};
2525

2626
// Where a writer puts its output, instead of a buffer from malloc: say, in a
27-
// buffer of a language binding, to hand it over without a copy.
27+
// buffer of a language binding, to hand it over without a copy. A writer
28+
// keeps a copy of it (the functions and the context must stay valid, not the
29+
// struct itself).
2830
struct allocator {
2931
// Returns a buffer of at least size bytes that begins with the used first
3032
// bytes of p (p is null at first), or null if it cannot.
@@ -63,7 +65,8 @@ struct msgpack_options {
6365
class msgpack_writer {
6466
public:
6567
explicit msgpack_writer(const msgpack_options& options = {});
66-
// With the allocator, release gives up a buffer from allocator.reallocate.
68+
// With the allocator (which we copy), release gives up a buffer from
69+
// allocator.reallocate.
6770
msgpack_writer(const msgpack_options& options, const allocator& allocator);
6871
~msgpack_writer();
6972
msgpack_writer(const msgpack_writer&) = delete;
@@ -82,8 +85,8 @@ class msgpack_writer {
8285
// The MessagePack written so far.
8386
std::string_view view() const;
8487
// Gives up the MessagePack written so far: a buffer from malloc (to free
85-
// with free), or from the allocator, of *size bytes. The writer is left
86-
// empty.
88+
// with free), or from the allocator, of *size bytes, followed by a zero
89+
// byte. The writer is left empty.
8790
char* release(size_t* size);
8891
// Forgets what was written (keeps the buffer).
8992
void clear();
@@ -128,6 +131,8 @@ struct json_options {
128131
class json_writer {
129132
public:
130133
explicit json_writer(const json_options& options = {});
134+
// With the allocator (which we copy), release gives up a buffer from
135+
// allocator.reallocate.
131136
json_writer(const json_options& options, const allocator& allocator);
132137
~json_writer();
133138
json_writer(const json_writer&) = delete;

‎src/buffer.h‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,13 @@ namespace internal {
1818
class buffer {
1919
public:
2020
buffer() = default;
21-
explicit buffer(const allocator* a) : allocator_(a) {}
21+
// We copy the allocator: it need not outlive the buffer.
22+
explicit buffer(const allocator* a) {
23+
if (a != nullptr) {
24+
allocator_ = *a;
25+
custom_ = true;
26+
}
27+
}
2228
buffer(const buffer&) = delete;
2329
buffer& operator=(const buffer&) = delete;
2430
~buffer() { free(data_); }
@@ -40,13 +46,18 @@ class buffer {
4046
// Sets the size (to at most the size and the room reserved).
4147
void resize(size_t size) { size_ = size; }
4248
// Gives up the buffer, followed by a zero byte. We shrink it when much of
43-
// it is unused.
49+
// it is unused, before we write the zero: an allocator keeps only the used
50+
// bytes.
4451
char* release(size_t* size) {
45-
*reserve(1) = '\0';
52+
reserve(1);
4653
if (capacity_ - size_ > 4096 && capacity_ - size_ > size_ / 4) {
4754
char* p = reallocate(data_, size_ + 1);
48-
if (p != nullptr) data_ = p;
55+
if (p != nullptr) {
56+
data_ = p;
57+
capacity_ = size_ + 1;
58+
}
4959
}
60+
data_[size_] = '\0';
5061
char* p = data_;
5162
*size = size_;
5263
data_ = nullptr;
@@ -56,15 +67,14 @@ class buffer {
5667

5768
private:
5869
char* reallocate(char* p, size_t capacity) {
59-
return allocator_ == nullptr
60-
? static_cast<char*>(std::realloc(p, capacity))
61-
: allocator_->reallocate(allocator_->context, p, size_, capacity);
70+
return custom_ ? allocator_.reallocate(allocator_.context, p, size_, capacity)
71+
: static_cast<char*>(std::realloc(p, capacity));
6272
}
6373
void free(char* p) {
64-
if (allocator_ == nullptr) {
74+
if (!custom_) {
6575
std::free(p);
6676
} else if (p != nullptr) {
67-
allocator_->deallocate(allocator_->context, p);
77+
allocator_.deallocate(allocator_.context, p);
6878
}
6979
}
7080
void grow(size_t n) {
@@ -76,7 +86,8 @@ class buffer {
7686
capacity_ = capacity;
7787
}
7888

79-
const allocator* allocator_{nullptr};
89+
allocator allocator_{};
90+
bool custom_{false}; // allocator_ is the caller's (otherwise: malloc)
8091
char* data_{nullptr};
8192
size_t size_{0};
8293
size_t capacity_{0};

‎tests/basic_tests.cpp‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include <cstdio>
33
#include <cstdlib>
44
#include <cstring>
5+
#include <memory>
56
#include <string>
67
#include <vector>
78

@@ -418,6 +419,67 @@ static void test_allocator() {
418419
std::free(p);
419420
}
420421

422+
// An allocator that keeps only the used bytes (as the contract allows) and
423+
// fills the rest with 'X'.
424+
static char* strict_reallocate(void*, char* p, size_t used, size_t size) {
425+
char* q = static_cast<char*>(std::malloc(size));
426+
if (q == nullptr) return nullptr;
427+
std::memset(q, 'X', size);
428+
if (p != nullptr) {
429+
std::memcpy(q, p, used);
430+
std::free(p);
431+
}
432+
return q;
433+
}
434+
static void strict_deallocate(void*, char* p) { std::free(p); }
435+
436+
static void test_strict_allocator() {
437+
const allocator strict{strict_reallocate, strict_deallocate, nullptr};
438+
// 2,500 bytes of JSON string: we reserve about 7.5 kB, write 2,503 bytes of
439+
// MessagePack, and shrink the buffer in release.
440+
const std::string json = "\"" + std::string(2500, 'a') + "\"";
441+
simdjson::padded_string padded(json);
442+
msgpack_writer w({}, strict);
443+
CHECK(w.write_json(padded).ok());
444+
size_t size;
445+
char* p = w.release(&size);
446+
CHECK(size == 2503);
447+
CHECK(p[size] == '\0');
448+
CHECK(std::string_view(p + 3, 2500) == std::string(2500, 'a'));
449+
std::free(p);
450+
// The same for JSON, and for many documents (the buffer grows).
451+
json_writer jw({}, strict);
452+
std::string mp;
453+
CHECK(json_to_msgpack(padded, &mp).ok());
454+
for (int i = 0; i < 100; i++) CHECK(jw.write_msgpack_stream(mp.data(), mp.size()).ok());
455+
p = jw.release(&size);
456+
CHECK(size == 100 * (json.size() + 1));
457+
CHECK(p[size] == '\0');
458+
CHECK(std::strlen(p) == size);
459+
std::free(p);
460+
}
461+
462+
// The writers copy the allocator: it may go away before them.
463+
static void test_allocator_lifetime() {
464+
std::unique_ptr<msgpack_writer> w;
465+
std::unique_ptr<json_writer> jw;
466+
{
467+
allocator temporary{strict_reallocate, strict_deallocate, nullptr};
468+
w.reset(new msgpack_writer({}, temporary));
469+
jw.reset(new json_writer({}, temporary));
470+
std::memset(&temporary, 0, sizeof(temporary));
471+
}
472+
std::string big = "[";
473+
for (int i = 0; i < 10000; i++) big += (i ? ",\"" : "\"") + std::to_string(i) + "\"";
474+
big += "]";
475+
simdjson::padded_string padded(big);
476+
CHECK(w->write_json(padded).ok()); // grows
477+
CHECK(jw->write_msgpack(w->view().data(), w->view().size()).ok());
478+
CHECK(jw->view() == big);
479+
w.reset(); // deallocates
480+
jw.reset();
481+
}
482+
421483
static void test_c_api() {
422484
char* out = nullptr;
423485
size_t length = 0;
@@ -622,6 +684,8 @@ int main() {
622684
test_ndjson();
623685
test_round_trip();
624686
test_allocator();
687+
test_strict_allocator();
688+
test_allocator_lifetime();
625689
test_c_api();
626690
test_random_round_trips();
627691
test_msgpack_mutations();

0 commit comments

Comments
 (0)