From c5c8067d1c3faa4a528d2fbefe428ab3fd2c060d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sat, 14 Nov 2026 00:06:59 -0500 Subject: [PATCH 1/2] map_many: drop_none=True leaves out the results that are None When fn keeps only a few documents (a filter, a count), the threads no longer build a list entry for each of the others, and the calling thread no longer collects and frees them. Counting the stars of 7.6 GB of GH Archive events with fn returning the repository or None (90,403 results out of 2.2 million) on big4, 64 threads, 3.14.10t: 73.6 to 88.0 GB/s end to end, Counter included. --- README.md | 14 ++++++++++++- src/fastsimdjson.cpp | 47 +++++++++++++++++++++++++++++--------------- tests/test_stream.py | 23 ++++++++++++++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 804225c..73e21d2 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,19 @@ other threads, and it must be safe to call from several threads at once. The worker threads are started by the first call that needs them and kept for later calls (they wait, detached from the interpreter, between calls, and stop at exit). `map_many` scales best: each document is built, used and -freed in the same thread, and only `fn`'s results come back. `loads_many` must return every +freed in the same thread, and only `fn`'s results come back. +With `drop_none=True`, `map_many` leaves out the results that are `None`: +when `fn` keeps only a few documents, the threads do not build a list +entry for each of the others, and the calling thread has fewer to collect. + +```python +def starred(event): + kind, repo = event + return repo if kind == "WatchEvent" else None + +names = fastsimdjson.map_many(starred, data, threads=64, drop_none=True, + fields=["/type", "/repo/name"]) +``` `loads_many` must return every document, and the Python objects of all of them, to the calling thread. With the GIL (the default build of Python), the calling thread does all the work: `threads` has no effect on speed. diff --git a/src/fastsimdjson.cpp b/src/fastsimdjson.cpp index 5ca3f91..6ca08b7 100644 --- a/src/fastsimdjson.cpp +++ b/src/fastsimdjson.cpp @@ -4216,6 +4216,7 @@ struct ParallelRun : PoolTask { StreamObject *whole; PyObject *fn; // borrowed, or nullptr + bool drop_none = false; // map_many(drop_none=True): fn's None results are not kept char delimiter = '\n'; bool snap_after = true; // false: the next slice starts on the delimiter struct Chunk { @@ -4340,6 +4341,10 @@ struct ParallelRun : PoolTask { break; } v = r; + if (drop_none && v == Py_None) { + Py_DECREF(v); + continue; + } } try { c.items.push_back(v); @@ -4617,7 +4622,7 @@ void pool_after_fork() { // All the results (documents, or fn of each) of the stream `whole`, as a // list, using up to `threads` threads. -PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { +PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads, bool drop_none) { PyObject *result = PyList_New(0); if (result == nullptr) { return nullptr; @@ -4628,6 +4633,7 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { ParallelRun run; run.whole = whole; run.fn = fn; + run.drop_none = drop_none; // json_seq snaps onto the record separator so the next slice starts with // it. Stage 1 finds a scalar value from that separator; a slice that // started after it would drop the separator and mis-parse the scalar. @@ -4708,6 +4714,10 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { break; } v = r; + if (drop_none && v == Py_None) { + Py_DECREF(v); + continue; + } } int rc = PyList_Append(result, v); Py_DECREF(v); @@ -4785,7 +4795,7 @@ PyObject *pop_keyword(PyObject *kwargs, const char *name) { // all results instead. With fields (not for parse_many), the values at the // fields' pointers come instead of the documents. PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject *fn, - bool force_list, bool allow_fields) { + bool force_list, bool allow_fields, bool drop_none = false) { Py_ssize_t threads; bool given; PyObject *rest = take_threads(kwargs, &threads, &given); @@ -4851,7 +4861,7 @@ PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject * "threads requires the format 'whitespace', 'lines' or 'json_seq'"); return nullptr; } - PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1); + PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1, drop_none); Py_DECREF(self); return r; } @@ -4865,8 +4875,9 @@ PyObject *parse_many(PyObject *, PyObject *args, PyObject *kwargs) { } // map_many(fn, data, *, format, batch_size, threads, lazy=False, fields, -// default): [fn(doc) for doc in loads_many(data)] (parse_many with -// lazy=True), with fn called in the threads. +// default, drop_none=False): [fn(doc) for doc in loads_many(data)] +// (parse_many with lazy=True), with fn called in the threads; with +// drop_none, the results that are None are left out. PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { if (PyTuple_GET_SIZE(args) < 1) { PyErr_SetString(PyExc_TypeError, "map_many() missing required argument 'fn'"); @@ -4877,20 +4888,23 @@ PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { PyErr_SetString(PyExc_TypeError, "map_many() fn must be callable"); return nullptr; } - bool lazy = false; + bool lazy = false, drop_none = false; PyObject *rest = kwargs ? PyDict_Copy(kwargs) : nullptr; if (kwargs != nullptr && rest == nullptr) { return nullptr; } if (rest != nullptr) { - PyObject *l = PyDict_GetItemString(rest, "lazy"); // borrowed - if (l != nullptr) { - int t = PyObject_IsTrue(l); - if (t < 0 || PyDict_DelItemString(rest, "lazy") < 0) { - Py_DECREF(rest); - return nullptr; + for (auto [name, flag] : {std::pair{"lazy", &lazy}, + std::pair{"drop_none", &drop_none}}) { + PyObject *l = PyDict_GetItemString(rest, name); // borrowed + if (l != nullptr) { + int t = PyObject_IsTrue(l); + if (t < 0 || PyDict_DelItemString(rest, name) < 0) { + Py_DECREF(rest); + return nullptr; + } + *flag = t != 0; } - lazy = t != 0; } } PyObject *tail = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); @@ -4898,7 +4912,7 @@ PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { Py_XDECREF(rest); return nullptr; } - PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true); + PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true, drop_none); Py_DECREF(tail); Py_XDECREF(rest); return r; @@ -6064,8 +6078,9 @@ PyMethodDef methods[] = { "values of the input becomes one line of NDJSON."}, {"map_many", reinterpret_cast(reinterpret_cast(map_many)), METH_VARARGS | METH_KEYWORDS, - "map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type): the " - "list of fn(doc) for the documents of data, with fn called in the threads."}, + "map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type, " + "drop_none=False): the list of fn(doc) for the documents of data, with fn called in " + "the threads; with drop_none, the results that are None are left out."}, {"parse", parse, METH_O, "Parse JSON lazily: objects and arrays are returned as read-only views " "(Object, Array) whose values are converted on access."}, diff --git a/tests/test_stream.py b/tests/test_stream.py index 4597127..0c20c24 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -622,3 +622,26 @@ def test_long_record_then_threads(): assert len(out) == 2 * len(objs) + 1 and out[len(objs)] == big del out assert fastsimdjson.loads_many(data, threads=4) == objs + + +@pytest.mark.parametrize("threads", [None, 1, 4]) +def test_map_many_drop_none(threads): + objs = [{"i": i, "keep": i % 7 == 0} for i in range(30000)] + data = ndjson(objs).encode() + pick = lambda d: d["i"] if d["keep"] else None + want = [o["i"] for o in objs if o["keep"]] + assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=True) == want + assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=False) == \ + [pick(o) for o in objs] + assert fastsimdjson.map_many(lambda t: t[0] if t[1] else None, data, threads=threads, + fields=["/i", "/keep"], drop_none=True) == want + assert fastsimdjson.map_many(lambda d: None, data, threads=threads, drop_none=True) == [] + # Falsy results other than None are kept. + assert fastsimdjson.map_many(lambda d: d["i"] % 2 and 0, data, threads=threads, + drop_none=True) == [0] * len(objs) + # An exception from fn still propagates; json's fallback still applies. + with pytest.raises(ZeroDivisionError): + fastsimdjson.map_many(lambda d: 1 // (d["i"] - 20000), data, threads=threads, + drop_none=True) + tail = data + b'{"i": 1e400, "keep": true}\n' + assert fastsimdjson.map_many(pick, tail, threads=threads, drop_none=True)[-1] == float("inf") From f8d8abe3a111af59fd2b99a7fab0be8461f8444a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sat, 14 Nov 2026 00:29:42 -0500 Subject: [PATCH 2/2] README: close the drop_none example's code fence on its own line --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 73e21d2..bb8ef61 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,11 @@ other threads, and it must be safe to call from several threads at once. The worker threads are started by the first call that needs them and kept for later calls (they wait, detached from the interpreter, between calls, and stop at exit). `map_many` scales best: each document is built, used and -freed in the same thread, and only `fn`'s results come back. +freed in the same thread, and only `fn`'s results come back. `loads_many` must return every +document, and the Python objects of all of them, to the calling thread. +With the GIL (the default build of Python), the calling thread does all the +work: `threads` has no effect on speed. + With `drop_none=True`, `map_many` leaves out the results that are `None`: when `fn` keeps only a few documents, the threads do not build a list entry for each of the others, and the calling thread has fewer to collect. @@ -362,10 +366,7 @@ def starred(event): names = fastsimdjson.map_many(starred, data, threads=64, drop_none=True, fields=["/type", "/repo/name"]) -``` `loads_many` must return every -document, and the Python objects of all of them, to the calling thread. -With the GIL (the default build of Python), the calling thread does all the -work: `threads` has no effect on speed. +``` ### Only some fields: `fields=`