diff --git a/README.md b/README.md index feef8d8..47c74b1 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,19 @@ document, and the Python objects of all of them, to the calling thread. With the GIL (the default build of Python), the calling thread does all the work: `threads` has no effect on speed. +With `drop_none=True`, `map_many` leaves out the results that are `None`: +when `fn` keeps only a few documents, the threads do not build a list +entry for each of the others, and the calling thread has fewer to collect. + +```python +def starred(event): + kind, repo = event + return repo if kind == "WatchEvent" else None + +names = fastsimdjson.map_many(starred, data, threads=64, drop_none=True, + fields=["/type", "/repo/name"]) +``` + ### Only some fields: `fields=` When you need a few values from each document, `fields` extracts them diff --git a/src/fastsimdjson.cpp b/src/fastsimdjson.cpp index 9746196..ba803c0 100644 --- a/src/fastsimdjson.cpp +++ b/src/fastsimdjson.cpp @@ -4731,6 +4731,7 @@ struct ParallelRun : PoolTask { StreamObject *whole; PyObject *fn; // borrowed, or nullptr + bool drop_none = false; // map_many(drop_none=True): fn's None results are not kept char delimiter = '\n'; bool snap_after = true; // false: the next slice starts on the delimiter struct Chunk { @@ -4855,6 +4856,10 @@ struct ParallelRun : PoolTask { break; } v = r; + if (drop_none && v == Py_None) { + Py_DECREF(v); + continue; + } } try { c.items.push_back(v); @@ -5132,7 +5137,7 @@ void pool_after_fork() { // All the results (documents, or fn of each) of the stream `whole`, as a // list, using up to `threads` threads. -PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { +PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads, bool drop_none) { PyObject *result = PyList_New(0); if (result == nullptr) { return nullptr; @@ -5143,6 +5148,7 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { ParallelRun run; run.whole = whole; run.fn = fn; + run.drop_none = drop_none; // json_seq snaps onto the record separator so the next slice starts with // it. Stage 1 finds a scalar value from that separator; a slice that // started after it would drop the separator and mis-parse the scalar. @@ -5223,6 +5229,10 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) { break; } v = r; + if (drop_none && v == Py_None) { + Py_DECREF(v); + continue; + } } int rc = PyList_Append(result, v); Py_DECREF(v); @@ -5300,7 +5310,7 @@ PyObject *pop_keyword(PyObject *kwargs, const char *name) { // all results instead. With fields (not for parse_many), the values at the // fields' pointers come instead of the documents. PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject *fn, - bool force_list, bool allow_fields) { + bool force_list, bool allow_fields, bool drop_none = false) { Py_ssize_t threads; bool given; PyObject *rest = take_threads(kwargs, &threads, &given); @@ -5371,7 +5381,7 @@ PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject * "threads requires the format 'whitespace', 'lines' or 'json_seq'"); return nullptr; } - PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1); + PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1, drop_none); Py_DECREF(self); return r; } @@ -5385,8 +5395,9 @@ PyObject *parse_many(PyObject *, PyObject *args, PyObject *kwargs) { } // map_many(fn, data, *, format, batch_size, threads, lazy=False, fields, -// default): [fn(doc) for doc in loads_many(data)] (parse_many with -// lazy=True), with fn called in the threads. +// default, drop_none=False): [fn(doc) for doc in loads_many(data)] +// (parse_many with lazy=True), with fn called in the threads; with +// drop_none, the results that are None are left out. PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { if (PyTuple_GET_SIZE(args) < 1) { PyErr_SetString(PyExc_TypeError, "map_many() missing required argument 'fn'"); @@ -5397,20 +5408,23 @@ PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { PyErr_SetString(PyExc_TypeError, "map_many() fn must be callable"); return nullptr; } - bool lazy = false; + bool lazy = false, drop_none = false; PyObject *rest = kwargs ? PyDict_Copy(kwargs) : nullptr; if (kwargs != nullptr && rest == nullptr) { return nullptr; } if (rest != nullptr) { - PyObject *l = PyDict_GetItemString(rest, "lazy"); // borrowed - if (l != nullptr) { - int t = PyObject_IsTrue(l); - if (t < 0 || PyDict_DelItemString(rest, "lazy") < 0) { - Py_DECREF(rest); - return nullptr; + for (auto [name, flag] : {std::pair{"lazy", &lazy}, + std::pair{"drop_none", &drop_none}}) { + PyObject *l = PyDict_GetItemString(rest, name); // borrowed + if (l != nullptr) { + int t = PyObject_IsTrue(l); + if (t < 0 || PyDict_DelItemString(rest, name) < 0) { + Py_DECREF(rest); + return nullptr; + } + *flag = t != 0; } - lazy = t != 0; } } PyObject *tail = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); @@ -5418,7 +5432,7 @@ PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) { Py_XDECREF(rest); return nullptr; } - PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true); + PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true, drop_none); Py_DECREF(tail); Py_XDECREF(rest); return r; @@ -6584,8 +6598,9 @@ PyMethodDef methods[] = { "values of the input becomes one line of NDJSON."}, {"map_many", reinterpret_cast(reinterpret_cast(map_many)), METH_VARARGS | METH_KEYWORDS, - "map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type): the " - "list of fn(doc) for the documents of data, with fn called in the threads."}, + "map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type, " + "drop_none=False): the list of fn(doc) for the documents of data, with fn called in " + "the threads; with drop_none, the results that are None are left out."}, {"parse", parse, METH_O, "Parse JSON lazily: objects and arrays are returned as read-only views " "(Object, Array) whose values are converted on access."}, diff --git a/tests/test_stream.py b/tests/test_stream.py index 29fa53a..6b18838 100644 --- a/tests/test_stream.py +++ b/tests/test_stream.py @@ -742,6 +742,29 @@ def test_long_record_then_threads(): assert fastsimdjson.loads_many(data, threads=4) == objs +@pytest.mark.parametrize("threads", [None, 1, 4]) +def test_map_many_drop_none(threads): + objs = [{"i": i, "keep": i % 7 == 0} for i in range(30000)] + data = ndjson(objs).encode() + pick = lambda d: d["i"] if d["keep"] else None + want = [o["i"] for o in objs if o["keep"]] + assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=True) == want + assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=False) == \ + [pick(o) for o in objs] + assert fastsimdjson.map_many(lambda t: t[0] if t[1] else None, data, threads=threads, + fields=["/i", "/keep"], drop_none=True) == want + assert fastsimdjson.map_many(lambda d: None, data, threads=threads, drop_none=True) == [] + # Falsy results other than None are kept. + assert fastsimdjson.map_many(lambda d: d["i"] % 2 and 0, data, threads=threads, + drop_none=True) == [0] * len(objs) + # An exception from fn still propagates; json's fallback still applies. + with pytest.raises(ZeroDivisionError): + fastsimdjson.map_many(lambda d: 1 // (d["i"] - 20000), data, threads=threads, + drop_none=True) + tail = data + b'{"i": 1e400, "keep": true}\n' + assert fastsimdjson.map_many(pick, tail, threads=threads, drop_none=True)[-1] == float("inf") + + def test_readonly_memoryview_in_place(tmp_path): import mmap objs = [{"id": i, "name": f"n{i}", "tags": ["a", "b"][: i % 3]} for i in range(20000)]