Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,19 @@ document, and the Python objects of all of them, to the calling thread.
With the GIL (the default build of Python), the calling thread does all the
work: `threads` has no effect on speed.

With `drop_none=True`, `map_many` leaves out the results that are `None`:
when `fn` keeps only a few documents, the threads do not build a list
entry for each of the others, and the calling thread has fewer to collect.

```python
def starred(event):
kind, repo = event
return repo if kind == "WatchEvent" else None

names = fastsimdjson.map_many(starred, data, threads=64, drop_none=True,
fields=["/type", "/repo/name"])
```

### Only some fields: `fields=`

When you need a few values from each document, `fields` extracts them
Expand Down
47 changes: 31 additions & 16 deletions src/fastsimdjson.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4731,6 +4731,7 @@ struct ParallelRun : PoolTask {

StreamObject *whole;
PyObject *fn; // borrowed, or nullptr
bool drop_none = false; // map_many(drop_none=True): fn's None results are not kept
char delimiter = '\n';
bool snap_after = true; // false: the next slice starts on the delimiter
struct Chunk {
Expand Down Expand Up @@ -4855,6 +4856,10 @@ struct ParallelRun : PoolTask {
break;
}
v = r;
if (drop_none && v == Py_None) {
Py_DECREF(v);
continue;
}
}
try {
c.items.push_back(v);
Expand Down Expand Up @@ -5132,7 +5137,7 @@ void pool_after_fork() {

// All the results (documents, or fn of each) of the stream `whole`, as a
// list, using up to `threads` threads.
PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) {
PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads, bool drop_none) {
PyObject *result = PyList_New(0);
if (result == nullptr) {
return nullptr;
Expand All @@ -5143,6 +5148,7 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) {
ParallelRun run;
run.whole = whole;
run.fn = fn;
run.drop_none = drop_none;
// json_seq snaps onto the record separator so the next slice starts with
// it. Stage 1 finds a scalar value from that separator; a slice that
// started after it would drop the separator and mis-parse the scalar.
Expand Down Expand Up @@ -5223,6 +5229,10 @@ PyObject *run_stream(StreamObject *whole, PyObject *fn, size_t threads) {
break;
}
v = r;
if (drop_none && v == Py_None) {
Py_DECREF(v);
continue;
}
}
int rc = PyList_Append(result, v);
Py_DECREF(v);
Expand Down Expand Up @@ -5300,7 +5310,7 @@ PyObject *pop_keyword(PyObject *kwargs, const char *name) {
// all results instead. With fields (not for parse_many), the values at the
// fields' pointers come instead of the documents.
PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject *fn,
bool force_list, bool allow_fields) {
bool force_list, bool allow_fields, bool drop_none = false) {
Py_ssize_t threads;
bool given;
PyObject *rest = take_threads(kwargs, &threads, &given);
Expand Down Expand Up @@ -5371,7 +5381,7 @@ PyObject *stream_or_list(PyObject *args, PyObject *kwargs, bool lazy, PyObject *
"threads requires the format 'whitespace', 'lines' or 'json_seq'");
return nullptr;
}
PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1);
PyObject *r = run_stream(whole, fn, given ? size_t(threads) : 1, drop_none);
Py_DECREF(self);
return r;
}
Expand All @@ -5385,8 +5395,9 @@ PyObject *parse_many(PyObject *, PyObject *args, PyObject *kwargs) {
}

// map_many(fn, data, *, format, batch_size, threads, lazy=False, fields,
// default): [fn(doc) for doc in loads_many(data)] (parse_many with
// lazy=True), with fn called in the threads.
// default, drop_none=False): [fn(doc) for doc in loads_many(data)]
// (parse_many with lazy=True), with fn called in the threads; with
// drop_none, the results that are None are left out.
PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) {
if (PyTuple_GET_SIZE(args) < 1) {
PyErr_SetString(PyExc_TypeError, "map_many() missing required argument 'fn'");
Expand All @@ -5397,28 +5408,31 @@ PyObject *map_many(PyObject *, PyObject *args, PyObject *kwargs) {
PyErr_SetString(PyExc_TypeError, "map_many() fn must be callable");
return nullptr;
}
bool lazy = false;
bool lazy = false, drop_none = false;
PyObject *rest = kwargs ? PyDict_Copy(kwargs) : nullptr;
if (kwargs != nullptr && rest == nullptr) {
return nullptr;
}
if (rest != nullptr) {
PyObject *l = PyDict_GetItemString(rest, "lazy"); // borrowed
if (l != nullptr) {
int t = PyObject_IsTrue(l);
if (t < 0 || PyDict_DelItemString(rest, "lazy") < 0) {
Py_DECREF(rest);
return nullptr;
for (auto [name, flag] : {std::pair<const char *, bool *>{"lazy", &lazy},
std::pair<const char *, bool *>{"drop_none", &drop_none}}) {
PyObject *l = PyDict_GetItemString(rest, name); // borrowed
if (l != nullptr) {
int t = PyObject_IsTrue(l);
if (t < 0 || PyDict_DelItemString(rest, name) < 0) {
Py_DECREF(rest);
return nullptr;
}
*flag = t != 0;
}
lazy = t != 0;
}
}
PyObject *tail = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args));
if (tail == nullptr) {
Py_XDECREF(rest);
return nullptr;
}
PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true);
PyObject *r = stream_or_list(tail, rest, lazy, fn, true, true, drop_none);
Py_DECREF(tail);
Py_XDECREF(rest);
return r;
Expand Down Expand Up @@ -6584,8 +6598,9 @@ PyMethodDef methods[] = {
"values of the input becomes one line of NDJSON."},
{"map_many", reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)(void)>(map_many)),
METH_VARARGS | METH_KEYWORDS,
"map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type): the "
"list of fn(doc) for the documents of data, with fn called in the threads."},
"map_many(fn, data, *, format, batch_size, threads, lazy, fields, default, type, "
"drop_none=False): the list of fn(doc) for the documents of data, with fn called in "
"the threads; with drop_none, the results that are None are left out."},
{"parse", parse, METH_O,
"Parse JSON lazily: objects and arrays are returned as read-only views "
"(Object, Array) whose values are converted on access."},
Expand Down
23 changes: 23 additions & 0 deletions tests/test_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,29 @@ def test_long_record_then_threads():
assert fastsimdjson.loads_many(data, threads=4) == objs


@pytest.mark.parametrize("threads", [None, 1, 4])
def test_map_many_drop_none(threads):
objs = [{"i": i, "keep": i % 7 == 0} for i in range(30000)]
data = ndjson(objs).encode()
pick = lambda d: d["i"] if d["keep"] else None
want = [o["i"] for o in objs if o["keep"]]
assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=True) == want
assert fastsimdjson.map_many(pick, data, threads=threads, drop_none=False) == \
[pick(o) for o in objs]
assert fastsimdjson.map_many(lambda t: t[0] if t[1] else None, data, threads=threads,
fields=["/i", "/keep"], drop_none=True) == want
assert fastsimdjson.map_many(lambda d: None, data, threads=threads, drop_none=True) == []
# Falsy results other than None are kept.
assert fastsimdjson.map_many(lambda d: d["i"] % 2 and 0, data, threads=threads,
drop_none=True) == [0] * len(objs)
# An exception from fn still propagates; json's fallback still applies.
with pytest.raises(ZeroDivisionError):
fastsimdjson.map_many(lambda d: 1 // (d["i"] - 20000), data, threads=threads,
drop_none=True)
tail = data + b'{"i": 1e400, "keep": true}\n'
assert fastsimdjson.map_many(pick, tail, threads=threads, drop_none=True)[-1] == float("inf")


def test_readonly_memoryview_in_place(tmp_path):
import mmap
objs = [{"id": i, "name": f"n{i}", "tags": ["a", "b"][: i % 3]} for i in range(20000)]
Expand Down
Loading