Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion .github/workflows/buildwheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,25 @@ jobs:
path: ./wheelhouse/*.whl
if-no-files-found: error

build_pyodide:
name: Build Pyodide (WebAssembly) wheel
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Build wheel
uses: pypa/cibuildwheel@v4.3.1
env:
CIBW_PLATFORM: pyodide

- uses: actions/upload-artifact@v4
with:
name: wheel-artifact-pyodide
path: ./wheelhouse/*.whl
if-no-files-found: error

upload_pypi:
needs: [build_wheels, build_sdist]
needs: [build_wheels, build_pyodide, build_sdist]
runs-on: ubuntu-latest
# Only upload to PyPI when triggered by a tag (not manual workflow_dispatch)
if: startsWith(github.ref, 'refs/tags/')
Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,16 @@ jobs:

- name: Test
run: python -m pytest tests -v

pyodide:
# WebAssembly: no SIMD kernels (the portable code paths), 32-bit, and
# no threads. cibuildwheel builds the wheel and runs the tests in Node.
name: Pyodide (WebAssembly)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Build and test
uses: pypa/cibuildwheel@v4.3.1
env:
CIBW_PLATFORM: pyodide
14 changes: 14 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ archs = ["x86_64", "universal2", "arm64"]
# are provided separately in the CI matrix, so each builds natively).
archs = ["native"]

[tool.cibuildwheel.pyodide]
# WebAssembly wheels, one per Pyodide ABI (cp313: Pyodide 0.28 and 0.29;
# cp314: Pyodide 314; cp315). In a Pyodide venv, the pytest script starts
# with a shebang to a shell script, which macOS cannot run: python -m pytest.
test-command = "python -m pytest {project}/tests -q"

[tool.cibuildwheel.windows]
# Build for 64-bit Windows (32-bit is rarely needed nowadays)
archs = ["AMD64"]

[tool.pyodide.build]
# Pyodide's default flags, but -O3 where they end with -Oz (size): dumps and
# dumpb are up to twice as fast, loads up to 15%. ($(...) variables are not
# expanded here, so the flags are spelled out.)
cflags = "-g0 -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -O3"
cxxflags = "-g0 -fPIC -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -O3"
ldflags = "-g0 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -s SIDE_MODULE=1 -O3"
32 changes: 32 additions & 0 deletions src/fastsimdjson.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>

// Pyodide's pyconfig.h defines _REENTRANT, which enables threads in simdjson:
// parse_many would then start a thread, which fails without pthreads, in a
// noexcept function (a crash).
#if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)
#undef _REENTRANT
#endif

#include <algorithm>
#include <atomic>
#include <cmath>
Expand Down Expand Up @@ -2177,6 +2184,26 @@ inline char *escape_into(char *p, const char *s, size_t len, bool escape_del) {
p = write_escaped_ascii(p, uint8_t(s[i]));
i++;
}
#else
// No SIMD (WebAssembly, other targets): 8 bytes at a time, so that under
// 16 bytes are left for the tail.
while (i + 16 <= len) {
uint64_t w = load_u64(s + i);
if (!word_needs_escape(w, escape_del)) {
memcpy(p, &w, 8);
p += 8;
i += 8;
continue;
}
for (size_t end = i + 8; i < end; i++) {
uint8_t c = uint8_t(s[i]);
if (byte_needs_escape(c, escape_del)) {
p = write_escaped_ascii(p, c);
} else {
*p++ = char(c);
}
}
}
#endif
// The tail (under 16 bytes) as two overlapping words: fixed-size loads and
// stores, no call to memcpy.
Expand Down Expand Up @@ -2813,6 +2840,11 @@ inline size_t first_escape(const char *s, size_t n) {
#endif
}
}
#else
// No SIMD: skip 8 bytes at a time; the byte loop below finds the exact one.
while (i + 8 <= n && !word_needs_escape(load_u64(s + i), false)) {
i += 8;
}
#endif
while (i < n && !byte_needs_escape(uint8_t(s[i]), false)) {
i++;
Expand Down
21 changes: 21 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import sys

import pytest

# Under Pyodide (WebAssembly), Python can start neither threads nor
# processes: the tests that need them are skipped.
EMSCRIPTEN = sys.platform == "emscripten"


def pytest_configure(config):
config.addinivalue_line("markers", "requires_threads: starts Python threads")
config.addinivalue_line("markers", "requires_processes: starts processes (fork, subprocess)")


def pytest_collection_modifyitems(config, items):
if not EMSCRIPTEN:
return
for item in items:
for name in ("requires_threads", "requires_processes"):
if item.get_closest_marker(name):
item.add_marker(pytest.mark.skip(reason=f"{name}: not available under Pyodide"))
12 changes: 10 additions & 2 deletions tests/test_dumps.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,10 @@ def test_circular():
d = {}
d["self"] = [d]
same_result(lambda m: m.dumps(d))
same_result(lambda m: m.dumps(d, check_circular=False))
# Unbounded recursion: under Pyodide, json too overflows the JavaScript
# stack (a fatal error) before any RecursionError.
if sys.platform != "emscripten":
same_result(lambda m: m.dumps(d, check_circular=False))
# Repeated (not circular) references are fine.
x = [1, 2]
assert fastsimdjson.dumps([x, x, {"k": x}]) == json.dumps([x, x, {"k": x}])
Expand Down Expand Up @@ -227,7 +230,10 @@ def test_deep_nesting():
# Where nesting runs out depends on the Python version (a counter before
# 3.14, the stack size since) and on each encoder's frames: the output
# must be correct, or the error a RecursionError, never a crash.
for depth in (1000, 10000, 100000, 1000000):
# Under Pyodide, the JavaScript stack runs out (a fatal error, for json
# too) before Python raises RecursionError: only the shallower depths.
depths = (1000,) if sys.platform == "emscripten" else (1000, 10000, 100000, 1000000)
for depth in depths:
deep = []
for _ in range(depth):
deep = [deep]
Expand Down Expand Up @@ -417,6 +423,7 @@ def test_dumpb_matches_orjson_cases():
orjson_outcome(orjson.dumps, obj, option=option), (obj, option)


@pytest.mark.requires_threads
def test_thread_exit_releases_caches():
# A key that enters dumpb's key cache (seen twice) is referenced by the
# cache; when the thread exits, its caches are released.
Expand Down Expand Up @@ -519,6 +526,7 @@ def default(o):
_error(lambda: fastsimdjson.dumpb(bad[2]))


@pytest.mark.requires_threads
def test_dumpb_many_under_mutation():
# Another thread changes the records while they are written: the output
# is still one valid document per record (free-threaded Python).
Expand Down
1 change: 1 addition & 0 deletions tests/test_lazy.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ def test_release():
assert fastsimdjson.parse(b"[3]").as_list() == [3]


@pytest.mark.requires_threads
def test_threads():
docs = [json.dumps({"t": t, "v": list(range(t * 10))}) for t in range(8)]
made = [fastsimdjson.parse(t) for t in docs]
Expand Down
5 changes: 5 additions & 0 deletions tests/test_loads.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,10 @@ def test_deep_nesting():
assert v == []
# simdjson's default depth is 1024. Past that, loads follows json.loads:
# the value if Python accepts it, otherwise the same exception class.
# (Under Pyodide, json.loads overflows the JavaScript stack, a fatal
# error, on 5000 levels.)
if sys.platform == "emscripten":
return
deep = "[" * 5000 + "]" * 5000
try:
expected = json.loads(deep)
Expand Down Expand Up @@ -306,6 +310,7 @@ def test_release_drops_key_cache():
assert next(iter(a)) is not next(iter(c))


@pytest.mark.requires_threads
def test_release_is_per_thread():
import threading

Expand Down
1 change: 1 addition & 0 deletions tests/test_msgpack.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ def test_large():
assert json.loads(fastsimdjson.msgpack_to_json(packed)) == value


@pytest.mark.requires_threads
def test_concurrent():
# Python threads at once (the GIL is released while we convert).
rng = random.Random(5)
Expand Down
5 changes: 5 additions & 0 deletions tests/test_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,7 @@ def boom(d):
_POOL_DATA = b"\n".join(b'{"n": %d, "s": "%s"}' % (i, b"x" * (i % 50)) for i in range(40000))


@pytest.mark.requires_threads
def test_threads_concurrent_callers():
import threading

Expand All @@ -368,6 +369,7 @@ def call():
assert errors == []


@pytest.mark.requires_processes
@pytest.mark.skipif(not hasattr(os, "fork"), reason="no fork")
def test_threads_after_fork():
import subprocess
Expand All @@ -389,6 +391,7 @@ def test_threads_after_fork():
assert r.returncode == 0 and r.stdout.strip() == "ok", r.stderr


@pytest.mark.requires_processes
def test_threads_exit_cleanly():
import subprocess
import sys
Expand Down Expand Up @@ -575,6 +578,7 @@ def test_release_after_threads():
assert fastsimdjson.map_many(len, data, threads=8) == [len(o) for o in objs]


@pytest.mark.requires_threads
def test_release_inside_map_many():
data, objs = _pool_input()

Expand All @@ -589,6 +593,7 @@ def body():
_no_hang(body)


@pytest.mark.requires_threads
def test_release_during_threaded_runs():
import threading

Expand Down
5 changes: 4 additions & 1 deletion tests/test_typed.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ def test_plan_is_cached():
assert loads(b"1", type=t) == 1


@pytest.mark.requires_threads
def test_threads():
# Threads that describe and use the same types at once (on free-threaded
# Python, really at once).
Expand Down Expand Up @@ -544,7 +545,9 @@ def test_deep_nesting_does_not_crash():
for _ in range(n - 1):
tree = tree.kids[0]
assert tree.kids == []
for n in (20000, 200000):
# Under Pyodide, json.loads overflows the JavaScript stack (a fatal
# error) on these depths before any RecursionError.
for n in () if sys.platform == "emscripten" else (20000, 200000):
data = b'{"kids":[' * n + b']}' * n
try:
loads(data, type=Tree)
Expand Down
Loading