Skip to content

Commit 96bc30b

Browse files
committed
JSON to MessagePack and back, with simdjson
Converts JSON (one document or newline-delimited) to the most compact MessagePack with simdjson's On-Demand parser, and MessagePack (one value or a sequence) to JSON, on one thread or many. C++17 and C interfaces, output into a buffer of the caller's allocator, one header and one source file. Tests, benchmarks against yyjson, RapidJSON + mpack and nlohmann::json.
0 parents  commit 96bc30b

21 files changed

Lines changed: 7117 additions & 0 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Builds with CMake and runs the tests: GCC and clang (with sanitizers) on
2+
# Linux, Apple clang on macOS, MSVC on Windows.
3+
name: ci
4+
5+
on:
6+
push:
7+
branches: [main]
8+
pull_request:
9+
workflow_dispatch:
10+
11+
jobs:
12+
test:
13+
strategy:
14+
fail-fast: false
15+
matrix:
16+
include:
17+
- os: ubuntu-latest
18+
cxx: g++
19+
- os: ubuntu-latest
20+
cxx: clang++
21+
flags: -fsanitize=address,undefined -fno-sanitize-recover=all
22+
build_type: Debug
23+
- os: ubuntu-latest
24+
cxx: clang++
25+
flags: -fsanitize=thread
26+
build_type: RelWithDebInfo
27+
- os: macos-latest
28+
cxx: c++
29+
- os: windows-latest
30+
runs-on: ${{ matrix.os }}
31+
steps:
32+
- uses: actions/checkout@v4
33+
- name: Configure
34+
shell: bash
35+
run: |
36+
args=(-DCMAKE_BUILD_TYPE=${{ matrix.build_type || 'Release' }})
37+
if [ -n "${{ matrix.cxx }}" ]; then args+=(-DCMAKE_CXX_COMPILER=${{ matrix.cxx }}); fi
38+
if [ -n "${{ matrix.flags }}" ]; then args+=("-DCMAKE_CXX_FLAGS=${{ matrix.flags }}"); fi
39+
cmake -B build "${args[@]}"
40+
- name: Build
41+
run: cmake --build build --config ${{ matrix.build_type || 'Release' }} -j 4
42+
- name: Test
43+
run: ctest --test-dir build -C ${{ matrix.build_type || 'Release' }} --output-on-failure

‎.github/workflows/release.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Writes the library as one header and one source file (tools/amalgamate.py),
2+
# tests them with simdjson's single files on Linux, macOS and Windows, and, for
3+
# a tag v*, attaches them to the GitHub release (with singleheader.zip, which
4+
# holds both, as simdjson does).
5+
name: single files and release
6+
7+
on:
8+
push:
9+
branches: [main]
10+
tags: ["v*"]
11+
pull_request:
12+
workflow_dispatch:
13+
14+
env:
15+
SIMDJSON_VERSION: v5.0.7
16+
17+
jobs:
18+
test:
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
os: [ubuntu-latest, macos-latest, windows-latest]
23+
runs-on: ${{ matrix.os }}
24+
steps:
25+
- uses: actions/checkout@v4
26+
with:
27+
fetch-depth: 0 # git describe gives the version in the banner
28+
- name: Write the single files
29+
run: python3 tools/amalgamate.py singleheader
30+
- name: Get simdjson's single files
31+
shell: bash
32+
run: |
33+
for f in simdjson.h simdjson.cpp; do
34+
curl -sSfL -o singleheader/$f \
35+
https://raw.githubusercontent.com/simdjson/simdjson/$SIMDJSON_VERSION/singleheader/$f
36+
done
37+
- uses: ilammy/msvc-dev-cmd@v1
38+
if: runner.os == 'Windows'
39+
- name: Build and run the tests (GCC or clang)
40+
if: runner.os != 'Windows'
41+
working-directory: singleheader
42+
run: |
43+
c++ -std=c++17 -O2 -Wall -I. ../tests/basic_tests.cpp simdjson_msgpack.cpp simdjson.cpp -o basic_tests
44+
./basic_tests
45+
c++ -std=c++17 -O2 -Wall -I. ../examples/example.cpp simdjson_msgpack.cpp simdjson.cpp -o example
46+
./example
47+
- name: Build and run the tests (MSVC)
48+
if: runner.os == 'Windows'
49+
working-directory: singleheader
50+
shell: cmd
51+
run: |
52+
cl /nologo /std:c++17 /O2 /EHsc /I. ..\tests\basic_tests.cpp simdjson_msgpack.cpp simdjson.cpp /Fe:basic_tests.exe || exit /b 1
53+
basic_tests.exe || exit /b 1
54+
cl /nologo /std:c++17 /O2 /EHsc /I. ..\examples\example.cpp simdjson_msgpack.cpp simdjson.cpp /Fe:example.exe || exit /b 1
55+
example.exe
56+
- uses: actions/upload-artifact@v4
57+
if: runner.os == 'Linux'
58+
with:
59+
name: singleheader
60+
path: |
61+
singleheader/simdjson_msgpack.h
62+
singleheader/simdjson_msgpack.cpp
63+
64+
release:
65+
if: startsWith(github.ref, 'refs/tags/v')
66+
needs: test
67+
runs-on: ubuntu-latest
68+
permissions:
69+
contents: write
70+
steps:
71+
- uses: actions/download-artifact@v4
72+
with:
73+
name: singleheader
74+
path: singleheader
75+
- name: Attach the files to the release
76+
working-directory: singleheader
77+
env:
78+
GH_TOKEN: ${{ github.token }}
79+
GH_REPO: ${{ github.repository }}
80+
TAG: ${{ github.ref_name }}
81+
run: |
82+
zip -j singleheader.zip simdjson_msgpack.h simdjson_msgpack.cpp
83+
gh release view "$TAG" > /dev/null 2>&1 || gh release create "$TAG" --title "$TAG" --generate-notes
84+
gh release upload "$TAG" simdjson_msgpack.h simdjson_msgpack.cpp singleheader.zip --clobber

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
build/
2+
build-*/
3+
singleheader/

‎CMakeLists.txt‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
cmake_minimum_required(VERSION 3.16)
2+
project(simdjson_msgpack VERSION 0.1.0 LANGUAGES CXX)
3+
4+
set(CMAKE_CXX_STANDARD 17)
5+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
6+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
7+
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
8+
set(CMAKE_BUILD_TYPE Release)
9+
endif()
10+
11+
option(SIMDJSON_MSGPACK_TESTS "Build the tests" ON)
12+
option(SIMDJSON_MSGPACK_BENCHMARKS "Build the benchmarks" ON)
13+
option(SIMDJSON_MSGPACK_EXAMPLES "Build the examples" ON)
14+
# The benchmark of other libraries (nlohmann/json, yyjson, RapidJSON, mpack),
15+
# which CMake downloads.
16+
option(SIMDJSON_MSGPACK_COMPETITION "Benchmark other libraries too" OFF)
17+
18+
include(FetchContent)
19+
FetchContent_Declare(simdjson
20+
URL https://github.com/simdjson/simdjson/archive/refs/tags/v5.0.7.tar.gz
21+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
22+
set(SIMDJSON_DEVELOPER_MODE OFF CACHE BOOL "" FORCE)
23+
FetchContent_MakeAvailable(simdjson)
24+
25+
# zmij (third_party/zmij) writes floating-point numbers.
26+
add_library(zmij OBJECT third_party/zmij/zmij.cc)
27+
28+
set(SIMDJSON_MSGPACK_SOURCES src/to_msgpack.cpp src/to_json.cpp src/c_api.cpp
29+
$<TARGET_OBJECTS:zmij>)
30+
31+
add_library(simdjson_msgpack STATIC ${SIMDJSON_MSGPACK_SOURCES})
32+
target_include_directories(simdjson_msgpack PUBLIC include PRIVATE third_party/zmij)
33+
target_link_libraries(simdjson_msgpack PUBLIC simdjson::simdjson)
34+
35+
# A shared library with the C entry points, for bindings and the Python tests (ctypes).
36+
add_library(simdjson_msgpack_c SHARED ${SIMDJSON_MSGPACK_SOURCES})
37+
target_include_directories(simdjson_msgpack_c PUBLIC include PRIVATE third_party/zmij)
38+
target_link_libraries(simdjson_msgpack_c PRIVATE simdjson::simdjson)
39+
40+
if(SIMDJSON_MSGPACK_TESTS)
41+
enable_testing()
42+
add_executable(basic_tests tests/basic_tests.cpp)
43+
target_link_libraries(basic_tests PRIVATE simdjson_msgpack)
44+
add_test(NAME basic_tests COMMAND basic_tests)
45+
endif()
46+
47+
if(SIMDJSON_MSGPACK_BENCHMARKS)
48+
add_executable(bench benchmarks/bench.cpp)
49+
target_link_libraries(bench PRIVATE simdjson_msgpack)
50+
if(SIMDJSON_MSGPACK_COMPETITION)
51+
FetchContent_Declare(nlohmann_json
52+
URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
53+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
54+
FetchContent_Declare(yyjson
55+
URL https://github.com/ibireme/yyjson/archive/refs/tags/0.12.0.tar.gz
56+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
57+
FetchContent_Declare(rapidjson
58+
URL https://github.com/Tencent/rapidjson/archive/24b5e7a8b27f42fa16b96fc70aade9106cf7102f.tar.gz
59+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
60+
FetchContent_Declare(mpack
61+
URL https://github.com/ludocode/mpack/releases/download/v1.1.1/mpack-amalgamation-1.1.1.tar.gz
62+
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
63+
FetchContent_MakeAvailable(yyjson)
64+
FetchContent_Populate(nlohmann_json) # header only
65+
FetchContent_Populate(rapidjson) # header only
66+
FetchContent_Populate(mpack)
67+
add_library(mpack STATIC ${mpack_SOURCE_DIR}/src/mpack/mpack.c)
68+
target_include_directories(mpack PUBLIC ${mpack_SOURCE_DIR}/src/mpack)
69+
target_compile_definitions(mpack PUBLIC MPACK_EXTENSIONS=1)
70+
set_target_properties(mpack PROPERTIES LINKER_LANGUAGE C)
71+
enable_language(C)
72+
add_executable(bench_competition benchmarks/bench_competition.cpp)
73+
target_include_directories(bench_competition PRIVATE
74+
${nlohmann_json_SOURCE_DIR}/single_include ${rapidjson_SOURCE_DIR}/include)
75+
target_link_libraries(bench_competition PRIVATE simdjson_msgpack yyjson mpack)
76+
endif()
77+
endif()
78+
79+
if(SIMDJSON_MSGPACK_EXAMPLES)
80+
add_executable(example examples/example.cpp)
81+
target_link_libraries(example PRIVATE simdjson_msgpack)
82+
if(SIMDJSON_MSGPACK_TESTS)
83+
add_test(NAME example COMMAND example)
84+
endif()
85+
endif()

0 commit comments

Comments
 (0)