diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml index 1a574af..0c7ce6e 100644 --- a/.github/workflows/test_ci.yml +++ b/.github/workflows/test_ci.yml @@ -26,15 +26,22 @@ jobs: windows-latest, # x86_64 windows-11-arm, # arm64 - ] + ] + rust: [ + 1.85.0, # MSRV of this project + stable + ] + max-parallel: 4 steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ matrix.rust }} - - name: Build and Test on ${{ matrix.os }} + - name: Build and Test on ${{ matrix.os }}, with Rust ${{ matrix.rust }} run: make test - - name: Test decds CLI on Linux/Mac + - name: Test `decds` CLI on ${{ matrix.os }}, with Rust ${{ matrix.rust }} if: runner.os == 'Linux' || runner.os == 'macOS' - run: bash scripts/test_decds_on_linux.sh + run: bash scripts/test_decds_on_unix.sh diff --git a/Cargo.lock b/Cargo.lock index 219cc09..6cecada 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,39 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "0.6.19" @@ -52,6 +85,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayref" version = "0.3.9" @@ -70,6 +115,21 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets 0.52.6", +] + [[package]] name = "bincode" version = "2.0.1" @@ -110,6 +170,38 @@ dependencies = [ "serde", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.28" @@ -166,6 +258,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "clru" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" + [[package]] name = "colorchoice" version = "1.0.4" @@ -178,6 +276,29 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" +[[package]] +name = "console" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "console_static_text" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55d8a913e62f6444b79e038be3eb09839e9cfc34d55d85f9336460710647d2f6" +dependencies = [ + "unicode-width 0.1.14", + "vte", +] + [[package]] name = "const-hex" version = "1.14.1" @@ -206,6 +327,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -231,20 +361,84 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "decds" -version = "0.1.0" +version = "0.2.0" dependencies = [ "blake3", "clap", + "console", + "console_static_text", "const-hex", "decds-lib", + "num_cpus", "rand", + "tokio", + "vergen-gix", ] [[package]] name = "decds-lib" -version = "0.1.0" +version = "0.2.0" dependencies = [ "bincode", "blake3", @@ -255,6 +449,67 @@ dependencies = [ "serde", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "divan" version = "0.1.21" @@ -280,12 +535,33 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "errno" version = "0.3.13" @@ -297,140 +573,1479 @@ dependencies = [ ] [[package]] -name = "getrandom" -version = "0.3.3" +name = "faster-hex" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "a2a2b11eda1d40935b26cf18f6833c526845ae8c41e58d09af6adeb6f0269183" dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi", + "serde", ] [[package]] -name = "heck" -version = "0.5.0" +name = "faster-hex" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless", + "serde", +] [[package]] -name = "hex" -version = "0.4.3" +name = "fastrand" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "is_terminal_polyfill" -version = "1.70.1" +name = "filetime" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "flate2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +dependencies = [ + "crc32fast", + "miniz_oxide", +] [[package]] -name = "libc" -version = "0.2.174" +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "linux-raw-sys" -version = "0.9.4" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] -name = "num-traits" -version = "0.2.19" +name = "form_urlencoded" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ - "autocfg", + "percent-encoding", ] [[package]] -name = "once_cell_polyfill" -version = "1.70.1" +name = "generic-array" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "getrandom" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ - "zerocopy", + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] -name = "proc-macro2" -version = "1.0.95" +name = "gimli" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "gix" +version = "0.71.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a61e71ec6817fc3c9f12f812682cfe51ee6ea0d2e27e02fc3849c35524617435" dependencies = [ - "unicode-ident", + "gix-actor", + "gix-attributes", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-features 0.41.1", + "gix-filter", + "gix-fs 0.14.0", + "gix-glob", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils 0.2.0", + "gix-validate 0.9.4", + "gix-worktree", + "once_cell", + "parking_lot", + "signal-hook", + "smallvec", + "thiserror", ] [[package]] -name = "proptest" -version = "1.7.0" +name = "gix-actor" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" +checksum = "f438c87d4028aca4b82f82ba8d8ab1569823cfb3e5bc5fa8456a71678b2a20e7" dependencies = [ - "bitflags", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "unarray", + "bstr", + "gix-date", + "gix-utils 0.2.0", + "itoa", + "thiserror", + "winnow", ] [[package]] -name = "quote" -version = "1.0.40" +name = "gix-attributes" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "e4e25825e0430aa11096f8b65ced6780d4a96a133f81904edceebb5344c8dd7f" dependencies = [ - "proc-macro2", + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror", + "unicode-bom", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "gix-bitmap" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "b1db9765c69502650da68f0804e3dc2b5f8ccc6a2d104ca6c85bc40700d37540" +dependencies = [ + "thiserror", +] [[package]] -name = "rand" -version = "0.9.1" +name = "gix-chunk" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "0b1f1d8764958699dc764e3f727cef280ff4d1bd92c107bbf8acd85b30c1bd6f" dependencies = [ - "rand_chacha", - "rand_core", + "thiserror", ] [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "gix-command" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "c0378995847773a697f8e157fe2963ecf3462fe64be05b7b3da000b3b472def8" dependencies = [ - "ppv-lite86", - "rand_core", + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", ] [[package]] -name = "rand_core" -version = "0.9.3" +name = "gix-commitgraph" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043cbe49b7a7505150db975f3cb7c15833335ac1e26781f615454d9d640a28fe" +dependencies = [ + "bstr", + "gix-chunk", + "gix-hash 0.17.0", + "memmap2", + "thiserror", +] + +[[package]] +name = "gix-config" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6f830bf746604940261b49abf7f655d2c19cadc9f4142ae9379e3a316e8cfa" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features 0.41.1", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "memchr", + "once_cell", + "smallvec", + "thiserror", + "unicode-bom", + "winnow", +] + +[[package]] +name = "gix-config-value" +version = "0.14.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc2c844c4cf141884678cabef736fd91dd73068b9146e6f004ba1a0457944b6" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror", +] + +[[package]] +name = "gix-date" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daa30058ec7d3511fbc229e4f9e696a35abd07ec5b82e635eff864a2726217e4" +dependencies = [ + "bstr", + "itoa", + "jiff", + "thiserror", +] + +[[package]] +name = "gix-diff" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c975dad2afc85e4e233f444d1efbe436c3cdcf3a07173984509c436d00a3f8" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "imara-diff", + "thiserror", +] + +[[package]] +name = "gix-dir" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5879497bd3815d8277ed864ec8975290a70de5b62bb92d2d666a4cefc5d4793b" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs 0.14.0", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils 0.2.0", + "gix-worktree", + "thiserror", +] + +[[package]] +name = "gix-discover" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fb8a4349b854506a3915de18d3341e5f1daa6b489c8affc9ca0d69efe86781" +dependencies = [ + "bstr", + "dunce", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror", +] + +[[package]] +name = "gix-features" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016d6050219458d14520fe22bdfdeb9cb71631dec9bc2724767c983f60109634" +dependencies = [ + "crc32fast", + "flate2", + "gix-path", + "gix-trace", + "gix-utils 0.2.0", + "libc", + "once_cell", + "prodash", + "thiserror", + "walkdir", +] + +[[package]] +name = "gix-features" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f4399af6ec4fd9db84dd4cf9656c5c785ab492ab40a7c27ea92b4241923fed" +dependencies = [ + "gix-trace", + "gix-utils 0.3.0", + "libc", + "prodash", +] + +[[package]] +name = "gix-filter" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb2b2bbffdc5cc9b2b82fc82da1b98163c9b423ac2b45348baa83a947ac9ab89" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash 0.17.0", + "gix-object", + "gix-packetline-blocking", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils 0.2.0", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-fs" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "951e886120dc5fa8cac053e5e5c89443f12368ca36811b2e43d1539081f9c111" +dependencies = [ + "bstr", + "fastrand", + "gix-features 0.41.1", + "gix-path", + "gix-utils 0.2.0", + "thiserror", +] + +[[package]] +name = "gix-fs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a0637149b4ef24d3ea55f81f77231401c8463fae6da27331c987957eb597c7" +dependencies = [ + "bstr", + "fastrand", + "gix-features 0.42.1", + "gix-path", + "gix-utils 0.3.0", + "thiserror", +] + +[[package]] +name = "gix-glob" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20972499c03473e773a2099e5fd0c695b9b72465837797a51a43391a1635a030" +dependencies = [ + "bitflags", + "bstr", + "gix-features 0.41.1", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "834e79722063958b03342edaa1e17595cd2939bb2b3306b3225d0815566dcb49" +dependencies = [ + "faster-hex 0.9.0", + "gix-features 0.41.1", + "sha1-checked", + "thiserror", +] + +[[package]] +name = "gix-hash" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d4900562c662852a6b42e2ef03442eccebf24f047d8eab4f23bc12ef0d785d8" +dependencies = [ + "faster-hex 0.10.0", + "gix-features 0.42.1", + "sha1-checked", + "thiserror", +] + +[[package]] +name = "gix-hashtable" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b5cb3c308b4144f2612ff64e32130e641279fcf1a84d8d40dad843b4f64904" +dependencies = [ + "gix-hash 0.18.0", + "hashbrown 0.14.5", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a27c8380f493a10d1457f756a3f81924d578fc08d6535e304dfcafbf0261d18" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-index" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "855bece2d4153453aa5d0a80d51deea1ce8cd6a3b4cf213da85ac344ccb908a7" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features 0.41.1", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils 0.2.0", + "gix-validate 0.9.4", + "hashbrown 0.14.5", + "itoa", + "libc", + "memmap2", + "rustix 0.38.44", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-lock" +version = "17.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "570f8b034659f256366dc90f1a24924902f20acccd6a15be96d44d1269e7a796" +dependencies = [ + "gix-tempfile", + "gix-utils 0.3.0", + "thiserror", +] + +[[package]] +name = "gix-object" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4943fcdae6ffc135920c9ea71e0362ed539182924ab7a85dd9dac8d89b0dd69a" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features 0.41.1", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-path", + "gix-utils 0.2.0", + "gix-validate 0.9.4", + "itoa", + "smallvec", + "thiserror", + "winnow", +] + +[[package]] +name = "gix-odb" +version = "0.68.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50306d40dcc982eb6b7593103f066ea6289c7b094cb9db14f3cd2be0b9f5e610" +dependencies = [ + "arc-swap", + "gix-date", + "gix-features 0.41.1", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror", +] + +[[package]] +name = "gix-pack" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b65fffb09393c26624ca408d32cfe8776fb94cd0a5cdf984905e1d2f39779cb" +dependencies = [ + "clru", + "gix-chunk", + "gix-features 0.41.1", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-object", + "gix-path", + "memmap2", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-packetline" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123844a70cf4d5352441dc06bab0da8aef61be94ec239cb631e0ba01dc6d3a04" +dependencies = [ + "bstr", + "faster-hex 0.9.0", + "gix-trace", + "thiserror", +] + +[[package]] +name = "gix-packetline-blocking" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecf3ea2e105c7e45587bac04099824301262a6c43357fad5205da36dbb233b3" +dependencies = [ + "bstr", + "faster-hex 0.9.0", + "gix-trace", + "thiserror", +] + +[[package]] +name = "gix-path" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6279d323d925ad4790602105ae27df4b915e7a7d81e4cdba2603121c03ad111" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate 0.10.0", + "home", + "once_cell", + "thiserror", +] + +[[package]] +name = "gix-pathspec" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fef8422c3c9066d649074b24025125963f85232bfad32d6d16aea9453b82ec14" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror", +] + +[[package]] +name = "gix-protocol" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5678ddae1d62880bc30e2200be1b9387af3372e0e88e21f81b4e7f8367355b5a" +dependencies = [ + "bstr", + "gix-date", + "gix-features 0.41.1", + "gix-hash 0.17.0", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils 0.2.0", + "maybe-async", + "thiserror", + "winnow", +] + +[[package]] +name = "gix-quote" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b005c550bf84de3b24aa5e540a23e6146a1c01c7d30470e35d75a12f827f969" +dependencies = [ + "bstr", + "gix-utils 0.2.0", + "thiserror", +] + +[[package]] +name = "gix-ref" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2e1f7eb6b7ce82d2d19961f74bd637bab3ea79b1bc7bfb23dbefc67b0415d8b" +dependencies = [ + "gix-actor", + "gix-features 0.41.1", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils 0.2.0", + "gix-validate 0.9.4", + "memmap2", + "thiserror", + "winnow", +] + +[[package]] +name = "gix-refspec" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8587b21e2264a6e8938d940c5c99662779c13a10741a5737b15fc85c252ffc" +dependencies = [ + "bstr", + "gix-hash 0.17.0", + "gix-revision", + "gix-validate 0.9.4", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-revision" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "342caa4e158df3020cadf62f656307c3948fe4eacfdf67171d7212811860c3e9" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "thiserror", +] + +[[package]] +name = "gix-revwalk" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc7c3d7e5cdc1ab8d35130106e4af0a4f9f9eca0c81f4312b690780e92bde0d" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-sec" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47aeb0f13de9ef2f3033f5ff218de30f44db827ac9f1286f9ef050aacddd5888" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "gix-shallow" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0598aacfe1d52575a21c9492fee086edbb21e228ec36c819c42ab923f434c3" +dependencies = [ + "bstr", + "gix-hash 0.17.0", + "gix-lock", + "thiserror", +] + +[[package]] +name = "gix-status" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605a6d0eb5891680c46e24b2ee7a63ef7bd39cb136dc7c7e55172960cf68b2f5" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features 0.41.1", + "gix-filter", + "gix-fs 0.14.0", + "gix-hash 0.17.0", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror", +] + +[[package]] +name = "gix-submodule" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c7390c2059505c365e9548016d4edc9f35749c6a9112b7b1214400bbc68da2" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror", +] + +[[package]] +name = "gix-tempfile" +version = "17.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c750e8c008453a2dba67a2b0d928b7716e05da31173a3f5e351d5457ad4470aa" +dependencies = [ + "dashmap", + "gix-fs 0.15.0", + "libc", + "once_cell", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ccaf54b0b1743a695b482ca0ab9d7603744d8d10b2e5d1a332fef337bee658" + +[[package]] +name = "gix-transport" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3f68c2870bfca8278389d2484a7f2215b67d0b0cc5277d3c72ad72acf41787e" +dependencies = [ + "bstr", + "gix-command", + "gix-features 0.41.1", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror", +] + +[[package]] +name = "gix-traverse" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c0b049f8bdb61b20016694102f7b507f2e1727e83e9c5e6dad4f7d84ff7384" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash 0.17.0", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-url" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dfe23f93f1ddb84977d80bb0dd7aa09d1bf5d5afc0c9b6820cccacc25ae860" +dependencies = [ + "bstr", + "gix-features 0.41.1", + "gix-path", + "percent-encoding", + "thiserror", + "url", +] + +[[package]] +name = "gix-utils" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189f8724cf903e7fd57cfe0b7bc209db255cacdcb22c781a022f52c3a774f8d0" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-utils" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5351af2b172caf41a3728eb4455326d84e0d70fe26fc4de74ab0bd37df4191c5" +dependencies = [ + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b5f1253109da6c79ed7cf6e1e38437080bb6d704c76af14c93e2f255234084" +dependencies = [ + "bstr", + "thiserror", +] + +[[package]] +name = "gix-validate" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b9e00cacde5b51388d28ed746c493b18a6add1f19b5e01d686b3b9ece66d4d" +dependencies = [ + "bstr", + "thiserror", +] + +[[package]] +name = "gix-worktree" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7760dbc4b79aa274fed30adc0d41dca6b917641f26e7867c4071b1fb4dc727b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-features 0.41.1", + "gix-fs 0.14.0", + "gix-glob", + "gix-hash 0.17.0", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate 0.9.4", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +dependencies = [ + "foldhash", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "imara-diff" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17d34b7d42178945f775e84bc4c36dde7c1c6cdfea656d3354d009056f2bb3d2" +dependencies = [ + "hashbrown 0.15.4", +] + +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jiff" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", + "windows-sys 0.59.0", +] + +[[package]] +name = "jiff-static" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libredox" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "maybe-async" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "memmap2" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "29.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04bb108f648884c23b98a0e940ebc2c93c0c3b89f04dbaf7eb8256ce617d1bc" +dependencies = [ + "log", + "parking_lot", +] + +[[package]] +name = "proptest" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" +dependencies = [ + "bitflags", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "unarray", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ @@ -438,82 +2053,294 @@ dependencies = [ ] [[package]] -name = "rand_xorshift" -version = "0.4.0" +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rlnc" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd5e3738e68f190aef9b75a355cfe7112aad801434a857117a5001ac5d704d6" +dependencies = [ + "rand", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.9.4", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest", + "sha1", +] + +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ - "rand_core", + "libc", + "signal-hook-registry", ] [[package]] -name = "rayon" -version = "1.10.0" +name = "signal-hook-registry" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ - "either", - "rayon-core", + "libc", ] [[package]] -name = "rayon-core" -version = "1.12.1" +name = "slab" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "libc", + "windows-sys 0.52.0", ] [[package]] -name = "regex-lite" -version = "0.1.6" +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "regex-syntax" -version = "0.8.5" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "rlnc" -version = "0.4.0" +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c3be8084b2485662e800617d171bb673e27d7347f0b7fc7018b214a933dcbb" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ - "rand", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "rustix" -version = "1.0.7" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.0.7", "windows-sys 0.59.0", ] [[package]] -name = "serde" -version = "1.0.219" +name = "terminal_size" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" dependencies = [ - "serde_derive", + "rustix 1.0.7", + "windows-sys 0.59.0", ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "thiserror" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", @@ -521,68 +2348,256 @@ dependencies = [ ] [[package]] -name = "shlex" -version = "1.3.0" +name = "time" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] [[package]] -name = "strsim" -version = "0.11.1" +name = "time-core" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] -name = "syn" -version = "2.0.104" +name = "time-macros" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "num-conv", + "time-core", ] [[package]] -name = "terminal_size" -version = "0.4.2" +name = "tinystr" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ - "rustix", - "windows-sys 0.59.0", + "displaydoc", + "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "unarray" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" + [[package]] name = "unty" version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vergen" +version = "9.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b2bf58be11fc9414104c6d3a2e464163db5ef74b12296bda593cac37b6e4777" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen-lib", +] + +[[package]] +name = "vergen-gix" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f8dfe6eb333a1397e596164ae7326f68e4b95267b41aedc6aa3c81f3426a010" +dependencies = [ + "anyhow", + "derive_builder", + "gix", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "virtue" version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" +[[package]] +name = "vte" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a0b683b20ef64071ff03745b14391751f6beab06a54347885459b77a3f2caa5" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.14.2+wasi-0.2.4" @@ -592,6 +2607,24 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -738,6 +2771,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" @@ -747,6 +2789,36 @@ dependencies = [ "bitflags", ] +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.26" @@ -766,3 +2838,57 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 1ebcc73..9f993bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["decds-lib", "decds-bin"] resolver = "3" [workspace.package] -version = "0.1.0" +version = "0.2.0" edition = "2024" rust-version = "1.85.0" authors = ["Anjan Roy "] @@ -21,19 +21,24 @@ categories = ["cryptography", "compression", "encoding"] [workspace.dependencies] blake3 = { version = "=1.8.2", features = ["serde"] } -rlnc = { version = "=0.4.0" } +rlnc = { version = "=0.7.0" } rand = "=0.9.1" serde = { version = "=1.0.219", features = ["derive"] } bincode = { version = "=2.0.1", features = ["serde"] } rayon = "=1.10.0" clap = { version = "=4.5.41", features = ["derive"] } const-hex = "=1.14.1" +num_cpus = "=1.17.0" +tokio = { version = "=1.46.1", features = ["full"] } +console_static_text = "=0.8.3" +console = "=0.16.0" [profile.optimized] inherits = "release" codegen-units = 1 lto = "thin" panic = "abort" +opt-level = 3 [profile.test-release] inherits = "release" diff --git a/Makefile b/Makefile index 334033e..34b5418 100644 --- a/Makefile +++ b/Makefile @@ -26,9 +26,13 @@ bench: ## Run all benchmarks coverage: ## Generates HTML code coverage report, using `cargo-tarpaulin` cargo tarpaulin -t 600 --profile test-release --out Html +.PHONY: build +build: ## Builds `decds` executable, placing in `./target/optimized/decds` + RUSTFLAGS='-C target-cpu=native' cargo build --profile optimized + .PHONY: install -install: # Installs `decds` executable in `$HOME/.cargo/bin` - cargo install --profile optimized --path decds-bin --locked +install: ## Installs `decds` executable in `$HOME/.cargo/bin` + RUSTFLAGS='-C target-cpu=native' cargo install --profile optimized --path decds-bin --locked .PHONY: clean clean: ## Removes cargo target directory diff --git a/README.md b/README.md index f97b806..ed89858 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,13 @@ test decds-lib/src/lib.rs - (line 59) ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.71s ``` +For testing functional correctness of `decds` CLI, run following command on Unix-like environment. + +```bash +# I've tested it on major Linux distributions and MacOS +bash scripts/test_decds_on_unix.sh +``` + ## Code Coverage To generate a detailed code coverage report in HTML format, use [cargo-tarpaulin](https://github.com/xd009642/tarpaulin): @@ -98,32 +105,48 @@ This will create an HTML coverage report at `tarpaulin-report.html` that you can ```bash Coverage Results: +Coverage Results: || Tested/Total Lines: -|| decds-bin/src/utils.rs: 0/19 -|| decds-lib/src/blob.rs: 58/94 +|| decds-bin/src/handlers/handle_break.rs: 0/57 +|| decds-bin/src/utils.rs: 0/24 +|| decds-lib/src/blob.rs: 79/121 || decds-lib/src/chunk.rs: 12/16 -|| decds-lib/src/chunkset.rs: 29/36 +|| decds-lib/src/chunkset.rs: 28/33 || decds-lib/src/errors.rs: 0/18 || decds-lib/src/merkle_tree.rs: 32/33 || -60.65% coverage, 131/216 lines covered +50.00% coverage, 151/302 lines covered ``` ## Installation For hands-on experience, install `decds` on your `$HOME/.cargo/bin`. ```bash -cargo install --profile optimized --git https://github.com/itzmeanjan/decds.git --locked +# ------------------------------ System-wide installation ------------------------------ + +RUSTFLAGS='-C target-cpu=native' cargo install --profile optimized --git https://github.com/itzmeanjan/decds.git --locked -# or +# Or, first clone and then install it yourself. git clone https://github.com/itzmeanjan/decds.git pushd decds make install popd -# now, try following, assuming $HOME/.cargo/bin is on your $PATH. +# Now, try following, assuming $HOME/.cargo/bin is on your $PATH. decds -V + +# For uninstalling `decds` executable from your system. +rm $(which decds) + +# ----------------------------- Using from build directory ----------------------------- + +git clone https://github.com/itzmeanjan/decds.git +cd decds +make build + +# Use executable from build directory +./target/optimized/decds -V ``` ## Usage diff --git a/decds-bin/Cargo.toml b/decds-bin/Cargo.toml index b4104e6..4b75e6d 100644 --- a/decds-bin/Cargo.toml +++ b/decds-bin/Cargo.toml @@ -2,6 +2,7 @@ name = "decds" description = "A CLI for Distributed Erasure-Coded Data Storage System" resolver = "3" +build = "build.rs" version = { workspace = true } edition = { workspace = true } rust-version = { workspace = true } @@ -17,4 +18,11 @@ clap = { workspace = true } rand = { workspace = true } const-hex = { workspace = true } blake3 = { workspace = true } -decds-lib = { version = "=0.1.0", path = "../decds-lib" } +decds-lib = { path = "../decds-lib" } +num_cpus = { workspace = true } +tokio = { workspace = true } +console_static_text = { workspace = true } +console = { workspace = true } + +[build-dependencies] +vergen-gix = { version = "1.0.9", features = ["build"] } diff --git a/decds-bin/build.rs b/decds-bin/build.rs new file mode 100644 index 0000000..a955d28 --- /dev/null +++ b/decds-bin/build.rs @@ -0,0 +1,9 @@ +use vergen_gix::{BuildBuilder, Emitter, GixBuilder}; + +fn main() -> Result<(), Box> { + let build = BuildBuilder::default().build_date(true).build()?; + let gitcl = GixBuilder::default().sha(true).build()?; + + Emitter::default().add_instructions(&build)?.add_instructions(&gitcl)?.emit()?; + Ok(()) +} diff --git a/decds-bin/src/errors.rs b/decds-bin/src/errors.rs index c6e3072..3fed502 100644 --- a/decds-bin/src/errors.rs +++ b/decds-bin/src/errors.rs @@ -1,4 +1,4 @@ -#[derive(Debug, PartialEq)] +#[derive(PartialEq)] pub enum DecdsCLIError { FailedToReadProofCarryingChunk(String), } @@ -10,3 +10,11 @@ impl std::fmt::Display for DecdsCLIError { } } } + +impl std::fmt::Debug for DecdsCLIError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl std::error::Error for DecdsCLIError {} diff --git a/decds-bin/src/handlers/handle_break.rs b/decds-bin/src/handlers/handle_break.rs index c170c6b..cebdfe4 100644 --- a/decds-bin/src/handlers/handle_break.rs +++ b/decds-bin/src/handlers/handle_break.rs @@ -1,106 +1,442 @@ use crate::utils::{format_bytes, get_target_directory_path}; -use decds_lib::{Blob, BlobHeader, DECDS_NUM_ERASURE_CODED_SHARES, ProofCarryingChunk}; -use std::{path::PathBuf, process::exit}; +use console::Term; +use console_static_text::{ConsoleSize, ConsoleStaticText}; +use decds_lib::{BlobBuilder, BlobHeader, DECDS_NUM_ERASURE_CODED_SHARES, MerkleTree, ProofCarryingChunk}; +use std::{ + io::Read, + path::PathBuf, + process::exit, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::Instant, +}; +use tokio::task::JoinSet; pub fn handle_break_command(blob_path: &PathBuf, opt_target_dir: &Option) { - match std::fs::read(blob_path) { - Ok(blob_bytes) => { - println!("Read {:?}", blob_path); - println!("Size {}", format_bytes(blob_bytes.len())); - - match Blob::new(blob_bytes) { - Ok(erasure_coded) => { - let metadata = erasure_coded.get_blob_header(); - println!("BLAKE3 Digest: {}", metadata.get_blob_digest()); - println!("Blob root commitment: {}", metadata.get_root_commitment()); - println!("Number of chunksets: {}", metadata.get_num_chunksets()); - println!("Number of chunks: {}", metadata.get_num_chunks()); - - let mut rng = rand::rng(); - let target_dir_path = get_target_directory_path(blob_path, opt_target_dir, &mut rng); - - if let Err(e) = std::fs::DirBuilder::new().recursive(true).create(&target_dir_path) { - eprintln!("Error: {}", e); + match tokio::runtime::Builder::new_multi_thread().enable_all().build() { + Ok(rt) => { + rt.block_on(async move { + let mut rng = rand::rng(); + let target_dir_path = get_target_directory_path(blob_path, opt_target_dir, &mut rng); + + if let Err(e) = std::fs::DirBuilder::new().recursive(true).create(&target_dir_path) { + eprintln!("Failed to create target directory {:?} to put chunks: {:?}", target_dir_path, e); + exit(1); + } + + println!("Reading {:?}", blob_path); + println!("Writing blob metadata and erasure-coded chunks in {:?}", target_dir_path); + + let metadata = read_blob_and_partially_chunkify(blob_path.to_owned(), target_dir_path.to_owned()).await; + + println!("Blob size {}", format_bytes(metadata.get_blob_size())); + println!("Blob BLAKE3 digest: {}", metadata.get_blob_digest()); + println!("Blob root commitment: {}", metadata.get_root_commitment()); + println!("Blob number of chunksets: {}", metadata.get_num_chunksets()); + println!("Blob number of chunks: {}", metadata.get_num_chunks()); + + write_blob_metadata(&target_dir_path, &metadata).await; + finalize_proof_carrying_chunks(metadata, target_dir_path.to_owned()).await; + + println!("Erasure-coded chunks placed in {:?}", target_dir_path); + }); + } + Err(e) => { + eprintln!("Failed to create a `tokio` async runtime: {:?}", e); + exit(1); + } + } +} + +async fn read_blob_and_partially_chunkify(blob_path: PathBuf, target_dir_path: PathBuf) -> BlobHeader { + let blob_bytes = Arc::new(Mutex::new(Vec::::new())); + let (blob_bytes_tx, blob_bytes_rx) = (blob_bytes.clone(), blob_bytes.clone()); + let (blob_eof_notifier_tx, blob_eof_notifier_rx) = tokio::sync::oneshot::channel::(); + + let chunks = Arc::new(Mutex::new(Vec::::new())); + let (chunks_tx, chunks_rx) = (chunks.clone(), chunks.clone()); + let (chunks_end_notifier_tx, chunks_end_notifier_rx) = tokio::sync::oneshot::channel::(); + + let blob_reader_task = tokio::task::spawn_blocking(move || read_blob_data(blob_path, blob_bytes_tx, blob_eof_notifier_tx)); + let blob_builder_task = tokio::task::spawn_blocking(move || build_blob(blob_bytes_rx, blob_eof_notifier_rx, chunks_tx, chunks_end_notifier_tx)); + let chunk_writer_task = tokio::task::spawn(write_partial_chunks(target_dir_path, chunks_rx, chunks_end_notifier_rx)); + + if let Err(e) = blob_reader_task.await { + eprintln!("Blob reader task failed to finish: {:?}", e); + exit(1); + } + + if let Err(e) = chunk_writer_task.await { + eprintln!("Partial chunk writer task failed to finish: {:?}", e); + exit(1); + } + + match blob_builder_task.await { + Ok(metadata) => metadata, + Err(e) => { + eprintln!("Blob builder task failed to finish: {:?}", e); + exit(1); + } + } +} + +fn read_blob_data(blob_path: PathBuf, blob_bytes_tx: Arc>>, blob_eof_notifier_tx: tokio::sync::oneshot::Sender) { + match std::fs::OpenOptions::new().read(true).open(&blob_path) { + Ok(fd) => { + const ONE_MB: usize = 1usize << 20; + const TEN_MB: usize = 10 * ONE_MB; + const SIXTEEN_MB: usize = 16 * ONE_MB; + + let mut has_reached_eof = false; + let mut buffered_fd = std::io::BufReader::with_capacity(SIXTEEN_MB, fd); + let mut buffer = vec![0u8; TEN_MB]; + + 'OUTER: loop { + let mut buffer_offset = 0; + + 'INNER: while buffer_offset < buffer.len() { + if let Ok(n) = buffered_fd.read(&mut buffer[buffer_offset..]) { + buffer_offset += n; + + if n == 0 { + has_reached_eof = true; + break 'INNER; + } + } + } + + if buffer_offset > 0 { + match blob_bytes_tx.lock() { + Ok(mut blob_bytes) => { + blob_bytes.extend_from_slice(&buffer[..buffer_offset]); + } + Err(e) => { + eprintln!("Failed to acquire lock to send blob bytes to blob builder task: {:?}", e); + exit(1); + } + } + } + + if has_reached_eof { + if let Err(e) = blob_eof_notifier_tx.send(true) { + eprintln!("Failed to inform blob builder task that blob reading is complete: {:?}", e); exit(1); } - println!("Writing blob metadata and erasure-coded chunks..."); + break 'OUTER; + } + } + } + Err(e) => { + eprintln!("Failed to open input blob file {:?}: {:?}", blob_path, e); + exit(1); + } + } +} - write_blob_metadata(&target_dir_path, metadata); - (0..DECDS_NUM_ERASURE_CODED_SHARES).for_each(|share_id| { - write_blob_share(&target_dir_path, share_id, erasure_coded.get_share(share_id).unwrap()); - }); +fn build_blob( + blob_bytes_rx: Arc>>, + mut blob_eof_notifier_rx: tokio::sync::oneshot::Receiver, + chunks_tx: Arc>>, + chunks_end_notifier_tx: tokio::sync::oneshot::Sender, +) -> BlobHeader { + let mut blob_builder = BlobBuilder::init(); + let mut has_reached_eof = false; - println!("Erasure-coded chunks placed in {:?}", &target_dir_path); + let mut progress = ConsoleStaticText::new(|| { + let (rows, cols) = Term::stdout().size(); + ConsoleSize { + rows: Some(rows), + cols: Some(cols), + } + }); + let now = Instant::now(); + + loop { + match blob_eof_notifier_rx.try_recv() { + Ok(_) => has_reached_eof = true, + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + eprintln!("Blob reader task exited without notifing EOF"); + exit(1); + } + } + + match blob_bytes_rx.try_lock() { + Ok(mut blob_bytes) => { + let extracted_blob_bytes = std::mem::take(&mut *blob_bytes); + + if let Some(mut chunks) = blob_builder.update(&extracted_blob_bytes) { + match chunks_tx.lock() { + Ok(mut partial_chunks) => { + partial_chunks.append(&mut chunks); + } + Err(e) => { + eprintln!("Failed to acquire lock to send erasure-coded chunks to chunk writer task: {:?}", e); + exit(1); + } + }; + + progress.eprint(&format!( + "Processed {} in {:?}...", + format_bytes(blob_builder.num_bytes_absorbed_so_far()), + now.elapsed() + )); + } + } + Err(_) => {} + } + + if has_reached_eof { + break; + } + } + + match blob_builder.finalize() { + Ok((mut chunks, blob_header)) => { + match chunks_tx.lock() { + Ok(mut partial_chunks) => { + partial_chunks.append(&mut chunks); } Err(e) => { - eprintln!("Error: {}", e); + eprintln!("Failed to acquire lock to send erasure-coded chunks to chunk writer task: {:?}", e); exit(1); } + }; + + if let Err(e) = chunks_end_notifier_tx.send(true) { + eprintln!("Failed to inform chunk writer task that no more chunks are coming: {:?}", e); + exit(1); } + + progress.eprint_clear(); + blob_header } Err(e) => { - eprintln!("Error: {}", e); + eprintln!("Failed to finalize the blob builder: {:?}", e); + exit(1); + } + } +} + +async fn write_partial_chunks( + target_dir_path: PathBuf, + chunks_rx: Arc>>, + mut chunks_end_notifier_rx: tokio::sync::oneshot::Receiver, +) { + let max_num_pending_spawned_tasks: usize = num_cpus::get() * 4; + + let mut join_handles = JoinSet::new(); + let mut has_reached_eof = false; + + loop { + match chunks_end_notifier_rx.try_recv() { + Ok(_) => has_reached_eof = true, + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} + Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + eprintln!("Blob builder task exited without notifing EOF"); + exit(1); + } + } + + match chunks_rx.try_lock() { + Ok(mut partial_chunks) => { + let extracted_partial_chunks = std::mem::take(&mut *partial_chunks); + + for chunk in extracted_partial_chunks { + let mut blob_share_path = target_dir_path.clone(); + + join_handles.spawn(async move { + match chunk.to_bytes() { + Ok(bytes) => { + blob_share_path.push(format!("chunkset.{}", chunk.get_chunkset_id())); + + if let Err(e) = tokio::fs::create_dir_all(&blob_share_path).await { + eprintln!("Failed to create directory {:?} for putting chunks: {:?}", blob_share_path, e); + exit(1); + } + + blob_share_path.push(format!("share{:02}.data", chunk.get_local_chunk_id())); + + if let Err(e) = tokio::fs::write(&blob_share_path, bytes).await { + eprintln!("Failed to write partial chunk to file {:?}: {:?}", blob_share_path, e); + exit(1); + } + } + Err(e) => { + eprintln!("Failed to serialize partial chunk with chunk-id {}: {:?}", chunk.get_global_chunk_id(), e); + exit(1); + } + } + }); + } + } + Err(_) => {} + } + + if join_handles.len() > max_num_pending_spawned_tasks { + let mut idx = 0; + let num_tasks_to_be_joined = join_handles.len() - max_num_pending_spawned_tasks; + + while idx < num_tasks_to_be_joined { + if let Err(e) = unsafe { join_handles.join_next().await.unwrap_unchecked() } { + eprintln!("Partial chunk writer sub-task failed to finish: {:?}", e); + exit(1); + } + + idx += 1; + } + } + + if has_reached_eof { + break; + } + } + + while let Some(task_result) = join_handles.join_next().await { + if let Err(e) = task_result { + eprintln!("Partial chunk writer sub-task failed to finish: {:?}", e); exit(1); } } } -fn write_blob_metadata(target_dir: &PathBuf, metadata: &BlobHeader) { +async fn write_blob_metadata(target_dir: &PathBuf, metadata: &BlobHeader) { let mut blob_metadata_path = target_dir.clone(); blob_metadata_path.push("metadata.commit"); match metadata.to_bytes() { Ok(bytes) => { - if let Err(e) = std::fs::write(blob_metadata_path, bytes) { - eprintln!("Error: {}", e); + if let Err(e) = tokio::fs::write(&blob_metadata_path, bytes).await { + eprintln!("Failed to write blob metadata to file {:?}: {:?}", blob_metadata_path, e); exit(1); } } Err(e) => { - eprintln!("Error: {}", e); + eprintln!("Failed to serialize blob metadata: {:?}", e); exit(1); } } } -fn write_blob_share(target_dir: &PathBuf, share_id: usize, share: Vec) { - let mut blob_share_path = target_dir.clone(); +async fn finalize_proof_carrying_chunks(metadata: BlobHeader, target_dir_path: PathBuf) { + if metadata.get_num_chunksets() == 1 { + return; + } + + let chunkset_root_commitments = (0..metadata.get_num_chunksets()) + .map(|chunkset_id| unsafe { metadata.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) + .collect(); + let merkle_tree = Arc::new(unsafe { MerkleTree::new(chunkset_root_commitments).unwrap_unchecked() }); + + let max_num_pending_spawned_tasks: usize = num_cpus::get() * 4; + let mut join_handles = JoinSet::new(); + + let mut progress = ConsoleStaticText::new(|| { + let (rows, cols) = Term::stdout().size(); + ConsoleSize { + rows: Some(rows), + cols: Some(cols), + } + }); + let num_finalized_chunks = Arc::new(AtomicUsize::new(0)); + let now = Instant::now(); + + for chunkset_id in 0..metadata.get_num_chunksets() { + for share_id in 0..DECDS_NUM_ERASURE_CODED_SHARES { + let mut blob_share_path = target_dir_path.clone(); + let merkle_tree_cloned = merkle_tree.clone(); + let num_finalized_chunks_clone = num_finalized_chunks.clone(); - for (chunkset_id, chunk) in share.iter().enumerate() { - blob_share_path.push(format!("chunkset.{}", chunkset_id)); + join_handles.spawn(async move { + blob_share_path.push(format!("chunkset.{}", chunkset_id)); + blob_share_path.push(format!("share{:02}.data", share_id)); - match blob_share_path.try_exists() { - Ok(ok) => { - if !ok { - if let Err(e) = std::fs::create_dir(&blob_share_path) { - eprintln!("Error: {}", e); + let mut chunk = match tokio::fs::read(&blob_share_path).await { + Ok(bytes) => match ProofCarryingChunk::from_bytes(&bytes) { + Ok((chunk, n)) => { + if n != bytes.len() { + eprintln!("Chunk file {:?} is {} bytes longer than it should be", blob_share_path, bytes.len() - n); + exit(1); + } else { + chunk + } + } + Err(e) => { + eprintln!("Failed to deserialize partial chunk from file {:?}: {:?}", blob_share_path, e); + exit(1); + } + }, + Err(e) => { + eprintln!("Failed to read partial chunk file {:?}: {:?}", blob_share_path, e); + exit(1); + } + }; + + match tokio::task::spawn_blocking(move || unsafe { merkle_tree_cloned.generate_proof(chunkset_id).unwrap_unchecked() }).await { + Ok(blob_level_proof) => { + chunk.append_proof_to_blob_root(&blob_level_proof); + } + Err(e) => { + eprintln!( + "Failed to generate blob-level Merkle proof-of-inclusion for chunk-id {}: {:?}", + chunk.get_global_chunk_id(), + e + ); exit(1); } } - } - Err(e) => { - eprintln!("Error: {}", e); - exit(1); - } - }; - blob_share_path.push(format!("share{:02}.data", share_id)); + if let Ok(bytes) = chunk.to_bytes() { + if let Err(e) = tokio::fs::write(&blob_share_path, bytes).await { + eprintln!("Failed to write complete chunk to file {:?}: {:?}", blob_share_path, e); + exit(1); + } + + // Update atomic counter, which keeps track of how many sub-tasks completed its job + // of updating a partial chunk with blob-level proof-of-inclusion. + num_finalized_chunks_clone.fetch_add(1, Ordering::Relaxed); + } + }); + } - match chunk.to_bytes() { - Ok(bytes) => { - if let Err(e) = std::fs::write(&blob_share_path, bytes) { - eprintln!("Error: {}", e); + if join_handles.len() > max_num_pending_spawned_tasks { + let mut idx = 0; + let num_tasks_to_be_joined = join_handles.len() - max_num_pending_spawned_tasks; + + while idx < num_tasks_to_be_joined { + if let Err(e) = unsafe { join_handles.join_next().await.unwrap_unchecked() } { + eprintln!("Partial chunk writer sub-task failed to finish: {:?}", e); exit(1); } + + idx += 1; } - Err(e) => { - eprintln!("Error: {}", e); - exit(1); - } - }; + } - blob_share_path.pop(); - blob_share_path.pop(); + progress.eprint(&format!( + "Finalized chunks {}/{} in {:?}...", + num_finalized_chunks.load(Ordering::Relaxed), + metadata.get_num_chunks(), + now.elapsed() + )); } + + while let Some(task_result) = join_handles.join_next().await { + if let Err(e) = task_result { + eprintln!("Partial chunk writer sub-task failed to finish: {:?}", e); + exit(1); + } + + progress.eprint(&format!( + "Finalized chunks {}/{} in {:?}...", + num_finalized_chunks.load(Ordering::Relaxed), + metadata.get_num_chunks(), + now.elapsed() + )); + } + + progress.eprint_clear(); } diff --git a/decds-bin/src/handlers/handle_repair.rs b/decds-bin/src/handlers/handle_repair.rs index 8b13283..f475217 100644 --- a/decds-bin/src/handlers/handle_repair.rs +++ b/decds-bin/src/handlers/handle_repair.rs @@ -15,7 +15,7 @@ pub fn handle_repair_command(chunk_dir_path: &PathBuf, opt_target_dir: &Option

{ - if blob_metadata.validate_chunk(&chunk) { - num_valid_shares += 1; - format!("{}- {}\t✅", indent, blob_share_path.file_name().unwrap().to_str().unwrap()) - } else { - format!( - "{}- {}\t🚫\tError: proof verification failed", - indent, - blob_share_path.file_name().unwrap().to_str().unwrap() - ) + let share_stat_log = if let Ok(ok) = blob_share_path.try_exists() { + if ok { + match read_proof_carrying_chunk(&blob_share_path) { + Ok(chunk) => { + if blob_metadata.validate_chunk(&chunk) { + num_valid_shares += 1; + format!("{}- {}\t✅", indent, blob_share_path.file_name().unwrap().to_str().unwrap()) + } else { + format!( + "{}- {}\t🚫\tError: proof verification failed", + indent, + blob_share_path.file_name().unwrap().to_str().unwrap() + ) + } + } + Err(e) => { + format!("{}- {}\t🚫\tError: {}", indent, blob_share_path.file_name().unwrap().to_str().unwrap(), e) } } - Err(e) => { - format!("{}- {}\t🚫\tError: {}", indent, blob_share_path.file_name().unwrap().to_str().unwrap(), e) - } + } else { + format!( + "{}- {}\t🚫\tError: chunk not present", + indent, + blob_share_path.file_name().unwrap().to_str().unwrap() + ) } } else { format!( diff --git a/decds-bin/src/main.rs b/decds-bin/src/main.rs index debf4cf..9622b0d 100644 --- a/decds-bin/src/main.rs +++ b/decds-bin/src/main.rs @@ -6,7 +6,7 @@ use clap::{Parser, Subcommand}; use std::path::PathBuf; #[derive(Parser)] -#[command(name = "decds", version, about, long_about = None)] +#[command(name = "decds", version = concat!(env!("CARGO_PKG_VERSION"), " ", "(", env!("VERGEN_GIT_SHA"), " ", env!("VERGEN_BUILD_DATE"), ")"), about, long_about = None)] struct DecdsCLI { #[command(subcommand)] command: DecdsCommand, diff --git a/decds-bin/src/utils.rs b/decds-bin/src/utils.rs index cadfc39..f2eb8ff 100644 --- a/decds-bin/src/utils.rs +++ b/decds-bin/src/utils.rs @@ -5,16 +5,17 @@ use std::{path::PathBuf, process::exit, str::FromStr}; use crate::errors::DecdsCLIError; pub fn format_bytes(bytes: usize) -> String { - let suffixes = ["B", "KB", "MB", "GB"]; + const SUFFIXES: [&str; 7] = ["B", "KB", "MB", "GB", "TB", "PB", "EB"]; + let mut index = 0; let mut size = bytes as f64; - while size >= 1024.0 && index < suffixes.len() - 1 { - size /= 1024.0; + while size >= 1024f64 && index < SUFFIXES.len() - 1 { + size /= 1024f64; index += 1; } - format!("{:.1}{}", size, suffixes[index]) + format!("{:.2}{}", size, SUFFIXES[index]) } pub fn read_blob_metadata(blob_metadata_path: &PathBuf) -> BlobHeader { diff --git a/decds-lib/benches/build_blob.rs b/decds-lib/benches/build_blob.rs index de7221b..ff2e860 100644 --- a/decds-lib/benches/build_blob.rs +++ b/decds-lib/benches/build_blob.rs @@ -1,4 +1,4 @@ -use decds_lib::Blob; +use decds_lib::BlobBuilder; use rand::Rng; use std::{fmt::Debug, time::Duration}; @@ -51,5 +51,9 @@ fn build_blob(bencher: divan::Bencher, rlnc_config: &BlobConfig) { (0..rlnc_config.data_byte_len).map(|_| rng.random()).collect::>() }) .input_counter(|data| divan::counter::BytesCount::new(data.len())) - .bench_values(|data| divan::black_box(Blob::new(divan::black_box(data)))); + .bench_values(|data| { + let mut blob_builder = BlobBuilder::init(); + divan::black_box(blob_builder.update(&data)); + divan::black_box(blob_builder.finalize()) + }); } diff --git a/decds-lib/benches/repair_blob.rs b/decds-lib/benches/repair_blob.rs index 70a243c..5f00f04 100644 --- a/decds-lib/benches/repair_blob.rs +++ b/decds-lib/benches/repair_blob.rs @@ -1,6 +1,7 @@ -use decds_lib::{Blob, DECDS_NUM_ERASURE_CODED_SHARES, ProofCarryingChunk, RepairingBlob}; +use decds_lib::{BlobBuilder, MerkleTree, RepairingBlob}; use rand::{Rng, seq::SliceRandom}; -use std::{fmt::Debug, time::Duration}; +use rayon::prelude::*; +use std::{collections::HashMap, fmt::Debug, time::Duration}; #[global_allocator] static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); @@ -46,15 +47,36 @@ fn repair_blob(bencher: divan::Bencher, rlnc_config: &BlobConfig) { .with_inputs(|| { let mut rng = rand::rng(); let data = (0..rlnc_config.data_byte_len).map(|_| rng.random()).collect::>(); - let blob = unsafe { Blob::new(data).unwrap_unchecked() }; + let (mut chunks, blob_header) = { + let mut all_chunks = Vec::new(); - let blob_header = blob.get_blob_header(); - let mut blob_shares = (0..(DECDS_NUM_ERASURE_CODED_SHARES - 4)) - .flat_map(|share_id| unsafe { blob.get_share(share_id).unwrap_unchecked() }) - .collect::>(); - blob_shares.shuffle(&mut rng); + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&data) { + all_chunks.extend(chunks); + } - (blob_header.to_owned(), blob_shares) + let (chunks, blob_header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, blob_header) + }; + + let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) + .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) + .collect(); + + let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); + let merkle_proofs = (0..blob_header.get_num_chunksets()) + .into_par_iter() + .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) + .collect::>>(); + + chunks.par_iter_mut().for_each(|chunk| { + chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); + }); + + chunks.shuffle(&mut rng); + (blob_header.to_owned(), chunks) }) .input_counter(|(header, _)| divan::counter::BytesCount::new(header.get_blob_size())) .bench_values(|(header, chunks)| { diff --git a/decds-lib/src/blob.rs b/decds-lib/src/blob.rs index c420c76..915c9d8 100644 --- a/decds-lib/src/blob.rs +++ b/decds-lib/src/blob.rs @@ -2,7 +2,7 @@ use crate::{ RepairingChunkSet, chunk::{self, ProofCarryingChunk}, chunkset::{self, ChunkSet}, - consts::{DECDS_BINCODE_CONFIG, DECDS_NUM_ERASURE_CODED_SHARES}, + consts::DECDS_BINCODE_CONFIG, errors::DecdsError, merkle_tree::MerkleTree, }; @@ -215,105 +215,178 @@ impl BlobHeader { } } -/// Represents a complete, erasure-coded blob of data, consisting of a `BlobHeader` and a collection of `ChunkSet`s, -/// each of which are holding 16 erasure-coded proof-of-inclusion carrying chunks. -pub struct Blob { - header: BlobHeader, - body: Vec, +/// `BlobBuilder` provides an incremental way to construct a `Blob` from a stream of data. +/// +/// This builder handles the division of input data into fixed-size `ChunkSet`s, prepares RLNC-based erasure-coded chunks, +/// computes BLAKE3 digest of blob, and generates Merkle inclusion (in respective `Chunkset`s) proof for proof-carrying chunks. +pub struct BlobBuilder { + hasher: blake3::Hasher, + num_bytes_absorbed: usize, + num_chunksets: usize, + offset: usize, + buffer: Vec, + chunkset_root_commitments: Vec, } -impl Blob { - /// Creates a new `Blob` from raw byte data. +impl BlobBuilder { + /// Initializes a new `BlobBuilder` - ready to build a blob. + pub fn init() -> Self { + BlobBuilder { + hasher: blake3::Hasher::new(), + num_bytes_absorbed: 0, + num_chunksets: 0, + offset: 0, + buffer: vec![0u8; ChunkSet::BYTE_LENGTH], + chunkset_root_commitments: vec![], + } + } + + pub fn num_bytes_absorbed_so_far(&self) -> usize { + self.num_bytes_absorbed + } + + /// Updates the `BlobBuilder` with new data. + /// + /// This method absorbs the provided `data` into the internal buffer. If enough + /// data accumulates to form a complete `ChunkSet`, it is processed (erasure-coded, + /// Merkle-proofed) and its resulting `ProofCarryingChunk`s are returned. /// - /// This involves: - /// 1. Calculating the blob's digest and padding its length to a multiple of `ChunkSet::BYTE_LENGTH`. - /// 2. Dividing the data into `ChunkSet`s and erasure-coding them individually. - /// 3. Building a Merkle tree over the chunksets' root commitments to create the blob's root commitment. - /// 4. Appending blob-level Merkle proofs to each chunk within the chunksets. + /// You can call this method arbitrary number of times, before calling `Self::finalize`. + /// Note, you must call this method atleast once with non-empty input data, to not get + /// an error from `Self::finalize` - as you can't build a blob over empty input data. /// /// # Arguments /// - /// * `data` - The raw `Vec` representing the blob's content. + /// * `data` - A byte slice containing the new data to be processed. /// /// # Returns /// - /// Returns a `Result` which is: - /// - `Ok(Self)` containing the newly created `Blob` if successful. - /// - `Err(DecdsError::EmptyDataForBlob)` if the input `data` is empty. - /// - Other `DecdsError` types may be returned from underlying `ChunkSet::new` or `MerkleTree::new` calls. - pub fn new(mut data: Vec) -> Result { + /// An `Option>`. + /// - `Some(Vec)` if one or more `ChunkSet`s were completed and their chunks generated. These chunks carry Merkle proof-of-inclusion in respective Chunkset. + /// - `None` if no complete `ChunkSet` was formed or if the input `data` was empty. + pub fn update(&mut self, data: &[u8]) -> Option> { if data.is_empty() { - return Err(DecdsError::EmptyDataForBlob); + return None; } - let blob_digest = blake3::hash(&data); - let blob_length = data.len(); + self.hasher.update(data); + self.num_bytes_absorbed += data.len(); - let num_chunksets = blob_length.div_ceil(chunkset::ChunkSet::BYTE_LENGTH); - let zero_padded_blob_len = num_chunksets * chunkset::ChunkSet::BYTE_LENGTH; - data.resize(zero_padded_blob_len, 0); + let total_num_bytes = self.offset + data.len(); + let num_chunksets = total_num_bytes / ChunkSet::BYTE_LENGTH; - let mut chunksets = (0..num_chunksets) - .into_par_iter() - .map(|chunkset_id| { - let offset = chunkset_id * chunkset::ChunkSet::BYTE_LENGTH; - let till = offset + chunkset::ChunkSet::BYTE_LENGTH; + if num_chunksets == 0 { + self.buffer[self.offset..total_num_bytes].copy_from_slice(data); + self.offset = total_num_bytes; - unsafe { chunkset::ChunkSet::new(chunkset_id, data[offset..till].to_vec()).unwrap_unchecked() } - }) - .collect::>(); + return None; + } else { + let remaining_num_bytes = total_num_bytes - num_chunksets * ChunkSet::BYTE_LENGTH; + let dont_use_from_idx = data.len() - remaining_num_bytes; - let merkle_leaves = chunksets.iter().map(|chunkset| chunkset.get_root_commitment()).collect::>(); - let merkle_tree = MerkleTree::new(merkle_leaves)?; - let commitment = merkle_tree.get_root_commitment(); + let mut chunks = Vec::with_capacity(num_chunksets * ChunkSet::NUM_ERASURE_CODED_CHUNKS); - chunksets.par_iter_mut().enumerate().for_each(|(chunkset_idx, chunkset)| { - let blob_proof = unsafe { merkle_tree.generate_proof(chunkset_idx).unwrap_unchecked() }; - chunkset.append_blob_inclusion_proof(&blob_proof); - }); + if num_chunksets == 1 { + self.buffer[self.offset..].copy_from_slice(&data[..dont_use_from_idx]); - Ok(Blob { - header: BlobHeader { - byte_length: blob_length, - num_chunksets, - digest: blob_digest, - root_commitment: commitment, - chunkset_root_commitments: chunksets.iter().map(|chunkset| chunkset.get_root_commitment()).collect(), - }, - body: chunksets, - }) - } + let chunkset_id = self.num_chunksets; + let owned_buffer = std::mem::replace(&mut self.buffer, vec![0u8; ChunkSet::BYTE_LENGTH]); + let chunkset = unsafe { chunkset::ChunkSet::new(chunkset_id, owned_buffer).unwrap_unchecked() }; - /// Returns a reference to the `BlobHeader` of this blob. - pub fn get_blob_header(&self) -> &BlobHeader { - &self.header + chunks.extend((0..ChunkSet::NUM_ERASURE_CODED_CHUNKS).map(|chunk_id| unsafe { chunkset.get_chunk(chunk_id).unwrap_unchecked().clone() })); + self.chunkset_root_commitments.push(chunkset.get_root_commitment()); + + self.num_chunksets += 1; + } else { + let mut working_mem = vec![0u8; num_chunksets * ChunkSet::BYTE_LENGTH]; + working_mem[..self.offset].copy_from_slice(&self.buffer[..self.offset]); + working_mem[self.offset..].copy_from_slice(&data[..dont_use_from_idx]); + + let mut chunkset_root_commitments = Vec::with_capacity(num_chunksets); + let mut nested_chunks: Vec> = Vec::with_capacity(num_chunksets); + + working_mem + .par_chunks_exact(ChunkSet::BYTE_LENGTH) + .enumerate() + .map(|(data_chunk_idx, data_chunk)| { + let chunkset_id = self.num_chunksets + data_chunk_idx; + let chunkset = unsafe { chunkset::ChunkSet::new(chunkset_id, data_chunk.to_vec()).unwrap_unchecked() }; + + ( + chunkset.get_root_commitment(), + (0..ChunkSet::NUM_ERASURE_CODED_CHUNKS) + .map(|chunk_id| unsafe { chunkset.get_chunk(chunk_id).unwrap_unchecked().clone() }) + .collect(), + ) + }) + .unzip_into_vecs(&mut chunkset_root_commitments, &mut nested_chunks); + + self.chunkset_root_commitments.append(&mut chunkset_root_commitments); + chunks.extend(nested_chunks.into_iter().flatten()); + + self.num_chunksets += num_chunksets; + } + + if remaining_num_bytes > 0 { + self.buffer[..remaining_num_bytes].copy_from_slice(&data[dont_use_from_idx..]); + self.offset = remaining_num_bytes; + } + + Some(chunks) + } } - /// Retrieves a specific "share" (a collection of erasure-coded chunks, one from each chunkset) - /// based on the `share_id`. - /// - /// Each share represents a vertical slice through the blob's chunksets. - /// - /// # Arguments + /// Finalizes the `BlobBuilder`, processing any remaining buffered data + /// and constructing the `BlobHeader`. /// - /// * `share_id` - The ID of the share to retrieve (`0` to `DECDS_NUM_ERASURE_CODED_SHARES - 1`). + /// This method pads any incomplete `ChunkSet` in the buffer with zeros, + /// processes it, computes the final blob digest, and builds the top-level + /// Merkle tree over chunkset root commitments, yielding the `BlobHeader` + /// and at max 16 proof-carrying chunks, if there was an incomplete chunkset, + /// which needed to be built. /// /// # Returns /// /// Returns a `Result` which is: - /// - `Ok(Vec)` containing a vector of proof-carrying chunks for the requested share. - /// - `Err(DecdsError::InvalidErasureCodedShareId)` if `share_id` is out of bounds. - pub fn get_share(&self, share_id: usize) -> Result, DecdsError> { - if share_id >= DECDS_NUM_ERASURE_CODED_SHARES { - return Err(DecdsError::InvalidErasureCodedShareId(share_id)); + /// - `Ok((Vec, BlobHeader))` containing either 0 or 16 `ProofCarryingChunk`s from last chunkset and the `BlobHeader` for the complete blob. + /// - `Err(DecdsError::EmptyDataForBlob)` if no data was ever absorbed by the builder. + /// - Other `DecdsError` types may be returned from underlying `MerkleTree::new` calls. + pub fn finalize(mut self) -> Result<(Vec, BlobHeader), DecdsError> { + if self.num_bytes_absorbed == 0 { + return Err(DecdsError::EmptyDataForBlob); } - Ok((0..self.header.num_chunksets) - .map(|chunkset_id| unsafe { - let chunkset = &self.body[chunkset_id]; - chunkset.get_chunk(share_id).unwrap_unchecked().clone() - }) - .collect::>()) + let chunks = if self.offset != 0 { + self.buffer[self.offset..].fill(0); + + let chunkset_id = self.num_chunksets; + let chunkset = unsafe { chunkset::ChunkSet::new(chunkset_id, self.buffer).unwrap_unchecked() }; + + self.chunkset_root_commitments.push(chunkset.get_root_commitment()); + self.num_chunksets += 1; + + (0..ChunkSet::NUM_ERASURE_CODED_CHUNKS) + .map(|chunk_id| unsafe { chunkset.get_chunk(chunk_id).unwrap_unchecked().clone() }) + .collect() + } else { + Vec::new() + }; + + let blob_digest = self.hasher.finalize(); + + let merkle_tree = MerkleTree::new(self.chunkset_root_commitments.clone())?; + let blob_root_commitment = merkle_tree.get_root_commitment(); + + Ok(( + chunks, + BlobHeader { + byte_length: self.num_bytes_absorbed, + num_chunksets: self.num_chunksets, + digest: blob_digest, + root_commitment: blob_root_commitment, + chunkset_root_commitments: self.chunkset_root_commitments, + }, + )) } } @@ -475,9 +548,11 @@ impl RepairingBlob { #[cfg(test)] mod tests { - use crate::{BlobHeader, ProofCarryingChunk, RepairingBlob, blob::Blob, chunkset::ChunkSet, consts, errors::DecdsError}; + use crate::{BlobHeader, RepairingBlob, blob::BlobBuilder, chunkset::ChunkSet, errors::DecdsError, merkle_tree::MerkleTree}; use blake3; use rand::Rng; + use rayon::prelude::*; + use std::collections::HashMap; #[test] fn prop_test_blob_preparation_and_commitment_works() { @@ -492,14 +567,35 @@ mod tests { let blob_byte_len = rng.random_range(MIN_BLOB_DATA_BYTE_LEN..=MAX_BLOB_DATA_BYTE_LEN); let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).expect("Must be able to prepare blob"); - let blob_header = blob.get_blob_header(); + let (mut chunks, blob_header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } - assert!( - (0..consts::DECDS_NUM_ERASURE_CODED_SHARES) - .flat_map(|share_id| blob.get_share(share_id).expect("Must be able to get erasure coded shares")) - .all(|share| blob_header.validate_chunk(&share)) - ); + let (chunks, blob_header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, blob_header) + }; + + let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) + .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) + .collect(); + + let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); + let merkle_proofs = (0..blob_header.get_num_chunksets()) + .into_par_iter() + .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) + .collect::>>(); + + chunks.par_iter_mut().for_each(|chunk| { + chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); + }); + + assert!(chunks.iter().all(|chunk| { blob_header.validate_chunk(chunk) })); }); } @@ -507,11 +603,22 @@ mod tests { fn test_get_chunkset_commitment() { let mut rng = rand::rng(); - let blob_byte_len = (ChunkSet::BYTE_LENGTH * 2) + (ChunkSet::BYTE_LENGTH / 2); + let blob_byte_len = ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2; let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).unwrap(); - let header = blob.get_blob_header(); + let (_, header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; // Valid chunkset ID let commitment = header.get_chunkset_commitment(0); @@ -530,11 +637,22 @@ mod tests { let mut rng = rand::rng(); // Blob size: 2.5 chunksets -> 2 full, 1 half - let blob_byte_len = (ChunkSet::BYTE_LENGTH * 2) + (ChunkSet::BYTE_LENGTH / 2); + let blob_byte_len = ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2; let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).unwrap(); - let header = blob.get_blob_header(); + let (_, header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; // Full chunkset assert_eq!(header.get_chunkset_size(0).unwrap(), ChunkSet::BYTE_LENGTH); @@ -554,11 +672,22 @@ mod tests { fn test_get_byte_range_for_chunkset() { let mut rng = rand::rng(); - let blob_byte_len = (ChunkSet::BYTE_LENGTH * 2) + (ChunkSet::BYTE_LENGTH / 2); + let blob_byte_len = ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2; let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).unwrap(); - let header = blob.get_blob_header(); + let (_, header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; // First chunkset assert_eq!(header.get_byte_range_for_chunkset(0).unwrap(), (0, ChunkSet::BYTE_LENGTH)); @@ -583,11 +712,22 @@ mod tests { fn test_get_chunkset_ids_for_byte_range() { let mut rng = rand::rng(); - let blob_byte_len = (ChunkSet::BYTE_LENGTH * 2) + (ChunkSet::BYTE_LENGTH / 2); + let blob_byte_len = ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2; let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).unwrap(); - let header = blob.get_blob_header(); + let (_, header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; // Range within a single chunkset assert_eq!(header.get_chunkset_ids_for_byte_range(0..10).unwrap(), vec![0]); @@ -636,8 +776,19 @@ mod tests { let blob_byte_len = ChunkSet::BYTE_LENGTH * 3; let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data).unwrap(); - let original_header = blob.get_blob_header().clone(); + let (_, original_header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; let serialized_header = original_header.to_bytes().expect("Header serialization failed"); let (deserialized_header, bytes_read) = BlobHeader::from_bytes(&serialized_header).expect("Header deserialization failed"); @@ -651,38 +802,29 @@ mod tests { #[test] fn test_blob_new_empty_data() { - assert_eq!(Blob::new(Vec::new()).err(), Some(DecdsError::EmptyDataForBlob)); + assert_eq!(BlobBuilder::init().finalize().err(), Some(DecdsError::EmptyDataForBlob)); } #[test] - fn test_blob_get_share_invalid_id() { + fn test_repairing_blob_new() { let mut rng = rand::rng(); - let blob_data: Vec = (0..(ChunkSet::BYTE_LENGTH * 2)).map(|_| rng.random()).collect(); - let blob = Blob::new(blob_data).unwrap(); + let blob_byte_len = ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2; + let blob_data: Vec = (0..blob_byte_len).map(|_| rng.random()).collect(); - // Test with an invalid share ID (out of bounds) - let invalid_share_id = consts::DECDS_NUM_ERASURE_CODED_SHARES; - assert_eq!( - blob.get_share(invalid_share_id).unwrap_err(), - DecdsError::InvalidErasureCodedShareId(invalid_share_id) - ); + let (_, header) = { + let mut all_chunks = Vec::new(); - // Test with a very large invalid share ID - let large_invalid_share_id = consts::DECDS_NUM_ERASURE_CODED_SHARES + 100; - assert_eq!( - blob.get_share(large_invalid_share_id).unwrap_err(), - DecdsError::InvalidErasureCodedShareId(large_invalid_share_id) - ); - } + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } - #[test] - fn test_repairing_blob_new() { - let mut rng = rand::rng(); + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); - let blob_data: Vec = (0..(ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2)).map(|_| rng.random()).collect(); - let blob = Blob::new(blob_data).unwrap(); - let header = blob.get_blob_header().clone(); + (all_chunks, header) + }; let repairer = RepairingBlob::new(header.clone()); @@ -702,18 +844,40 @@ mod tests { fn test_repairing_blob_add_chunk() { let mut rng = rand::rng(); - let blob_data: Vec = (0..(ChunkSet::BYTE_LENGTH * 2)).map(|_| rng.random()).collect(); // Two full chunksets - let blob = Blob::new(blob_data).unwrap(); + let blob_data: Vec = (0..ChunkSet::BYTE_LENGTH * 2).map(|_| rng.random()).collect(); // Two full chunksets - let blob_header = blob.get_blob_header().clone(); - let mut repairer = RepairingBlob::new(blob_header.clone()); + let (mut chunks, blob_header) = { + let mut all_chunks = Vec::new(); - let all_chunks: Vec = (0..consts::DECDS_NUM_ERASURE_CODED_SHARES) - .flat_map(|share_id| blob.get_share(share_id).unwrap()) + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; + + let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) + .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) .collect(); + let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); + let merkle_proofs = (0..blob_header.get_num_chunksets()) + .into_par_iter() + .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) + .collect::>>(); + + chunks.par_iter_mut().for_each(|chunk| { + chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); + }); + + let mut repairer = RepairingBlob::new(blob_header.clone()); + // Test valid chunk addition - let chunk_to_add = &all_chunks[0]; + let chunk_to_add = &chunks[0]; assert!(repairer.add_chunk(chunk_to_add).is_ok()); // Simulate an invalid chunk proof by creating a new header with a different root commitment @@ -728,11 +892,11 @@ mod tests { // Add enough chunks to make a chunkset ready for repair let mut repairer_ready = RepairingBlob::new(blob_header.clone()); - let chunkset_id = all_chunks[0].get_chunkset_id(); + let chunkset_id = chunks[0].get_chunkset_id(); - for chunk in &all_chunks { + for chunk in &chunks { if chunk.get_chunkset_id() == chunkset_id { - repairer_ready.add_chunk(chunk).unwrap(); + let _ = repairer_ready.add_chunk(chunk); if repairer_ready.is_chunkset_ready_to_repair(chunkset_id).unwrap() { break; @@ -743,9 +907,9 @@ mod tests { assert!(repairer_ready.is_chunkset_ready_to_repair(chunkset_id).unwrap()); // Try adding another chunk to a chunkset already ready for repair - let extra_chunk = &all_chunks + let extra_chunk = &chunks .iter() - .find(|c| c.get_chunkset_id() == chunkset_id && c.get_global_chunk_id() != all_chunks[0].get_global_chunk_id()) + .find(|c| c.get_chunkset_id() == chunkset_id && c.get_global_chunk_id() != chunks[0].get_global_chunk_id()) .unwrap(); assert_eq!( @@ -769,15 +933,38 @@ mod tests { let mut rng = rand::rng(); let blob_data: Vec = (0..(ChunkSet::BYTE_LENGTH * 2 + ChunkSet::BYTE_LENGTH / 2)).map(|_| rng.random()).collect(); - let blob = Blob::new(blob_data.clone()).unwrap(); + let original_blob_data_copy = blob_data.clone(); - let blob_header = blob.get_blob_header().clone(); - let mut repairer = RepairingBlob::new(blob_header.clone()); + let (mut chunks, blob_header) = { + let mut all_chunks = Vec::new(); + + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, header) + }; - let all_chunks: Vec = (0..consts::DECDS_NUM_ERASURE_CODED_SHARES) - .flat_map(|share_id| blob.get_share(share_id).unwrap()) + let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) + .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) .collect(); + let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); + let merkle_proofs = (0..blob_header.get_num_chunksets()) + .into_par_iter() + .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) + .collect::>>(); + + chunks.par_iter_mut().for_each(|chunk| { + chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); + }); + + let mut repairer = RepairingBlob::new(blob_header.clone()); + // Test `ChunksetNotYetReadyToRepair` let chunkset_id_0 = 0; assert_eq!( @@ -786,9 +973,9 @@ mod tests { ); // Add enough chunks for the first chunkset - for chunk in &all_chunks { + for chunk in &chunks { if chunk.get_chunkset_id() == chunkset_id_0 { - repairer.add_chunk(chunk).unwrap(); + let _ = repairer.add_chunk(chunk); if repairer.is_chunkset_ready_to_repair(chunkset_id_0).unwrap() { break; @@ -799,7 +986,7 @@ mod tests { // Test successful repair let repaired_data_0 = repairer.get_repaired_chunkset(chunkset_id_0).unwrap(); - let expected_data_0 = blob_data[0..ChunkSet::BYTE_LENGTH].to_vec(); + let expected_data_0 = original_blob_data_copy[0..ChunkSet::BYTE_LENGTH].to_vec(); assert_eq!(repaired_data_0, expected_data_0); assert!(repairer.is_chunkset_already_repaired(chunkset_id_0).unwrap()); @@ -813,9 +1000,9 @@ mod tests { // Test for a partial last chunkset let chunkset_id_2 = 2; - for chunk in &all_chunks { + for chunk in &chunks { if chunk.get_chunkset_id() == chunkset_id_2 { - repairer.add_chunk(chunk).unwrap(); + let _ = repairer.add_chunk(chunk); if repairer.is_chunkset_ready_to_repair(chunkset_id_2).unwrap() { break; @@ -825,7 +1012,7 @@ mod tests { assert!(repairer.is_chunkset_ready_to_repair(chunkset_id_2).unwrap()); let repaired_data_2 = repairer.get_repaired_chunkset(chunkset_id_2).unwrap(); - let expected_data_2 = blob_data[ChunkSet::BYTE_LENGTH * 2..].to_vec(); + let expected_data_2 = original_blob_data_copy[ChunkSet::BYTE_LENGTH * 2..].to_vec(); assert_eq!(repaired_data_2, expected_data_2); // Test invalid chunkset ID diff --git a/decds-lib/src/chunk.rs b/decds-lib/src/chunk.rs index 206b08d..e436119 100644 --- a/decds-lib/src/chunk.rs +++ b/decds-lib/src/chunk.rs @@ -138,7 +138,7 @@ impl ProofCarryingChunk { /// # Arguments /// /// * `blob_proof` - A slice of `blake3::Hash` representing the proof to append. - pub(crate) fn append_proof_to_blob_root(&mut self, blob_proof: &[blake3::Hash]) { + pub fn append_proof_to_blob_root(&mut self, blob_proof: &[blake3::Hash]) { self.proof.extend_from_slice(blob_proof); } diff --git a/decds-lib/src/chunkset.rs b/decds-lib/src/chunkset.rs index fbfce0a..599a81a 100644 --- a/decds-lib/src/chunkset.rs +++ b/decds-lib/src/chunkset.rs @@ -87,19 +87,6 @@ impl ChunkSet { pub fn get_chunk(&self, chunk_id: usize) -> Result<&chunk::ProofCarryingChunk, DecdsError> { self.chunks.get(chunk_id).ok_or(DecdsError::InvalidErasureCodedShareId(chunk_id)) } - - /// Appends a Merkle proof for the blob inclusion to all `ProofCarryingChunk`s within this `ChunkSet`. - /// This extends the chunkset-level proof to a blob-level proof for each chunk. - /// - /// # Arguments - /// - /// * `blob_proof` - A slice of `blake3::Hash` representing the Merkle path from the chunkset's - /// root commitment to the blob's root commitment. - pub(crate) fn append_blob_inclusion_proof(&mut self, blob_proof: &[blake3::Hash]) { - if !blob_proof.is_empty() { - self.chunks.iter_mut().for_each(|chunk| chunk.append_proof_to_blob_root(blob_proof)); - } - } } /// A structure designed to help incrementally reconstruct the original data of a `ChunkSet` @@ -214,7 +201,7 @@ mod tests { DecdsError, chunk::ProofCarryingChunk, chunkset::{ChunkSet, RepairingChunkSet}, - merkle_tree::{MerkleTree, tests::flip_a_bit}, + merkle_tree::tests::flip_a_bit, }; use rand::{Rng, seq::SliceRandom}; @@ -273,7 +260,7 @@ mod tests { let mut chunk_idx = 0; while !repairing_chunkset.is_ready_to_repair() { - repairing_chunkset.add_chunk(chunks[chunk_idx]).unwrap(); + let _ = repairing_chunkset.add_chunk(chunks[chunk_idx]); chunk_idx += 1; } @@ -314,71 +301,6 @@ mod tests { ); } - #[test] - fn test_chunkset_append_blob_inclusion_proof_unit() { - let mut rng = rand::rng(); - - // 1. Create a base ChunkSet - let data_for_chunkset = (0..ChunkSet::BYTE_LENGTH).map(|_| rng.random()).collect::>(); - let mut chunkset_1 = ChunkSet::new(1, data_for_chunkset.clone()).expect("Must be able to build erasure-coded ChunkSet"); - let chunkset_1_commitment = chunkset_1.get_root_commitment(); - - // 2. Create mock blob-level Merkle tree leaves (chunkset roots) - // This mock tree will have chunkset_1_commitment at index 1 - let mock_blob_leaves = vec![ - blake3::hash(b"dummy_chunkset_root_0"), // Leaf 0 - chunkset_1_commitment, // Leaf 1 (our chunkset_1's commitment) - blake3::hash(b"dummy_chunkset_root_2"), // Leaf 2 - blake3::hash(b"dummy_chunkset_root_3"), // Leaf 3 - ]; - - // 3. Build a mock blob MerkleTree - let mock_blob_merkle_tree = MerkleTree::new(mock_blob_leaves).expect("Must be able to build mock blob Merkle Tree"); - let mock_blob_root_commitment = mock_blob_merkle_tree.get_root_commitment(); - - // 4. Generate the blob_proof for chunkset_1_commitment at its index (1) - let blob_proof_for_chunkset_1 = mock_blob_merkle_tree.generate_proof(1).expect("Must be able to generate blob proof"); - - // Take a chunk for validation BEFORE appending the blob proof - let chunk_before_append = chunkset_1.get_chunk(0).unwrap().clone(); - // It should NOT validate against the blob root commitment yet because it doesn't have the blob proof - assert!(!chunk_before_append.validate_inclusion_in_blob(mock_blob_root_commitment)); - - // 5. Call the method under test: append_blob_inclusion_proof - chunkset_1.append_blob_inclusion_proof(&blob_proof_for_chunkset_1); - - // 6. Verify the outcome using a chunk from the modified chunkset - let chunk_after_append = chunkset_1.get_chunk(0).unwrap(); - - // 7. Assert that validate_inclusion_in_blob now returns true - assert!(chunk_after_append.validate_inclusion_in_blob(mock_blob_root_commitment)); - - // Test with an empty blob_proof (should not change anything, i.e., validation still works) - chunkset_1.append_blob_inclusion_proof(&[]); - let chunk_after_empty_append = chunkset_1.get_chunk(0).unwrap(); - assert!(chunk_after_empty_append.validate_inclusion_in_blob(mock_blob_root_commitment)); - - // Negative test: Tamper the proof and verify it fails - let mut tampered_blob_proof = blob_proof_for_chunkset_1.clone(); - if !tampered_blob_proof.is_empty() { - // Flip a bit in the first hash of the proof to tamper it - let random_byte_index = rng.random_range(0..blake3::OUT_LEN); - let random_bit_index = rng.random_range(0..u8::BITS) as usize; - - let mut bytes = [0u8; blake3::OUT_LEN]; - bytes.copy_from_slice(tampered_blob_proof[0].as_bytes()); - bytes[random_byte_index] = flip_a_bit(bytes[random_byte_index], random_bit_index); - - tampered_blob_proof[0] = blake3::Hash::from_bytes(bytes); - } - - let mut chunkset_1 = ChunkSet::new(1, data_for_chunkset).expect("Must be able to build erasure-coded ChunkSet"); - chunkset_1.append_blob_inclusion_proof(&tampered_blob_proof); - - let tampered_chunk = chunkset_1.get_chunk(0).unwrap(); - assert!(!tampered_chunk.validate_inclusion_in_blob(mock_blob_root_commitment)); - } - #[test] fn test_repairing_chunkset_new() { let mut rng = rand::rng(); @@ -444,8 +366,19 @@ mod tests { let mut repairing_chunkset = RepairingChunkSet::new(0, chunkset.get_root_commitment()); // Add fewer than NUM_ORIGINAL_CHUNKS chunks - for i in 0..(ChunkSet::NUM_ORIGINAL_CHUNKS - 1) { - repairing_chunkset.add_chunk(chunkset.get_chunk(i).unwrap()).unwrap(); + let mut chunk_idx = 0; + let mut useful_count = 0; + + while chunk_idx < ChunkSet::NUM_ERASURE_CODED_CHUNKS { + let chunk = chunkset.get_chunk(chunk_idx).expect("Must be able to lookup chunk by id"); + if let Ok(_) = repairing_chunkset.add_chunk(chunk) { + useful_count += 1; + } + + chunk_idx += 1; + if useful_count == ChunkSet::NUM_ORIGINAL_CHUNKS - 1 { + break; + } } assert!(!repairing_chunkset.is_ready_to_repair()); @@ -461,9 +394,9 @@ mod tests { let mut repairing_chunkset = RepairingChunkSet::new(0, chunkset.get_root_commitment()); let mut chunk_idx = 0; - while !repairing_chunkset.is_ready_to_repair() { + while !repairing_chunkset.is_ready_to_repair() && chunk_idx < ChunkSet::NUM_ERASURE_CODED_CHUNKS { let chunk = chunkset.get_chunk(chunk_idx).expect("Must be able to lookup chunk by id"); - repairing_chunkset.add_chunk(chunk).expect("Must be able to add valid chunk"); + let _ = repairing_chunkset.add_chunk(chunk); chunk_idx += 1; } diff --git a/decds-lib/src/errors.rs b/decds-lib/src/errors.rs index 7c8dea5..205c2b2 100644 --- a/decds-lib/src/errors.rs +++ b/decds-lib/src/errors.rs @@ -1,6 +1,7 @@ use crate::{chunkset::ChunkSet, consts}; -#[derive(Debug, PartialEq)] +/// Errors encountered by `decds` during blob building and repairing. +#[derive(PartialEq)] pub enum DecdsError { /// Returned when trying to create a blob with empty data. EmptyDataForBlob, @@ -82,3 +83,11 @@ impl std::fmt::Display for DecdsError { } } } + +impl std::fmt::Debug for DecdsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl std::error::Error for DecdsError {} diff --git a/decds-lib/src/lib.rs b/decds-lib/src/lib.rs index 7e1e90b..d08c847 100644 --- a/decds-lib/src/lib.rs +++ b/decds-lib/src/lib.rs @@ -1,6 +1,6 @@ //! # DECDS-lib: Decentralized Erasure-Coded Data Storage Library //! -//! `decds-lib` provides functionalities for creating, storing, retrieving, and repairing arbitrary size data blobs +//! `decds-lib` provides functionalities for disseminating, verifying and reconstructing arbitrary size data blobs //! using Random Linear Network Coding (RLNC) based erasure-coding and binary Merkle trees for data integrity verification. //! //! This library is designed to enable decentralized data storage solutions by breaking down @@ -8,89 +8,96 @@ //! //! ## How to Use //! -//! ### 1. Create a Blob +//! To build a blob, first initialize the `BlobBuilder` struct, and incrementally keep putting data into it +//! by calling `BlobBuilder::update` function arbirary many times, possibly returning some `ProofCarryingChunk`s. +//! Once all data has been absorbed by the `BlobBuilder`` instance, it can be finalized, which returns the `BlobHeader`, +//! holding all necessary metadata for chunk validation and reconstruction. //! -//! To store data, first create a `Blob` from a `Vec`. The `Blob::new` function -//! takes your raw data, divides it into chunksets, applies erasure coding, and builds -//! Merkle trees for both chunkset-level and blob-level integrity. +//! Note, the `ProofCarryingChunk`s which are generated during blob building, does only carry proof of +//! inclusion in respective `ChunkSet`s. Once the full blob is built, the proof carried by each chunk +//! needs to be extended for validating blob inclusion, during reconstruction. Following example demonstrates that. //! -//! ```rust -//! use decds_lib::Blob; -//! use rand::Rng; // Assuming `rand` is available in your project for example data -//! -//! let mut rng = rand::thread_rng(); -//! let original_data: Vec = (0..1024 * 1024 * 50).map(|_| rng.random()).collect(); // 50MB of random data -//! let blob = Blob::new(original_data).expect("Failed to create blob"); -//! println!("Blob created with size: {} bytes", blob.get_blob_header().get_blob_size()); -//! ``` +//! To reconstruct the original blob data, you need to collect *enough* `ProofCarryingChunk`s for all chunksets. +//! You initialize a `RepairingBlob` with the `BlobHeader`, which is the source of truth for validating `ProofCarryingChunk`s, +//! and then add chunks to it. Once *enough* chunks for a specific chunkset are collected, you can retrieve +//! its repaired data. Each chunkset requires at least 10 valid chunks to be recovered. To recover the full blob, +//! you need to collect at least 10 valid chunks per chunkset. //! -//! ### 2. Retrieve Erasure-Coded Shares (Proof-Carrying Chunks) -//! -//! Once a `Blob` is created, you can retrieve its erasure-coded shares. Each share is a `Vec`, -//! where each `ProofCarryingChunk` is a verifiable piece of data. -//! You need `DECDS_NUM_ERASURE_CODED_SHARES` total shares per chunkset, but only `ChunkSet::NUM_ORIGINAL_CHUNKS` -//! (which is 10) are needed to reconstruct the original data of that chunkset. +//! For more details see README in `decds` repository @ . //! //! ```rust -//! use decds_lib::{Blob, DECDS_NUM_ERASURE_CODED_SHARES}; -//! use rand::Rng; -//! -//! let mut rng = rand::thread_rng(); -//! let original_data: Vec = (0..1024).map(|_| rng.random()).collect(); -//! let blob = Blob::new(original_data).expect("Failed to create blob"); +//! use decds_lib::{BlobBuilder, BlobHeader, ProofCarryingChunk, RepairingBlob, DECDS_NUM_ERASURE_CODED_SHARES, DecdsError, MerkleTree}; +//! use rand::{Rng, seq::SliceRandom}; +//! use rayon::prelude::*; +//! use std::collections::HashMap; //! -//! let first_share = blob.get_share(0).expect("Failed to get share"); -//! println!("Retrieved {} chunks for share 0.", first_share.len()); +//! let mut rng = rand::rng(); //! -//! // You can iterate through all available shares: -//! for share_id in 0..DECDS_NUM_ERASURE_CODED_SHARES { -//! let share = blob.get_share(share_id).expect("Failed to get share"); -//! // Do something with the share, e.g., send it to a storage node -//! } -//! ``` +//! const ONE_MB: usize = 1usize << 20; +//! let original_data: Vec = (0..42 * ONE_MB).map(|_| rng.random()).collect(); // 42MB of random data +//! let original_data_copy = original_data.clone(); //! -//! ### 3. Repair/Reconstruct a Blob //! -//! To reconstruct the original blob data, you need to collect enough `ProofCarryingChunk`s. -//! You initialize a `RepairingBlob` with the `BlobHeader` (which can be serialized/deserialized), -//! and then add chunks to it. Once enough chunks for a specific chunkset are collected, -//! you can retrieve its repaired data. +//! // Build the blob and collect all generated chunks. +//! // Remember these proof-carrying chunks don't carry a proof-of-inclusion +//! // in the blob, rather they carry only proof-of-inclusion in corresponding chunkset +//! // that they belong to. +//! let (mut chunks, blob_header) = { +//! let mut all_chunks = Vec::new(); //! -//! ```rust -//! use decds_lib::{Blob, BlobHeader, ProofCarryingChunk, RepairingBlob, DECDS_NUM_ERASURE_CODED_SHARES, DecdsError}; -//! use rand::{Rng, seq::SliceRandom}; +//! let mut blob_builder = BlobBuilder::init(); +//! if let Some(chunks) = blob_builder.update(&original_data) { +//! all_chunks.extend(chunks); +//! } //! -//! let mut rng = rand::thread_rng(); -//! let original_data: Vec = (0..1024 * 1024 * 50).map(|_| rng.random()).collect(); // 50MB of random data -//! let original_data_copy = original_data.clone(); +//! let (final_chunks, blob_header) = blob_builder.finalize().expect("Failed to finalize blob"); +//! all_chunks.extend(final_chunks); //! -//! let blob = Blob::new(original_data).expect("Failed to create blob"); -//! let blob_header = blob.get_blob_header().clone(); +//! (all_chunks, blob_header) +//! }; //! -//! // Collect all chunks from the blob (simulate receiving them from storage) -//! let mut all_chunks: Vec = (0..DECDS_NUM_ERASURE_CODED_SHARES) -//! .flat_map(|share_id| blob.get_share(share_id).unwrap()) +//! // We have to collect root commitments of all chunksets, as we will build a Merkle tree on them, +//! // considering they are the leaf nodes of the Merkle tree. +//! let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) +//! .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) //! .collect(); //! -//! // Simulate data loss by shuffling and taking only a subset (but enough for repair) -//! // In a real scenario, you'd receive chunks from various sources -//! all_chunks.shuffle(&mut rng); -//! +//! // This is the Merkle tree build on chunkset root commitments, giving us a blob level root commitment. +//! let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); +//! // Let's generate Merkle proof-of-inclusion for each chunkset. We will extend each chunk's proof to include +//! // corresponding chunkset's inclusion proof in the blob root commitment. +//! let merkle_proofs = (0..blob_header.get_num_chunksets()) +//! .into_par_iter() +//! .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) +//! .collect::>>(); +//! // Extend each proof-carrying chunk to include proof-of-inclusion in the blob. +//! // And this completes blob building phase. +//! chunks.par_iter_mut().for_each(|chunk| { +//! chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); +//! }); +//! +//! // Simulate data loss and reordering by shuffling and taking only a subset (but enough for repair). +//! // In a real scenario, you'd receive chunks from various sources. +//! chunks.shuffle(&mut rng); +//! +//! // Let's try to repair the full blob. //! let mut repairer = RepairingBlob::new(blob_header.clone()); //! let num_chunksets = blob_header.get_num_chunksets(); //! -//! // Add chunks to the repairer until all chunksets are repaired +//! // Add chunks to the repairer until all chunksets are repaired. //! let mut chunk_idx = 0; //! let mut repaired_chunksets_count = 0; //! //! while repaired_chunksets_count < num_chunksets { -//! if chunk_idx >= all_chunks.len() { +//! if chunk_idx >= chunks.len() { //! println!("Not enough chunks to repair the entire blob!"); //! break; //! } -//! let chunk = &all_chunks[chunk_idx]; +//! +//! let chunk = &chunks[chunk_idx]; //! let chunkset_id = chunk.get_chunkset_id(); -//! // Try to add the chunk, handling various repair states +//! +//! // Try to add the chunk, handling various repair states. //! match repairer.add_chunk(chunk) { //! Ok(_) => { //! if repairer.is_chunkset_ready_to_repair(chunkset_id).expect("Failed to check chunkset repair status") { @@ -103,18 +110,21 @@ //! match e { //! DecdsError::ChunksetReadyToRepair(_) | DecdsError::ChunksetAlreadyRepaired(_) | DecdsError::InvalidProofInChunk(_) => { //! // Chunk is redundant, already repaired, or invalid; simply skip it. -//! // In a real system, invalid chunks would indicate a security issue. +//! // In case we get invalid proof-of-inclusion, it means some intermediary must have tampered with it. //! }, //! _ => { -//! eprintln!("Error adding chunk: {}", e); +//! eprintln!("Error adding chunk: {:?}", e); //! std::process::exit(1); //! }, //! } //! }, //! } +//! //! chunk_idx += 1; //! } //! +//! // As we didn't tamper with any proof-carrying chunks, it must be possible for us to recover the +//! // whole blob. Let's recover and compare. //! let final_repaired_data = (0..blob_header.get_num_chunksets()).flat_map(|chunkset_id| { //! repairer.get_repaired_chunkset(chunkset_id).expect("Failed to get repaired chunkset") //! }).collect::>(); @@ -133,8 +143,9 @@ mod merkle_tree; #[cfg(test)] mod tests; -pub use blob::{Blob, BlobHeader, RepairingBlob}; +pub use blob::{BlobBuilder, BlobHeader, RepairingBlob}; pub use chunk::ProofCarryingChunk; pub use chunkset::RepairingChunkSet; pub use consts::DECDS_NUM_ERASURE_CODED_SHARES; pub use errors::DecdsError; +pub use merkle_tree::MerkleTree; diff --git a/decds-lib/src/tests.rs b/decds-lib/src/tests.rs index 6072a44..8e3fd1c 100644 --- a/decds-lib/src/tests.rs +++ b/decds-lib/src/tests.rs @@ -1,5 +1,7 @@ -use crate::{Blob, ProofCarryingChunk, RepairingBlob, consts, errors::DecdsError}; +use crate::{BlobBuilder, DecdsError, MerkleTree, RepairingBlob}; use rand::{Rng, seq::SliceRandom}; +use rayon::prelude::*; +use std::collections::HashMap; #[test] fn prop_test_blob_building_and_repairing_works() { @@ -14,16 +16,38 @@ fn prop_test_blob_building_and_repairing_works() { let blob_byte_len = rng.random_range(MIN_BLOB_DATA_BYTE_LEN..=MAX_BLOB_DATA_BYTE_LEN); let blob_data = (0..blob_byte_len).map(|_| rng.random()).collect::>(); - let blob = Blob::new(blob_data.clone()).expect("Must be able to prepare blob"); + let (mut chunks, blob_header) = { + let mut all_chunks = Vec::new(); - let blob_header = blob.get_blob_header().to_owned(); - let mut chunk_shares = (0..consts::DECDS_NUM_ERASURE_CODED_SHARES) - .flat_map(|share_id| unsafe { blob.get_share(share_id).unwrap_unchecked() }) - .collect::>(); - chunk_shares.shuffle(&mut rng); + let mut blob_builder = BlobBuilder::init(); + if let Some(chunks) = blob_builder.update(&blob_data) { + all_chunks.extend(chunks); + } + + let (chunks, blob_header) = blob_builder.finalize().expect("Must be able to prepare blob"); + all_chunks.extend(chunks); + + (all_chunks, blob_header) + }; + + let chunkset_root_commitments = (0..blob_header.get_num_chunksets()) + .map(|chunkset_id| unsafe { blob_header.get_chunkset_commitment(chunkset_id).unwrap_unchecked() }) + .collect(); + + let merkle_tree = MerkleTree::new(chunkset_root_commitments).expect("Must be able to build Merkle tree"); + let merkle_proofs = (0..blob_header.get_num_chunksets()) + .into_par_iter() + .map(|chunkset_id| unsafe { (chunkset_id, merkle_tree.generate_proof(chunkset_id).unwrap_unchecked()) }) + .collect::>>(); + + chunks.par_iter_mut().for_each(|chunk| { + chunk.append_proof_to_blob_root(&merkle_proofs[&chunk.get_chunkset_id()]); + }); + + chunks.shuffle(&mut rng); let mut repairer = RepairingBlob::new(blob_header.clone()); - let mut shares = chunk_shares.iter(); + let mut shares = chunks.iter(); loop { if let Some(share) = shares.next() { diff --git a/scripts/test_decds_on_linux.sh b/scripts/test_decds_on_linux.sh deleted file mode 100755 index 287833f..0000000 --- a/scripts/test_decds_on_linux.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/bash - -# Generate random 256MB data blob -dd if=/dev/urandom of=random.data bs=1M count=256 - -# Break blob into chunksets and verify each chunk's validity -cargo run --profile optimized -- break -b random.data -o broken -cargo run --profile optimized -- verify broken - -# Repair chunksets and check SHA256 digest of original data blob and repaired data blob -cargo run --profile optimized -- repair -c broken -o repairing-with-16 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-16/repaired.data" | sha256sum --check - -# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 -# Repairing with 15 valid chunks for chunkset-15 - must work! -dd if=/dev/urandom of=broken/chunkset.15/share00.data bs=1 seek=1 count=1 conv=notrunc -cargo run --profile optimized -- repair -c broken -o repairing-with-15 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-15/repaired.data" | sha256sum --check - -# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 -# Repairing with 14 valid chunks for chunkset-15 - must work! -dd if=/dev/urandom of=broken/chunkset.15/share02.data bs=1 seek=11 count=1 conv=notrunc -cargo run --profile optimized -- repair -c broken -o repairing-with-14 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-14/repaired.data" | sha256sum --check - -# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 -# Repairing with 13 valid chunks for chunkset-15 - must work! -dd if=/dev/urandom of=broken/chunkset.15/share04.data bs=1 seek=111 count=1 conv=notrunc -cargo run --profile optimized -- repair -c broken -o repairing-with-13 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-13/repaired.data" | sha256sum --check - -# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 -# Repairing with 12 valid chunks for chunkset-15 - must work! -dd if=/dev/urandom of=broken/chunkset.15/share15.data bs=1 seek=1 count=1 conv=notrunc -cargo run --profile optimized -- repair -c broken -o repairing-with-12 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-12/repaired.data" | sha256sum --check - -# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 -# Repairing with 11 valid chunks for chunkset-15 - must work! -dd if=/dev/urandom of=broken/chunkset.15/share12.data bs=1 seek=100 count=1 conv=notrunc -cargo run --profile optimized -- repair -c broken -o repairing-with-11 -echo "$(sha256sum random.data | awk '{print $1}') repairing-with-11/repaired.data" | sha256sum --check - -# Note: -# It should ideally be possible to recover a chunkset with 10 valid chunks. -# Though it is possible that all possible unique permutations of 10 valid chunks don't -# result in successful recovery of the chunkset - because some of those chunks might be -# linearly dependent. So we need to collect 10 linearly independent chunks for each chunkset. -# A safe bet is collecting minimum 11 chunks per chunkset, to be *almost* sure, that set will -# have 10 linearly independent i.e. useful chunks. - -# Mutate a single byte of two proof-carrying chunks belonging to chunkset-15. -# Now chunkset-15 should have 9 valid proof-carrying chunks, which should not suffice for recovering that chunkset. -dd if=/dev/urandom of=broken/chunkset.15/share09.data bs=1 seek=781 count=1 conv=notrunc -dd if=/dev/urandom of=broken/chunkset.15/share07.data bs=1 seek=223 count=1 conv=notrunc - -# Now trying to repair, with 9 valid chunks for chunkset-15, it should fail with return code 1, as chunkset-15 can't be recovered. -cargo run --profile optimized -- repair -c broken -o repairing-with-9 | tee console.out; test ${PIPESTATUS[0]} -eq 1 - -# Clean up -rm -rf random.data broken repairing-with* console.out diff --git a/scripts/test_decds_on_unix.sh b/scripts/test_decds_on_unix.sh new file mode 100755 index 0000000..6012078 --- /dev/null +++ b/scripts/test_decds_on_unix.sh @@ -0,0 +1,121 @@ +#!/usr/bin/bash + +# Trap failure of any following commands +set -e + +# Detect operating system +case "$(uname -s)" in + Linux*) OS=Linux;; + Darwin*) OS=Mac;; + *) echo "Unsupported OS"; exit 1;; +esac + +# Check for required commands +if ! command -v dd >/dev/null 2>&1; then + echo "Error: dd command not found" + exit 1 +fi +if ! command -v make >/dev/null 2>&1; then + echo "Error: make command not found" + exit 1 +fi +if ! command -v awk >/dev/null 2>&1; then + echo "Error: awk command not found" + exit 1 +fi + +# Set SHA256 command based on OS +if [ "$OS" = "Linux" ]; then + if command -v sha256sum >/dev/null 2>&1; then + SHA256CMD="sha256sum" + else + echo "Error: sha256sum not found on Linux" + exit 1 + fi +elif [ "$OS" = "Mac" ]; then + if command -v shasum >/dev/null 2>&1; then + SHA256CMD="shasum -a 256" + else + echo "Error: shasum not found on macOS" + exit 1 + fi +fi + +# Function to verify checksum (handles Linux and macOS differences) +verify_checksum() { + local hash="$1" + local file="$2" + if [ "$OS" = "Linux" ]; then + echo "$hash $file" | $SHA256CMD --check + elif [ "$OS" = "Mac" ]; then + # Use a temporary file to ensure correct format + echo "$hash $file" > /tmp/checksum.txt + $SHA256CMD -c /tmp/checksum.txt + rm -f /tmp/checksum.txt + fi +} + +# Generate random 1GB data blob +dd if=/dev/urandom of=random.data bs=1M count=1024 +ORIGINAL_HASH=$($SHA256CMD random.data | awk '{print $1}') + +# Build `decds` executable +make build +echo "Using $(./target/optimized/decds -V)" + +# Break blob into chunksets and verify each chunk's validity +time ./target/optimized/decds break -b random.data -o broken +time ./target/optimized/decds verify broken + +# Repair chunksets and check SHA256 digest of original data blob and repaired data blob +time ./target/optimized/decds repair -c broken -o repairing-with-16 +verify_checksum "$ORIGINAL_HASH" "repairing-with-16/repaired.data" + +# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 +# Repairing with 15 valid chunks for chunkset-15 - must work! +dd if=/dev/urandom of=broken/chunkset.15/share00.data bs=1 seek=1 count=1 conv=notrunc +time ./target/optimized/decds repair -c broken -o repairing-with-15 +verify_checksum "$ORIGINAL_HASH" "repairing-with-15/repaired.data" + +# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 +# Repairing with 14 valid chunks for chunkset-15 - must work! +dd if=/dev/urandom of=broken/chunkset.15/share02.data bs=1 seek=11 count=1 conv=notrunc +time ./target/optimized/decds repair -c broken -o repairing-with-14 +verify_checksum "$ORIGINAL_HASH" "repairing-with-14/repaired.data" + +# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 +# Repairing with 13 valid chunks for chunkset-15 - must work! +dd if=/dev/urandom of=broken/chunkset.15/share04.data bs=1 seek=111 count=1 conv=notrunc +time ./target/optimized/decds repair -c broken -o repairing-with-13 +verify_checksum "$ORIGINAL_HASH" "repairing-with-13/repaired.data" + +# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 +# Repairing with 12 valid chunks for chunkset-15 - must work! +dd if=/dev/urandom of=broken/chunkset.15/share15.data bs=1 seek=77 count=1 conv=notrunc +time ./target/optimized/decds repair -c broken -o repairing-with-12 +verify_checksum "$ORIGINAL_HASH" "repairing-with-12/repaired.data" + +# Mutate a single byte of a proof-carrying chunk belonging to chunkset-15 +# Repairing with 11 valid chunks for chunkset-15 - must work! +dd if=/dev/urandom of=broken/chunkset.15/share12.data bs=1 seek=175 count=1 conv=notrunc +time ./target/optimized/decds repair -c broken -o repairing-with-11 +verify_checksum "$ORIGINAL_HASH" "repairing-with-11/repaired.data" + +# Note: +# It should ideally be possible to recover a chunkset with 10 valid chunks. +# Though it is possible that all possible unique permutations of 10 valid chunks don't +# result in successful recovery of the chunkset - because some of those chunks might be +# linearly dependent. So we need to collect 10 linearly independent chunks for each chunkset. +# A safe bet is collecting minimum 11 chunks per chunkset, to be *almost* sure, that set will +# have 10 linearly independent i.e. useful chunks. + +# Mutate a single byte of two proof-carrying chunks belonging to chunkset-15. +# Now chunkset-15 should have 9 valid proof-carrying chunks, which should not suffice for recovering that chunkset. +dd if=/dev/urandom of=broken/chunkset.15/share09.data bs=1 seek=781 count=1 conv=notrunc +dd if=/dev/urandom of=broken/chunkset.15/share07.data bs=1 seek=223 count=1 conv=notrunc + +# Now trying to repair, with 9 valid chunks for chunkset-15, it should fail with return code 1, as chunkset-15 can't be recovered. +time ./target/optimized/decds repair -c broken -o repairing-with-9 | tee console.out; test ${PIPESTATUS[0]} -eq 1 + +# Clean up +rm -rf random.data broken repairing-with* console.out /tmp/checksum.txt