diff --git a/Cargo.lock b/Cargo.lock index f966c3e..60e4da8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -719,9 +719,9 @@ dependencies = [ [[package]] name = "borsh_utils" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c0cd2efcc60ce44ca958baa626593a23006b5418b1fa8f038c23aa85f7a97f" +checksum = "1a9716bbbca7bd5eb81c46151ef82ce10ebcf428f63397371284b2b705c28cb9" dependencies = [ "bitcoin", "borsh", @@ -1135,6 +1135,7 @@ dependencies = [ name = "fabric-resolver" version = "0.2.3" dependencies = [ + "borsh", "dashmap", "hex", "libveritas", @@ -2035,9 +2036,9 @@ dependencies = [ [[package]] name = "libveritas" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9b646e5e793b4a806c5f17646e7b6a1eb9f1ab89b5539f3249e99f02c175e95" +checksum = "6509a45e4b44e68fbcf5748dccb74d15f50fd02919c784bf5155dc8792d42580" dependencies = [ "base64 0.22.1", "borsh", @@ -2054,9 +2055,9 @@ dependencies = [ [[package]] name = "libveritas_testutil" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a6fca4cc7c1d13baf31337c776a2bc0b18bdcce80a9c365c8cbc3f8102e740" +checksum = "8b61380363c1103e67fe02f7b0d5e2eea75ade8fa74a185c3bd6f875909371ed" dependencies = [ "bitcoin", "borsh", @@ -2480,7 +2481,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.11+spec-1.1.0", ] [[package]] @@ -2765,8 +2766,10 @@ dependencies = [ "libveritas", "libveritas_testutil", "rand 0.9.4", + "relay", "reqwest", "rusqlite", + "serde", "serde_json", "sha2", "spacedb", @@ -2775,6 +2778,7 @@ dependencies = [ "spaces_nums", "spaces_protocol", "tokio", + "toml", "tower-http", "tracing", "url", @@ -3334,6 +3338,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3403,9 +3416,9 @@ checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "sip7" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b684197991554bec0801fb44663974608d59ac6df4f95d2d84a3955e82499a85" +checksum = "e0ed9b8980298850d71acd2c96f55bef13a4e4d28d2976c503a86c69c791fd34" dependencies = [ "base64 0.22.1", "hex", @@ -3477,9 +3490,9 @@ dependencies = [ [[package]] name = "spaces_checkpoint" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044392ea966b553c792fcf85b72e4da08ea2c8ea37ded4eeb43cb491be528de4" +checksum = "611add09bc1b52a4739a71bb15bcbdb162449c668bf0c74f8c4a48835c4e5c60" dependencies = [ "anyhow", "clap", @@ -3497,9 +3510,9 @@ dependencies = [ [[package]] name = "spaces_client" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d817eddfabcd6d3115f1c5c9b19a0f64ead6542d216a79dac1afe9d7c19ffb4" +checksum = "a27ec4e8bc75f0f2e87beb778702149efd341a0b10efe42493b3de32a670d64c" dependencies = [ "anyhow", "base64 0.22.1", @@ -3532,9 +3545,9 @@ dependencies = [ [[package]] name = "spaces_nums" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2015555e35781fc95ac0eb8e50bef6e70929248ea61d4101b195b4d0abb13e1d" +checksum = "d6b842f5b10f200e3c9548489657da4e889897183c72d35673f27d20c2ecb3be" dependencies = [ "bech32", "bitcoin", @@ -3548,9 +3561,9 @@ dependencies = [ [[package]] name = "spaces_protocol" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f46b7eb14311193304301a394a30e5f7bdfd7fd9f70a5c8113610bf2b5cc2c" +checksum = "69f412ceb3e0a1ce5ca00a8cc7bb875cd614ba7ac809e4085502ec397e354a4a" dependencies = [ "bitcoin", "borsh", @@ -3560,9 +3573,9 @@ dependencies = [ [[package]] name = "spaces_wallet" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703c9b4b9d4fa4c7097bf46a471699604f1bc3102559609ab66e97c8965258b0" +checksum = "b6ba9c85e7685c47229c8430cb9496c67799af42b7c9bc64f12e15245a601aa2" dependencies = [ "anyhow", "bdk_wallet_backport", @@ -3854,6 +3867,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -3863,6 +3897,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.11+spec-1.1.0" @@ -3870,9 +3918,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.1", ] [[package]] @@ -3881,9 +3929,15 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.1", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.4.13" @@ -4527,6 +4581,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 449fbf5..adb6d75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,11 @@ authors = ["Buffrr "] [workspace.dependencies] fabric-resolver = { path = "fabric", version = "0.2.3" } -libveritas = { version = "0.2" } -libveritas_testutil = { version = "0.2" } +libveritas = { version = "0.3.1" } +libveritas_testutil = { version = "0.3.1" } -spaces_client = { version = "0.1" } -spaces_protocol = { version = "0.1" } -spaces_nums = { version = "0.1" } -spaces_checkpoint = { version = "0.1" } +spaces_client = { version = "0.2.1" } +spaces_protocol = { version = "0.2.1" } +spaces_nums = { version = "0.2.1" } +spaces_checkpoint = { version = "0.2.1" } spacedb = { version = "0.1", features = ["hash-idx"] } diff --git a/README.md b/README.md index acb09d0..38510e4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Certificate relay network for the [Spaces protocol](https://spacesprotocol.org). Certrelay consists of two components: -- **relay** — HTTP server that verifies certificates, stores them in SQLite, and gossips with peers +- **relay** — HTTP server that verifies certificates, stores them in SQLite, and syncs with peers (pull-based replication with poke notifications) - **fabric** — Client library available in Rust, JavaScript, Go, Python, Kotlin, and Swift The protocol is plain HTTP — relays are queryable from browsers, mobile apps, and any language with an HTTP client. All verification is done client-side against Bitcoin's chain state. @@ -38,12 +38,83 @@ No external Bitcoin node required. Data is stored in `~/.certrelay` by default. | `--data-dir` | `CERTRELAY_DATA_DIR` | `~/.certrelay` | Data directory | | `--bind` | `CERTRELAY_BIND` | `127.0.0.1` | Bind address | | `--port` | `CERTRELAY_PORT` | `7778` (mainnet) / `7779` (other) | Listen port | -| `--self-url` | `CERTRELAY_SELF_URL` | - | Public URL for peer announcements | +| `--self-url` | `CERTRELAY_SELF_URL` | - | Public URL for peer announcements (also enables poke sending) | +| `--config` | `CERTRELAY_CONFIG` | - | Path to a TOML config file (see below) | | `--spaced-rpc-url` | `CERTRELAY_SPACED_RPC_URL` | - | External spaced RPC (skips embedded node) | -| `--remote-ip-header` | `CERTRELAY_REMOTE_IP_HEADER` | - | Header for client IP behind reverse proxy | +| `--remote-ip-header` | `CERTRELAY_REMOTE_IP_HEADER` | - | Header for client IP behind reverse proxy (rightmost entry is used) | | `--is-bootstrap` | `CERTRELAY_BOOTSTRAP` | `false` | Run as a bootstrap node | +| `--anchor-refresh` | `CERTRELAY_ANCHOR_REFRESH` | `300` | Anchor refresh interval in seconds | +| `--allow-private-peers` | `CERTRELAY_ALLOW_PRIVATE_PEERS` | `false` | Accept peers on private/loopback addresses (local development only) | | `--skip-checkpoint-sync` | - | `false` | Skip checkpoint download, sync from scratch | +### Configuration file + +Rate limits, sync tuning, peer table sizes, concurrency caps, and storage +retention live in an optional TOML file passed via `--config` (or +`CERTRELAY_CONFIG`). Every field is optional and defaults to the values shown — +a config file only needs the settings being changed. Unknown keys are rejected +to catch typos. + +```toml +[rate_limits] +message_per_min = 60 # /message (client publishes) +proof_per_min = 30 # /query, /chain-proof (trigger proof generation) +read_per_min = 120 # /hints, /reverse, /addrs, /anchors, /peers, /stats +announce_per_min = 5 # /announce +sync_per_min = 60 # /sync, /sync/summary (pages are the unit) +poke_per_min = 30 # /poke +space_per_min = 100 # per-space content updates (replacements only) +handle_period_secs = 300 # per-handle content cap period (replacements only) +handle_burst = 3 # per-handle burst within the period + +[sync] +interval_secs = 45 # pull round cadence (plus jitter) +jitter_secs = 15 +page_limit = 1000 # rows requested per /sync page (max 1000) +peers_per_round = 2 # peers pulled from per round +max_pages_per_peer = 200 # page budget per peer per round +poke_debounce_ms = 2000 # coalescing window for outgoing pokes +poke_cooldown_ms = 5000 # min gap between poke-triggered pulls per peer + +[peers] +max_unverified = 1000 # announced-but-unverified peer slots +max_verified = 100 # verified peer slots +verified_ttl_secs = 600 # verified peers expire without a liveness refresh + +[limits] +max_message_size = 524288 # /message body cap in bytes (512 KB) +proof_concurrency = 6 # concurrent chain-proof generations (503 beyond) +verify_concurrency = 4 # concurrent message verifications (503 beyond) + +[retention] +max_storage_bytes = 10737418240 # handle payload budget (10 GB); 0 = unlimited +entitlement_per_epoch = 10000 # handles per (space, epoch) counted as paid-for +evict_low_water_pct = 90 # evict down to this % of the budget +sweep_interval_secs = 30 # pressure check cadence +eviction_batch = 1000 # rows deleted per transaction +``` + +Notes on the less obvious knobs: + +- **Content limits are churn-only.** `space_per_min` and the `handle_*` caps + charge only when an existing record is *replaced*; the first insert of a + handle is always free so relays can bootstrap-sync a whole network's data. +- **Retention never rejects data under budget.** A space's *entitlement* is + `entitlement_per_epoch × epochs it committed on-chain` — commitments cost + Bitcoin transactions, so storage beyond entitlement is storage nobody paid + for. Only when `max_storage_bytes` is exceeded does the relay evict, most + over-entitled space first (oldest and least-queried rows first), and stop + admitting *new* handles for over-entitled spaces until pressure clears. + Size the budget to your disk; a small relay stays functional by shedding + the heaviest spaces, a big relay can hold everything. + +### Monitoring + +- `GET /health` — unmetered liveness check for load balancers and peers. +- `GET /stats` — JSON counters: message intake, sync progress (including last + successful sync per peer — the key signal that replication is healthy), + pokes, rate-limit rejections, storage totals versus budget, and evictions. + ### Public relay behind a reverse proxy ```bash diff --git a/fabric/Cargo.toml b/fabric/Cargo.toml index 10075af..20bf0c6 100644 --- a/fabric/Cargo.toml +++ b/fabric/Cargo.toml @@ -27,6 +27,7 @@ client = ["reqwest", "rand", "tokio"] signing = ["secp256k1"] [dependencies] +borsh = { version = "1.6", features = ["derive"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" libveritas = { workspace = true } diff --git a/fabric/go/go.mod b/fabric/go/go.mod index 39e1fcb..882e7af 100644 --- a/fabric/go/go.mod +++ b/fabric/go/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( github.com/btcsuite/btcd/btcec/v2 v2.3.6 - github.com/spacesprotocol/libveritas-go v0.2.0 + github.com/spacesprotocol/libveritas-go v0.3.1 ) require ( diff --git a/fabric/go/go.sum b/fabric/go/go.sum index 0a1668a..620a083 100644 --- a/fabric/go/go.sum +++ b/fabric/go/go.sum @@ -8,5 +8,5 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/spacesprotocol/libveritas-go v0.2.0 h1:QJ/QYb3ixZu+PGQB1MgXaqVwY8NvcqjDE1dtqF2HFVQ= -github.com/spacesprotocol/libveritas-go v0.2.0/go.mod h1:HXnX2FNL43ueJuedsQwX9GF5jRHYZLqXYfFLWz900H8= +github.com/spacesprotocol/libveritas-go v0.3.1 h1:MN7DkFvuySiv006iE5xBZc0SUAwi7QB3aO1iuWjKeQ8= +github.com/spacesprotocol/libveritas-go v0.3.1/go.mod h1:HXnX2FNL43ueJuedsQwX9GF5jRHYZLqXYfFLWz900H8= diff --git a/fabric/js/fabric-react-native/package.json b/fabric/js/fabric-react-native/package.json index 01a19b8..f19569d 100644 --- a/fabric/js/fabric-react-native/package.json +++ b/fabric/js/fabric-react-native/package.json @@ -22,8 +22,8 @@ "build": "tsc" }, "dependencies": { - "@spacesprotocol/fabric-core": "*", - "@spacesprotocol/react-native-libveritas": "0.0.0-dev.20260407170644", + "@spacesprotocol/fabric-core": "^0.1.0", + "@spacesprotocol/react-native-libveritas": "^0.3.1", "uniffi-bindgen-react-native": "0.29.3-1" }, "devDependencies": { diff --git a/fabric/js/fabric-web/package.json b/fabric/js/fabric-web/package.json index c97b88c..32a67af 100644 --- a/fabric/js/fabric-web/package.json +++ b/fabric/js/fabric-web/package.json @@ -26,8 +26,8 @@ }, "dependencies": { "@noble/curves": "^1.8.0", - "@spacesprotocol/fabric-core": "*", - "@spacesprotocol/libveritas": "^0.2.0" + "@spacesprotocol/fabric-core": "^0.1.0", + "@spacesprotocol/libveritas": "^0.3.1" }, "devDependencies": { "@types/node": "^25.5.0", diff --git a/fabric/js/package-lock.json b/fabric/js/package-lock.json index c264373..e103456 100644 --- a/fabric/js/package-lock.json +++ b/fabric/js/package-lock.json @@ -27,8 +27,8 @@ "version": "0.1.0", "license": "MIT", "dependencies": { - "@spacesprotocol/fabric-core": "*", - "@spacesprotocol/react-native-libveritas": "0.0.0-dev.20260407170644", + "@spacesprotocol/fabric-core": "^0.1.0", + "@spacesprotocol/react-native-libveritas": "^0.3.1", "uniffi-bindgen-react-native": "0.29.3-1" }, "devDependencies": { @@ -41,8 +41,8 @@ "license": "MIT", "dependencies": { "@noble/curves": "^1.8.0", - "@spacesprotocol/fabric-core": "*", - "@spacesprotocol/libveritas": "^0.1.4" + "@spacesprotocol/fabric-core": "^0.1.0", + "@spacesprotocol/libveritas": "^0.3.1" }, "bin": { "fabric": "dist/cli.js" @@ -972,14 +972,14 @@ "link": true }, "node_modules/@spacesprotocol/libveritas": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/@spacesprotocol/libveritas/-/libveritas-0.1.4.tgz", - "integrity": "sha512-65HvMZE5750AW83rQRQdWln3YWwu6Lcs7SBhtXGBUcmyjRNOZ0NfGGIvUCzqhUYAvCvyqSt+A89Zx3K50f/ssQ==" + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@spacesprotocol/libveritas/-/libveritas-0.3.1.tgz", + "integrity": "sha512-xcM/syYCnr5TKvSXq5zNSOmq6vcZDyPzpEMxJsABr0DuQMXWBPlm7Bn7ea3ri576/Z3wOqmHwxj59q4UnSg1Hg==" }, "node_modules/@spacesprotocol/react-native-libveritas": { - "version": "0.0.0-dev.20260407170644", - "resolved": "https://registry.npmjs.org/@spacesprotocol/react-native-libveritas/-/react-native-libveritas-0.0.0-dev.20260407170644.tgz", - "integrity": "sha512-EZ4ZFmd/ik8DeIqq8gBN0ZInTNDF9QkLUeeHkzEe6C7MGEb7aXW8hegi2n+voyfd8XV6hozXaCaapKyUzCWl6Q==", + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@spacesprotocol/react-native-libveritas/-/react-native-libveritas-0.3.1.tgz", + "integrity": "sha512-MT8QX0yHX7YBgUJHYK6KTkzzkjigVCNhcTdipXXnHvXfI0wmhdxlfvzS3Sa0uhtVvU1J6fyhVK8vJ/ULUaohjg==", "license": "MIT", "peerDependencies": { "react": "*", diff --git a/fabric/kotlin/build.gradle.kts b/fabric/kotlin/build.gradle.kts index 518cb52..424b3b6 100644 --- a/fabric/kotlin/build.gradle.kts +++ b/fabric/kotlin/build.gradle.kts @@ -19,9 +19,9 @@ dependencies { // compileOnly — consumers provide the right variant at runtime: // Android: org.spacesprotocol:libveritas (AAR) // JVM: org.spacesprotocol:libveritas-jvm (JAR) - compileOnly("org.spacesprotocol:libveritas-jvm:0.2.0") + compileOnly("org.spacesprotocol:libveritas-jvm:0.3.1") // CLI needs it at runtime - runtimeOnly("org.spacesprotocol:libveritas-jvm:0.2.0") + runtimeOnly("org.spacesprotocol:libveritas-jvm:0.3.1") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3") compileOnly("fr.acinq.secp256k1:secp256k1-kmp:0.17.3") diff --git a/fabric/python/pyproject.toml b/fabric/python/pyproject.toml index 0d1c1cd..d559276 100644 --- a/fabric/python/pyproject.toml +++ b/fabric/python/pyproject.toml @@ -3,7 +3,7 @@ name = "fabric-resolver" version = "0.1.0" description = "Python client for the certrelay fabric network" requires-python = ">=3.10" -dependencies = ["libveritas>=0.2.0"] +dependencies = ["libveritas>=0.3.1"] [project.optional-dependencies] signing = ["coincurve"] diff --git a/fabric/rust/src/lib.rs b/fabric/rust/src/lib.rs index 25e7178..84a389b 100644 --- a/fabric/rust/src/lib.rs +++ b/fabric/rust/src/lib.rs @@ -297,6 +297,86 @@ pub struct AddrEntry { pub rev: String, } +/// One stored handle row served by `GET /sync` (borsh-encoded inside [`SyncPage`]). +/// +/// `cert` and `zone` are opaque borsh blobs passed through exactly as stored. +/// The metadata fields mirror the serving relay's table columns and are +/// **claims** used only for duplicate pre-filtering — a puller must never store +/// them; real values are re-derived from the zone after full verification. +#[derive(Clone, Debug, borsh::BorshSerialize, borsh::BorshDeserialize)] +pub struct SyncRecord { + pub handle: String, + pub epoch_height: u32, + pub seq: u64, + pub delegate_seq: u64, + pub cert: Vec, + pub zone: Vec, +} + +impl SyncRecord { + /// The space portion of the handle (`"alice@bitcoin"` -> `"@bitcoin"`). + pub fn space_name(&self) -> &str { + match self.handle.find(['@', '#']) { + Some(i) => &self.handle[i..], + None => &self.handle, + } + } +} + +/// A page of sync records. `next_cursor` is `None` when the page is empty +/// (nothing beyond the requested cursor). +#[derive(Clone, Debug, Default, borsh::BorshSerialize, borsh::BorshDeserialize)] +pub struct SyncPage { + pub records: Vec, + pub next_cursor: Option, +} + +/// Body of `POST /poke` (JSON): "I have new data up to `cursor` — pull me." +/// +/// Content-free by design: a poke can never transfer state, so it can never +/// amplify. `url` must already be a verified peer of the receiver (poke is not +/// discovery), and `cursor` is a claim checked against the receiver's +/// watermark — the watermark itself only advances from real sync pages. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Poke { + /// The sender's own base URL (where to pull from). + pub url: String, + /// The sender's latest sync cursor. + pub cursor: String, +} + +/// Response of `GET /sync/summary` (JSON, curl-friendly). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncSummary { + /// Total handle rows stored. + pub count: u64, + /// Cursor of the newest stored row, if any. + pub latest_cursor: Option, +} + +/// Sync cursor: position in a relay's strictly-increasing write sequence. +/// +/// Cursors are **peer-local** (they encode the serving relay's private write +/// counter): echo them back to the relay that issued them and compare only +/// cursors from the same relay. Serialized as a decimal string; treat the +/// format as opaque. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct SyncCursor(pub u64); + +impl fmt::Display for SyncCursor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl FromStr for SyncCursor { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + s.parse().map(Self).map_err(|_| "invalid cursor") + } +} + impl AnchorSet { pub fn from_anchors(anchors: Vec) -> Self { Self { entries: anchors } diff --git a/fabric/swift/Package.swift b/fabric/swift/Package.swift index ec0a041..ae818c5 100644 --- a/fabric/swift/Package.swift +++ b/fabric/swift/Package.swift @@ -9,7 +9,7 @@ let package = Package( .executable(name: "fabric", targets: ["FabricCLI"]), ], dependencies: [ - .package(url: "https://github.com/spacesprotocol/libveritas-swift.git", exact: "0.2.0"), + .package(url: "https://github.com/spacesprotocol/libveritas-swift.git", exact: "0.3.1"), .package(url: "https://github.com/21-DOT-DEV/swift-secp256k1.git", exact: "0.17.0"), ], targets: [ diff --git a/relay/Cargo.toml b/relay/Cargo.toml index eff19dd..e67004d 100644 --- a/relay/Cargo.toml +++ b/relay/Cargo.toml @@ -21,7 +21,9 @@ governor = "0.8" anyhow = "1.0" axum = { version = "0.8", features = ["json"] } borsh = { version = "1.6", features = ["derive"] } +serde = { version = "1", features = ["derive"] } serde_json = "1" +toml = "0.8" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } rusqlite = { version = "0.31", features = ["bundled"] } tokio = { version = "1.37", features = ["sync", "rt-multi-thread", "macros"] } @@ -39,5 +41,6 @@ url = "2" testutil = [] [dev-dependencies] +relay = { path = ".", features = ["testutil"] } libveritas_testutil = {workspace = true} spaces_nums = { workspace = true } diff --git a/relay/src/app.rs b/relay/src/app.rs index 4e2708c..a16da0a 100644 --- a/relay/src/app.rs +++ b/relay/src/app.rs @@ -52,13 +52,22 @@ struct Args { #[arg(long, env = "CERTRELAY_REMOTE_IP_HEADER")] remote_ip_header: Option, - /// Anchor refresh interval in seconds (default: 1800 = 30 minutes) + /// Anchor refresh interval in seconds (default: 300 = 5 minutes) #[arg(long, default_value = "300", env = "CERTRELAY_ANCHOR_REFRESH")] anchor_refresh: u64, /// Skip downloading a checkpoint and sync from scratch #[arg(long)] skip_checkpoint_sync: bool, + + /// Accept peers with private/loopback addresses (local development only) + #[arg(long, env = "CERTRELAY_ALLOW_PRIVATE_PEERS")] + allow_private_peers: bool, + + /// Path to a TOML config file for rate limits, sync tuning, peer table + /// sizes, and concurrency caps (all fields optional) + #[arg(long, env = "CERTRELAY_CONFIG")] + config: Option, } fn default_data_dir() -> PathBuf { @@ -169,11 +178,24 @@ pub async fn run( spaced_url = Some(spaced_auth_url); } + let settings = match &args.config { + Some(path) => { + let s = crate::settings::FileConfig::load(path)?; + tracing::info!("loaded config from {}", path.display()); + s + } + None => crate::settings::FileConfig::default(), + }; + let sync_config = settings.sync_config(); + let mut config = Config::new(data_dir, args.chain); config.spaced_url = spaced_url; config.is_bootstrap = args.is_bootstrap; config.self_url = args.self_url; config.remote_ip_header = args.remote_ip_header; + config.allow_private_peers = args.allow_private_peers; + config.peer_config = settings.peer_config(); + config.settings = settings; let relay = Relay::new(config)?; @@ -211,37 +233,34 @@ pub async fn run( } }); - // Periodically verify unverified peers when we need more verified ones - tokio::spawn({ - let state = relay.state().clone(); - async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(10)); - loop { - interval.tick().await; - let candidate = { - let mut peers = state.peers.lock().await; - peers.demote_expired(); - if !peers.needs_peers() { - continue; - } - peers.next_candidate().map(|s| s.to_string()) - }; - if let Some(url) = candidate { - let check_url = format!("{}/peers", url); - match state.http_client.head(&check_url).send().await { - Ok(resp) if resp.status().is_success() => { - let mut peers = state.peers.lock().await; - peers.mark_alive(&url); - tracing::debug!("verified peer: {}", url); - } - _ => { - tracing::debug!("peer health check failed: {}", url); - } - } - } - } - } - }); + // Storage retention: entitlement-weighted eviction under the disk budget + tokio::spawn(crate::retention::run_retention_loop( + relay.state().clone(), + relay.state().retention.clone(), + )); + + // Peer-table maintenance: proactive refresh of verified peers, candidate + // verification, and rate-limiter map cleanup + tokio::spawn(crate::sync::run_peer_maintenance_loop( + relay.state().clone(), + std::time::Duration::from_secs(10), + 3, + )); + + // Pull-based propagation: periodically sync stored handles from peers, + // send pokes when we store new data, and pull promptly when poked. + tokio::spawn(crate::sync::run_sync_loop( + relay.state().clone(), + sync_config.clone(), + )); + tokio::spawn(crate::sync::run_poke_send_loop( + relay.state().clone(), + sync_config.clone(), + )); + tokio::spawn(crate::sync::run_poke_sync_loop( + relay.state().clone(), + sync_config, + )); // Periodically re-announce to verified peers and discover new ones tokio::spawn({ @@ -275,14 +294,9 @@ pub async fn run( let listener = tokio::net::TcpListener::bind(&bind_addr).await?; tracing::info!("relay listening on {}", listener.local_addr()?); - let mut shutdown_rx = shutdown.subscribe(); - tokio::select! { - result = relay.run(listener) => result, - _ = shutdown_rx.recv() => { - tracing::info!("shutdown signal received"); - Ok(()) - } - } + relay + .run_with_shutdown(listener, shutdown.subscribe()) + .await } async fn refresh_anchors(state: &AppState) -> anyhow::Result<()> { @@ -290,7 +304,7 @@ async fn refresh_anchors(state: &AppState) -> anyhow::Result<()> { let anchor_store = AnchorSets::from_anchors(anchors.clone()); anchors.truncate(ROOT_ANCHORS_COUNT as _); let new_veritas = create_relay_veritas(anchors)?; - *state.handler.veritas.lock().unwrap() = new_veritas; + *state.handler.veritas.write().unwrap() = new_veritas; *state.handler.anchor_store.lock().unwrap() = anchor_store; Ok(()) } diff --git a/relay/src/handler.rs b/relay/src/handler.rs index 3481c76..b5e0d0c 100644 --- a/relay/src/handler.rs +++ b/relay/src/handler.rs @@ -1,6 +1,6 @@ use crate::anchor::AnchorSets; use crate::spaced::SpacedClient; -use crate::store::{HandleRecord, SqliteStore}; +use crate::store::{BulkStoreResult, HandleRecord, SqliteStore}; use governor::DefaultKeyedRateLimiter; use governor::Quota; use libveritas::builder::{DataUpdateRequest, MessageBuilder}; @@ -14,12 +14,12 @@ use spaces_protocol::slabel::SLabel; use std::collections::{HashMap, HashSet}; use std::num::NonZeroU32; use std::str::FromStr; -use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, RwLock}; /// Certificate handler that verifies messages and stores zones/handles. pub struct Handler { - pub veritas: Mutex, + pub veritas: RwLock, pub anchor_store: Mutex, pub store: SqliteStore, pub dev_mode: bool, @@ -34,7 +34,7 @@ pub struct Handler { impl Handler { pub fn new(veritas: Veritas, store: SqliteStore, anchor_store: AnchorSets) -> Self { Self { - veritas: Mutex::new(veritas), + veritas: RwLock::new(veritas), anchor_store: Mutex::new(anchor_store), store, dev_mode: false, @@ -44,12 +44,30 @@ impl Handler { handle_rate: governor::RateLimiter::keyed( Quota::with_period(std::time::Duration::from_secs(300)) .unwrap() - .allow_burst(NonZeroU32::new(1).unwrap()), + .allow_burst(NonZeroU32::new(3).unwrap()), ), msg_count: AtomicU64::new(0), } } + /// Override the per-space and per-handle content rate limits (config file). + pub fn with_content_rates( + mut self, + space_per_min: u32, + handle_period: std::time::Duration, + handle_burst: u32, + ) -> Self { + self.space_rate = governor::RateLimiter::keyed(Quota::per_minute( + NonZeroU32::new(space_per_min.max(1)).unwrap(), + )); + self.handle_rate = governor::RateLimiter::keyed( + Quota::with_period(handle_period) + .unwrap_or_else(|| Quota::with_period(std::time::Duration::from_secs(300)).unwrap()) + .allow_burst(NonZeroU32::new(handle_burst.max(1)).unwrap()), + ); + self + } + pub async fn resolve( &self, chain: &SpacedClient, @@ -153,7 +171,7 @@ impl Handler { } let mut res = resolver::HintsResponse { - anchor_tip: self.veritas.lock().expect("lock").newest_anchor(), + anchor_tip: self.veritas.read().expect("lock").newest_anchor(), hints: vec![], }; @@ -204,8 +222,35 @@ impl Handler { /// Handle an incoming certificate message. /// /// Verifies the message against the current chain state, updates stored zones, - /// and stores any new handle records. - pub fn handle_message(&self, msg: msg::Message) -> anyhow::Result<()> { + /// and stores any new handle records. Returns the store result so callers can + /// distinguish new data (`stored > 0`) from duplicates and stale updates. + pub fn handle_message(&self, msg: msg::Message) -> anyhow::Result { + self.handle_message_opts(msg, &HashSet::new(), true) + } + + /// [`Handler::handle_message`] with a set of spaces whose **first + /// inserts** must be skipped (retention admission gate under storage + /// pressure). Updates to existing handles are never gated. + pub fn handle_message_gated( + &self, + msg: msg::Message, + gated_spaces: &HashSet, + ) -> anyhow::Result { + self.handle_message_opts(msg, gated_spaces, true) + } + + /// Full-control variant: `charge_content_limits = false` exempts the + /// message from the per-space/per-handle velocity caps. Used by sync + /// ingest — silently rate-dropping synced records would advance the + /// watermark past data that was never stored (permanent loss); sync CPU + /// is already bounded by the failed-space budget and verify semaphore, + /// and storage by retention. + pub fn handle_message_opts( + &self, + msg: msg::Message, + gated_spaces: &HashSet, + charge_content_limits: bool, + ) -> anyhow::Result { // Periodically clean up expired rate limiter entries if self .msg_count @@ -234,7 +279,7 @@ impl Handler { }; let res = self .veritas - .lock() + .read() .unwrap() .verify_with_options(&ctx, msg, options)?; @@ -256,7 +301,32 @@ impl Handler { // Max offchain records size per handle (1 KB) const MAX_RECORDS_SIZE: usize = 1024; + // Bulk-load existing hint rows: powers the duplicate pre-skip and the + // churn-only rate limits (first insert of a handle is free; replacing + // an existing row is what gets rate limited — otherwise bootstrap sync + // would trip handle_rate on a network's worth of first-time handles). + let all_handles: Vec = res.certificates().map(|c| c.subject.to_string()).collect(); + let handle_refs: Vec<&str> = all_handles.iter().map(|s| s.as_str()).collect(); + let existing: HashMap)> = self + .store + .get_handle_hints(&handle_refs)? + .into_iter() + .map(|r| { + ( + r.handle, + ( + r.epoch_height, + r.offchain_seq, + r.delegate_offchain_seq, + r.zone_hash, + ), + ) + }) + .collect(); + // Bundle each certificate with its zone and epoch height + let mut preskipped = 0usize; + let mut gated = 0usize; let mut revs: Vec<(String, String)> = Vec::new(); // canonical_handle -> (rev_name, vec of (addr_name, addr_value)) let mut addr_index: HashMap)> = HashMap::new(); @@ -303,8 +373,37 @@ impl Handler { let space = cert.subject.space()?.to_string(); - // Rate limit per space (100 handle updates/min) and per handle (1 per 5 min) - if !self.dev_mode { + let epoch_height = epoch_map.get(&space).copied().unwrap_or(0); + let offchain_seq = zone.records.seq().unwrap_or(0); + let delegate_offchain_seq = match &zone.delegate { + ProvableOption::Exists { value: d } => d.records.seq().unwrap_or(0), + _ => 0, + }; + + // Duplicate pre-skip: identical metadata AND identical zone bytes + // means a redelivery (multi-path sync, republish) — skip it + // without charging the content rate limits. The hash check + // matters: a better zone can carry unchanged seqs (e.g. a + // pending->finalized commitment transition). + let stored = existing.get(&handle_str); + if let Some((e, s, d, hash)) = stored + && (*e, *s, *d) == (epoch_height, offchain_seq, delegate_offchain_seq) + && borsh::to_vec(*zone).is_ok_and(|z| crate::store::zone_hash(&z) == *hash) + { + preskipped += 1; + return None; + } + + // Retention admission gate: under storage pressure, spaces + // over their entitlement get no new handles (updates pass). + if stored.is_none() && gated_spaces.contains(&space) { + gated += 1; + return None; + } + + // Rate limit per space (100 handle updates/min) and per handle + // (1 per 5 min) — churn only: first insert of a handle is free. + if charge_content_limits && !self.dev_mode && stored.is_some() { if self.space_rate.check_key(&space).is_err() { tracing::warn!("{}: space rate limited, skipping", space); return None; @@ -314,12 +413,6 @@ impl Handler { return None; } } - let epoch_height = epoch_map.get(&space).copied().unwrap_or(0); - let offchain_seq = zone.records.seq().unwrap_or(0); - let delegate_offchain_seq = match &zone.delegate { - ProvableOption::Exists { value: d } => d.records.seq().unwrap_or(0), - _ => 0, - }; Some(HandleRecord { cert, zone: (*zone).clone(), @@ -330,7 +423,9 @@ impl Handler { }) .collect(); - let result = self.store.update_handles(&updates)?; + let mut result = self.store.update_handles(&updates)?; + result.skipped += preskipped; + result.gated += gated; tracing::debug!( "stored {} handles, skipped {} (existing zone was better)", result.stored, @@ -356,15 +451,25 @@ impl Handler { } } - Ok(()) + Ok(result) } } -/// Check if a wire-format epoch hint is verifiable by a zone. -/// Only checks height - the root is used by clients, not the relay. +/// Check whether a client's cached epoch hint lets it verify the served zone +/// without the relay including the ZK receipt. +/// +/// libveritas restores a stripped receipt from the client's cache only when the +/// served commitment's `state_root` matches the cached one (see +/// `Zone::update_receipt_cache`), so the relay may drop the receipt only when the +/// roots match. If the client's cached commitment is strictly newer than the one +/// being served, the client discards the served (older) zone in favor of its +/// cache and never asks for the receipt, so dropping it is safe there too. fn epoch_hint_verifiable_by(hint: &resolver::EpochHint, zone: &Zone) -> bool { if let ProvableOption::Exists { value: c } = &zone.commitment { - c.onchain.block_height >= hint.height + if hint.height > c.onchain.block_height { + return true; // client already verified a newer commitment + } + hint.height == c.onchain.block_height && hint.root == hex::encode(c.onchain.state_root) } else { false } diff --git a/relay/src/http.rs b/relay/src/http.rs index 15ab24c..11c406c 100644 --- a/relay/src/http.rs +++ b/relay/src/http.rs @@ -31,25 +31,34 @@ use crate::spaced::SpacedClient; pub type IpRateLimiter = RateLimiter, DefaultClock>; /// Configuration for rate limits. +/// +/// Buckets follow the cheap/expensive split: `read` covers indexed lookups, +/// `proof` covers endpoints that trigger proof generation. #[derive(Clone)] pub struct RateLimitConfig { - /// Quota for /message endpoint + /// Quota for /message (client publish intake) pub message: Quota, - /// Quota for /query and /chain-proof endpoints (both trigger proof generation) - pub query: Quota, - /// Quota for /announce endpoint + /// Quota for /query and /chain-proof (both trigger proof generation) + pub proof: Quota, + /// Quota for cheap reads: /hints, /reverse, /addrs, /anchors, /peers + pub read: Quota, + /// Quota for /announce pub announce: Quota, - /// Quota for /peers endpoint - pub peers: Quota, + /// Quota for /sync and /sync/summary (pages are the unit) + pub sync: Quota, + /// Quota for /poke + pub poke: Quota, } impl Default for RateLimitConfig { fn default() -> Self { Self { - message: Quota::per_minute(NonZeroU32::new(10).unwrap()), - query: Quota::per_minute(NonZeroU32::new(15).unwrap()), + message: Quota::per_minute(NonZeroU32::new(60).unwrap()), + proof: Quota::per_minute(NonZeroU32::new(30).unwrap()), + read: Quota::per_minute(NonZeroU32::new(120).unwrap()), announce: Quota::per_minute(NonZeroU32::new(5).unwrap()), - peers: Quota::per_minute(NonZeroU32::new(10).unwrap()), + sync: Quota::per_minute(NonZeroU32::new(60).unwrap()), + poke: Quota::per_minute(NonZeroU32::new(30).unwrap()), } } } @@ -57,18 +66,37 @@ impl Default for RateLimitConfig { /// Rate limiters for each endpoint type. pub struct RateLimiters { pub message: Arc, - pub query: Arc, + pub proof: Arc, + pub read: Arc, pub announce: Arc, - pub peers: Arc, + pub sync: Arc, + pub poke: Arc, } impl RateLimiters { pub fn new(config: &RateLimitConfig) -> Self { Self { message: Arc::new(RateLimiter::dashmap(config.message)), - query: Arc::new(RateLimiter::dashmap(config.query)), + proof: Arc::new(RateLimiter::dashmap(config.proof)), + read: Arc::new(RateLimiter::dashmap(config.read)), announce: Arc::new(RateLimiter::dashmap(config.announce)), - peers: Arc::new(RateLimiter::dashmap(config.peers)), + sync: Arc::new(RateLimiter::dashmap(config.sync)), + poke: Arc::new(RateLimiter::dashmap(config.poke)), + } + } + + /// Evict stale per-IP entries; call periodically or the maps grow forever. + pub fn cleanup(&self) { + for limiter in [ + &self.message, + &self.proof, + &self.read, + &self.announce, + &self.sync, + &self.poke, + ] { + limiter.retain_recent(); + limiter.shrink_to_fit(); } } } @@ -76,6 +104,25 @@ impl RateLimiters { /// Default max message size (512 KB). pub const DEFAULT_MAX_MESSAGE_SIZE: usize = 512 * 1024; +/// Connect timeout for outbound requests to peers. +pub const OUTBOUND_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); +/// Total timeout for outbound requests to peers. +pub const OUTBOUND_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +/// Max bytes accepted when reading a peer's /peers response. +pub const MAX_PEERS_RESPONSE_SIZE: usize = 64 * 1024; +/// Max peer entries processed from a single /peers response. +pub const MAX_PEERS_PER_RESPONSE: usize = 256; + +/// Max rows in one /sync page. +pub const MAX_SYNC_PAGE_ROWS: usize = 1000; +/// Soft byte cap for one /sync page (stops adding rows once exceeded). +pub const MAX_SYNC_PAGE_BYTES: usize = 2 * 1024 * 1024; + +/// Concurrent chain-proof generations (global, identity-independent cap). +pub const PROOF_CONCURRENCY: usize = 6; +/// Concurrent message verifications (global, identity-independent cap). +pub const VERIFY_CONCURRENCY: usize = 4; + /// Default bootstrap relay URLs. pub const BOOTSTRAP_RELAYS: &[&str] = &[ "https://relay-cosmos.spacesprotocol.org", @@ -99,6 +146,28 @@ pub struct AppState { /// HTTP header to read the client IP from (e.g. "x-forwarded-for", "cf-connecting-ip"). /// If None, uses the socket address directly. pub remote_ip_header: Option, + /// Accept peers with private/loopback addresses (local development and tests). + pub allow_private_peers: bool, + /// Signal that new data was stored — wakes the poke-send loop, which + /// coalesces bursts into one poke per peer per debounce window. + pub poke_dirty: tokio::sync::Notify, + /// Queue of peer URLs to sync with soon (fed by validated /poke requests, + /// drained by the poke-sync loop with per-peer cooldowns). + pub poke_sync_tx: tokio::sync::mpsc::UnboundedSender, + /// Receiver half, taken once by the poke-sync loop. + pub poke_sync_rx: Mutex>>, + /// Global cap on concurrent proof generation. HTTP handlers try-acquire + /// and return 503 when saturated; background sync waits its turn. + pub proof_sem: Arc, + /// Global cap on concurrent message verification (CPU-bound). + pub verify_sem: Arc, + /// Observability counters served by GET /stats. + pub stats: crate::stats::Stats, + /// Retention policy (storage budget + entitlement) for the admission + /// gate and eviction sweep. + pub retention: crate::retention::RetentionConfig, + /// Approximate recently-queried handles, spared during eviction. + pub query_heat: std::sync::Mutex, } impl AppState { @@ -112,17 +181,29 @@ impl AppState { peer_config: PeerConfig, rate_config: RateLimitConfig, ) -> Self { + let (poke_sync_tx, poke_sync_rx) = tokio::sync::mpsc::unbounded_channel(); Self { handler, chain, peers: Mutex::new(PeerTable::new(peer_config)), limiters: RateLimiters::new(&rate_config), max_message_size: DEFAULT_MAX_MESSAGE_SIZE, - http_client: reqwest::Client::new(), + http_client: outbound_client_builder() + .build() + .expect("failed to build http client"), self_url: None, capabilities: 0, is_bootstrap: false, remote_ip_header: None, + allow_private_peers: false, + poke_dirty: tokio::sync::Notify::new(), + poke_sync_tx, + poke_sync_rx: Mutex::new(Some(poke_sync_rx)), + proof_sem: Arc::new(tokio::sync::Semaphore::new(PROOF_CONCURRENCY)), + verify_sem: Arc::new(tokio::sync::Semaphore::new(VERIFY_CONCURRENCY)), + stats: crate::stats::Stats::default(), + retention: crate::retention::RetentionConfig::default(), + query_heat: std::sync::Mutex::new(crate::retention::QueryHeat::default()), } } @@ -131,6 +212,49 @@ impl AppState { self.self_url = Some(url); self } + + /// Client for contacting a peer URL with the address policy enforced at + /// connection time. For DNS-named peers the host is resolved, every + /// address checked against the IP policy, and the vetted addresses pinned + /// into the client — closing the resolve-then-fetch TOCTOU (DNS + /// rebinding). IP-literal peers already passed the syntactic policy, so + /// the shared client is used. Errors mean the peer's address is not + /// allowed (or unresolvable) and it should not be contacted. + pub async fn peer_client(&self, url: &str) -> anyhow::Result { + if self.allow_private_peers { + return Ok(self.http_client.clone()); + } + crate::peer::validate_peer_url(url, false).map_err(|e| anyhow::anyhow!(e))?; + let parsed = url::Url::parse(url)?; + match parsed.host() { + Some(url::Host::Domain(domain)) => { + let port = parsed.port_or_known_default().unwrap_or(443); + let addrs: Vec = + tokio::net::lookup_host((domain, port)).await?.collect(); + if addrs.is_empty() { + anyhow::bail!("peer host did not resolve: {}", url); + } + if !addrs.iter().all(|a| crate::peer::ip_is_public(&a.ip())) { + anyhow::bail!("peer resolves to a disallowed address: {}", url); + } + Ok(outbound_client_builder() + .resolve_to_addrs(domain, &addrs) + .build()?) + } + Some(_) => Ok(self.http_client.clone()), + None => anyhow::bail!("peer url has no host: {}", url), + } + } +} + +/// Base builder for all peer-facing clients: bounded timeouts and **no +/// redirect following** — a 302 from a peer must never steer a request at +/// internal services (spaced/yuki RPC, cloud metadata). +fn outbound_client_builder() -> reqwest::ClientBuilder { + reqwest::Client::builder() + .connect_timeout(OUTBOUND_CONNECT_TIMEOUT) + .timeout(OUTBOUND_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) } /// Build the router with all routes. @@ -147,22 +271,28 @@ pub fn router(state: Arc) -> Router { .route("/chain-proof", post(handle_chain_proof)) .route("/reverse", get(handle_reverse)) .route("/addrs", get(handle_addrs)) + .route("/sync", get(handle_sync)) + .route("/sync/summary", get(handle_sync_summary)) + .route("/poke", post(handle_poke)) + .route("/health", get(handle_health)) + .route("/stats", get(handle_stats)) .layer(cors) .with_state(state) } /// Extract the client IP from the configured header, falling back to socket address. /// -/// If `remote_ip_header` is set, reads that header and parses the first IP -/// (handles comma-separated lists like X-Forwarded-For). +/// If `remote_ip_header` is set, reads that header and parses the **last** IP. +/// For append-style headers (X-Forwarded-For) the rightmost entry is the one +/// written by our own trusted proxy — leftmost entries are client-controlled +/// and would let anyone rotate fake IPs past the per-IP limits. Single-value +/// overwrite headers (CF-Connecting-IP) are unaffected. fn client_ip(addr: &SocketAddr, headers: &HeaderMap, header_name: &Option) -> IpAddr { if let Some(name) = header_name && let Some(value) = headers.get(name.as_str()).and_then(|v| v.to_str().ok()) { - // Take the first entry (leftmost = original client for XFF-style headers, - // and the only value for single-value headers like CF-Connecting-IP) - let first = value.split(',').next().unwrap_or("").trim(); - if let Ok(ip) = first.parse::() { + let last = value.rsplit(',').next().unwrap_or("").trim(); + if let Ok(ip) = last.parse::() { return ip; } } @@ -172,7 +302,8 @@ fn client_ip(addr: &SocketAddr, headers: &HeaderMap, header_name: &Option>, ConnectInfo(addr): ConnectInfo, @@ -180,7 +311,9 @@ async fn handle_message( body: Bytes, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); + crate::stats::bump(&state.stats.messages_received); if state.limiters.message.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_message); return (StatusCode::TOO_MANY_REQUESTS, "rate limited".to_string()); } @@ -203,17 +336,161 @@ async fn handle_message( } }; - // Verify and store - if let Err(e) = state.handler.handle_message(msg) { - tracing::warn!("failed to handle message: {}", e); - return (StatusCode::BAD_REQUEST, format!("rejected: {}", e)); + // Global verify cap: shed load instead of queueing unbounded CPU work + let Ok(_permit) = Arc::clone(&state.verify_sem).try_acquire_owned() else { + crate::stats::bump(&state.stats.busy_rejections); + return (StatusCode::SERVICE_UNAVAILABLE, "busy".to_string()); + }; + + // Retention admission gate: under storage pressure, spaces over their + // entitlement accept no new handles (updates still pass). + let mut gated_spaces = std::collections::HashSet::new(); + for bundle in &msg.spaces { + let space = bundle.subject.to_string(); + if !gated_spaces.contains(&space) + && crate::retention::first_insert_gated(&state, &state.retention, &space) + .unwrap_or(false) + { + gated_spaces.insert(space); + } } - gossip_message(state, body).await; + // Verify and store on the blocking pool: ZK receipt verification is + // CPU-bound and must not stall the async runtime. + let blocking_state = Arc::clone(&state); + let result = tokio::task::spawn_blocking(move || { + blocking_state + .handler + .handle_message_gated(msg, &gated_spaces) + }) + .await; + match result { + Ok(Ok(result)) => { + crate::stats::bump_by(&state.stats.admission_gated, result.gated as u64); + if result.stored > 0 { + crate::stats::bump(&state.stats.messages_accepted); + state.poke_dirty.notify_one(); + } else { + crate::stats::bump(&state.stats.messages_deduped); + } + } + Ok(Err(e)) => { + crate::stats::bump(&state.stats.messages_rejected); + tracing::warn!("failed to handle message: {}", e); + return (StatusCode::BAD_REQUEST, format!("rejected: {}", e)); + } + Err(e) => { + tracing::error!("message verification task failed: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "internal error".to_string(), + ); + } + } (StatusCode::OK, "ok".to_string()) } +/// GET/HEAD /health - Unmetered liveness check (peer health checks and load +/// balancers target this so they never contend with rate-limited endpoints). +async fn handle_health() -> impl IntoResponse { + (StatusCode::OK, "ok") +} + +/// GET /stats - Observability counters (JSON), plus live peer/semaphore state. +async fn handle_stats( + State(state): State>, + ConnectInfo(addr): ConnectInfo, + headers: HeaderMap, +) -> impl IntoResponse { + let ip = client_ip(&addr, &headers, &state.remote_ip_header); + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); + return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); + } + + let mut snapshot = state.stats.snapshot(); + let (verified, unverified) = { + let peers = state.peers.lock().await; + (peers.verified_count(), peers.unverified_count()) + }; + snapshot["peers"] = serde_json::json!({ + "verified": verified, + "unverified": unverified, + }); + if let Ok((rows, bytes)) = state.handler.store.storage_totals() { + snapshot["storage"] = serde_json::json!({ + "rows": rows, + "bytes": bytes, + "budget_bytes": state.retention.max_storage_bytes, + }); + } + snapshot["concurrency"] = serde_json::json!({ + "proof_permits_available": state.proof_sem.available_permits(), + "verify_permits_available": state.verify_sem.available_permits(), + }); + axum::Json(snapshot).into_response() +} + +/// POST /poke - A peer signals it has new data; schedule a pull from it. +/// +/// Body: JSON [`resolver::Poke`]. Content-free fast propagation: the poke +/// carries no records, only "pull me." Only verified peers are acted on, a +/// cursor at or behind our watermark is dropped, and the actual pull runs +/// through the same rate-paced sync path as the interval loop — so a poke +/// flood cannot make us do more work than the steady-state maximum. +async fn handle_poke( + State(state): State>, + ConnectInfo(addr): ConnectInfo, + headers: HeaderMap, + body: Bytes, +) -> impl IntoResponse { + let ip = client_ip(&addr, &headers, &state.remote_ip_header); + if state.limiters.poke.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_poke); + return (StatusCode::TOO_MANY_REQUESTS, "rate limited"); + } + + crate::stats::bump(&state.stats.pokes_received); + let poke: resolver::Poke = match serde_json::from_slice(&body) { + Ok(p) => p, + Err(_) => return (StatusCode::BAD_REQUEST, "invalid poke format"), + }; + if poke.url.len() > 256 { + return (StatusCode::BAD_REQUEST, "invalid url"); + } + // Normalize before ANY keyed use: https://x, https://x/, https://x// all + // pass verification but would otherwise be distinct watermark/cooldown + // keys, bypassing the cursor dedup and per-peer cooldown. + let poke_url = crate::peer::normalize_url(&poke.url); + let Ok(cursor) = poke.cursor.parse::() else { + return (StatusCode::BAD_REQUEST, "invalid cursor"); + }; + + // Poke is not discovery: act only on peers we already verified. + // Respond "ok" either way so the response doesn't leak table membership. + if !state.peers.lock().await.is_verified(&poke_url) { + return (StatusCode::OK, "ok"); + } + + // Claimed-cursor dedup: at or behind our watermark means nothing new. + // (Watermarks only ever advance from real sync pages, never from here.) + let watermark = state + .handler + .store + .get_watermark(&poke_url) + .ok() + .flatten() + .and_then(|c| c.parse::().ok()); + if watermark.is_some_and(|w| cursor <= w) { + return (StatusCode::OK, "ok"); + } + + crate::stats::bump(&state.stats.pokes_accepted); + let _ = state.poke_sync_tx.send(poke_url); + (StatusCode::OK, "ok") +} + /// POST /announce - Announce a peer URL with capabilities. /// /// Body: JSON Announcement { url, capabilities } @@ -225,6 +502,7 @@ async fn handle_announce( ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); if state.limiters.announce.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_announce); return (StatusCode::TOO_MANY_REQUESTS, "rate limited"); } @@ -239,6 +517,12 @@ async fn handle_announce( if announcement.url.is_empty() || announcement.url.len() > 256 { return (StatusCode::BAD_REQUEST, "invalid url"); } + if let Err(reason) = + crate::peer::validate_peer_url(&announcement.url, state.allow_private_peers) + { + tracing::debug!("rejected announce {}: {}", announcement.url, reason); + return (StatusCode::BAD_REQUEST, reason); + } let peer = PeerInfo { source_ip: ip, @@ -267,7 +551,8 @@ async fn handle_peers( headers: HeaderMap, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.peers.check_key(&ip).is_err() { + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); } @@ -291,7 +576,8 @@ async fn handle_query( axum::extract::Query(params): axum::extract::Query>, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.query.check_key(&ip).is_err() { + if state.limiters.proof.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_proof); return (StatusCode::TOO_MANY_REQUESTS, vec![]).into_response(); } @@ -312,6 +598,14 @@ async fn handle_query( .filter(|s| !s.is_empty()) .collect(); + // Requested handles are "hot": spared during retention eviction. + { + let mut heat = state.query_heat.lock().unwrap(); + for h in &handles { + heat.touch(h); + } + } + const MAX_HANDLES: usize = 6; if handles.len() > MAX_HANDLES { return ( @@ -367,8 +661,17 @@ async fn handle_query( let mut resp_headers = HeaderMap::new(); resp_headers.insert("cache-control", "public, max-age=300".parse().unwrap()); + // Global proof cap: shed load instead of queueing proof generation + let Ok(_permit) = state.proof_sem.try_acquire() else { + crate::stats::bump(&state.stats.busy_rejections); + return (StatusCode::SERVICE_UNAVAILABLE, vec![]).into_response(); + }; + match state.handler.resolve(&state.chain, queries).await { - Ok(msg) => (resp_headers, msg.to_bytes()).into_response(), + Ok(msg) => { + crate::stats::bump(&state.stats.proofs_served); + (resp_headers, msg.to_bytes()).into_response() + } Err(e) => { tracing::warn!("failed to resolve query: {}", e); (StatusCode::INTERNAL_SERVER_ERROR, vec![]).into_response() @@ -390,7 +693,8 @@ async fn handle_anchors( axum::extract::Query(params): axum::extract::Query>, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.peers.check_key(&ip).is_err() { + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); } @@ -445,7 +749,8 @@ async fn handle_hints( axum::extract::Query(params): axum::extract::Query>, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.query.check_key(&ip).is_err() { + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); } @@ -487,7 +792,8 @@ async fn handle_reverse( axum::extract::Query(params): axum::extract::Query>, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.query.check_key(&ip).is_err() { + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); } @@ -528,7 +834,8 @@ async fn handle_addrs( axum::extract::Query(params): axum::extract::Query>, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.query.check_key(&ip).is_err() { + if state.limiters.read.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_read); return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); } @@ -565,6 +872,84 @@ async fn handle_addrs( } } +/// GET /sync?cursor=&limit= - Page of stored handle rows. +/// +/// Returns a borsh-encoded [`resolver::SyncPage`] ordered by +/// `(updated_at, handle)`. Serving is one indexed SELECT streaming blobs +/// exactly as stored — no proof generation. The cursor is peer-local: echo it +/// back to this relay only. +async fn handle_sync( + State(state): State>, + ConnectInfo(addr): ConnectInfo, + headers: HeaderMap, + axum::extract::Query(params): axum::extract::Query>, +) -> impl IntoResponse { + let ip = client_ip(&addr, &headers, &state.remote_ip_header); + if state.limiters.sync.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_sync); + return (StatusCode::TOO_MANY_REQUESTS, vec![]).into_response(); + } + + let cursor = match params.get("cursor").filter(|c| !c.is_empty()) { + Some(raw) => match raw.parse::() { + Ok(c) => Some(c), + Err(e) => return (StatusCode::BAD_REQUEST, e.as_bytes().to_vec()).into_response(), + }, + None => None, + }; + let limit = params + .get("limit") + .and_then(|l| l.parse::().ok()) + .unwrap_or(MAX_SYNC_PAGE_ROWS) + .clamp(1, MAX_SYNC_PAGE_ROWS); + + // Page reads copy up to several MB of blobs under the connection mutex — + // keep that off the async runtime. + let blocking_state = Arc::clone(&state); + let page = tokio::task::spawn_blocking(move || { + blocking_state + .handler + .store + .sync_page(cursor, limit, MAX_SYNC_PAGE_BYTES) + }) + .await + .unwrap_or_else(|e| Err(anyhow::anyhow!("sync page task failed: {e}"))); + match page { + Ok(page) => match borsh::to_vec(&page) { + Ok(bytes) => (StatusCode::OK, bytes).into_response(), + Err(e) => { + tracing::warn!("failed to serialize sync page: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, vec![]).into_response() + } + }, + Err(e) => { + tracing::warn!("sync page failed: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, vec![]).into_response() + } + } +} + +/// GET /sync/summary - Row count and newest cursor (JSON, curl-friendly). +async fn handle_sync_summary( + State(state): State>, + ConnectInfo(addr): ConnectInfo, + headers: HeaderMap, +) -> impl IntoResponse { + let ip = client_ip(&addr, &headers, &state.remote_ip_header); + if state.limiters.sync.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_sync); + return (StatusCode::TOO_MANY_REQUESTS, "rate limited").into_response(); + } + + match state.handler.store.sync_summary() { + Ok(summary) => axum::Json(summary).into_response(), + Err(e) => { + tracing::warn!("sync summary failed: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "summary failed").into_response() + } + } +} + /// POST /chain-proof - Build a chain proof from a ChainProofRequest. /// /// Body: JSON ChainProofRequest @@ -576,7 +961,8 @@ async fn handle_chain_proof( body: Bytes, ) -> impl IntoResponse { let ip = client_ip(&addr, &headers, &state.remote_ip_header); - if state.limiters.query.check_key(&ip).is_err() { + if state.limiters.proof.check_key(&ip).is_err() { + crate::stats::bump(&state.stats.rl_proof); return (StatusCode::TOO_MANY_REQUESTS, vec![]).into_response(); } @@ -603,8 +989,17 @@ async fn handle_chain_proof( .into_response(); } + // Global proof cap: shed load instead of queueing proof generation + let Ok(_permit) = state.proof_sem.try_acquire() else { + crate::stats::bump(&state.stats.busy_rejections); + return (StatusCode::SERVICE_UNAVAILABLE, vec![]).into_response(); + }; + match state.chain.prove(&request).await { - Ok(proof) => (StatusCode::OK, proof.to_bytes()).into_response(), + Ok(proof) => { + crate::stats::bump(&state.stats.proofs_served); + (StatusCode::OK, proof.to_bytes()).into_response() + } Err(e) => { tracing::warn!("failed to build chain proof: {}", e); (StatusCode::INTERNAL_SERVER_ERROR, vec![]).into_response() @@ -612,53 +1007,6 @@ async fn handle_chain_proof( } } -/// Gossip a message to up to 4 random verified peers. -async fn gossip_message(state: Arc, msg_bytes: Bytes) { - use rand::seq::IndexedRandom; - - let peer_list: Vec = { - let peers = state.peers.lock().await; - peers.peers_info() - }; - - let targets: Vec<_> = peer_list - .choose_multiple(&mut rand::rng(), 4) - .cloned() - .collect(); - - for peer in targets { - let state = Arc::clone(&state); - let msg_bytes = msg_bytes.clone(); - - tokio::spawn(async move { - let url = format!("{}/message", peer.url); - let result = state - .http_client - .post(&url) - .body(msg_bytes.to_vec()) - .header("Content-Type", "application/octet-stream") - .send() - .await; - - let mut peers = state.peers.lock().await; - match result { - Ok(resp) if resp.status().is_success() => { - peers.mark_alive(&peer.url); - tracing::trace!("gossip to {} succeeded", peer.url); - } - Ok(resp) => { - peers.deprioritize(&peer.url); - tracing::debug!("gossip to {} failed: {}", peer.url, resp.status()); - } - Err(e) => { - peers.deprioritize(&peer.url); - tracing::debug!("gossip to {} failed: {}", peer.url, e); - } - } - }); - } -} - /// Bootstrap from the default bootstrap relays. /// Does nothing if this node is a bootstrap node itself. pub async fn bootstrap(state: &Arc) { @@ -685,6 +1033,8 @@ pub async fn bootstrap_from( state: &Arc, bootstrap_url: &str, ) -> anyhow::Result> { + let client = state.peer_client(bootstrap_url).await?; + // Announce ourselves if we have a self URL if let Some(ref self_url) = state.self_url { let announcement = Announcement { @@ -692,18 +1042,27 @@ pub async fn bootstrap_from( capabilities: state.capabilities, }; let url = format!("{}/announce", bootstrap_url); - let _ = state - .http_client - .post(&url) - .json(&announcement) - .send() - .await; + let _ = client.post(&url).json(&announcement).send().await; } - // Fetch their peer list + // Fetch their peer list, bounding how much we read from an untrusted body let url = format!("{}/peers", bootstrap_url); - let resp = state.http_client.get(&url).send().await?; - let peers: Vec = resp.json().await?; + let resp = client.get(&url).send().await?; + if let Some(len) = resp.content_length() + && len > MAX_PEERS_RESPONSE_SIZE as u64 + { + anyhow::bail!("peers response too large: {} bytes", len); + } + let body = resp.bytes().await?; + if body.len() > MAX_PEERS_RESPONSE_SIZE { + anyhow::bail!("peers response too large: {} bytes", body.len()); + } + let mut peers: Vec = serde_json::from_slice(&body)?; + peers.truncate(MAX_PEERS_PER_RESPONSE); + peers.retain(|p| { + p.url.len() <= 256 + && crate::peer::validate_peer_url(&p.url, state.allow_private_peers).is_ok() + }); // Add discovered peers to our table { diff --git a/relay/src/lib.rs b/relay/src/lib.rs index f62264d..a81f872 100644 --- a/relay/src/lib.rs +++ b/relay/src/lib.rs @@ -5,8 +5,12 @@ pub mod handler; pub mod http; pub mod peer; pub mod relay; +pub mod retention; +pub mod settings; pub mod spaced; +pub mod stats; pub mod store; +pub mod sync; pub use resolver::anchor; @@ -21,10 +25,16 @@ pub use peer::{AnnounceResult, PeerConfig, PeerTable}; pub use handler::Handler; pub use relay::{Config, Relay, ServiceRunner}; pub use resolver::{Announcement, EpochHint, PeerInfo, Query, QueryRequest, capabilities}; +pub use retention::{QueryHeat, RetentionConfig, run_retention_loop}; +pub use settings::FileConfig; pub use spaced::SpacedClient; pub use spaces_client::config::ExtendedNetwork; use spaces_nums::RootAnchor; pub use store::SqliteStore; +pub use sync::{ + SyncConfig, SyncIngest, ingest_sync_records, run_peer_maintenance_loop, run_poke_send_loop, + run_poke_sync_loop, run_sync_loop, sync_round, sync_with_peer, +}; // Create veritas with disabled name expansion fn create_relay_veritas(anchors: Vec) -> Result { diff --git a/relay/src/peer.rs b/relay/src/peer.rs index 6bf956b..2ebbe4a 100644 --- a/relay/src/peer.rs +++ b/relay/src/peer.rs @@ -6,6 +6,82 @@ use std::{ pub use resolver::{PeerInfo, capabilities}; +/// Syntactic policy check for a peer URL: http(s) scheme, a host, no +/// credentials, and — unless `allow_private` — no private/reserved IP literal. +/// Plain http and bare-IP URLs are allowed by design: peer data re-verifies +/// locally, so the network must not depend on DNS/CA infrastructure. +/// DNS-named hosts are additionally resolve-checked in [`peer_addr_allowed`]. +pub fn validate_peer_url(raw: &str, allow_private: bool) -> Result<(), &'static str> { + let parsed = url::Url::parse(raw).map_err(|_| "invalid url")?; + if parsed.scheme() != "http" && parsed.scheme() != "https" { + return Err("url scheme must be http or https"); + } + if !parsed.username().is_empty() || parsed.password().is_some() { + return Err("url must not contain credentials"); + } + match parsed.host() { + None => Err("url must have a host"), + Some(url::Host::Domain(_)) => Ok(()), + Some(url::Host::Ipv4(ip)) if allow_private || ip_is_public(&IpAddr::V4(ip)) => Ok(()), + Some(url::Host::Ipv6(ip)) if allow_private || ip_is_public(&IpAddr::V6(ip)) => Ok(()), + Some(_) => Err("url resolves to a private address"), + } +} + +/// Resolve a peer URL's host and check every address against the IP policy. +/// Returns false on resolution failure — an unresolvable peer can re-announce. +pub async fn peer_addr_allowed(raw: &str, allow_private: bool) -> bool { + if validate_peer_url(raw, allow_private).is_err() { + return false; + } + if allow_private { + return true; + } + let Ok(parsed) = url::Url::parse(raw) else { + return false; + }; + match parsed.host() { + Some(url::Host::Domain(domain)) => { + let port = parsed.port_or_known_default().unwrap_or(443); + match tokio::net::lookup_host((domain, port)).await { + Ok(mut addrs) => addrs.all(|a| ip_is_public(&a.ip())), + Err(_) => false, + } + } + Some(url::Host::Ipv4(_)) | Some(url::Host::Ipv6(_)) => true, // checked above + None => false, + } +} + +/// True if the address is publicly routable (not private, loopback, +/// link-local, CGNAT, documentation, or otherwise reserved). +pub(crate) fn ip_is_public(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let o = v4.octets(); + !(v4.is_private() + || v4.is_loopback() + || v4.is_link_local() + || v4.is_broadcast() + || v4.is_documentation() + || v4.is_unspecified() + || o[0] == 0 + || (o[0] == 100 && (o[1] & 0xc0) == 64)) // CGNAT 100.64/10 + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return ip_is_public(&IpAddr::V4(v4)); + } + let s = v6.segments(); + !(v6.is_loopback() + || v6.is_unspecified() + || (s[0] & 0xfe00) == 0xfc00 // unique local fc00::/7 + || (s[0] & 0xffc0) == 0xfe80 // link-local fe80::/10 + || (s[0] == 0x2001 && s[1] == 0xdb8)) // documentation + } + } +} + pub struct PeerTable { /// IP -> announced URL (one slot per IP) ip_slots: HashMap, @@ -183,6 +259,22 @@ impl PeerTable { } } + /// True if the URL is a verified, non-stale peer. + pub fn is_verified(&self, url: &str) -> bool { + let url = normalize_url(url); + self.verified + .get(&url) + .is_some_and(|e| Instant::now().duration_since(e.last_seen) < self.config.verified_ttl) + } + + /// Remove a peer entirely (e.g., its address failed the IP policy). + pub fn remove(&mut self, url: &str) { + let url = normalize_url(url); + self.unverified.remove(&url); + self.verified.remove(&url); + self.ip_slots.retain(|_, u| *u != url); + } + /// Deprioritize a URL after a failed health check. /// Bumps it to the back of the line instead of removing it. pub fn deprioritize(&mut self, url: &str) { @@ -229,6 +321,34 @@ impl PeerTable { .map(|(url, _)| url.as_str()) } + /// Up to `n` unverified candidates to health-check, least-recently-seen + /// first, so simultaneous expiries don't drain the verified list one + /// candidate at a time. + pub fn next_candidates(&self, n: usize) -> Vec { + let mut entries: Vec<(&String, Instant)> = self + .unverified + .iter() + .map(|(url, e)| (url, e.last_seen)) + .collect(); + entries.sort_by_key(|(_, t)| *t); + entries + .into_iter() + .take(n) + .map(|(u, _)| u.clone()) + .collect() + } + + /// Verified peers past half their TTL: due for a proactive liveness + /// refresh so quiet periods (no sync traffic) don't expire them. + pub fn refresh_candidates(&self) -> Vec { + let now = Instant::now(); + self.verified + .iter() + .filter(|(_, e)| now.duration_since(e.last_seen) >= self.config.verified_ttl / 2) + .map(|(url, _)| url.clone()) + .collect() + } + /// True if we need more verified peers. pub fn needs_peers(&self) -> bool { let active = self @@ -260,7 +380,7 @@ impl PeerTable { } } -fn normalize_url(url: &str) -> String { +pub(crate) fn normalize_url(url: &str) -> String { url.trim().trim_end_matches('/').to_string() } @@ -371,6 +491,56 @@ mod tests { assert_eq!(result, AnnounceResult::AlreadyVerified); } + #[test] + fn url_policy() { + // Allowed: http and https, domains and public IP literals, with ports + for ok in [ + "https://relay.example.com", + "http://relay.example.com:7778", + "http://8.8.8.8:7778", + ] { + assert!(validate_peer_url(ok, false).is_ok(), "{ok} should pass"); + } + + // Rejected without allow_private + for bad in [ + "http://127.0.0.1:7778", + "http://10.0.0.5", + "http://192.168.1.10:7778", + "http://169.254.169.254/latest/meta-data", + "http://100.64.0.1", + "http://[::1]:7778", + "http://[fc00::1]", + "ftp://relay.example.com", + "file:///etc/passwd", + "http://user:pass@relay.example.com", + "not a url", + ] { + assert!(validate_peer_url(bad, false).is_err(), "{bad} should fail"); + } + + // allow_private admits loopback/private, still rejects bad schemes + assert!(validate_peer_url("http://127.0.0.1:7778", true).is_ok()); + assert!(validate_peer_url("ftp://127.0.0.1", true).is_err()); + } + + #[test] + fn remove_clears_everywhere() { + let mut table = PeerTable::new(config()); + table.announce(&peer(1, "https://relay1.com")); + table.announce(&peer(2, "https://relay2.com")); + table.mark_alive("https://relay2.com"); + + table.remove("https://relay1.com"); + table.remove("https://relay2.com"); + assert_eq!(table.unverified_count(), 0); + assert_eq!(table.verified_count(), 0); + + // Re-announcing after removal works (ip slot was freed) + table.announce(&peer(1, "https://relay1.com")); + assert_eq!(table.unverified_count(), 1); + } + #[test] fn peers_info_includes_capabilities() { let mut table = PeerTable::new(config()); diff --git a/relay/src/relay.rs b/relay/src/relay.rs index ee8dcb9..8153942 100644 --- a/relay/src/relay.rs +++ b/relay/src/relay.rs @@ -46,6 +46,13 @@ pub struct Config { pub remote_ip_header: Option, /// Accept fake ZK receipts (for testing only). pub dev_mode: bool, + /// Accept peers with private/loopback addresses (local development and tests). + pub allow_private_peers: bool, + /// Loaded file configuration (rate limits, content caps, concurrency). + pub settings: crate::settings::FileConfig, + /// Mocked chain proof + anchors (tests only; replaces the spaced RPC). + #[cfg(any(test, feature = "testutil"))] + pub mock_chain: Option<(libveritas::msg::ChainProof, Vec)>, } impl Config { @@ -63,6 +70,10 @@ impl Config { peer_config: PeerConfig::default(), remote_ip_header: None, dev_mode: false, + allow_private_peers: false, + settings: crate::settings::FileConfig::default(), + #[cfg(any(test, feature = "testutil"))] + mock_chain: None, } } } @@ -108,15 +119,39 @@ impl Relay { let anchor_store = AnchorSets::from_anchors(config.anchors); let store = SqliteStore::open(&config.db_path)?; + #[cfg(any(test, feature = "testutil"))] + let chain = match config.mock_chain { + Some(mock) => SpacedClient::mock(mock), + None => SpacedClient::new(rpc_client), + }; + #[cfg(not(any(test, feature = "testutil")))] let chain = SpacedClient::new(rpc_client); - let mut handler = Handler::new(veritas, store, anchor_store); + let rl = &config.settings.rate_limits; + let mut handler = Handler::new(veritas, store, anchor_store).with_content_rates( + rl.space_per_min, + std::time::Duration::from_secs(rl.handle_period_secs), + rl.handle_burst, + ); handler.dev_mode = config.dev_mode; - let mut state = AppState::new(handler, chain, config.peer_config); - state.max_message_size = config.max_message_size; + let mut state = AppState::with_rate_limits( + handler, + chain, + config.peer_config, + config.settings.rate_limit_config(), + ); + state.proof_sem = Arc::new(tokio::sync::Semaphore::new( + config.settings.limits.proof_concurrency.max(1), + )); + state.verify_sem = Arc::new(tokio::sync::Semaphore::new( + config.settings.limits.verify_concurrency.max(1), + )); + state.max_message_size = config.settings.limits.max_message_size; + state.retention = config.settings.retention_config(); state.capabilities = config.capabilities; state.is_bootstrap = config.is_bootstrap; state.remote_ip_header = config.remote_ip_header; + state.allow_private_peers = config.allow_private_peers; if let Some(url) = config.self_url { state = state.with_self_url(url); @@ -140,6 +175,25 @@ impl Relay { .await?; Ok(()) } + + /// Run the HTTP server, finishing in-flight requests when the shutdown + /// signal fires instead of dropping them mid-response. + pub async fn run_with_shutdown( + self, + listener: tokio::net::TcpListener, + mut shutdown: tokio::sync::broadcast::Receiver<()>, + ) -> anyhow::Result<()> { + let router = http::router(self.state); + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = shutdown.recv().await; + }) + .await?; + Ok(()) + } } /// Runs yuki (Bitcoin light client) and spaced in dedicated threads, diff --git a/relay/src/retention.rs b/relay/src/retention.rs new file mode 100644 index 0000000..a2cb7a5 --- /dev/null +++ b/relay/src/retention.rs @@ -0,0 +1,225 @@ +//! Storage retention: entitlement-weighted eviction under a disk budget. +//! +//! Nothing verified is hard-rejected in normal operation. Each space's +//! *entitlement* is `entitlement_per_epoch x epochs it was seen committing` — +//! commitments cost on-chain transactions, so storage beyond entitlement is +//! storage nobody paid for. When the payload budget is exceeded, a background +//! sweep evicts from the most over-entitled space first (oldest, coldest rows +//! first), and an admission gate stops first-inserts for over-entitled spaces +//! so evicted rows aren't endlessly re-pulled from peers. Under budget, both +//! mechanisms are inert. + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use crate::AppState; + +/// Tuning for retention. All values TOML-configurable (`[retention]`). +#[derive(Clone)] +pub struct RetentionConfig { + /// Budget for handle payload bytes (cert + zone blobs). 0 = unlimited. + pub max_storage_bytes: u64, + /// Handles per (space, epoch) counted as paid-for. + pub entitlement_per_epoch: u64, + /// Evict down to this percentage of the budget (hysteresis). + pub low_water_pct: u8, + /// Pressure check cadence. + pub sweep_interval: Duration, + /// Rows deleted per transaction. + pub eviction_batch: usize, + /// Max batches per sweep (bounds one sweep's runtime; the next sweep + /// continues). + pub max_batches_per_sweep: usize, +} + +impl Default for RetentionConfig { + fn default() -> Self { + Self { + max_storage_bytes: 10 * 1024 * 1024 * 1024, + entitlement_per_epoch: 10_000, + low_water_pct: 90, + sweep_interval: Duration::from_secs(30), + eviction_batch: 1_000, + max_batches_per_sweep: 20, + } + } +} + +/// Approximate recently-queried set: two generational HashSets swapped when +/// the active one fills. O(1) touch on the query path, no DB writes, no +/// persistence — eviction is rare and approximate, so losing heat on restart +/// is fine. +pub struct QueryHeat { + cap: usize, + active: HashSet, + prev: HashSet, +} + +impl QueryHeat { + pub fn new(cap: usize) -> Self { + Self { + cap: cap.max(1), + active: HashSet::new(), + prev: HashSet::new(), + } + } + + pub fn touch(&mut self, handle: &str) { + if self.active.len() >= self.cap { + self.prev = std::mem::take(&mut self.active); + } + self.active.insert(handle.to_string()); + } + + pub fn is_hot(&self, handle: &str) -> bool { + self.active.contains(handle) || self.prev.contains(handle) + } +} + +impl Default for QueryHeat { + fn default() -> Self { + Self::new(50_000) + } +} + +/// True when a first-insert for `space` should be skipped: only under +/// storage pressure AND when the space already exceeds its entitlement. +/// Inert (always false) with no budget or under budget. +pub fn first_insert_gated( + state: &AppState, + config: &RetentionConfig, + space: &str, +) -> anyhow::Result { + if config.max_storage_bytes == 0 { + return Ok(false); + } + let (_, bytes) = state.handler.store.storage_totals()?; + if bytes <= config.max_storage_bytes { + return Ok(false); + } + let (stored, epochs) = state.handler.store.space_usage(space)?; + Ok(stored > epochs.max(1) * config.entitlement_per_epoch) +} + +/// Run pressure checks forever. Spawned as a background task at startup. +pub async fn run_retention_loop(state: Arc, config: RetentionConfig) { + if config.max_storage_bytes == 0 { + return; + } + let mut ticker = tokio::time::interval(config.sweep_interval); + loop { + ticker.tick().await; + match sweep(&state, &config).await { + Ok(0) => {} + Ok(evicted) => tracing::info!("retention: evicted {} handles", evicted), + Err(e) => tracing::warn!("retention sweep failed: {}", e), + } + } +} + +/// One sweep: evict batches from the most over-entitled space until the +/// low-water mark (or the per-sweep batch budget) is reached. Returns rows +/// evicted. Public so tests can drive it directly. +pub async fn sweep(state: &Arc, config: &RetentionConfig) -> anyhow::Result { + let mut evicted = 0; + let low_water = config.max_storage_bytes * config.low_water_pct.min(100) as u64 / 100; + // Spaces that yielded no candidates this sweep (e.g. only their root row + // remains, which is never evicted) — move on to the next-worst victim. + let mut exhausted: HashSet = HashSet::new(); + + for _ in 0..config.max_batches_per_sweep { + let (_, bytes) = state.handler.store.storage_totals()?; + if bytes + <= if evicted == 0 { + config.max_storage_bytes + } else { + low_water + } + { + break; + } + + // Victim: worst stored/entitlement ratio; if nobody is over + // entitlement the budget still wins (a full disk kills the relay), + // so fall back to the largest space and tell the operator. + let usage = { + let blocking_state = Arc::clone(state); + tokio::task::spawn_blocking(move || blocking_state.handler.store.space_usage_all()) + .await?? + }; + let usage: Vec<_> = usage + .into_iter() + .filter(|(space, _, _)| !exhausted.contains(space)) + .collect(); + let over = usage + .iter() + .filter(|(_, stored, epochs)| *stored > epochs.max(&1) * config.entitlement_per_epoch) + .max_by(|a, b| { + let ra = a.1 as f64 / (a.2.max(1) * config.entitlement_per_epoch) as f64; + let rb = b.1 as f64 / (b.2.max(1) * config.entitlement_per_epoch) as f64; + ra.total_cmp(&rb) + }); + let victim = match over { + Some((space, _, _)) => space.clone(), + None => match usage.iter().max_by_key(|(_, stored, _)| *stored) { + Some((space, _, _)) => { + tracing::warn!( + "retention: over budget but every space is within entitlement — \ + evicting from {}; raise max_storage_bytes", + space + ); + space.clone() + } + None => break, + }, + }; + + // Oldest rows first (the root row is never a candidate), sparing + // recently-queried handles when possible — but never stalling: if + // everything is hot, evict anyway. + let candidates = { + let blocking_state = Arc::clone(state); + let victim_space = victim.clone(); + let limit = config.eviction_batch * 2; + tokio::task::spawn_blocking(move || { + blocking_state + .handler + .store + .eviction_candidates(&victim_space, limit) + }) + .await?? + }; + if candidates.is_empty() { + exhausted.insert(victim); + continue; + } + let batch: Vec = { + let heat = state.query_heat.lock().unwrap(); + let cold: Vec = candidates + .iter() + .filter(|h| !heat.is_hot(h)) + .take(config.eviction_batch) + .cloned() + .collect(); + if cold.is_empty() { + candidates.into_iter().take(config.eviction_batch).collect() + } else { + cold + } + }; + + let deleted = { + let blocking_state = Arc::clone(state); + tokio::task::spawn_blocking(move || blocking_state.handler.store.delete_handles(&batch)) + .await?? + }; + crate::stats::bump_by(&state.stats.evicted_rows, deleted as u64); + evicted += deleted; + if deleted == 0 { + break; + } + } + + Ok(evicted) +} diff --git a/relay/src/settings.rs b/relay/src/settings.rs new file mode 100644 index 0000000..9591526 --- /dev/null +++ b/relay/src/settings.rs @@ -0,0 +1,297 @@ +//! Optional TOML configuration file (`--config` / `CERTRELAY_CONFIG`). +//! +//! Every field has a default matching the built-in constants, so a config +//! file only needs the values being tuned. Unknown keys are rejected to +//! catch typos. +//! +//! ```toml +//! [rate_limits] +//! message_per_min = 60 +//! proof_per_min = 30 +//! read_per_min = 120 +//! announce_per_min = 5 +//! sync_per_min = 60 +//! poke_per_min = 30 +//! space_per_min = 100 +//! handle_period_secs = 300 +//! handle_burst = 3 +//! +//! [sync] +//! interval_secs = 45 +//! jitter_secs = 15 +//! page_limit = 1000 +//! peers_per_round = 2 +//! max_pages_per_peer = 200 +//! poke_debounce_ms = 2000 +//! poke_cooldown_ms = 5000 +//! +//! [peers] +//! max_unverified = 1000 +//! max_verified = 100 +//! verified_ttl_secs = 600 +//! +//! [limits] +//! max_message_size = 524288 +//! proof_concurrency = 6 +//! verify_concurrency = 4 +//! ``` + +use std::num::NonZeroU32; +use std::path::Path; +use std::time::Duration; + +use serde::Deserialize; + +use crate::http::{RateLimitConfig, VERIFY_CONCURRENCY}; +use crate::peer::PeerConfig; +use crate::sync::SyncConfig; +use crate::{DEFAULT_MAX_MESSAGE_SIZE, Quota, http::PROOF_CONCURRENCY}; + +#[derive(Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct FileConfig { + pub rate_limits: RateLimitsSection, + pub sync: SyncSection, + pub peers: PeersSection, + pub limits: LimitsSection, + pub retention: RetentionSection, +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RetentionSection { + /// Budget for handle payload bytes. 0 = unlimited (no eviction, no gate). + pub max_storage_bytes: u64, + /// Handles per (space, epoch) counted as paid-for. + pub entitlement_per_epoch: u64, + pub evict_low_water_pct: u8, + pub sweep_interval_secs: u64, + pub eviction_batch: usize, +} + +impl Default for RetentionSection { + fn default() -> Self { + let d = crate::retention::RetentionConfig::default(); + Self { + max_storage_bytes: d.max_storage_bytes, + entitlement_per_epoch: d.entitlement_per_epoch, + evict_low_water_pct: d.low_water_pct, + sweep_interval_secs: d.sweep_interval.as_secs(), + eviction_batch: d.eviction_batch, + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RateLimitsSection { + pub message_per_min: u32, + pub proof_per_min: u32, + pub read_per_min: u32, + pub announce_per_min: u32, + pub sync_per_min: u32, + pub poke_per_min: u32, + /// Per-space content velocity cap (churn only). + pub space_per_min: u32, + /// Per-handle content cap period (churn only). + pub handle_period_secs: u64, + /// Per-handle burst within the period. + pub handle_burst: u32, +} + +impl Default for RateLimitsSection { + fn default() -> Self { + Self { + message_per_min: 60, + proof_per_min: 30, + read_per_min: 120, + announce_per_min: 5, + sync_per_min: 60, + poke_per_min: 30, + space_per_min: 100, + handle_period_secs: 300, + handle_burst: 3, + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SyncSection { + pub interval_secs: u64, + pub jitter_secs: u64, + pub page_limit: usize, + pub peers_per_round: usize, + pub max_pages_per_peer: usize, + pub poke_debounce_ms: u64, + pub poke_cooldown_ms: u64, +} + +impl Default for SyncSection { + fn default() -> Self { + let d = SyncConfig::default(); + Self { + interval_secs: d.interval.as_secs(), + jitter_secs: d.jitter.as_secs(), + page_limit: d.page_limit, + peers_per_round: d.peers_per_round, + max_pages_per_peer: d.max_pages_per_peer, + poke_debounce_ms: d.poke_debounce.as_millis() as u64, + poke_cooldown_ms: d.poke_cooldown.as_millis() as u64, + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct PeersSection { + pub max_unverified: usize, + pub max_verified: usize, + pub verified_ttl_secs: u64, +} + +impl Default for PeersSection { + fn default() -> Self { + let d = PeerConfig::default(); + Self { + max_unverified: d.max_unverified, + max_verified: d.max_verified, + verified_ttl_secs: d.verified_ttl.as_secs(), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct LimitsSection { + pub max_message_size: usize, + pub proof_concurrency: usize, + pub verify_concurrency: usize, +} + +impl Default for LimitsSection { + fn default() -> Self { + Self { + max_message_size: DEFAULT_MAX_MESSAGE_SIZE, + proof_concurrency: PROOF_CONCURRENCY, + verify_concurrency: VERIFY_CONCURRENCY, + } + } +} + +fn per_minute(n: u32) -> Quota { + Quota::per_minute(NonZeroU32::new(n.max(1)).expect("nonzero")) +} + +impl FileConfig { + pub fn load(path: &Path) -> anyhow::Result { + let raw = std::fs::read_to_string(path) + .map_err(|e| anyhow::anyhow!("could not read config {}: {}", path.display(), e))?; + toml::from_str(&raw) + .map_err(|e| anyhow::anyhow!("invalid config {}: {}", path.display(), e)) + } + + pub fn rate_limit_config(&self) -> RateLimitConfig { + let r = &self.rate_limits; + RateLimitConfig { + message: per_minute(r.message_per_min), + proof: per_minute(r.proof_per_min), + read: per_minute(r.read_per_min), + announce: per_minute(r.announce_per_min), + sync: per_minute(r.sync_per_min), + poke: per_minute(r.poke_per_min), + } + } + + pub fn sync_config(&self) -> SyncConfig { + let s = &self.sync; + SyncConfig { + interval: Duration::from_secs(s.interval_secs), + jitter: Duration::from_secs(s.jitter_secs), + page_limit: s.page_limit.clamp(1, crate::http::MAX_SYNC_PAGE_ROWS), + peers_per_round: s.peers_per_round, + max_pages_per_peer: s.max_pages_per_peer, + poke_debounce: Duration::from_millis(s.poke_debounce_ms), + poke_cooldown: Duration::from_millis(s.poke_cooldown_ms), + // Serving pages overshoot by up to one record plus appended root + // records; derive the acceptance slack from the configured max + // message size so a raised limit can't wedge the puller. + max_page_bytes: 2 * crate::http::MAX_SYNC_PAGE_BYTES + self.limits.max_message_size, + } + } + + pub fn retention_config(&self) -> crate::retention::RetentionConfig { + let r = &self.retention; + crate::retention::RetentionConfig { + max_storage_bytes: r.max_storage_bytes, + entitlement_per_epoch: r.entitlement_per_epoch.max(1), + low_water_pct: r.evict_low_water_pct.min(100), + sweep_interval: Duration::from_secs(r.sweep_interval_secs.max(1)), + eviction_batch: r.eviction_batch.max(1), + ..crate::retention::RetentionConfig::default() + } + } + + pub fn peer_config(&self) -> PeerConfig { + let p = &self.peers; + PeerConfig { + max_unverified: p.max_unverified, + max_verified: p.max_verified, + verified_ttl: Duration::from_secs(p.verified_ttl_secs), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_match_builtins() { + let cfg = FileConfig::default(); + assert_eq!(cfg.limits.max_message_size, DEFAULT_MAX_MESSAGE_SIZE); + assert_eq!(cfg.sync.interval_secs, 45); + assert_eq!(cfg.peers.verified_ttl_secs, 600); + assert_eq!(cfg.rate_limits.handle_burst, 3); + } + + /// The README's example config must always parse against the real + /// schema, so the docs can't drift. + #[test] + fn readme_example_parses() { + let readme = include_str!("../../README.md"); + let start = readme + .find("```toml") + .expect("README should contain a toml example") + + "```toml".len(); + let end = start + + readme[start..] + .find("```") + .expect("unterminated toml block"); + let example = &readme[start..end]; + + let cfg: FileConfig = toml::from_str(example).expect("README config must parse"); + // The example documents the defaults — they must match the code + assert_eq!(cfg.rate_limits.message_per_min, 60); + assert_eq!(cfg.sync.interval_secs, SyncSection::default().interval_secs); + assert_eq!( + cfg.retention.max_storage_bytes, + RetentionSection::default().max_storage_bytes + ); + assert_eq!( + cfg.limits.max_message_size, + LimitsSection::default().max_message_size + ); + } + + #[test] + fn parses_partial_file_and_rejects_typos() { + let cfg: FileConfig = + toml::from_str("[sync]\ninterval_secs = 10\n").expect("partial config"); + assert_eq!(cfg.sync.interval_secs, 10); + assert_eq!(cfg.sync.page_limit, 1000, "unset fields keep defaults"); + + let err = toml::from_str::("[sync]\nintervall_secs = 10\n"); + assert!(err.is_err(), "unknown keys must be rejected"); + } +} diff --git a/relay/src/stats.rs b/relay/src/stats.rs new file mode 100644 index 0000000..f682af1 --- /dev/null +++ b/relay/src/stats.rs @@ -0,0 +1,113 @@ +//! In-memory counters for runner observability, exposed by `GET /stats`. +//! +//! Dependency-free by design: atomics snapshotted into JSON. Runners +//! `curl /stats`; a Prometheus endpoint can be layered on later. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Default)] +pub struct Stats { + // /message intake + pub messages_received: AtomicU64, + pub messages_accepted: AtomicU64, + pub messages_deduped: AtomicU64, + pub messages_rejected: AtomicU64, + + // Sync pulls (this relay acting as a client) + pub sync_pages_pulled: AtomicU64, + pub sync_records_stored: AtomicU64, + pub sync_records_prefiltered: AtomicU64, + pub sync_failed_spaces: AtomicU64, + pub sync_errors: AtomicU64, + + // Poke + pub pokes_received: AtomicU64, + pub pokes_accepted: AtomicU64, + pub pokes_sent: AtomicU64, + + // Proof generation served to clients + pub proofs_served: AtomicU64, + + // Rate-limit rejections per bucket + pub rl_message: AtomicU64, + pub rl_proof: AtomicU64, + pub rl_read: AtomicU64, + pub rl_announce: AtomicU64, + pub rl_sync: AtomicU64, + pub rl_poke: AtomicU64, + + // Saturation rejections (semaphores full) + pub busy_rejections: AtomicU64, + + // Retention + pub evicted_rows: AtomicU64, + pub admission_gated: AtomicU64, + + /// Unix seconds of the last successful sync, per peer. The single most + /// important operational signal: a stalled sync loop shows up here. + pub last_sync_success: Mutex>, +} + +impl Stats { + pub fn record_sync_success(&self, peer_url: &str) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + self.last_sync_success + .lock() + .unwrap() + .insert(peer_url.to_string(), now); + } + + /// Snapshot every counter into a JSON object. + pub fn snapshot(&self) -> serde_json::Value { + let g = |a: &AtomicU64| a.load(Ordering::Relaxed); + serde_json::json!({ + "messages": { + "received": g(&self.messages_received), + "accepted": g(&self.messages_accepted), + "deduped": g(&self.messages_deduped), + "rejected": g(&self.messages_rejected), + }, + "sync": { + "pages_pulled": g(&self.sync_pages_pulled), + "records_stored": g(&self.sync_records_stored), + "records_prefiltered": g(&self.sync_records_prefiltered), + "failed_spaces": g(&self.sync_failed_spaces), + "errors": g(&self.sync_errors), + "last_success_by_peer": self.last_sync_success.lock().unwrap().clone(), + }, + "pokes": { + "received": g(&self.pokes_received), + "accepted": g(&self.pokes_accepted), + "sent": g(&self.pokes_sent), + }, + "proofs_served": g(&self.proofs_served), + "rate_limited": { + "message": g(&self.rl_message), + "proof": g(&self.rl_proof), + "read": g(&self.rl_read), + "announce": g(&self.rl_announce), + "sync": g(&self.rl_sync), + "poke": g(&self.rl_poke), + }, + "busy_rejections": g(&self.busy_rejections), + "retention": { + "evicted_rows": g(&self.evicted_rows), + "admission_gated": g(&self.admission_gated), + }, + }) + } +} + +/// Relaxed increment helper. +pub fn bump(counter: &AtomicU64) { + counter.fetch_add(1, Ordering::Relaxed); +} + +pub fn bump_by(counter: &AtomicU64, n: u64) { + counter.fetch_add(n, Ordering::Relaxed); +} diff --git a/relay/src/store.rs b/relay/src/store.rs index 1e29dfb..f92764f 100644 --- a/relay/src/store.rs +++ b/relay/src/store.rs @@ -24,11 +24,37 @@ CREATE TABLE IF NOT EXISTS handles ( epoch_height INTEGER NOT NULL, offchain_seq INTEGER NOT NULL DEFAULT 0, delegate_offchain_seq INTEGER NOT NULL DEFAULT 0, + -- Strictly-increasing write sequence (from sync_counter); /sync pages in + -- this order. Never reused, so a peer's watermark can't miss same-second + -- writes the way a timestamp cursor could. + sync_seq INTEGER NOT NULL DEFAULT 0, + -- sha256 of zone_data: exact-duplicate detection for ingest pre-filtering + -- (the seq metadata alone can't distinguish a better zone with unchanged + -- seqs, e.g. a pending->finalized commitment transition). + zone_hash BLOB NOT NULL DEFAULT x'', updated_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_handles_space ON handles(space); +-- Sync pagination order (see /sync). +CREATE INDEX IF NOT EXISTS idx_handles_sync_seq ON handles(sync_seq); + +-- Singleton write counter feeding handles.sync_seq. +CREATE TABLE IF NOT EXISTS sync_counter ( + id INTEGER PRIMARY KEY CHECK (id = 1), + seq INTEGER NOT NULL +); +INSERT OR IGNORE INTO sync_counter (id, seq) VALUES (1, 0); + +-- Per-peer sync progress: the last cursor fully processed from that peer. +-- Cursors are peer-local; never compared across peers. +CREATE TABLE IF NOT EXISTS sync_watermarks ( + peer_url TEXT PRIMARY KEY, + cursor TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + -- Reverse records: maps a numeric identity to its preferred human-readable name. -- Updated when a message with a Sig record containing a non-empty rev is stored. CREATE TABLE IF NOT EXISTS reverse ( @@ -49,6 +75,51 @@ CREATE TABLE IF NOT EXISTS addrs ( ); CREATE INDEX IF NOT EXISTS idx_addrs_lookup ON addrs(name, addr); + +-- Eviction deletes addr rows by handle in bulk. +CREATE INDEX IF NOT EXISTS idx_addrs_handle ON addrs(handle); + +-- Reverse rows are cleaned up by name when their target handle is evicted. +CREATE INDEX IF NOT EXISTS idx_reverse_name ON reverse(name); + +-- Eviction victim ordering within a space (oldest updated first). +CREATE INDEX IF NOT EXISTS idx_handles_space_updated ON handles(space, updated_at); + +-- Per-(space, epoch) handle counts: O(1) entitlement accounting for +-- retention. Rows persist at handles = 0 after eviction — an epoch the space +-- was ever seen committing keeps counting toward its entitlement (otherwise +-- eviction would shrink entitlement and spiral). +CREATE TABLE IF NOT EXISTS space_epoch_counts ( + space TEXT NOT NULL, + epoch_height INTEGER NOT NULL, + handles INTEGER NOT NULL, + PRIMARY KEY (space, epoch_height) +); + +-- Singleton totals for the handles table payload (rows + blob bytes). +CREATE TABLE IF NOT EXISTS storage_totals ( + id INTEGER PRIMARY KEY CHECK (id = 1), + rows INTEGER NOT NULL, + bytes INTEGER NOT NULL +); +INSERT OR IGNORE INTO storage_totals (id, rows, bytes) VALUES (1, 0, 0); + +-- Keep the accounting true for every insert/replace/delete path (REPLACE +-- fires the delete trigger for the displaced row via recursive_triggers). +CREATE TRIGGER IF NOT EXISTS handles_count_insert AFTER INSERT ON handles BEGIN + INSERT INTO space_epoch_counts (space, epoch_height, handles) + VALUES (NEW.space, NEW.epoch_height, 1) + ON CONFLICT (space, epoch_height) DO UPDATE SET handles = handles + 1; + UPDATE storage_totals SET rows = rows + 1, + bytes = bytes + length(NEW.cert_data) + length(NEW.zone_data) WHERE id = 1; +END; + +CREATE TRIGGER IF NOT EXISTS handles_count_delete AFTER DELETE ON handles BEGIN + UPDATE space_epoch_counts SET handles = handles - 1 + WHERE space = OLD.space AND epoch_height = OLD.epoch_height; + UPDATE storage_totals SET rows = rows - 1, + bytes = bytes - length(OLD.cert_data) - length(OLD.zone_data) WHERE id = 1; +END; "#; /// Result of a bulk store operation. @@ -58,6 +129,8 @@ pub struct BulkStoreResult { pub stored: usize, /// Number of handles skipped (existing zone was better). pub skipped: usize, + /// First-inserts skipped by the retention admission gate. + pub gated: usize, /// Handles that were actually stored (not skipped). pub stored_handles: Vec, } @@ -80,6 +153,14 @@ pub struct HandleHintRow { pub epoch_height: u32, pub offchain_seq: u64, pub delegate_offchain_seq: u64, + /// sha256 of the stored zone_data blob. + pub zone_hash: Vec, +} + +/// sha256 of a zone blob, matching `handles.zone_hash`. +pub fn zone_hash(zone_data: &[u8]) -> Vec { + use sha2::Digest; + sha2::Sha256::digest(zone_data).to_vec() } /// SQLite-backed store for handles. @@ -90,22 +171,115 @@ pub struct SqliteStore { impl SqliteStore { /// Open or create a SQLite database at the given path. pub fn open(path: impl AsRef) -> anyhow::Result { - let conn = Connection::open(path.as_ref())?; - conn.execute_batch(SCHEMA)?; - Ok(Self { - conn: Mutex::new(conn), - }) + Self::init(Connection::open(path.as_ref())?) } /// Create an in-memory database (useful for testing). pub fn in_memory() -> anyhow::Result { - let conn = Connection::open_in_memory()?; + Self::init(Connection::open_in_memory()?) + } + + fn init(conn: Connection) -> anyhow::Result { + // WAL keeps readers unblocked during writes; busy_timeout prevents + // immediate SQLITE_BUSY errors; NORMAL sync is durable enough under + // WAL (data re-syncs from peers in the worst case anyway). + let _ = conn.query_row("PRAGMA journal_mode = WAL", [], |_| Ok(())); + conn.pragma_update(None, "busy_timeout", 5000)?; + conn.pragma_update(None, "synchronous", "NORMAL")?; + // INSERT OR REPLACE must fire the delete trigger for the displaced + // row, or the storage accounting drifts on every update. + conn.pragma_update(None, "recursive_triggers", true)?; + + // Column migration must run before SCHEMA: the sync_seq index in + // SCHEMA would fail against a pre-sync handles table. + Self::migrate_columns(&conn)?; conn.execute_batch(SCHEMA)?; + Self::backfill_sync(&conn)?; + Self::backfill_storage(&conn)?; Ok(Self { conn: Mutex::new(conn), }) } + /// Add columns introduced after the first production deploy to an + /// existing handles table (no-op for fresh or already-migrated DBs). + fn migrate_columns(conn: &Connection) -> anyhow::Result<()> { + let has_handles: bool = conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'handles'", + [], + |r| r.get::<_, i64>(0).map(|n| n > 0), + )?; + if !has_handles { + return Ok(()); + } + + let mut stmt = conn.prepare("SELECT name FROM pragma_table_info('handles')")?; + let columns: Vec = stmt + .query_map([], |r| r.get(0))? + .collect::>()?; + + if !columns.iter().any(|c| c == "sync_seq") { + conn.execute( + "ALTER TABLE handles ADD COLUMN sync_seq INTEGER NOT NULL DEFAULT 0", + [], + )?; + } + if !columns.iter().any(|c| c == "zone_hash") { + conn.execute( + "ALTER TABLE handles ADD COLUMN zone_hash BLOB NOT NULL DEFAULT x''", + [], + )?; + } + Ok(()) + } + + /// Assign sync sequence numbers and zone hashes to pre-migration rows. + /// + /// New rows always get `sync_seq >= 1` at insert, so `sync_seq = 0` + /// identifies exactly the legacy rows; the backfill is idempotent and + /// crash-safe (single transaction). + fn backfill_sync(conn: &Connection) -> anyhow::Result<()> { + // Chunked so the migration never loads the whole table (with zone + // blobs) into memory. sync_seq = 0 marks unmigrated rows, so each + // chunk is one transaction and a crash resumes where it left off. + const CHUNK: usize = 1000; + let mut migrated = 0usize; + loop { + let chunk: Vec<(String, Vec)> = { + let mut stmt = conn.prepare( + "SELECT handle, zone_data FROM handles WHERE sync_seq = 0 + ORDER BY updated_at, handle LIMIT ?", + )?; + let rows = stmt.query_map(params![CHUNK as i64], |r| Ok((r.get(0)?, r.get(1)?)))?; + rows.collect::>()? + }; + if chunk.is_empty() { + break; + } + + let tx = conn.unchecked_transaction()?; + let base: i64 = tx.query_row("SELECT seq FROM sync_counter WHERE id = 1", [], |r| { + r.get(0) + })?; + for (i, (handle, zone_data)) in chunk.iter().enumerate() { + tx.execute( + "UPDATE handles SET sync_seq = ?, zone_hash = ? WHERE handle = ?", + params![base + 1 + i as i64, zone_hash(zone_data), handle], + )?; + } + tx.execute( + "UPDATE sync_counter SET seq = ? WHERE id = 1", + params![base + chunk.len() as i64], + )?; + tx.commit()?; + migrated += chunk.len(); + } + if migrated > 0 { + tracing::info!("migrated {} pre-sync handle rows", migrated); + } + Ok(()) + } + fn now() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -165,13 +339,29 @@ impl SqliteStore { let handles: Vec<&str> = entries.iter().map(|e| e.handle.as_str()).collect(); let existing_zones = Self::get_zones_inner(&conn, &handles)?; + // seq is a unix-seconds timestamp; reject records claiming a seq more than + // 6h in the future (accidental far-future clocks or deliberate freshness + // pinning). + let max_seq = (now + 6 * 3600) as u64; + // Filter to entries where the incoming zone is better (or new) let to_store: Vec<_> = entries .into_iter() .zip(updates.iter()) - .filter(|(e, update)| match existing_zones.get(e.handle.as_str()) { - Some(existing) => update.zone.is_better_than(existing).unwrap_or(false), - None => true, + .filter(|(e, update)| { + if e.offchain_seq > max_seq || e.delegate_offchain_seq > max_seq { + tracing::warn!( + "{}: rejecting update, seq {} exceeds max {} (>6h in future)", + e.handle, + e.offchain_seq.max(e.delegate_offchain_seq), + max_seq + ); + return false; + } + match existing_zones.get(e.handle.as_str()) { + Some(existing) => update.zone.is_better_than(existing).unwrap_or(false), + None => true, + } }) .map(|(e, _)| e) .collect(); @@ -182,22 +372,34 @@ impl SqliteStore { return Ok(BulkStoreResult { stored: 0, skipped, + gated: 0, stored_handles: vec![], }); } + // Counter bump + row inserts commit atomically: a crash can't leave + // rows claiming sequence numbers the counter doesn't cover. + let tx = conn.unchecked_transaction()?; + let seq_base: i64 = tx.query_row("SELECT seq FROM sync_counter WHERE id = 1", [], |r| { + r.get(0) + })?; + tx.execute( + "UPDATE sync_counter SET seq = ? WHERE id = 1", + params![seq_base + to_store.len() as i64], + )?; + // Bulk INSERT let placeholders: Vec = to_store .iter() - .map(|_| "(?, ?, ?, ?, ?, ?, ?, ?)".to_string()) + .map(|_| "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)".to_string()) .collect(); let query = format!( - "INSERT OR REPLACE INTO handles (handle, space, cert_data, zone_data, epoch_height, offchain_seq, delegate_offchain_seq, updated_at) VALUES {}", + "INSERT OR REPLACE INTO handles (handle, space, cert_data, zone_data, epoch_height, offchain_seq, delegate_offchain_seq, sync_seq, zone_hash, updated_at) VALUES {}", placeholders.join(", ") ); - let mut params: Vec> = Vec::with_capacity(to_store.len() * 8); - for e in &to_store { + let mut params: Vec> = Vec::with_capacity(to_store.len() * 10); + for (i, e) in to_store.iter().enumerate() { params.push(Box::new(e.handle.clone())); params.push(Box::new(e.space.clone())); params.push(Box::new(e.cert_data.clone())); @@ -205,20 +407,195 @@ impl SqliteStore { params.push(Box::new(e.epoch_height)); params.push(Box::new(e.offchain_seq as i64)); params.push(Box::new(e.delegate_offchain_seq as i64)); + params.push(Box::new(seq_base + 1 + i as i64)); + params.push(Box::new(zone_hash(&e.zone_data))); params.push(Box::new(now)); } let param_refs: Vec<&dyn rusqlite::ToSql> = params.iter().map(|p| p.as_ref()).collect(); - conn.execute(&query, param_refs.as_slice())?; + tx.execute(&query, param_refs.as_slice())?; + tx.commit()?; let stored_handles = to_store.iter().map(|e| e.handle.clone()).collect(); Ok(BulkStoreResult { stored: to_store.len(), skipped, + gated: 0, stored_handles, }) } + /// Rebuild the storage accounting tables from a full scan when they are + /// empty but handles exist (upgrade of a pre-retention database). + /// Idempotent: a fresh or already-accounted DB is a no-op. + fn backfill_storage(conn: &Connection) -> anyhow::Result<()> { + let (acct_rows,): (i64,) = + conn.query_row("SELECT rows FROM storage_totals WHERE id = 1", [], |r| { + Ok((r.get(0)?,)) + })?; + if acct_rows != 0 { + return Ok(()); + } + let actual: i64 = conn.query_row("SELECT COUNT(*) FROM handles", [], |r| r.get(0))?; + if actual == 0 { + return Ok(()); + } + + tracing::info!("backfilling storage accounting for {} handle rows", actual); + let tx = conn.unchecked_transaction()?; + tx.execute( + "INSERT OR REPLACE INTO space_epoch_counts (space, epoch_height, handles) + SELECT space, epoch_height, COUNT(*) FROM handles GROUP BY space, epoch_height", + [], + )?; + tx.execute( + "UPDATE storage_totals SET + rows = (SELECT COUNT(*) FROM handles), + bytes = (SELECT COALESCE(SUM(length(cert_data) + length(zone_data)), 0) FROM handles) + WHERE id = 1", + [], + )?; + tx.commit()?; + Ok(()) + } + + // ========================================================================= + // Retention accounting + // ========================================================================= + + /// Current handle-table payload totals: (rows, bytes). O(1). + pub fn storage_totals(&self) -> anyhow::Result<(u64, u64)> { + let conn = self.conn.lock().unwrap(); + let (rows, bytes): (i64, i64) = conn.query_row( + "SELECT rows, bytes FROM storage_totals WHERE id = 1", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((rows.max(0) as u64, bytes.max(0) as u64)) + } + + /// One space's stored handle count and the number of epochs it was ever + /// seen committing (empty buckets still count toward entitlement). + pub fn space_usage(&self, space: &str) -> anyhow::Result<(u64, u64)> { + let conn = self.conn.lock().unwrap(); + let (stored, epochs): (i64, i64) = conn.query_row( + "SELECT COALESCE(SUM(handles), 0), COUNT(*) FROM space_epoch_counts WHERE space = ?", + params![space], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((stored.max(0) as u64, epochs.max(0) as u64)) + } + + /// Usage for every space with stored handles: (space, stored, epochs). + /// Scans the counts table (spaces x epochs rows — small), not handles. + pub fn space_usage_all(&self) -> anyhow::Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT space, SUM(handles), COUNT(*) FROM space_epoch_counts + GROUP BY space HAVING SUM(handles) > 0", + )?; + let rows = stmt.query_map([], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, i64>(1)?.max(0) as u64, + r.get::<_, i64>(2)?.max(0) as u64, + )) + })?; + rows.collect::, _>>().map_err(Into::into) + } + + /// The oldest-updated handles of a space (eviction candidates). The root + /// row (`handle = space`) is never a candidate: without it every + /// remaining sub-handle becomes unresolvable and unverifiable, and the + /// space cannot recover (peers won't re-serve a row below our watermark). + pub fn eviction_candidates(&self, space: &str, limit: usize) -> anyhow::Result> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT handle FROM handles WHERE space = ?1 AND handle != ?1 + ORDER BY updated_at ASC LIMIT ?2", + )?; + let rows = stmt.query_map(params![space, limit as i64], |r| r.get(0))?; + rows.collect::, _>>().map_err(Into::into) + } + + /// Delete handles (and their address-index and reverse rows) in one + /// transaction. The accounting triggers keep counts and totals true. + pub fn delete_handles(&self, handles: &[String]) -> anyhow::Result { + if handles.is_empty() { + return Ok(0); + } + let conn = self.conn.lock().unwrap(); + let tx = conn.unchecked_transaction()?; + let placeholders: Vec<&str> = handles.iter().map(|_| "?").collect(); + let params: Vec<&dyn rusqlite::ToSql> = + handles.iter().map(|h| h as &dyn rusqlite::ToSql).collect(); + + // Reverse rows pointing at deleted handles would resolve to nothing. + // Extract the exact (num_id, rev name) pairs from the zones being + // deleted — mirroring how set_revs created them — so handles with a + // Sig but no Addr records are covered, and reverse rows of other + // num_ids sharing a display name are never touched. + { + use libveritas::sip7::SIG_PRIMARY_ZONE; + let mut stmt = tx.prepare(&format!( + "SELECT zone_data FROM handles WHERE handle IN ({})", + placeholders.join(", ") + ))?; + let blobs = stmt.query_map(params.as_slice(), |r| r.get::<_, Vec>(0))?; + for blob in blobs { + let Ok(zone) = borsh::from_slice::(&blob?) else { + continue; + }; + if let Some(sig) = zone.records.sig() + && sig.flags & SIG_PRIMARY_ZONE == SIG_PRIMARY_ZONE + && let Some(num_id) = &zone.num_id + { + tx.execute( + "DELETE FROM reverse WHERE num_id = ? AND name = ?", + params![num_id.to_string(), sig.handle.to_string()], + )?; + } + } + } + tx.execute( + &format!( + "DELETE FROM addrs WHERE handle IN ({})", + placeholders.join(", ") + ), + params.as_slice(), + )?; + let deleted = tx.execute( + &format!( + "DELETE FROM handles WHERE handle IN ({})", + placeholders.join(", ") + ), + params.as_slice(), + )?; + tx.commit()?; + Ok(deleted) + } + + /// Test helper: rewrite a handle's row position to the end of the sync + /// stream (simulates a root republished after its sub-handles). + #[cfg(any(test, feature = "testutil"))] + pub fn bump_sync_seq(&self, handle: &str) -> anyhow::Result<()> { + let conn = self.conn.lock().unwrap(); + let tx = conn.unchecked_transaction()?; + let next: i64 = tx.query_row("SELECT seq + 1 FROM sync_counter WHERE id = 1", [], |r| { + r.get(0) + })?; + tx.execute( + "UPDATE handles SET sync_seq = ? WHERE handle = ?", + params![next, handle], + )?; + tx.execute( + "UPDATE sync_counter SET seq = ? WHERE id = 1", + params![next], + )?; + tx.commit()?; + Ok(()) + } + /// Get a single handle record. pub fn get_handle(&self, handle: &str) -> anyhow::Result> { let conn = self.conn.lock().unwrap(); @@ -334,7 +711,7 @@ impl SqliteStore { let placeholders: Vec<&str> = handles.iter().map(|_| "?").collect(); let query = format!( - "SELECT handle, epoch_height, offchain_seq, delegate_offchain_seq FROM handles WHERE handle IN ({})", + "SELECT handle, epoch_height, offchain_seq, delegate_offchain_seq, zone_hash FROM handles WHERE handle IN ({})", placeholders.join(", ") ); @@ -348,12 +725,203 @@ impl SqliteStore { epoch_height: row.get(1)?, offchain_seq: row.get::<_, i64>(2)? as u64, delegate_offchain_seq: row.get::<_, i64>(3)? as u64, + zone_hash: row.get(4)?, }) })?; rows.collect::, _>>().map_err(Into::into) } + // ========================================================================= + // Sync + // ========================================================================= + + /// Read a page of handle rows after `cursor` in `sync_seq` order. + /// + /// `limit` bounds the row count; `max_bytes` additionally stops the page + /// once the accumulated blob size exceeds it (always returns at least one + /// row if any exist past the cursor). + pub fn sync_page( + &self, + cursor: Option, + limit: usize, + max_bytes: usize, + ) -> anyhow::Result { + let conn = self.conn.lock().unwrap(); + let after = cursor.map(|c| c.0 as i64).unwrap_or(0); + + let mut stmt = conn.prepare( + "SELECT handle, epoch_height, offchain_seq, delegate_offchain_seq, cert_data, zone_data, sync_seq + FROM handles + WHERE sync_seq > ?1 + ORDER BY sync_seq + LIMIT ?2", + )?; + let rows = stmt.query_map(params![after, limit as i64], |row| { + Ok(( + resolver::SyncRecord { + handle: row.get(0)?, + epoch_height: row.get(1)?, + seq: row.get::<_, i64>(2)? as u64, + delegate_seq: row.get::<_, i64>(3)? as u64, + cert: row.get(4)?, + zone: row.get(5)?, + }, + row.get::<_, i64>(6)?, + )) + })?; + + let mut main: Vec<(resolver::SyncRecord, i64)> = Vec::new(); + let mut bytes = 0usize; + for row in rows { + let (record, sync_seq) = row?; + bytes += record.cert.len() + record.zone.len(); + main.push((record, sync_seq)); + if bytes > max_bytes { + break; + } + } + drop(stmt); + + // A root republished after its sub-handles carries a higher sync_seq, + // so a bootstrapping peer would reach the subs before the root and be + // unable to verify them. Append such roots (they land beyond this + // page) as extra records — the peer's duplicate pre-filter drops them + // again when their own row arrives in cursor order. next_cursor is + // unaffected: it tracks only the main selection. + let last_seq = main.last().map(|(_, s)| *s).unwrap_or(after); + let sub_spaces: std::collections::HashSet = main + .iter() + .filter(|(r, _)| r.handle != r.space_name()) + .map(|(r, _)| r.space_name().to_string()) + .collect(); + let mut roots: HashMap = HashMap::new(); + if !sub_spaces.is_empty() { + let space_list: Vec<&str> = sub_spaces.iter().map(|s| s.as_str()).collect(); + let placeholders: Vec<&str> = space_list.iter().map(|_| "?").collect(); + let query = format!( + "SELECT handle, epoch_height, offchain_seq, delegate_offchain_seq, cert_data, zone_data + FROM handles + WHERE handle IN ({}) AND handle = space AND sync_seq > ?", + placeholders.join(", ") + ); + let mut stmt = conn.prepare(&query)?; + let mut params: Vec<&dyn rusqlite::ToSql> = space_list + .iter() + .map(|s| s as &dyn rusqlite::ToSql) + .collect(); + params.push(&last_seq); + let found = stmt.query_map(params.as_slice(), |row| { + Ok(resolver::SyncRecord { + handle: row.get(0)?, + epoch_height: row.get(1)?, + seq: row.get::<_, i64>(2)? as u64, + delegate_seq: row.get::<_, i64>(3)? as u64, + cert: row.get(4)?, + zone: row.get(5)?, + }) + })?; + for root in found { + let root = root?; + roots.insert(root.handle.clone(), root); + } + } + + // Hard combined cap: appended roots (which can carry ~250 KB ZK + // receipts) count against the page too. If they don't fit, shrink the + // main selection from the tail — dropped rows are simply re-served + // next page — until rows + their required roots fit. One row plus one + // root always fits (both bounded by the message size cap), so the + // page always makes progress and can never exceed what pullers accept. + let hard_cap = max_bytes.saturating_mul(2); + let record_bytes = |r: &resolver::SyncRecord| r.cert.len() + r.zone.len(); + let total = |main: &[(resolver::SyncRecord, i64)], + roots: &HashMap| { + let needed: std::collections::HashSet<&str> = main + .iter() + .filter(|(r, _)| r.handle != r.space_name()) + .map(|(r, _)| r.space_name()) + .collect(); + main.iter().map(|(r, _)| record_bytes(r)).sum::() + + roots + .values() + .filter(|r| needed.contains(r.handle.as_str())) + .map(record_bytes) + .sum::() + }; + while main.len() > 1 && total(&main, &roots) > hard_cap { + let Some((popped, _)) = main.pop() else { + break; + }; + // A popped root may still be needed by its sub-handles earlier in + // the page — re-home it into the roots map (it only counts toward + // the cap, and only ships, while such a sub survives). Discarding + // it would send orphaned subs a bootstrapping peer cannot verify. + if popped.handle == popped.space_name() { + roots.entry(popped.handle.clone()).or_insert(popped); + } + } + + let mut page = resolver::SyncPage::default(); + let needed: std::collections::HashSet = main + .iter() + .filter(|(r, _)| r.handle != r.space_name()) + .map(|(r, _)| r.space_name().to_string()) + .collect(); + for (record, sync_seq) in main { + page.next_cursor = Some(resolver::SyncCursor(sync_seq as u64).to_string()); + page.records.push(record); + } + for (handle, root) in roots { + if needed.contains(handle.as_str()) { + page.records.push(root); + } + } + Ok(page) + } + + /// Row count and newest cursor, for /sync/summary. + /// + /// The cursor is the persistent write counter, not MAX(sync_seq): the + /// counter never regresses when the highest-seq rows are evicted, so + /// peers cannot misread eviction as a cursor-space reset (which would + /// trigger a full re-sync). Pullers that reach a region with no rows + /// left get an empty page and catch their watermark up. + pub fn sync_summary(&self) -> anyhow::Result { + let conn = self.conn.lock().unwrap(); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM handles", [], |r| r.get(0))?; + let counter: i64 = + conn.query_row("SELECT seq FROM sync_counter WHERE id = 1", [], |r| { + r.get(0) + })?; + Ok(resolver::SyncSummary { + count: count as u64, + latest_cursor: (counter > 0).then(|| resolver::SyncCursor(counter as u64).to_string()), + }) + } + + /// Last fully-processed sync cursor for a peer. + pub fn get_watermark(&self, peer_url: &str) -> anyhow::Result> { + let conn = self.conn.lock().unwrap(); + conn.query_row( + "SELECT cursor FROM sync_watermarks WHERE peer_url = ?", + params![peer_url], + |r| r.get(0), + ) + .optional() + .map_err(Into::into) + } + + /// Persist the sync cursor for a peer. + pub fn set_watermark(&self, peer_url: &str, cursor: &str) -> anyhow::Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT OR REPLACE INTO sync_watermarks (peer_url, cursor, updated_at) VALUES (?, ?, ?)", + params![peer_url, cursor, Self::now()], + )?; + Ok(()) + } + /// Get existing zones for handles (used internally for is_better_than comparison). fn get_zones_inner( conn: &Connection, @@ -399,12 +967,14 @@ impl SqliteStore { } let conn = self.conn.lock().unwrap(); let now = Self::now(); + let tx = conn.unchecked_transaction()?; for (num_id, name) in entries { - conn.execute( + tx.execute( "INSERT OR REPLACE INTO reverse (num_id, name, updated_at) VALUES (?, ?, ?)", params![num_id, name, now], )?; } + tx.commit()?; Ok(()) } @@ -447,17 +1017,16 @@ impl SqliteStore { entries: &[(&str, &str)], ) -> anyhow::Result<()> { let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM addrs WHERE handle = ?", params![handle])?; - if entries.is_empty() { - return Ok(()); - } + let tx = conn.unchecked_transaction()?; + tx.execute("DELETE FROM addrs WHERE handle = ?", params![handle])?; let now = Self::now(); for (name, addr) in entries { - conn.execute( + tx.execute( "INSERT INTO addrs (name, addr, handle, rev, updated_at) VALUES (?, ?, ?, ?, ?)", params![name, addr, handle, rev, now], )?; } + tx.commit()?; Ok(()) } @@ -476,6 +1045,308 @@ impl SqliteStore { mod tests { use super::*; + /// Opening a database created before the sync schema (production relays + /// with real handle rows) must add the new columns, backfill sync_seq in + /// (updated_at, handle) order, hash the zones, and seed the counter. + #[test] + fn test_migrates_pre_sync_database() { + let path = std::env::temp_dir().join(format!( + "certrelay-migration-test-{}.db", + std::process::id() + )); + let _ = std::fs::remove_file(&path); + + // Exact pre-sync production schema + { + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE handles ( + handle TEXT PRIMARY KEY, + space TEXT NOT NULL, + cert_data BLOB NOT NULL, + zone_data BLOB NOT NULL, + epoch_height INTEGER NOT NULL, + offchain_seq INTEGER NOT NULL DEFAULT 0, + delegate_offchain_seq INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL + ); + CREATE INDEX idx_handles_space ON handles(space); + CREATE TABLE reverse ( + num_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + "#, + ) + .unwrap(); + for (handle, ts) in [ + ("@bitcoin", 100), + ("alice@bitcoin", 200), + ("bob@bitcoin", 150), + ] { + conn.execute( + "INSERT INTO handles (handle, space, cert_data, zone_data, epoch_height, offchain_seq, delegate_offchain_seq, updated_at) + VALUES (?, '@bitcoin', x'01', ?, 870000, 5, 0, ?)", + params![handle, handle.as_bytes(), ts], + ) + .unwrap(); + } + } + + let store = SqliteStore::open(&path).unwrap(); + + // Backfill covers every row, ordered by (updated_at, handle): + // @bitcoin (100) -> 1, bob (150) -> 2, alice (200) -> 3 + let summary = store.sync_summary().unwrap(); + assert_eq!(summary.count, 3); + assert_eq!(summary.latest_cursor.as_deref(), Some("3")); + + let page = store.sync_page(None, 10, usize::MAX).unwrap(); + let order: Vec<&str> = page.records.iter().map(|r| r.handle.as_str()).collect(); + assert_eq!(order, ["@bitcoin", "bob@bitcoin", "alice@bitcoin"]); + + // zone_hash matches the stored blob; hint metadata survived + let hints = store.get_handle_hints(&["alice@bitcoin"]).unwrap(); + assert_eq!(hints[0].zone_hash, zone_hash(b"alice@bitcoin")); + assert_eq!(hints[0].offchain_seq, 5); + assert_eq!(hints[0].epoch_height, 870000); + + // New writes continue after the backfilled sequence range + { + let conn = store.conn.lock().unwrap(); + let seq: i64 = conn + .query_row("SELECT seq FROM sync_counter WHERE id = 1", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(seq, 3); + } + + // Storage accounting was backfilled from the legacy rows + let (rows, bytes) = store.storage_totals().unwrap(); + assert_eq!(rows, 3); + assert!(bytes > 0, "backfilled bytes should cover the blobs"); + assert_eq!(store.space_usage("@bitcoin").unwrap(), (3, 1)); + + // Re-opening must not re-migrate (idempotent) + drop(store); + let store = SqliteStore::open(&path).unwrap(); + assert_eq!( + store.sync_summary().unwrap().latest_cursor.as_deref(), + Some("3") + ); + assert_eq!(store.storage_totals().unwrap().0, 3, "no double backfill"); + + let _ = std::fs::remove_file(&path); + } + + fn raw_insert(store: &SqliteStore, handle: &str, space: &str, epoch: i64, blob: &[u8]) { + let conn = store.conn.lock().unwrap(); + conn.execute( + "INSERT OR REPLACE INTO handles (handle, space, cert_data, zone_data, epoch_height, updated_at) + VALUES (?, ?, x'01', ?, ?, 100)", + params![handle, space, blob, epoch], + ) + .unwrap(); + } + + /// Triggers keep storage accounting true across insert, replace, and + /// delete — including REPLACE displacing an old row (recursive_triggers). + #[test] + fn test_storage_accounting_triggers() { + let store = SqliteStore::in_memory().unwrap(); + assert_eq!(store.storage_totals().unwrap(), (0, 0)); + + raw_insert(&store, "@a", "@a", 100, &[0u8; 10]); + raw_insert(&store, "x@a", "@a", 100, &[0u8; 20]); + raw_insert(&store, "y@a", "@a", 200, &[0u8; 30]); + // cert x'01' = 1 byte each + assert_eq!(store.storage_totals().unwrap(), (3, 63)); + assert_eq!(store.space_usage("@a").unwrap(), (3, 2)); + + // REPLACE: old row's bytes must be subtracted, count unchanged + raw_insert(&store, "x@a", "@a", 100, &[0u8; 50]); + assert_eq!(store.storage_totals().unwrap(), (3, 93)); + assert_eq!(store.space_usage("@a").unwrap(), (3, 2)); + + // Delete decrements; the emptied epoch bucket still counts toward + // entitlement (epochs stays 2) + store.delete_handles(&["y@a".to_string()]).unwrap(); + assert_eq!(store.storage_totals().unwrap(), (2, 62)); + assert_eq!(store.space_usage("@a").unwrap(), (2, 2)); + + // space_usage_all only lists spaces with stored handles + raw_insert(&store, "@b", "@b", 300, &[0u8; 5]); + let mut all = store.space_usage_all().unwrap(); + all.sort(); + assert_eq!( + all, + vec![("@a".to_string(), 2, 2), ("@b".to_string(), 1, 1)] + ); + } + + /// Reverse cleanup is precise: it extracts (num_id, rev) from the deleted + /// zones, so reverse rows of *other* identities — even ones sharing a + /// display name with the deleted handle — are never collateral damage. + /// (Positive-path cleanup needs a real signed zone with a primary Sig and + /// num_id, exercised via the handler paths; here we verify the guardrails: + /// undecodable zones don't crash the delete and unrelated rows survive.) + #[test] + fn test_delete_handles_preserves_unrelated_reverse_rows() { + let store = SqliteStore::in_memory().unwrap(); + raw_insert(&store, "x@a", "@a", 100, &[0u8; 10]); + raw_insert(&store, "y@a", "@a", 100, &[0u8; 10]); + + // Reverse rows for other identities, one even sharing the deleted + // handle's name — none may be touched by deleting x@a (its raw zone + // blob carries no num_id). + store + .set_revs(&[("1", "x@a"), ("2", "x-pretty@a"), ("3", "y@a")]) + .unwrap(); + store + .set_addrs("x@a", "x-pretty@a", &[("btc", "bc1qxyz")]) + .unwrap(); + + let deleted = store.delete_handles(&["x@a".to_string()]).unwrap(); + assert_eq!(deleted, 1); + + let left = store.get_revs(&["1", "2", "3"]).unwrap(); + assert_eq!(left.len(), 3, "unrelated reverse rows must survive"); + assert!( + store.get_addrs("btc", "bc1qxyz").unwrap().is_empty(), + "addr index rows must be gone" + ); + } + + /// Appended roots count against a hard combined page cap (roots can carry + /// ~250 KB ZK receipts): when they don't fit, the main selection shrinks + /// from the tail and the dropped rows arrive on the next page — the page + /// never exceeds what pullers accept and never stops making progress. + #[test] + fn test_sync_page_trims_when_appended_roots_exceed_cap() { + let store = SqliteStore::in_memory().unwrap(); + // Two spaces, each: one sub (30-byte zone) + one root (60-byte zone, + // republished after the sub) + raw_insert(&store, "x@a", "@a", 1, &[0u8; 30]); + raw_insert(&store, "y@b", "@b", 1, &[0u8; 30]); + raw_insert(&store, "@a", "@a", 1, &[0u8; 60]); + raw_insert(&store, "@b", "@b", 1, &[0u8; 60]); + for h in ["x@a", "y@b", "@a", "@b"] { + store.bump_sync_seq(h).unwrap(); + } + + // Both subs fit max_bytes, but subs + both roots exceed the hard cap + // (2x max_bytes) — the page must shrink to one sub + its root + let max_bytes = 80; + let page = store.sync_page(None, 2, max_bytes).unwrap(); + let total: usize = page + .records + .iter() + .map(|r| r.cert.len() + r.zone.len()) + .sum(); + assert!( + total <= 2 * max_bytes, + "page ({total} bytes) must respect the hard cap" + ); + let names: Vec<&str> = page.records.iter().map(|r| r.handle.as_str()).collect(); + assert!(names.contains(&"x@a") && names.contains(&"@a")); + assert!(!names.contains(&"y@b"), "trimmed row waits for next page"); + assert_eq!(page.next_cursor.as_deref(), Some("1"), "cursor tracks trim"); + + // The trimmed row arrives on the following page, with its root + let cursor = page.next_cursor.unwrap().parse().ok(); + let page2 = store.sync_page(cursor, 2, max_bytes).unwrap(); + let names2: Vec<&str> = page2.records.iter().map(|r| r.handle.as_str()).collect(); + assert!(names2.contains(&"y@b") && names2.contains(&"@b")); + } + + /// A root row popped by the trim while its sub-handles survive earlier in + /// the page must be re-homed as an appended root — otherwise the page + /// ships orphaned subs that a bootstrapping peer counts as failed and + /// loses behind its watermark. + #[test] + fn test_sync_page_trim_rehomes_in_page_roots() { + let store = SqliteStore::in_memory().unwrap(); + // Seq order: y@b (sub), z@c (sub), @b (root, in page range), + // @c (root, beyond page — appended, and large enough to force a trim) + raw_insert(&store, "y@b", "@b", 1, &[0u8; 30]); + raw_insert(&store, "z@c", "@c", 1, &[0u8; 30]); + raw_insert(&store, "@b", "@b", 1, &[0u8; 60]); + raw_insert(&store, "@c", "@c", 1, &[0u8; 200]); + for h in ["y@b", "z@c", "@b", "@c"] { + store.bump_sync_seq(h).unwrap(); + } + + // All three in-range rows fit max_bytes; adding the appended @c root + // busts the hard cap, so the trim pops @b (root — must be re-homed) + // and then z@c (releasing @c). The page must keep y@b WITH @b. + let page = store.sync_page(None, 3, 130).unwrap(); + let names: Vec<&str> = page.records.iter().map(|r| r.handle.as_str()).collect(); + assert!(names.contains(&"y@b"), "surviving sub stays"); + assert!( + names.contains(&"@b"), + "popped in-page root must be re-homed, not discarded" + ); + assert!(!names.contains(&"z@c"), "trimmed sub waits for next page"); + assert_eq!(page.next_cursor.as_deref(), Some("1")); + let total: usize = page + .records + .iter() + .map(|r| r.cert.len() + r.zone.len()) + .sum(); + assert!(total <= 2 * 130, "cap still holds after re-homing"); + + // Next page re-serves the trimmed rows with their root + let page2 = store + .sync_page(Some(resolver::SyncCursor(1)), 3, 130) + .unwrap(); + let names2: Vec<&str> = page2.records.iter().map(|r| r.handle.as_str()).collect(); + assert!(names2.contains(&"z@c") && names2.contains(&"@c")); + } + + /// The advertised cursor comes from the persistent counter, so evicting + /// the newest rows can't look like a cursor-space reset to peers (which + /// would trigger a needless full re-sync). + #[test] + fn test_latest_cursor_survives_eviction_of_newest_row() { + let store = SqliteStore::in_memory().unwrap(); + raw_insert(&store, "a@s", "@s", 1, &[0u8; 4]); + raw_insert(&store, "b@s", "@s", 1, &[0u8; 4]); + store.bump_sync_seq("a@s").unwrap(); + store.bump_sync_seq("b@s").unwrap(); // b@s now holds the highest seq + + let before = store.sync_summary().unwrap().latest_cursor; + assert_eq!(before.as_deref(), Some("2")); + + store.delete_handles(&["b@s".to_string()]).unwrap(); + assert_eq!( + store.sync_summary().unwrap().latest_cursor, + before, + "eviction of the newest row must not regress the cursor" + ); + } + + /// Eviction candidates come back oldest-updated first. + #[test] + fn test_eviction_candidates_order() { + let store = SqliteStore::in_memory().unwrap(); + let conn = store.conn.lock().unwrap(); + for (handle, ts) in [("a@s", 300), ("b@s", 100), ("c@s", 200)] { + conn.execute( + "INSERT INTO handles (handle, space, cert_data, zone_data, epoch_height, updated_at) + VALUES (?, '@s', x'01', x'01', 1, ?)", + params![handle, ts], + ) + .unwrap(); + } + drop(conn); + assert_eq!( + store.eviction_candidates("@s", 2).unwrap(), + vec!["b@s".to_string(), "c@s".to_string()] + ); + } + #[test] fn test_open_in_memory() { let store = SqliteStore::in_memory().expect("create in-memory store"); diff --git a/relay/src/sync.rs b/relay/src/sync.rs new file mode 100644 index 0000000..5314426 --- /dev/null +++ b/relay/src/sync.rs @@ -0,0 +1,609 @@ +//! Pull-based propagation: periodically sync stored handles from verified peers. +//! +//! This is THE propagation mechanism (there is no push gossip). Each relay +//! pages `/sync` from peers it selected, re-verifies everything locally, and +//! tracks a persistent per-peer watermark so downtime resumes as a delta pull. +//! A relay with an empty database bootstraps through the same code path — the +//! watermark just starts at zero. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use libveritas::builder::{DataUpdateRequest, MessageBuilder}; +use libveritas::cert::Certificate; +use libveritas::{ProvableOption, Zone}; +use rand::seq::SliceRandom; +use resolver::{SyncCursor, SyncPage, SyncSummary}; + +use crate::AppState; + +/// Outcome of ingesting one batch of pulled sync records. +#[derive(Debug, Default)] +pub struct SyncIngest { + /// Handles actually stored (new or better than existing). + pub stored: usize, + /// Records skipped before any crypto (exact metadata match with our row). + pub prefiltered: usize, + /// Spaces whose records could not be verified this round (missing root, + /// stale anchor, bad data). Not retried within the round. + pub failed_spaces: usize, + /// First-insert records skipped by the retention admission gate + /// (before any prove/verify work). + pub gated: usize, + /// False when the failed-space budget tripped and the rest of the batch + /// was left unattempted — the caller must NOT advance its watermark past + /// this page (unattempted != processed). + pub incomplete: bool, +} + +/// Tuning for the sync loop. Millisecond-scale values are valid so the test +/// harness can drive convergence in-process. +#[derive(Clone)] +pub struct SyncConfig { + /// Base interval between rounds. + pub interval: Duration, + /// Random extra delay added per round (jitter). + pub jitter: Duration, + /// Rows requested per page. + pub page_limit: usize, + /// Peers contacted per round. + pub peers_per_round: usize, + /// Max pages pulled from one peer in one round (a partial bootstrap + /// resumes from the watermark next round). + pub max_pages_per_peer: usize, + /// Max bytes accepted for one page body. + pub max_page_bytes: usize, + /// Coalescing window for outgoing pokes: bursts of stores within this + /// window produce one poke per peer. + pub poke_debounce: Duration, + /// Minimum gap between poke-triggered syncs with the same peer. + pub poke_cooldown: Duration, +} + +impl Default for SyncConfig { + fn default() -> Self { + Self { + interval: Duration::from_secs(45), + jitter: Duration::from_secs(15), + page_limit: crate::http::MAX_SYNC_PAGE_ROWS, + peers_per_round: 2, + max_pages_per_peer: 200, + // Serving pages overshoot MAX_SYNC_PAGE_BYTES by up to one row + // plus appended root records — accept double, plus one message. + max_page_bytes: 2 * crate::http::MAX_SYNC_PAGE_BYTES + crate::DEFAULT_MAX_MESSAGE_SIZE, + poke_debounce: Duration::from_secs(2), + poke_cooldown: Duration::from_secs(5), + } + } +} + +/// Run sync rounds forever. Spawned as a background task at startup. +pub async fn run_sync_loop(state: Arc, config: SyncConfig) { + loop { + let jitter_ms = if config.jitter.is_zero() { + 0 + } else { + rand::random_range(0..config.jitter.as_millis() as u64) + }; + tokio::time::sleep(config.interval + Duration::from_millis(jitter_ms)).await; + sync_round(&state, &config).await; + } +} + +/// One round: pick up to `peers_per_round` random verified peers and sync +/// from each. +pub async fn sync_round(state: &Arc, config: &SyncConfig) { + let mut peer_urls: Vec = { + let peers = state.peers.lock().await; + peers.peers().iter().map(|s| s.to_string()).collect() + }; + peer_urls.shuffle(&mut rand::rng()); + peer_urls.truncate(config.peers_per_round); + + for url in peer_urls { + match sync_with_peer(state, &url, config).await { + Ok(stats) => { + if stats.stored > 0 || stats.failed_spaces > 0 { + tracing::info!( + "sync from {}: stored {}, prefiltered {}, failed spaces {}", + url, + stats.stored, + stats.prefiltered, + stats.failed_spaces + ); + } + if stats.stored > 0 { + // We have data our other peers may lack — poke them + // (multi-hop propagation). + state.poke_dirty.notify_one(); + } + state.stats.record_sync_success(&url); + state.peers.lock().await.mark_alive(&url); + } + Err(e) => { + crate::stats::bump(&state.stats.sync_errors); + tracing::debug!("sync from {} failed: {}", url, e); + state.peers.lock().await.deprioritize(&url); + } + } + } +} + +/// Peer-table maintenance: proactively refresh verified peers before their +/// TTL expires (decoupling liveness from data traffic) and verify several +/// unverified candidates per tick so simultaneous expiries recover quickly. +pub async fn run_peer_maintenance_loop( + state: Arc, + interval: Duration, + candidates_per_tick: usize, +) { + let mut ticker = tokio::time::interval(interval); + let mut tick: u64 = 0; + loop { + ticker.tick().await; + tick += 1; + + // Evict stale per-IP rate limiter entries periodically (~10 min at + // the default 10s tick) — these maps otherwise grow forever. + if tick.is_multiple_of(60) { + state.limiters.cleanup(); + } + + let (refresh, candidates) = { + let mut peers = state.peers.lock().await; + peers.demote_expired(); + let refresh = peers.refresh_candidates(); + let candidates = if peers.needs_peers() { + peers.next_candidates(candidates_per_tick) + } else { + vec![] + }; + (refresh, candidates) + }; + + for url in refresh.into_iter().chain(candidates) { + let state = Arc::clone(&state); + tokio::spawn(async move { + // peer_client resolve-checks and pins the address: a DNS-named + // peer pointing at a private/internal address (SSRF via + // /announce) is refused and removed. + let client = match state.peer_client(&url).await { + Ok(c) => c, + Err(e) => { + tracing::debug!("peer failed address policy, removing {}: {}", url, e); + state.peers.lock().await.remove(&url); + return; + } + }; + let check_url = format!("{}/health", url); + match client.head(&check_url).send().await { + Ok(resp) if resp.status().is_success() => { + state.peers.lock().await.mark_alive(&url); + tracing::debug!("peer alive: {}", url); + } + _ => { + tracing::debug!("peer health check failed: {}", url); + } + } + }); + } + } +} + +/// Send pokes to all verified peers whenever new data lands, coalescing +/// bursts into one poke per peer per debounce window. Requires `self_url` +/// (peers must know where to pull from); without it, propagation still +/// happens via peers' interval loops. +pub async fn run_poke_send_loop(state: Arc, config: SyncConfig) { + let Some(self_url) = state.self_url.clone() else { + return; + }; + loop { + state.poke_dirty.notified().await; + // Coalesce: everything stored during this window rides one poke. + tokio::time::sleep(config.poke_debounce).await; + + let cursor = match state.handler.store.sync_summary() { + Ok(s) => match s.latest_cursor { + Some(c) => c, + None => continue, + }, + Err(e) => { + tracing::warn!("poke send: sync summary failed: {}", e); + continue; + } + }; + let poke = resolver::Poke { + url: self_url.clone(), + cursor, + }; + let peer_urls: Vec = { + let peers = state.peers.lock().await; + peers.peers().iter().map(|s| s.to_string()).collect() + }; + for url in peer_urls { + let state = Arc::clone(&state); + let poke = poke.clone(); + crate::stats::bump(&state.stats.pokes_sent); + tokio::spawn(async move { + // Best-effort: a lost poke is recovered by the interval loop. + let Ok(client) = state.peer_client(&url).await else { + return; + }; + let _ = client + .post(format!("{}/poke", url)) + .json(&poke) + .send() + .await; + }); + } + } +} + +/// Drain poke-triggered sync requests, enforcing a per-peer cooldown so a +/// poke flood can't exceed the steady-state pull cadence. Serial by design: +/// one poke-triggered sync at a time bounds concurrent verify work. +pub async fn run_poke_sync_loop(state: Arc, config: SyncConfig) { + let Some(mut rx) = state.poke_sync_rx.lock().await.take() else { + tracing::warn!("poke sync receiver already taken"); + return; + }; + let mut last_synced: std::collections::HashMap = + std::collections::HashMap::new(); + + while let Some(url) = rx.recv().await { + // Cooldown: recently-synced peers wait for the interval loop. + let now = std::time::Instant::now(); + if last_synced + .get(&url) + .is_some_and(|t| now.duration_since(*t) < config.poke_cooldown) + { + continue; + } + last_synced.insert(url.clone(), now); + last_synced.retain(|_, t| now.duration_since(*t) < config.poke_cooldown * 4); + + match sync_with_peer(&state, &url, &config).await { + Ok(stats) => { + if stats.stored > 0 { + tracing::info!("poke sync from {}: stored {}", url, stats.stored); + // Cascade to our own peers. + state.poke_dirty.notify_one(); + } + state.stats.record_sync_success(&url); + state.peers.lock().await.mark_alive(&url); + } + Err(e) => { + crate::stats::bump(&state.stats.sync_errors); + tracing::debug!("poke sync from {} failed: {}", url, e); + state.peers.lock().await.deprioritize(&url); + } + } + } +} + +/// Verify and store records pulled from a peer via `/sync`. +/// +/// The peer's metadata fields are claims used only to skip exact duplicates; +/// surviving records are rebuilt into a message with a chain proof from **our +/// own** spaced and pushed through the same `handle_message` verification +/// path as a client publish, so nothing a peer serves is trusted. Proof +/// generation waits on the global proof semaphore, and verification runs on +/// the blocking pool under the global verify semaphore. +/// +/// Returns an error only on undecodable input (the caller should abort the +/// page and not advance its watermark); per-space verification failures are +/// counted and skipped. `max_failed_spaces` bounds the prove+verify CPU a +/// batch of unverifiable spaces can burn: once spent, the remaining spaces in +/// the batch are skipped (they self-heal via other peers or the next update). +pub async fn ingest_sync_records( + state: &Arc, + records: Vec, + max_failed_spaces: usize, +) -> anyhow::Result { + let mut out = SyncIngest::default(); + let store = &state.handler.store; + + // Pre-filter: drop records whose claimed metadata AND zone bytes exactly + // match our stored row — the common multi-peer duplicate. Anything else + // goes to full verification, which is authoritative. + let handles: Vec<&str> = records.iter().map(|r| r.handle.as_str()).collect(); + let existing: HashMap)> = store + .get_handle_hints(&handles)? + .into_iter() + .map(|r| { + ( + r.handle, + ( + r.epoch_height, + r.offchain_seq, + r.delegate_offchain_seq, + r.zone_hash, + ), + ) + }) + .collect(); + let survivors: Vec = records + .into_iter() + .filter(|r| { + let dup = existing.get(&r.handle).is_some_and(|(e, s, d, hash)| { + (*e, *s, *d) == (r.epoch_height, r.seq, r.delegate_seq) + && crate::store::zone_hash(&r.zone) == *hash + }); + if dup { + out.prefiltered += 1; + } + !dup + }) + .collect(); + + // Decode blobs; garbage fails the whole batch (untrusted page). + let mut decoded: Vec<(Certificate, Zone)> = Vec::new(); + for r in &survivors { + let cert: Certificate = borsh::from_slice(&r.cert) + .map_err(|e| anyhow::anyhow!("undecodable cert for {}: {}", r.handle, e))?; + let zone: Zone = borsh::from_slice(&r.zone) + .map_err(|e| anyhow::anyhow!("undecodable zone for {}: {}", r.handle, e))?; + if cert.subject.to_string() != r.handle { + anyhow::bail!("record handle {} does not match cert subject", r.handle); + } + decoded.push((cert, zone)); + } + + // Group by space; roots sort first within a group (single-label names). + let mut by_space: HashMap> = HashMap::new(); + for (cert, zone) in decoded { + let Some(space) = cert.subject.space().map(|s| s.to_string()) else { + continue; + }; + by_space.entry(space).or_default().push((cert, zone)); + } + + // One message per space: failure isolation, and the prove call is local. + for (space, mut group) in by_space { + // In-batch circuit breaker: a page full of bogus spaces stops burning + // prove+verify CPU once the failure budget is spent. The rest of the + // batch is unattempted, so the page must not be marked processed. + if out.failed_spaces >= max_failed_spaces { + tracing::warn!( + "failed-space budget spent ({}), aborting rest of batch", + out.failed_spaces + ); + out.incomplete = true; + break; + } + + // Retention admission gate, applied before any prove/verify work so + // re-pulls of evicted rows cost ~nothing: drop first-insert records + // of over-entitled spaces under storage pressure (updates pass). + if crate::retention::first_insert_gated(state, &state.retention, &space).unwrap_or(false) { + let before = group.len(); + group.retain(|(cert, _)| existing.contains_key(&cert.subject.to_string())); + out.gated += before - group.len(); + if group.is_empty() { + continue; + } + } + + let mut builder = MessageBuilder::new(); + + let root_idx = group + .iter() + .position(|(c, _)| c.subject.to_string() == space); + let (root_cert, root_zone) = match root_idx { + Some(i) => group.remove(i), + // Root not in this page — use our stored copy (it sorts before + // its sub-handles, so it normally synced in an earlier row). + None => match store.get_handle(&space)? { + Some(rec) => (rec.cert, rec.zone), + None => { + tracing::debug!("{}: no root available for sync batch, skipping", space); + out.failed_spaces += 1; + continue; + } + }, + }; + + for (cert, zone) in std::iter::once((root_cert, root_zone)).chain(group) { + builder.add_update(DataUpdateRequest { + handle: cert.subject.clone(), + records: Some(zone.records.clone()), + delegate_records: if let ProvableOption::Exists { value } = zone.delegate { + Some(value.records) + } else { + None + }, + }); + builder.add_cert(cert); + } + + // Infra failures (spaced down, task join) abort the whole batch — + // the watermark must not advance past records dropped for reasons + // unrelated to their validity. Only affirmative build/verification + // rejections count as failed spaces. + let proof = { + let _permit = state.proof_sem.acquire().await?; + state + .chain + .prove(&builder.chain_proof_request()) + .await + .map_err(|e| anyhow::anyhow!("chain proof failed (infra): {e}"))? + }; + let result = match builder.build(proof) { + Ok((msg, _unsigned)) => { + let _permit = state.verify_sem.acquire().await?; + let blocking_state = Arc::clone(state); + tokio::task::spawn_blocking(move || { + // Sync is exempt from the content velocity caps: a + // rate-dropped record would be silently lost behind the + // advancing watermark. CPU is bounded by the failed-space + // budget + verify semaphore, storage by retention. + blocking_state + .handler + .handle_message_opts(msg, &Default::default(), false) + }) + .await + .map_err(|e| anyhow::anyhow!("verify task failed (infra): {e}"))? + } + Err(e) => Err(anyhow::Error::from(e)), + }; + match result { + Ok(res) => out.stored += res.stored, + Err(e) => { + tracing::debug!("{}: sync batch failed verification: {}", space, e); + out.failed_spaces += 1; + } + } + } + + Ok(out) +} + +/// Failed-space budget per sync round with one peer: bounds the prove+verify +/// CPU a peer serving unverifiable data can burn before the round stops. +const MAX_FAILED_SPACES_PER_ROUND: usize = 32; + +/// Sync from one peer: check its summary against our watermark, then page +/// `/sync` until caught up (or the per-round page budget runs out). +/// +/// The watermark advances only after a page has been fully processed — +/// never from a summary claim — and is clamped to the peer's advertised +/// `latest_cursor`, so a bogus `next_cursor` (e.g. `u64::MAX`) can't poison +/// it past data that actually exists. +pub async fn sync_with_peer( + state: &Arc, + peer_url: &str, + config: &SyncConfig, +) -> anyhow::Result { + let mut totals = SyncIngest::default(); + let client = state.peer_client(peer_url).await?; + let watermark: Option = state + .handler + .store + .get_watermark(peer_url)? + .and_then(|c| c.parse().ok()); + + // Cheap freshness check: nothing new past our watermark → done. + let summary: SyncSummary = client + .get(format!("{}/sync/summary", peer_url)) + .send() + .await? + .error_for_status()? + .json() + .await?; + let latest: Option = summary.latest_cursor.and_then(|c| c.parse().ok()); + let latest = match (&watermark, latest) { + (_, None) => return Ok(totals), + (Some(mark), Some(latest)) if latest < *mark => { + // The peer's cursor space regressed below our watermark: it wiped + // or recreated its DB (fresh counter), or a past run stored a + // poisoned watermark. Start over — the pre-filter makes the + // re-pull cheap. + tracing::info!("{}: cursor space regressed, resetting watermark", peer_url); + state.handler.store.set_watermark(peer_url, "0")?; + return Ok(totals); + } + (Some(mark), Some(latest)) if latest == *mark => return Ok(totals), + (_, Some(latest)) => latest, + }; + + let mut cursor = watermark; + for _ in 0..config.max_pages_per_peer { + let mut req = client + .get(format!("{}/sync", peer_url)) + .query(&[("limit", config.page_limit.to_string())]); + if let Some(c) = &cursor { + req = req.query(&[("cursor", c.to_string())]); + } + let resp = req.send().await?.error_for_status()?; + if let Some(len) = resp.content_length() + && len > config.max_page_bytes as u64 + { + anyhow::bail!("sync page too large: {} bytes", len); + } + let body = resp.bytes().await?; + if body.len() > config.max_page_bytes { + anyhow::bail!("sync page too large: {} bytes", body.len()); + } + let page: SyncPage = borsh::from_slice(&body)?; + if page.records.is_empty() { + // Nothing exists between our cursor and the advertised latest + // (rows were evicted/deleted on the peer) — catch the watermark + // up so future rounds short-circuit on the summary. New writes + // always get sequences above `latest`, so nothing can be missed. + if cursor.unwrap_or_default() < latest { + state + .handler + .store + .set_watermark(peer_url, &latest.to_string())?; + } + break; + } + let next_cursor: SyncCursor = page + .next_cursor + .as_deref() + .ok_or_else(|| anyhow::anyhow!("sync page missing cursor"))? + .parse() + .map_err(|e| anyhow::anyhow!("bad sync cursor: {e}"))?; + // A stuck or rewinding cursor means a broken/malicious peer. + if cursor.as_ref().is_some_and(|c| next_cursor <= *c) { + anyhow::bail!("sync cursor did not advance"); + } + + let remaining_failures = + MAX_FAILED_SPACES_PER_ROUND.saturating_sub(totals.failed_spaces) + 1; + let stats = ingest_sync_records(state, page.records, remaining_failures).await?; + crate::stats::bump(&state.stats.sync_pages_pulled); + crate::stats::bump_by(&state.stats.sync_records_stored, stats.stored as u64); + crate::stats::bump_by( + &state.stats.sync_records_prefiltered, + stats.prefiltered as u64, + ); + crate::stats::bump_by(&state.stats.sync_failed_spaces, stats.failed_spaces as u64); + totals.stored += stats.stored; + totals.prefiltered += stats.prefiltered; + totals.failed_spaces += stats.failed_spaces; + + // The failed-space budget tripped mid-page: the tail of the page was + // never attempted, so advancing the watermark would silently discard + // it. Abort the round instead; a peer serving garbage at this volume + // also gets deprioritized by the caller. + if stats.incomplete { + anyhow::bail!( + "failed-space budget spent mid-page ({} failures) — not advancing", + totals.failed_spaces + ); + } + + // Page fully processed (every record stored, prefiltered, gated, or + // affirmatively rejected by verification) — safe to advance. Clamp to + // the advertised latest: rows written on the peer after our summary + // check are simply re-pulled next round. + let accepted = next_cursor.min(latest); + state + .handler + .store + .set_watermark(peer_url, &accepted.to_string())?; + cursor = Some(next_cursor); + + // Caught up to everything the summary advertised — rows written on + // the peer since then are next round's work. + if next_cursor >= latest { + break; + } + + // Round-level breaker: cumulative affirmative failures across fully + // processed pages still cut the round short (bounded CPU per round). + if totals.failed_spaces > MAX_FAILED_SPACES_PER_ROUND { + tracing::warn!( + "{}: too many failed spaces this round ({}), stopping", + peer_url, + totals.failed_spaces + ); + break; + } + } + + Ok(totals) +} diff --git a/relay/tests/integration_tests.rs b/relay/tests/integration_tests.rs index 49db249..91ca5ac 100644 --- a/relay/tests/integration_tests.rs +++ b/relay/tests/integration_tests.rs @@ -7,7 +7,10 @@ use libveritas::Veritas; use libveritas::msg::QueryContext; use libveritas_testutil::fixture::*; use relay::anchor::AnchorSets; -use relay::{AppState, Config, ExtendedNetwork, Handler, PeerInfo, Relay, SqliteStore}; +use relay::{ + AppState, Config, ExtendedNetwork, Handler, PeerInfo, Relay, SqliteStore, SyncConfig, + sync_with_peer, +}; use resolver::{AnchorSet, HintsResponse}; use spaces_protocol::slabel::SLabel; @@ -36,7 +39,7 @@ fn setup_handler(state: &ChainState) -> Handler { /// Replace the handler's Veritas with one built from the current chain state. fn sync_veritas(handler: &Handler, state: &ChainState) { - *handler.veritas.lock().unwrap() = build_veritas(state); + *handler.veritas.write().unwrap() = build_veritas(state); } /// Collect the test anchors from a ChainState. @@ -56,9 +59,45 @@ async fn start_relay(chain_state: &ChainState) -> (String, Arc) { config.spaced_url = Some("http://127.0.0.1:1".into()); config.anchors = test_anchors(chain_state); config.dev_mode = true; + config.allow_private_peers = true; let relay = Relay::new(config).unwrap(); - *relay.state().handler.veritas.lock().unwrap() = build_veritas(chain_state); + *relay.state().handler.veritas.write().unwrap() = build_veritas(chain_state); + *relay.state().handler.anchor_store.lock().unwrap() = + AnchorSets::from_anchors(test_anchors(chain_state)); + + let state = relay.state().clone(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let url = format!("http://{}", addr); + + tokio::spawn(async move { + relay.run(listener).await.unwrap(); + }); + + (url, state) +} + +/// Chain proof matching the current test chain state (what a relay's own +/// spaced would produce for any request). +fn chain_proof(state: &ChainState) -> libveritas::msg::ChainProof { + state.message(vec![]).chain +} + +/// Start a relay whose SpacedClient is mocked from the test chain state, so +/// sync ingestion can build chain proofs "locally". +async fn start_relay_mocked(chain_state: &ChainState) -> (String, Arc) { + let mut config = Config::new(PathBuf::from("/tmp/relay-test"), ExtendedNetwork::Testnet4); + config.db_path = PathBuf::from(":memory:"); + config.spaced_url = Some("http://127.0.0.1:1".into()); + config.anchors = test_anchors(chain_state); + config.dev_mode = true; + config.allow_private_peers = true; + config.mock_chain = Some((chain_proof(chain_state), test_anchors(chain_state))); + + let relay = Relay::new(config).unwrap(); + *relay.state().handler.veritas.write().unwrap() = build_veritas(chain_state); *relay.state().handler.anchor_store.lock().unwrap() = AnchorSets::from_anchors(test_anchors(chain_state)); @@ -528,8 +567,10 @@ async fn test_hints_endpoint() { assert!(hints.anchor_tip > 0, "anchor_tip should be > 0"); } +/// Propagation is pull-based: accepting a message must never push it to peers, +/// even when verified peers are present. #[tokio::test] -async fn test_gossip_propagation() { +async fn test_message_is_not_forwarded_to_peers() { let mut state = ChainState::new(); let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); runner.run(&mut state); @@ -562,15 +603,1011 @@ async fn test_gossip_propagation() { .unwrap(); assert_eq!(resp.status().as_u16(), 200); - // Wait for gossip tokio::time::sleep(Duration::from_millis(500)).await; + let alice = state_a.handler.store.get_handle("alice@sovereign").unwrap(); + assert!(alice.is_some(), "relay A should have stored the message"); + let root = state_b.handler.store.get_handle("@sovereign").unwrap(); - assert!(root.is_some(), "relay B should have @sovereign from gossip"); + assert!(root.is_none(), "relay B must not receive pushed messages"); let alice = state_b.handler.store.get_handle("alice@sovereign").unwrap(); + assert!(alice.is_none(), "relay B must not receive pushed messages"); +} + +/// /announce rejects URLs pointing at private or internal addresses. +#[tokio::test] +async fn test_announce_rejects_private_urls() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url, app_state) = start_relay(&state).await; + let client = reqwest::Client::new(); + + let announce = |peer_url: &str| { + let client = client.clone(); + let url = url.clone(); + let body = serde_json::json!({ "url": peer_url, "capabilities": 0 }); + async move { + client + .post(format!("{}/announce", url)) + .json(&body) + .send() + .await + .unwrap() + .status() + .as_u16() + } + }; + + // start_relay sets allow_private_peers, so loopback is accepted here — + // but bad schemes and credentials are always rejected + assert_eq!(announce("http://127.0.0.1:9999").await, 200); + assert_eq!(announce("ftp://relay.example.com").await, 400); + assert_eq!(announce("file:///etc/passwd").await, 400); + assert_eq!(announce("http://user:pass@relay.example.com").await, 400); + + // Flip to strict mode via a second relay? allow_private_peers is baked into + // AppState at construction, so exercise the validator directly instead. + assert!(relay::peer::validate_peer_url("http://169.254.169.254/", false).is_err()); + assert!(relay::peer::validate_peer_url("http://127.0.0.1:12888", false).is_err()); + assert!(relay::peer::validate_peer_url("https://relay.example.com", false).is_ok()); + + // The accepted loopback announce landed in the unverified table + let peers = app_state.peers.lock().await; + assert_eq!(peers.unverified_count(), 1); +} + +// ───────────────────────────────────────────────────────────────────────── +// Sync (pull-based propagation) +// ───────────────────────────────────────────────────────────────────────── + +/// Publish to relay A over HTTP, returning the stored handle names. +async fn publish(url: &str, state: &ChainState, runner: &mut FixtureRunner) { + let bundle = runner.build_bundle(); + let msg = state.message(vec![bundle]); + let resp = reqwest::Client::new() + .post(format!("{}/message", url)) + .body(msg.to_bytes()) + .header("content-type", "application/octet-stream") + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200, "publish should succeed"); +} + +/// Fresh-DB bootstrap: relay B pulls everything relay A has via /sync and +/// converges; a second sync transfers nothing new. +#[tokio::test] +async fn test_sync_bootstrap_convergence() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + assert!( + state_a + .handler + .store + .get_handle("alice@sovereign") + .unwrap() + .is_some() + ); + + // B starts empty, with a mocked local chain for proof building + let (_url_b, state_b) = start_relay_mocked(&state).await; + let stats = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert!(stats.stored > 0, "bootstrap should store handles"); + assert_eq!(stats.failed_spaces, 0, "no spaces should fail verification"); + + for key in ["@sovereign", "alice@sovereign", "bob@sovereign"] { + assert!( + state_b.handler.store.get_handle(key).unwrap().is_some(), + "{} should be on relay B after sync", + key + ); + } + + // Watermark is caught up: the next sync short-circuits on the summary + let again = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert_eq!(again.stored, 0); + assert_eq!(again.prefiltered, 0, "summary check should skip the pull"); +} + +/// Re-pulling already-synced rows (watermark reset) is caught by the metadata +/// pre-filter without storing or re-verifying anything. +#[tokio::test] +async fn test_sync_prefilter_skips_duplicates() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, _state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + + let (_url_b, state_b) = start_relay_mocked(&state).await; + let first = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert!(first.stored > 0); + + // Rewind the watermark: everything gets re-pulled, nothing gets re-stored + state_b.handler.store.set_watermark(&url_a, "0").unwrap(); + let second = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert_eq!(second.stored, 0, "duplicates must not store"); + assert_eq!( + second.prefiltered, first.stored, + "every previously-synced row should be pre-filtered" + ); +} + +/// After the first sync, only new data crosses the wire (delta pull from the +/// persisted watermark). +#[tokio::test] +async fn test_sync_watermark_delta() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, two_commits_both_finalized()); + runner.run_next(&mut state); // first step only + + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + + let (_url_b, state_b) = start_relay_mocked(&state).await; + let first = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert!(first.stored > 0); + + // The chain advances and more data lands on A after B's first sync + runner.run(&mut state); + *state_a.handler.veritas.write().unwrap() = build_veritas(&state); + publish(&url_a, &state, &mut runner).await; + + // B refreshes its chain view (anchors moved) and pulls the delta + *state_b.handler.veritas.write().unwrap() = build_veritas(&state); + *state_b.chain.mock_chain_proof.lock().unwrap() = + Some((chain_proof(&state), test_anchors(&state))); + + let delta = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert!(delta.stored > 0, "delta sync should store the new data"); + assert_eq!( + delta.prefiltered, 0, + "already-synced rows must not be re-pulled" + ); + assert!( + state_b + .handler + .store + .get_handle("charlie@two-finalized") + .unwrap() + .is_some(), + "second-commit handle should reach relay B" + ); + assert!( + state_b + .handler + .store + .get_handle("alice@two-finalized") + .unwrap() + .is_some(), + "first-commit handle must survive the delta sync" + ); +} + +/// /sync pages respect limit and cursor ordering; the final page is empty. +#[tokio::test] +async fn test_sync_pagination() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + + let store = &state_a.handler.store; + let total = store.sync_summary().unwrap().count as usize; + assert!(total >= 3, "fixture should store at least 3 handles"); + + let mut seen = 0; + let mut cursor = None; + loop { + let page = store.sync_page(cursor, 2, usize::MAX).unwrap(); + if page.records.is_empty() { + assert!(page.next_cursor.is_none()); + break; + } + assert!(page.records.len() <= 2, "limit must be respected"); + seen += page.records.len(); + let next: resolver::SyncCursor = page.next_cursor.unwrap().parse().unwrap(); + if let Some(prev) = cursor { + assert!(next > prev, "cursor must advance"); + } + cursor = Some(next); + } + assert_eq!(seen, total, "pagination must cover every row exactly once"); +} + +/// A peer serving garbage pages fails the sync without moving the watermark +/// or storing anything. +#[tokio::test] +async fn test_sync_garbage_page_rejected() { + use axum::{Router, routing::get}; + + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + // Fake peer: summary claims data, /sync serves garbage + let app = Router::new() + .route( + "/sync/summary", + get(|| async { axum::Json(serde_json::json!({ "count": 5, "latest_cursor": "5" })) }), + ) + .route("/sync", get(|| async { vec![0xFFu8; 64] })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let fake_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let (_url_b, state_b) = start_relay_mocked(&state).await; + let result = sync_with_peer(&state_b, &fake_url, &SyncConfig::default()).await; + assert!(result.is_err(), "garbage page must fail the sync"); + assert!( + state_b + .handler + .store + .get_watermark(&fake_url) + .unwrap() + .is_none(), + "watermark must not advance on garbage" + ); + assert_eq!(state_b.handler.store.sync_summary().unwrap().count, 0); +} + +/// Verified peers must survive past their TTL with zero data traffic — the +/// maintenance loop proactively refreshes them (regression: /peers used to +/// intermittently empty out in quiet periods). +#[tokio::test] +async fn test_verified_peers_survive_quiet_periods() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_b, _state_b) = start_relay(&state).await; + + // Relay A with a very short verified TTL + let mut config = Config::new(PathBuf::from("/tmp/relay-test"), ExtendedNetwork::Testnet4); + config.db_path = PathBuf::from(":memory:"); + config.spaced_url = Some("http://127.0.0.1:1".into()); + config.anchors = test_anchors(&state); + config.dev_mode = true; + config.allow_private_peers = true; + config.peer_config = relay::PeerConfig { + max_unverified: 1000, + max_verified: 100, + verified_ttl: Duration::from_millis(500), + }; + let relay_a = Relay::new(config).unwrap(); + let state_a = relay_a.state().clone(); + + { + let mut peers = state_a.peers.lock().await; + peers.announce(&PeerInfo { + source_ip: IpAddr::from([10, 0, 0, 2]), + url: url_b.clone(), + capabilities: 0, + }); + peers.mark_alive(&url_b); + } + + tokio::spawn(relay::run_peer_maintenance_loop( + state_a.clone(), + Duration::from_millis(50), + 3, + )); + + // Wait well past several TTLs with no data traffic at all + tokio::time::sleep(Duration::from_millis(1500)).await; + + let peers = state_a.peers.lock().await; + assert!( + peers.peers().contains(&url_b.as_str()), + "verified peer must stay listed across TTLs via proactive refresh" + ); +} + +// ───────────────────────────────────────────────────────────────────────── +// Poke (fast propagation) +// ───────────────────────────────────────────────────────────────────────── + +/// Fast SyncConfig for poke tests: interval loop effectively off, poke paths +/// tuned to milliseconds. +fn poke_test_config() -> SyncConfig { + SyncConfig { + interval: Duration::from_secs(3600), + jitter: Duration::ZERO, + poke_debounce: Duration::from_millis(50), + poke_cooldown: Duration::from_millis(100), + ..SyncConfig::default() + } +} + +/// Poll until `check` passes or the timeout elapses. +async fn wait_for(timeout: Duration, mut check: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if check() { + return true; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + false +} + +/// End-to-end fast propagation: publish to A → A pokes B → B pulls from A, +/// with no interval-loop involvement. +#[tokio::test] +async fn test_publish_pokes_peer_into_pulling() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + // A needs self_url (it goes in the poke body), so bind before building + let listener_a = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url_a = format!("http://{}", listener_a.local_addr().unwrap()); + + let mut config_a = Config::new(PathBuf::from("/tmp/relay-test"), ExtendedNetwork::Testnet4); + config_a.db_path = PathBuf::from(":memory:"); + config_a.spaced_url = Some("http://127.0.0.1:1".into()); + config_a.anchors = test_anchors(&state); + config_a.dev_mode = true; + config_a.allow_private_peers = true; + config_a.self_url = Some(url_a.clone()); + let relay_a = Relay::new(config_a).unwrap(); + *relay_a.state().handler.veritas.write().unwrap() = build_veritas(&state); + let state_a = relay_a.state().clone(); + tokio::spawn(async move { relay_a.run(listener_a).await.unwrap() }); + + let (url_b, state_b) = start_relay_mocked(&state).await; + + // A gossips pokes to B; B accepts pokes only from verified peers + { + let mut peers = state_a.peers.lock().await; + peers.announce(&PeerInfo { + source_ip: IpAddr::from([10, 0, 0, 2]), + url: url_b.clone(), + capabilities: 0, + }); + peers.mark_alive(&url_b); + } + { + let mut peers = state_b.peers.lock().await; + peers.announce(&PeerInfo { + source_ip: IpAddr::from([10, 0, 0, 1]), + url: url_a.clone(), + capabilities: 0, + }); + peers.mark_alive(&url_a); + } + + let cfg = poke_test_config(); + tokio::spawn(relay::run_poke_send_loop(state_a.clone(), cfg.clone())); + tokio::spawn(relay::run_poke_sync_loop(state_b.clone(), cfg)); + + publish(&url_a, &state, &mut runner).await; + + let converged = wait_for(Duration::from_secs(5), || { + state_b + .handler + .store + .get_handle("alice@sovereign") + .unwrap() + .is_some() + }) + .await; + assert!(converged, "poke should drive B to pull A's data within 5s"); +} + +/// Pokes from unknown peers and pokes with stale cursors must not trigger any +/// pull; a fresh cursor from a verified peer must. +#[tokio::test] +async fn test_poke_validation_gates_pulls() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + // Counting fake peer: valid summary, empty page + let hits = Arc::new(AtomicUsize::new(0)); + let hits_clone = hits.clone(); + let app = axum::Router::new() + .route( + "/sync/summary", + axum::routing::get(move || { + hits_clone.fetch_add(1, Ordering::SeqCst); + async { axum::Json(serde_json::json!({ "count": 1, "latest_cursor": "15" })) } + }), + ) + .route( + "/sync", + axum::routing::get(|| async { borsh::to_vec(&resolver::SyncPage::default()).unwrap() }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let fake_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let (url_b, state_b) = start_relay_mocked(&state).await; + tokio::spawn(relay::run_poke_sync_loop( + state_b.clone(), + poke_test_config(), + )); + + let client = reqwest::Client::new(); + let poke = |url: String, cursor: &str| { + let client = client.clone(); + let body = serde_json::json!({ "url": url, "cursor": cursor }); + let target = format!("{}/poke", url_b); + async move { + client + .post(target) + .json(&body) + .send() + .await + .unwrap() + .status() + .as_u16() + } + }; + + // Unknown peer: accepted (no membership leak) but ignored + assert_eq!(poke(fake_url.clone(), "15").await, 200); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + hits.load(Ordering::SeqCst), + 0, + "unknown peer must be ignored" + ); + + // Known peer, stale cursor: dropped against the watermark + { + let mut peers = state_b.peers.lock().await; + peers.announce(&PeerInfo { + source_ip: IpAddr::from([10, 0, 0, 9]), + url: fake_url.clone(), + capabilities: 0, + }); + peers.mark_alive(&fake_url); + } + state_b + .handler + .store + .set_watermark(&fake_url, "20") + .unwrap(); + assert_eq!(poke(fake_url.clone(), "15").await, 200); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + hits.load(Ordering::SeqCst), + 0, + "stale cursor must be dropped" + ); + + // URL variants (trailing slashes) must normalize to the same watermark + // key — a stale cursor stays dropped no matter how the URL is spelled + assert_eq!(poke(format!("{}//", fake_url), "15").await, 200); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + hits.load(Ordering::SeqCst), + 0, + "slash variants must not bypass the watermark dedup" + ); + + // Known peer, fresh cursor: triggers a pull + assert_eq!(poke(fake_url.clone(), "25").await, 200); + let pulled = wait_for(Duration::from_secs(3), || hits.load(Ordering::SeqCst) > 0).await; + assert!( + pulled, + "fresh cursor from verified peer must trigger a pull" + ); + + // Malformed pokes are rejected + let bad = client + .post(format!("{}/poke", url_b)) + .json(&serde_json::json!({ "url": fake_url, "cursor": "not-a-number" })) + .send() + .await + .unwrap(); + assert_eq!(bad.status().as_u16(), 400); +} + +/// A peer's next_cursor is clamped to its advertised latest_cursor, so a +/// bogus u64::MAX cursor can't poison the watermark; and a peer whose cursor +/// space regressed below our watermark (DB wipe / prior poisoning) gets its +/// watermark reset instead of never being pulled again. +#[tokio::test] +async fn test_sync_cursor_clamped_and_regression_resets() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + // Real records from relay A so ingestion verifies, but served by a fake + // peer that lies about the cursor + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + let mut page = state_a + .handler + .store + .sync_page(None, 100, usize::MAX) + .unwrap(); + let honest_latest = state_a + .handler + .store + .sync_summary() + .unwrap() + .latest_cursor + .unwrap(); + page.next_cursor = Some(u64::MAX.to_string()); + let page_bytes = borsh::to_vec(&page).unwrap(); + + let latest_for_summary = honest_latest.clone(); + let app = axum::Router::new() + .route( + "/sync/summary", + axum::routing::get(move || { + let latest = latest_for_summary.clone(); + async move { + axum::Json(serde_json::json!({ "count": 3, "latest_cursor": latest })) + } + }), + ) + .route( + "/sync", + axum::routing::get(move || { + let bytes = page_bytes.clone(); + async move { bytes } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let fake_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let (_url_b, state_b) = start_relay_mocked(&state).await; + let stats = sync_with_peer(&state_b, &fake_url, &SyncConfig::default()) + .await + .unwrap(); + assert!(stats.stored > 0, "records should still ingest"); + assert_eq!( + state_b.handler.store.get_watermark(&fake_url).unwrap(), + Some(honest_latest.clone()), + "watermark must be clamped to the advertised latest, not u64::MAX" + ); + + // Regression: watermark far beyond the peer's advertised latest → reset + state_b + .handler + .store + .set_watermark(&fake_url, &u64::MAX.to_string()) + .unwrap(); + sync_with_peer(&state_b, &fake_url, &SyncConfig::default()) + .await + .unwrap(); + assert_eq!( + state_b.handler.store.get_watermark(&fake_url).unwrap(), + Some("0".to_string()), + "regressed cursor space must reset the watermark for recovery" + ); +} + +/// A root republished after its sub-handles carries a higher sync_seq; /sync +/// must append it to pages containing the subs so a bootstrapping relay can +/// verify them (regression: subs were skipped and lost behind the watermark). +#[tokio::test] +async fn test_bootstrap_survives_root_republished_after_subs() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + + // Simulate the root being republished after its subs: move it to the + // end of the sync stream + state_a.handler.store.bump_sync_seq("@sovereign").unwrap(); + + // The first page no longer contains the root in cursor order — the + // server must append it so the subs are verifiable + let page = state_a + .handler + .store + .sync_page(None, 2, usize::MAX) + .unwrap(); + assert!( + page.records.iter().any(|r| r.handle == "@sovereign"), + "later-seq root must be appended to pages containing its subs" + ); + + // A fresh relay bootstrapping with tiny pages converges completely + let (_url_b, state_b) = start_relay_mocked(&state).await; + let cfg = SyncConfig { + page_limit: 2, + ..SyncConfig::default() + }; + let stats = sync_with_peer(&state_b, &url_a, &cfg).await.unwrap(); + assert_eq!(stats.failed_spaces, 0, "no space may fail on bootstrap"); + for key in ["@sovereign", "alice@sovereign", "bob@sovereign"] { + assert!( + state_b.handler.store.get_handle(key).unwrap().is_some(), + "{} must survive root-after-sub ordering", + key + ); + } +} + +/// Infra failures (spaced unreachable) must abort the page without advancing +/// the watermark — records dropped for reasons unrelated to their validity +/// must be re-pulled once the infra recovers. +#[tokio::test] +async fn test_sync_infra_failure_does_not_advance_watermark() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, _state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + + // B has NO mocked chain: its spaced URL points nowhere, so local proof + // building fails — an infra failure, not a data rejection + let (_url_b, state_b) = start_relay(&state).await; + let result = sync_with_peer(&state_b, &url_a, &SyncConfig::default()).await; + assert!(result.is_err(), "infra failure must fail the sync round"); assert!( - alice.is_some(), - "relay B should have alice@sovereign from gossip" + state_b + .handler + .store + .get_watermark(&url_a) + .unwrap() + .is_none(), + "watermark must not advance past records dropped by infra failures" + ); + assert_eq!(state_b.handler.store.sync_summary().unwrap().count, 0); +} + +/// Affirmative verification rejections are terminal: they count as failed +/// spaces and the watermark advances (retrying them forever would wedge). +#[tokio::test] +async fn test_sync_verification_reject_advances_watermark() { + let mut state_a = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state_a, single_commit_finalized()); + runner.run(&mut state_a); + + let (url_a, state_a_app) = start_relay(&state_a).await; + publish(&url_a, &state_a, &mut runner).await; + let latest_a = state_a_app + .handler + .store + .sync_summary() + .unwrap() + .latest_cursor + .unwrap(); + + // B lives on a different chain: A's records affirmatively fail + // verification against B's anchors + let mut state_b_chain = ChainState::new(); + let mut runner_b = FixtureRunner::new(&mut state_b_chain, staged_only()); + runner_b.run(&mut state_b_chain); + let (_url_b, state_b) = start_relay_mocked(&state_b_chain).await; + + let stats = sync_with_peer(&state_b, &url_a, &SyncConfig::default()) + .await + .unwrap(); + assert!(stats.failed_spaces > 0, "foreign-chain records must fail"); + assert_eq!(stats.stored, 0); + assert_eq!( + state_b.handler.store.get_watermark(&url_a).unwrap(), + Some(latest_a), + "affirmative rejections advance the watermark" ); } + +/// The in-batch failure budget gates space processing: with the budget spent, +/// no further prove/verify work happens even for valid records. +#[tokio::test] +async fn test_ingest_failure_budget_gates_processing() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url_a, state_a) = start_relay(&state).await; + publish(&url_a, &state, &mut runner).await; + let page = state_a + .handler + .store + .sync_page(None, 100, usize::MAX) + .unwrap(); + + let (_url_b, state_b) = start_relay_mocked(&state).await; + + // Budget of zero: nothing may be processed, valid or not — and the + // batch must be flagged incomplete so no watermark can advance past it + let gated = relay::ingest_sync_records(&state_b, page.records.clone(), 0) + .await + .unwrap(); + assert_eq!(gated.stored, 0, "spent budget must gate all processing"); + assert!( + gated.incomplete, + "unattempted batch must be flagged incomplete" + ); + + // Normal budget: the same records ingest fine + let ok = relay::ingest_sync_records(&state_b, page.records, 32) + .await + .unwrap(); + assert!(ok.stored > 0); +} + +/// Outbound requests must not follow redirects — a 302 from a peer must never +/// steer a request at internal services. +#[tokio::test] +async fn test_sync_does_not_follow_redirects() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + // "Internal" target that must never be reached + let internal_hits = Arc::new(AtomicUsize::new(0)); + let internal_hits_clone = internal_hits.clone(); + let internal = axum::Router::new().route( + "/sync/summary", + axum::routing::get(move || { + internal_hits_clone.fetch_add(1, Ordering::SeqCst); + async { "gotcha" } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let internal_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, internal).await.unwrap() }); + + // Malicious peer redirecting everything at the internal target + let redirect_to = format!("{}/sync/summary", internal_url); + let evil = axum::Router::new().route( + "/sync/summary", + axum::routing::get(move || { + let target = redirect_to.clone(); + async move { axum::response::Redirect::temporary(&target) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let evil_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, evil).await.unwrap() }); + + let (_url_b, state_b) = start_relay_mocked(&state).await; + let result = sync_with_peer(&state_b, &evil_url, &SyncConfig::default()).await; + assert!(result.is_err(), "redirected summary must fail the sync"); + assert_eq!( + internal_hits.load(Ordering::SeqCst), + 0, + "the redirect target must never be contacted" + ); +} + +// ───────────────────────────────────────────────────────────────────────── +// Retention (storage budget, eviction, admission gate) +// ───────────────────────────────────────────────────────────────────────── + +/// Under a tiny budget the sweep evicts from the over-entitled space until +/// under the low-water mark, keeping the accounting true. +#[tokio::test] +async fn test_retention_sweep_evicts_over_entitled_space() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url, app_state) = start_relay(&state).await; + publish(&url, &state, &mut runner).await; + + let (rows_before, bytes_before) = app_state.handler.store.storage_totals().unwrap(); + assert!(rows_before >= 3 && bytes_before > 0); + + // entitlement 1 handle/epoch makes the fixture space over-entitled; + // budget of 1 byte forces full pressure + let cfg = relay::RetentionConfig { + max_storage_bytes: 1, + entitlement_per_epoch: 1, + ..relay::RetentionConfig::default() + }; + let evicted = relay::retention::sweep(&app_state, &cfg).await.unwrap(); + assert_eq!( + evicted as u64, + rows_before - 1, + "everything except the root evicted under pressure" + ); + + let (rows_after, _) = app_state.handler.store.storage_totals().unwrap(); + assert_eq!(rows_after, 1, "only the root row survives"); + assert!( + app_state + .handler + .store + .get_handle("@sovereign") + .unwrap() + .is_some(), + "the root handle must never be evicted while the space has data" + ); + // Emptied epoch buckets still count toward entitlement + let (_, epochs) = app_state.handler.store.space_usage("@sovereign").unwrap(); + assert!(epochs >= 1, "entitlement epochs survive eviction"); +} + +/// Recently-queried handles are spared while cold rows are evicted first. +#[tokio::test] +async fn test_retention_spares_hot_handles() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url, app_state) = start_relay(&state).await; + publish(&url, &state, &mut runner).await; + + app_state + .query_heat + .lock() + .unwrap() + .touch("alice@sovereign"); + + // Evict exactly one row: it must be a cold one, not alice + let cfg = relay::RetentionConfig { + max_storage_bytes: 1, + entitlement_per_epoch: 1, + eviction_batch: 1, + max_batches_per_sweep: 1, + ..relay::RetentionConfig::default() + }; + let evicted = relay::retention::sweep(&app_state, &cfg).await.unwrap(); + assert_eq!(evicted, 1); + assert!( + app_state + .handler + .store + .get_handle("alice@sovereign") + .unwrap() + .is_some(), + "hot handle must be spared while cold rows exist" + ); +} + +/// Under storage pressure, first-inserts for an over-entitled space are +/// gated while updates to existing handles still pass. +#[tokio::test] +async fn test_admission_gate_blocks_new_handles_under_pressure() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, two_commits_both_finalized()); + runner.run_next(&mut state); // first step: alice + bob staged + + // Relay with pressure-inducing retention settings: budget 1 byte, + // entitlement 1 handle/epoch + let mut config = Config::new(PathBuf::from("/tmp/relay-test"), ExtendedNetwork::Testnet4); + config.db_path = PathBuf::from(":memory:"); + config.spaced_url = Some("http://127.0.0.1:1".into()); + config.anchors = test_anchors(&state); + config.dev_mode = true; + config.allow_private_peers = true; + config.settings.retention.max_storage_bytes = 1; + config.settings.retention.entitlement_per_epoch = 1; + let relay_a = Relay::new(config).unwrap(); + *relay_a.state().handler.veritas.write().unwrap() = build_veritas(&state); + let app_state = relay_a.state().clone(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { relay_a.run(listener).await.unwrap() }); + + // First publish: no pressure yet (empty DB) — everything admits + publish(&url, &state, &mut runner).await; + assert!( + app_state + .handler + .store + .get_handle("alice@two-finalized") + .unwrap() + .is_some() + ); + + // Chain advances; charlie appears. Now bytes > budget and the space is + // over-entitled: charlie (first insert) must be gated, existing handles + // must still update. + runner.run(&mut state); + *app_state.handler.veritas.write().unwrap() = build_veritas(&state); + publish(&url, &state, &mut runner).await; + + assert!( + app_state + .handler + .store + .get_handle("charlie@two-finalized") + .unwrap() + .is_none(), + "new handle must be gated under storage pressure" + ); + let alice = app_state + .handler + .store + .get_handle("alice@two-finalized") + .unwrap() + .expect("alice still stored"); + assert!( + alice.epoch_height > 0, + "existing handle updated past staging" + ); + assert!( + app_state + .stats + .admission_gated + .load(std::sync::atomic::Ordering::Relaxed) + > 0, + "gated first-inserts must be counted" + ); +} + +/// /stats reports message counters and /health is unmetered. +#[tokio::test] +async fn test_stats_and_health_endpoints() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let (url, _state) = start_relay(&state).await; + let client = reqwest::Client::new(); + + // /health responds without consuming any rate budget + for _ in 0..30 { + let resp = client.get(format!("{}/health", url)).send().await.unwrap(); + assert_eq!(resp.status().as_u16(), 200); + } + + publish(&url, &state, &mut runner).await; + + let stats: serde_json::Value = client + .get(format!("{}/stats", url)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(stats["messages"]["received"], 1); + assert_eq!(stats["messages"]["accepted"], 1); + assert!(stats["peers"]["verified"].is_number()); + assert!(stats["concurrency"]["verify_permits_available"].is_number()); + assert!(stats["sync"]["last_success_by_peer"].is_object()); +} + +/// Duplicate and stale messages are accepted but reported as skipped, not stored. +#[test] +fn test_duplicate_message_reports_nothing_stored() { + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + + let handler = setup_handler(&state); + let bundle = runner.build_bundle(); + let msg = state.message(vec![bundle]); + + let first = handler.handle_message(msg.clone()).unwrap(); + assert!(first.stored > 0, "first delivery should store handles"); + + let second = handler.handle_message(msg).unwrap(); + assert_eq!(second.stored, 0, "duplicate delivery should store nothing"); + assert!(second.skipped > 0, "duplicate delivery should be skipped"); +} diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 488e1f3..8c30bd4 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -55,7 +55,7 @@ pub fn setup_handler(state: &ChainState) -> Handler { /// Replace the handler's Veritas with one built from the current chain state. pub fn sync_veritas(handler: &Handler, state: &ChainState) { - *handler.veritas.lock().unwrap() = build_veritas(state); + *handler.veritas.write().unwrap() = build_veritas(state); } /// Build a QueryContext from the handler's store (mirrors what handler does internally). @@ -79,11 +79,14 @@ pub async fn start_relay(chain_state: &ChainState) -> (String, Arc) { handler.dev_mode = true; let chain = SpacedClient::mock(mock_chain_proof(chain_state)); + let generous = Quota::per_second(NonZeroU32::new(100).unwrap()); let rate_config = RateLimitConfig { - message: Quota::per_second(NonZeroU32::new(100).unwrap()), - query: Quota::per_second(NonZeroU32::new(100).unwrap()), - announce: Quota::per_second(NonZeroU32::new(100).unwrap()), - peers: Quota::per_second(NonZeroU32::new(100).unwrap()), + message: generous, + proof: generous, + read: generous, + announce: generous, + sync: generous, + poke: generous, }; let state = Arc::new(AppState::with_rate_limits( handler,