From 047adf934d207391b801e454b5f665d2c72e0cc3 Mon Sep 17 00:00:00 2001 From: Buffrr Date: Fri, 31 Jul 2026 19:26:19 +0200 Subject: [PATCH 1/6] fix: preserve owner records across a commitment finalize --- relay/src/store.rs | 53 ++++++++++++++--- relay/tests/integration_tests.rs | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/relay/src/store.rs b/relay/src/store.rs index f92764f..ac8b77e 100644 --- a/relay/src/store.rs +++ b/relay/src/store.rs @@ -344,11 +344,12 @@ impl SqliteStore { // pinning). let max_seq = (now + 6 * 3600) as u64; - // Filter to entries where the incoming zone is better (or new) - let to_store: Vec<_> = entries + // Filter to entries where the incoming zone is better (or new), and + // preserve owner records across a commitment upgrade (see below). + let to_store: Vec = entries .into_iter() .zip(updates.iter()) - .filter(|(e, update)| { + .filter_map(|(mut e, update)| { if e.offchain_seq > max_seq || e.delegate_offchain_seq > max_seq { tracing::warn!( "{}: rejecting update, seq {} exceeds max {} (>6h in future)", @@ -356,14 +357,50 @@ impl SqliteStore { e.offchain_seq.max(e.delegate_offchain_seq), max_seq ); - return false; + return None; } - match existing_zones.get(e.handle.as_str()) { - Some(existing) => update.zone.is_better_than(existing).unwrap_or(false), - None => true, + + let existing = match existing_zones.get(e.handle.as_str()) { + None => return Some(e), // new handle, nothing to preserve + Some(existing) => { + if !update.zone.is_better_than(existing).unwrap_or(false) { + return None; // stored zone is as good or better + } + existing + } + }; + + // A commitment upgrade (e.g. a temp -> final cert) can arrive + // carrying empty or stale owner records. `is_better_than` picks + // it on commitment height alone, which would silently drop the + // owner's records. Keep them when the same key still controls the + // handle (script_pubkey unchanged) and the stored records are + // fresher — they remain valid under the new commitment. A genuine + // owner update (higher records seq) or a key transfer (different + // script_pubkey) is left untouched. + if existing.script_pubkey == update.zone.script_pubkey + && !existing.records.is_empty() + && (update.zone.records.is_empty() + || existing.records.seq().unwrap_or(0) + > update.zone.records.seq().unwrap_or(0)) + { + let mut merged = update.zone.clone(); + merged.records = existing.records.clone(); + match borsh::to_vec(&merged) { + Ok(bytes) => { + e.zone_data = bytes; + e.offchain_seq = merged.records.seq().unwrap_or(0); + } + // Fall back to storing the incoming zone unmerged rather + // than dropping the update entirely. + Err(err) => { + tracing::warn!("{}: merged-zone re-serialize failed: {}", e.handle, err) + } + } } + + Some(e) }) - .map(|(e, _)| e) .collect(); let skipped = updates.len() - to_store.len(); diff --git a/relay/tests/integration_tests.rs b/relay/tests/integration_tests.rs index 91ca5ac..6462d3c 100644 --- a/relay/tests/integration_tests.rs +++ b/relay/tests/integration_tests.rs @@ -256,6 +256,104 @@ fn test_incremental_zone_replacement() { } } +/// A finalize upgrades a sub-handle's commitment (Unknown -> Exists) while the +/// operator publishes empty owner records. `is_better_than` picks the finalized +/// zone on the commitment upgrade alone — before records are ever compared — +/// which would silently drop the owner's records. The relay must preserve them +/// when the same key still controls the handle, and only drop them on a key +/// change. Regression for silent record deletion on finalize. +#[test] +fn test_finality_upgrade_preserves_owner_records() { + use libveritas::ProvableOption; + use relay::store::HandleRecord; + + let mut state = ChainState::new(); + let mut runner = FixtureRunner::new(&mut state, single_commit_finalized()); + runner.run(&mut state); + let handler = setup_handler(&state); + let bundle = runner.build_bundle(); + let msg = state.message(vec![bundle]); + handler.handle_message(msg).unwrap(); + + // Sub-handles carry owner records with a not-yet-Exists commitment (proven + // via the space tree); the space root carries the Exists commitment. + let alice = handler + .store + .get_handle("alice@sovereign") + .unwrap() + .unwrap(); + let bob = handler.store.get_handle("bob@sovereign").unwrap().unwrap(); + let root = handler.store.get_handle("@sovereign").unwrap().unwrap(); + assert!( + !alice.zone.records.is_empty(), + "precondition: alice has records" + ); + assert!( + matches!(root.zone.commitment, ProvableOption::Exists { .. }), + "precondition: the root carries an Exists commitment to graft" + ); + assert!( + !matches!(alice.zone.commitment, ProvableOption::Exists { .. }), + "precondition: the sub-handle commitment is not yet Exists" + ); + let alice_seq = alice.zone.records.seq(); + + // A distinct controlling key for the key-change (negative) case. + let other_key = { + let mut bytes = alice.zone.script_pubkey.clone().into_bytes(); + *bytes.last_mut().unwrap() ^= 0xff; + spaces_protocol::bitcoin::ScriptBuf::from_bytes(bytes) + }; + assert_ne!(alice.zone.script_pubkey, other_key); + + // Synthesize the finalize: upgrade the commitment to Exists (which wins + // is_better_than before records are looked at) and publish empty records. + let finalize = |src: &HandleRecord, script_pubkey| -> HandleRecord { + let mut zone = src.zone.clone(); + zone.commitment = root.zone.commitment.clone(); // Exists beats Unknown + zone.records = Default::default(); // operator publishes empty records + zone.script_pubkey = script_pubkey; + HandleRecord { + cert: src.cert.clone(), + zone, + epoch_height: src.epoch_height + 1, + offchain_seq: 0, + delegate_offchain_seq: src.delegate_offchain_seq, + } + }; + + // (a) Same key still controls it -> records preserved, commitment upgraded. + let alice_final = finalize(&alice, alice.zone.script_pubkey.clone()); + handler.store.update_handles(&[alice_final]).unwrap(); + let after = handler + .store + .get_handle("alice@sovereign") + .unwrap() + .unwrap(); + assert!( + !after.zone.records.is_empty(), + "owner records must survive the temp->final finalize" + ); + assert_eq!( + after.zone.records.seq(), + alice_seq, + "preserved records keep their seq" + ); + assert!( + matches!(after.zone.commitment, ProvableOption::Exists { .. }), + "the commitment upgrade must still be applied" + ); + + // (b) A key change (different script_pubkey) -> old records are dropped. + let bob_final = finalize(&bob, other_key); + handler.store.update_handles(&[bob_final]).unwrap(); + let bob_after = handler.store.get_handle("bob@sovereign").unwrap().unwrap(); + assert!( + bob_after.zone.records.is_empty(), + "records must NOT be preserved when the controlling key changes" + ); +} + #[test] fn test_all_fixtures() { let fixtures: Vec<(&str, Fixture, Vec<&str>)> = vec![ From 2456527b85b984eb81b809672b6ea044a672b28c Mon Sep 17 00:00:00 2001 From: Buffrr Date: Fri, 31 Jul 2026 21:27:13 +0200 Subject: [PATCH 2/6] fix: don't charge velocity limits for commitment finalizes The per-space/per-handle velocity limits are meant to cap cheap off-chain record churn (seq bumps). They were charged on every replacement, so a daily batch finalizing all previously-issued temp handles hit space_rate (100/min/space) and got silently dropped past the cap. --- relay/src/handler.rs | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/relay/src/handler.rs b/relay/src/handler.rs index b5e0d0c..7d80ef5 100644 --- a/relay/src/handler.rs +++ b/relay/src/handler.rs @@ -402,8 +402,16 @@ impl Handler { } // Rate limit per space (100 handle updates/min) and per handle - // (1 per 5 min) — churn only: first insert of a handle is free. - if charge_content_limits && !self.dev_mode && stored.is_some() { + // (1 per 5 min). Charge only cheap off-chain record churn: a + // replacement at the same-or-lower commitment epoch. A first + // insert (stored.is_none()) is free, and so is a commitment + // finalize / new epoch — that advances epoch_height and is + // already gated by on-chain transaction cost, so a daily batch + // finalizing every issued handle is never throttled by space_rate. + if charge_content_limits + && !self.dev_mode + && charges_churn(stored.map(|s| s.0), epoch_height) + { if self.space_rate.check_key(&space).is_err() { tracing::warn!("{}: space rate limited, skipping", space); return None; @@ -474,3 +482,32 @@ fn epoch_hint_verifiable_by(hint: &resolver::EpochHint, zone: &Zone) -> bool { false } } + +/// Whether an ingested record incurs the per-space / per-handle velocity limits. +/// +/// Only cheap off-chain record churn is charged: a replacement at the +/// same-or-lower commitment epoch. A first insert (`stored_epoch` is `None`) is +/// free, and so is a commitment finalize / new epoch — that advances the epoch +/// and is already gated by on-chain transaction cost, so a batch finalizing +/// every issued handle is never throttled by `space_rate`. +fn charges_churn(stored_epoch: Option, incoming_epoch: u32) -> bool { + stored_epoch.is_some_and(|e| incoming_epoch <= e) +} + +#[cfg(test)] +mod tests { + use super::charges_churn; + + #[test] + fn charges_only_same_or_lower_epoch_churn() { + // First insert of a handle: free. + assert!(!charges_churn(None, 100)); + // Commitment finalize / new epoch (epoch advances): free. + assert!(!charges_churn(Some(100), 101)); + assert!(!charges_churn(Some(100), 500)); + // Off-chain record churn at the same commitment epoch: charged. + assert!(charges_churn(Some(100), 100)); + // Defensive: a lower epoch is charged (is_better_than rejects it anyway). + assert!(charges_churn(Some(100), 99)); + } +} From f8653c9f3e393afde75c49c66737e6265a10dc8f Mon Sep 17 00:00:00 2001 From: Buffrr Date: Fri, 31 Jul 2026 21:39:23 +0200 Subject: [PATCH 3/6] fix: stop trusting source_ip from propagated peer lists --- fabric/rust/src/lib.rs | 9 +++- relay/src/http.rs | 11 ++++- relay/src/peer.rs | 98 +++++++++++++++++++++++++++++++++++------- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/fabric/rust/src/lib.rs b/fabric/rust/src/lib.rs index 84a389b..17463e2 100644 --- a/fabric/rust/src/lib.rs +++ b/fabric/rust/src/lib.rs @@ -260,7 +260,10 @@ impl Announcement { /// Information about a peer, returned from GET /peers. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PeerInfo { - /// The IP address that announced this peer. + /// The IP address that announced this peer. Informational only: a peers + /// list is remote-controlled, so receivers must not trust this field for + /// anything (defaults to the unspecified address when absent). + #[serde(default = "unspecified_ip")] pub source_ip: IpAddr, /// The URL where this peer can be reached. pub url: String, @@ -274,6 +277,10 @@ impl PeerInfo { } } +fn unspecified_ip() -> IpAddr { + IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) +} + /// A reverse record mapping a numeric identity to its human-readable name. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ReverseRecord { diff --git a/relay/src/http.rs b/relay/src/http.rs index 11c406c..286acb5 100644 --- a/relay/src/http.rs +++ b/relay/src/http.rs @@ -1064,11 +1064,18 @@ pub async fn bootstrap_from( && crate::peer::validate_peer_url(&p.url, state.allow_private_peers).is_ok() }); - // Add discovered peers to our table + // Add discovered peers to our table. The claimed source_ip in a peers + // list is remote-controlled and unverifiable — coerce it to the + // unspecified address so it can never claim an IP slot or displace + // entries attributed to real client IPs (see PeerTable::announce). { let mut peer_table = state.peers.lock().await; for peer in &peers { - peer_table.announce(peer); + peer_table.announce(&PeerInfo { + source_ip: std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + url: peer.url.clone(), + capabilities: peer.capabilities, + }); } } diff --git a/relay/src/peer.rs b/relay/src/peer.rs index 2ebbe4a..8819c57 100644 --- a/relay/src/peer.rs +++ b/relay/src/peer.rs @@ -167,23 +167,36 @@ impl PeerTable { return AnnounceResult::AlreadyVerified; } - // Remove this IP's previous announcement if it was a different URL - if let Some(old_url) = self.ip_slots.get(&source_ip) - && *old_url != url - { - let old_url = old_url.clone(); - // Remove old URL from unverified if no other IP points to it - let other_refs = self - .ip_slots - .iter() - .any(|(ip, u)| *ip != source_ip && *u == old_url); - if !other_refs { - self.unverified.remove(&old_url); + // An unspecified source IP means the announcement is unattributed + // (e.g. learned from a peers list, where the claimed IP is + // remote-controlled and unverifiable). Such entries never own an IP + // slot and never displace anything: they only fill spare capacity. + let unattributed = source_ip.is_unspecified(); + if unattributed { + if !self.unverified.contains_key(&url) + && self.unverified.len() >= self.config.max_unverified + { + return AnnounceResult::Unverified; // table full, don't evict for it + } + } else { + // Remove this IP's previous announcement if it was a different URL + if let Some(old_url) = self.ip_slots.get(&source_ip) + && *old_url != url + { + let old_url = old_url.clone(); + // Remove old URL from unverified if no other IP points to it + let other_refs = self + .ip_slots + .iter() + .any(|(ip, u)| *ip != source_ip && *u == old_url); + if !other_refs { + self.unverified.remove(&old_url); + } } - } - // Assign this IP's slot - self.ip_slots.insert(source_ip, url.clone()); + // Assign this IP's slot + self.ip_slots.insert(source_ip, url.clone()); + } // Upsert into unverified self.unverified @@ -541,6 +554,61 @@ mod tests { assert_eq!(table.unverified_count(), 1); } + fn unattributed(url: &str) -> PeerInfo { + PeerInfo { + source_ip: IpAddr::from([0, 0, 0, 0]), + url: url.to_string(), + capabilities: 0, + } + } + + /// Unattributed announcements (unspecified source IP, e.g. peers-list + /// propagation) never own an IP slot: they coexist instead of evicting + /// each other, and never displace an attributed entry. + #[test] + fn unattributed_announces_claim_no_slot() { + let mut table = PeerTable::new(config()); + table.announce(&peer(1, "https://relay1.com")); + table.announce(&unattributed("https://seed1.com")); + table.announce(&unattributed("https://seed2.com")); + + // No slot fights: all three coexist. + assert_eq!(table.unverified_count(), 3); + + // An attributed announce from a fresh IP doesn't collide with them. + table.announce(&peer(2, "https://relay1.com")); + assert_eq!(table.unverified_count(), 3); + } + + /// A full unverified table drops unattributed announcements instead of + /// letting them evict attributed entries; attributed announcements still + /// evict the oldest as before. + #[test] + fn unattributed_never_evicts_at_capacity() { + let mut table = PeerTable::new(config()); // max_unverified = 3 + table.announce(&peer(1, "https://relay1.com")); + table.announce(&peer(2, "https://relay2.com")); + table.announce(&peer(3, "https://relay3.com")); + + table.announce(&unattributed("https://evil.com")); + assert_eq!(table.unverified_count(), 3); + assert!(!table.next_candidates(3).iter().any(|u| u.contains("evil"))); + + // Refreshing an unattributed entry that's already present still works. + table.announce(&unattributed("https://relay2.com")); + assert_eq!(table.unverified_count(), 3); + + // An attributed announce still rotates the table normally. + table.announce(&peer(4, "https://relay4.com")); + assert_eq!(table.unverified_count(), 3); + assert!( + table + .next_candidates(3) + .iter() + .any(|u| u.contains("relay4")) + ); + } + #[test] fn peers_info_includes_capabilities() { let mut table = PeerTable::new(config()); From 5e331c9b709589deed0d7222d4e029a2a7176fee Mon Sep 17 00:00:00 2001 From: Buffrr Date: Fri, 31 Jul 2026 21:49:31 +0200 Subject: [PATCH 4/6] fix: peer discovery hardening, sync-failure demotion, standing seeds, URL canonicalization --- relay/src/app.rs | 12 +- relay/src/peer.rs | 224 +++++++++++++++++++++++++++++-- relay/src/settings.rs | 1 + relay/src/sync.rs | 14 +- relay/tests/integration_tests.rs | 2 + 5 files changed, 238 insertions(+), 15 deletions(-) diff --git a/relay/src/app.rs b/relay/src/app.rs index a16da0a..513c61c 100644 --- a/relay/src/app.rs +++ b/relay/src/app.rs @@ -240,11 +240,13 @@ pub async fn run( )); // Peer-table maintenance: proactive refresh of verified peers, candidate - // verification, and rate-limiter map cleanup + // verification, rate-limiter map cleanup, and standing seed candidates + // (so a fleet-wide restart can't strand a relay with an empty table) tokio::spawn(crate::sync::run_peer_maintenance_loop( relay.state().clone(), std::time::Duration::from_secs(10), 3, + BOOTSTRAP_RELAYS.iter().map(|s| s.to_string()).collect(), )); // Pull-based propagation: periodically sync stored handles from peers, @@ -266,9 +268,13 @@ pub async fn run( tokio::spawn({ let state = relay.state().clone(); async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(20 * 60)); loop { - interval.tick().await; + // Jittered so a fleet restarted together doesn't hit the + // seeds in lockstep every sweep. (Startup discovery is + // handled by bootstrap() and the maintenance loop's standing + // seed candidates, so no immediate first sweep is needed.) + let jitter = std::time::Duration::from_millis(rand::random_range(0..120_000)); + tokio::time::sleep(std::time::Duration::from_secs(20 * 60) + jitter).await; let mut urls: Vec = { let peers = state.peers.lock().await; peers.peers().iter().map(|s| s.to_string()).collect() diff --git a/relay/src/peer.rs b/relay/src/peer.rs index 8819c57..5260a0d 100644 --- a/relay/src/peer.rs +++ b/relay/src/peer.rs @@ -98,6 +98,11 @@ pub struct PeerConfig { pub max_unverified: usize, pub max_verified: usize, pub verified_ttl: Duration, + /// How long an unverified entry may sit without a successful health + /// check or a fresh announcement before it is dropped. Generous by + /// design: a peer offline for days can still come back on its own, and + /// one that misses the window simply re-announces. + pub unverified_ttl: Duration, } impl Default for PeerConfig { @@ -106,14 +111,24 @@ impl Default for PeerConfig { max_unverified: 1_000, max_verified: 1_00, verified_ttl: Duration::from_secs(600), + unverified_ttl: Duration::from_secs(3 * 24 * 3600), } } } +/// Consecutive sync failures before a verified peer is demoted back to +/// unverified (it must re-pass health checks and stops consuming sync slots). +const SYNC_FAILURES_BEFORE_DEMOTE: u32 = 3; + struct PeerEntry { source_ip: IpAddr, capabilities: u32, last_seen: Instant, + /// When this entry (re-)entered its current table. Unlike `last_seen`, + /// never bumped by failed checks — it anchors the unverified expiry. + added: Instant, + /// Consecutive sync failures while verified; any success resets it. + sync_failures: u32, } #[derive(Debug, PartialEq)] @@ -198,17 +213,21 @@ impl PeerTable { self.ip_slots.insert(source_ip, url.clone()); } - // Upsert into unverified + // Upsert into unverified. A fresh announcement is liveness evidence, + // so it also renews the expiry anchor. self.unverified .entry(url) .and_modify(|e| { e.last_seen = now; + e.added = now; e.capabilities = capabilities; }) .or_insert(PeerEntry { source_ip, capabilities, last_seen: now, + added: now, + sync_failures: 0, }); // Evict oldest if over capacity @@ -238,6 +257,7 @@ impl PeerTable { // If already verified, just refresh if let Some(entry) = self.verified.get_mut(&url) { entry.last_seen = now; + entry.sync_failures = 0; return; } @@ -254,6 +274,8 @@ impl PeerTable { source_ip: entry.source_ip, capabilities: entry.capabilities, last_seen: now, + added: now, + sync_failures: 0, }, ); @@ -288,15 +310,84 @@ impl PeerTable { self.ip_slots.retain(|_, u| *u != url); } - /// Deprioritize a URL after a failed health check. - /// Bumps it to the back of the line instead of removing it. - pub fn deprioritize(&mut self, url: &str) { + /// Make a locally-configured seed URL a standing candidate. Idempotent: + /// inserts into unverified only when the URL is not ourselves and not + /// already known (either table). Called every maintenance tick, so + /// discovery never depends on a bootstrap peer's list being populated at + /// the right moment — even a lost seed entry comes right back. + pub fn ensure_seed(&mut self, url: &str) { + let url = normalize_url(url); + if self.is_self(&url) || self.verified.contains_key(&url) { + return; + } + let now = Instant::now(); + // Seeds are unattributed (no real source IP is known behind their + // public proxied URL) but first-class: unlike propagated entries they + // may displace the oldest entry when the table is full. + self.unverified.entry(url).or_insert(PeerEntry { + source_ip: IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), + capabilities: 0, + last_seen: now, + added: now, + sync_failures: 0, + }); + while self.unverified.len() > self.config.max_unverified { + if let Some(oldest) = self + .unverified + .iter() + .min_by_key(|(_, e)| e.last_seen) + .map(|(url, _)| url.clone()) + { + self.unverified.remove(&oldest); + self.ip_slots.retain(|_, u| *u != oldest); + } else { + break; + } + } + } + + /// Record a failed sync attempt from a peer. + /// + /// Unverified: bumped to the back of the health-check line. Verified: + /// counted, and after `SYNC_FAILURES_BEFORE_DEMOTE` consecutive failures + /// the peer is demoted to unverified — a peer whose `/health` answers but + /// whose `/sync` is broken must not keep winning sync slots. Any sync + /// success or verified refresh resets the counter. + pub fn record_sync_failure(&mut self, url: &str) { let url = normalize_url(url); + let now = Instant::now(); if let Some(entry) = self.unverified.get_mut(&url) { - entry.last_seen = Instant::now(); + entry.last_seen = now; + return; + } + let Some(entry) = self.verified.get_mut(&url) else { + return; + }; + entry.sync_failures += 1; + if entry.sync_failures >= SYNC_FAILURES_BEFORE_DEMOTE { + let mut entry = self.verified.remove(&url).unwrap(); + tracing::info!( + "{}: demoting after {} sync failures", + url, + entry.sync_failures + ); + entry.last_seen = now; + entry.added = now; + entry.sync_failures = 0; + self.unverified.insert(url, entry); } } + /// Drop unverified entries with no fresh announcement or successful check + /// within `unverified_ttl`. A peer that comes back later re-announces. + pub fn expire_unverified(&mut self) { + let now = Instant::now(); + self.unverified + .retain(|_, e| now.duration_since(e.added) < self.config.unverified_ttl); + self.ip_slots + .retain(|_, u| self.unverified.contains_key(u) || self.verified.contains_key(u)); + } + /// Get list of verified, non-stale peer URLs. pub fn peers(&self) -> Vec<&str> { let now = Instant::now(); @@ -379,7 +470,9 @@ impl PeerTable { .verified .extract_if(|_, e| now.duration_since(e.last_seen) >= self.config.verified_ttl) .collect(); - for (url, entry) in expired { + for (url, mut entry) in expired { + entry.added = now; // fresh expiry anchor in the new table + entry.sync_failures = 0; self.unverified.entry(url).or_insert(entry); } } @@ -393,8 +486,15 @@ impl PeerTable { } } +/// Canonicalize a peer URL so trivial variants map to one table identity: +/// lowercased scheme/host, default ports elided, trailing slashes stripped +/// (the url crate handles the first two). Unparseable input falls back to +/// trim-only so existing behavior is preserved for it. pub(crate) fn normalize_url(url: &str) -> String { - url.trim().trim_end_matches('/').to_string() + match url::Url::parse(url.trim()) { + Ok(parsed) => parsed.as_str().trim_end_matches('/').to_string(), + Err(_) => url.trim().trim_end_matches('/').to_string(), + } } #[cfg(test)] @@ -406,6 +506,7 @@ mod tests { max_unverified: 3, max_verified: 2, verified_ttl: Duration::from_secs(600), + unverified_ttl: Duration::from_secs(3 * 24 * 3600), } } @@ -481,7 +582,7 @@ mod tests { } #[test] - fn deprioritize_sends_to_back() { + fn sync_failure_sends_unverified_to_back() { let mut table = PeerTable::new(config()); table.announce(&peer(1, "https://relay1.com")); table.announce(&peer(2, "https://relay2.com")); @@ -489,11 +590,114 @@ mod tests { // relay1 announced first, so it's the next candidate assert!(table.next_candidate().unwrap().contains("relay1")); - // deprioritize bumps it to the back - table.deprioritize("https://relay1.com"); + // a sync failure bumps it to the back + table.record_sync_failure("https://relay1.com"); assert!(table.next_candidate().unwrap().contains("relay2")); } + /// A verified peer whose syncs keep failing is demoted back to unverified + /// after the threshold — a working /health must not keep a peer with a + /// broken /sync in the rotation forever. Any success resets the count. + #[test] + fn repeated_sync_failures_demote_verified_peer() { + let mut table = PeerTable::new(config()); + table.announce(&peer(1, "https://relay1.com")); + table.mark_alive("https://relay1.com"); + + // Failures below the threshold, then a success: counter resets. + table.record_sync_failure("https://relay1.com"); + table.record_sync_failure("https://relay1.com"); + table.mark_alive("https://relay1.com"); + table.record_sync_failure("https://relay1.com"); + table.record_sync_failure("https://relay1.com"); + assert_eq!(table.peers(), vec!["https://relay1.com"]); + + // Third consecutive failure: demoted, must re-verify. + table.record_sync_failure("https://relay1.com"); + assert!(table.peers().is_empty()); + assert_eq!(table.unverified_count(), 1); + + // It can come back through the normal health-check path. + table.mark_alive("https://relay1.com"); + assert_eq!(table.peers(), vec!["https://relay1.com"]); + } + + /// Seeds are standing candidates: idempotent insert, never duplicating a + /// verified entry and never adding ourselves. + #[test] + fn ensure_seed_is_idempotent_and_skips_self_and_verified() { + let mut table = PeerTable::new(config()); + table.set_self_url("https://me.com"); + + table.ensure_seed("https://me.com"); + assert_eq!(table.unverified_count(), 0); + + table.ensure_seed("https://seed1.com/"); + table.ensure_seed("https://seed1.com"); + assert_eq!(table.unverified_count(), 1); + + table.mark_alive("https://seed1.com"); + table.ensure_seed("https://seed1.com"); + assert_eq!(table.unverified_count(), 0); + assert_eq!(table.peers(), vec!["https://seed1.com"]); + + // A full table still admits a seed (displacing the oldest) — unlike + // unattributed peers-list entries, seeds are first-class. + let mut table = PeerTable::new(config()); // max_unverified = 3 + table.announce(&peer(1, "https://relay1.com")); + table.announce(&peer(2, "https://relay2.com")); + table.announce(&peer(3, "https://relay3.com")); + table.ensure_seed("https://seed1.com"); + assert_eq!(table.unverified_count(), 3); + assert!(table.next_candidates(3).iter().any(|u| u.contains("seed1"))); + } + + /// Unverified entries expire after `unverified_ttl` with no fresh + /// announcement; failed checks (which bump `last_seen`) don't keep a dead + /// entry alive, while a re-announce does. + #[test] + fn unverified_entries_expire() { + let mut table = PeerTable::new(PeerConfig { + unverified_ttl: Duration::ZERO, // everything is instantly expired + ..config() + }); + table.announce(&peer(1, "https://relay1.com")); + table.record_sync_failure("https://relay1.com"); // bumps last_seen only + table.expire_unverified(); + assert_eq!(table.unverified_count(), 0); + + // A verified peer is untouched by unverified expiry. + table.announce(&peer(2, "https://relay2.com")); + table.mark_alive("https://relay2.com"); + table.expire_unverified(); + assert_eq!(table.peers(), vec!["https://relay2.com"]); + } + + /// Trivial URL variants (host case, default port, trailing slash) map to + /// one table identity. + #[test] + fn normalize_url_canonicalizes_variants() { + for v in [ + "https://Relay1.COM", + "https://relay1.com:443", + "https://relay1.com/", + " https://relay1.com ", + "HTTPS://relay1.com", + ] { + assert_eq!(normalize_url(v), "https://relay1.com", "variant: {v}"); + } + // Non-default ports and paths survive. + assert_eq!( + normalize_url("http://relay1.com:7778/base/"), + "http://relay1.com:7778/base" + ); + + let mut table = PeerTable::new(config()); + table.announce(&peer(1, "https://Relay1.com/")); + table.announce(&peer(2, "https://relay1.com:443")); + assert_eq!(table.unverified_count(), 1); + } + #[test] fn already_verified_refreshes() { let mut table = PeerTable::new(config()); diff --git a/relay/src/settings.rs b/relay/src/settings.rs index 9591526..c808507 100644 --- a/relay/src/settings.rs +++ b/relay/src/settings.rs @@ -238,6 +238,7 @@ impl FileConfig { max_unverified: p.max_unverified, max_verified: p.max_verified, verified_ttl: Duration::from_secs(p.verified_ttl_secs), + ..PeerConfig::default() } } } diff --git a/relay/src/sync.rs b/relay/src/sync.rs index 5314426..04f7cf1 100644 --- a/relay/src/sync.rs +++ b/relay/src/sync.rs @@ -124,7 +124,7 @@ pub async fn sync_round(state: &Arc, config: &SyncConfig) { Err(e) => { crate::stats::bump(&state.stats.sync_errors); tracing::debug!("sync from {} failed: {}", url, e); - state.peers.lock().await.deprioritize(&url); + state.peers.lock().await.record_sync_failure(&url); } } } @@ -133,10 +133,16 @@ pub async fn sync_round(state: &Arc, config: &SyncConfig) { /// Peer-table maintenance: proactively refresh verified peers before their /// TTL expires (decoupling liveness from data traffic) and verify several /// unverified candidates per tick so simultaneous expiries recover quickly. +/// +/// `seeds` are standing candidates re-asserted every tick, so joining the +/// mesh never depends on a bootstrap peer's list being populated at the +/// right moment (e.g. during a fleet-wide restart). Pass an empty list in +/// tests to keep them off the network. pub async fn run_peer_maintenance_loop( state: Arc, interval: Duration, candidates_per_tick: usize, + seeds: Vec, ) { let mut ticker = tokio::time::interval(interval); let mut tick: u64 = 0; @@ -152,7 +158,11 @@ pub async fn run_peer_maintenance_loop( let (refresh, candidates) = { let mut peers = state.peers.lock().await; + for seed in &seeds { + peers.ensure_seed(seed); + } peers.demote_expired(); + peers.expire_unverified(); let refresh = peers.refresh_candidates(); let candidates = if peers.needs_peers() { peers.next_candidates(candidates_per_tick) @@ -277,7 +287,7 @@ pub async fn run_poke_sync_loop(state: Arc, config: SyncConfig) { Err(e) => { crate::stats::bump(&state.stats.sync_errors); tracing::debug!("poke sync from {} failed: {}", url, e); - state.peers.lock().await.deprioritize(&url); + state.peers.lock().await.record_sync_failure(&url); } } } diff --git a/relay/tests/integration_tests.rs b/relay/tests/integration_tests.rs index 6462d3c..7467eb4 100644 --- a/relay/tests/integration_tests.rs +++ b/relay/tests/integration_tests.rs @@ -995,6 +995,7 @@ async fn test_verified_peers_survive_quiet_periods() { max_unverified: 1000, max_verified: 100, verified_ttl: Duration::from_millis(500), + ..relay::PeerConfig::default() }; let relay_a = Relay::new(config).unwrap(); let state_a = relay_a.state().clone(); @@ -1013,6 +1014,7 @@ async fn test_verified_peers_survive_quiet_periods() { state_a.clone(), Duration::from_millis(50), 3, + vec![], // no seeds: keep the test off the network )); // Wait well past several TTLs with no data traffic at all From 34033a5c8b531d829b7f43e7079a9c7208df0e32 Mon Sep 17 00:00:00 2001 From: Buffrr Date: Sat, 1 Aug 2026 13:12:02 +0200 Subject: [PATCH 5/6] fix: only reset the sync-failure counter on a sync success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync-failure demotion (record_sync_failure demotes a verified peer to unverified after 3 consecutive failures) was undercut because mark_alive cleared the counter, and mark_alive runs on a /health success too. A peer with a working /health but broken /sync had its count zeroed by every health refresh, so it only demoted if 3 sync failures landed between two refreshes — true for <=4 verified peers, false beyond that, leaving the broken peer winning sync slots forever (the exact case the demotion targets). mark_alive no longer clears sync_failures; a new mark_synced (called only on a real sync success) does. The health path keeps refreshing liveness without masking a broken /sync. Test updated to prove a health success does not reset the count and a sync success does. --- relay/src/peer.rs | 52 +++++++++++++++++++++++++++++++++++------------ relay/src/sync.rs | 4 ++-- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/relay/src/peer.rs b/relay/src/peer.rs index 5260a0d..7b9c945 100644 --- a/relay/src/peer.rs +++ b/relay/src/peer.rs @@ -254,10 +254,12 @@ impl PeerTable { let url = normalize_url(url); let now = Instant::now(); - // If already verified, just refresh + // If already verified, just refresh. A /health (or gossip) liveness + // signal refreshes last_seen but must NOT clear the sync-failure count — + // only a real sync success does that (via `mark_synced`), so a peer with + // a working /health but a broken /sync still demotes. if let Some(entry) = self.verified.get_mut(&url) { entry.last_seen = now; - entry.sync_failures = 0; return; } @@ -294,6 +296,18 @@ impl PeerTable { } } + /// Record a successful sync: mark the peer alive (promote / refresh) and + /// clear its sync-failure counter. Unlike a bare `/health` success + /// (`mark_alive`), a real sync success is what proves `/sync` works, so it + /// is the only signal that resets the demotion counter. + pub fn mark_synced(&mut self, url: &str) { + self.mark_alive(url); + let url = normalize_url(url); + if let Some(entry) = self.verified.get_mut(&url) { + entry.sync_failures = 0; + } + } + /// True if the URL is a verified, non-stale peer. pub fn is_verified(&self, url: &str) -> bool { let url = normalize_url(url); @@ -596,30 +610,42 @@ mod tests { } /// A verified peer whose syncs keep failing is demoted back to unverified - /// after the threshold — a working /health must not keep a peer with a - /// broken /sync in the rotation forever. Any success resets the count. + /// after 3 consecutive failures. A working /health (`mark_alive`) refreshes + /// liveness but must NOT reset the sync-failure count — otherwise a broken- + /// /sync peer stays in the rotation forever. Only a real sync success + /// (`mark_synced`) resets it. #[test] fn repeated_sync_failures_demote_verified_peer() { let mut table = PeerTable::new(config()); table.announce(&peer(1, "https://relay1.com")); table.mark_alive("https://relay1.com"); - // Failures below the threshold, then a success: counter resets. + // A /health success between failures does NOT rescue a broken /sync. table.record_sync_failure("https://relay1.com"); table.record_sync_failure("https://relay1.com"); + table.mark_alive("https://relay1.com"); // health ok, sync still broken + table.record_sync_failure("https://relay1.com"); // 3rd consecutive + assert!( + table.peers().is_empty(), + "a health success must not reset the sync-failure count" + ); + assert_eq!(table.unverified_count(), 1); + + // Re-verify via /health, then a real sync success resets the counter. table.mark_alive("https://relay1.com"); + assert_eq!(table.peers(), vec!["https://relay1.com"]); table.record_sync_failure("https://relay1.com"); table.record_sync_failure("https://relay1.com"); - assert_eq!(table.peers(), vec!["https://relay1.com"]); - - // Third consecutive failure: demoted, must re-verify. + table.mark_synced("https://relay1.com"); // sync ok — resets the count table.record_sync_failure("https://relay1.com"); + table.record_sync_failure("https://relay1.com"); + assert_eq!( + table.peers(), + vec!["https://relay1.com"], + "a sync success should have reset the count" + ); + table.record_sync_failure("https://relay1.com"); // 3rd since reset assert!(table.peers().is_empty()); - assert_eq!(table.unverified_count(), 1); - - // It can come back through the normal health-check path. - table.mark_alive("https://relay1.com"); - assert_eq!(table.peers(), vec!["https://relay1.com"]); } /// Seeds are standing candidates: idempotent insert, never duplicating a diff --git a/relay/src/sync.rs b/relay/src/sync.rs index 04f7cf1..1ddb71d 100644 --- a/relay/src/sync.rs +++ b/relay/src/sync.rs @@ -119,7 +119,7 @@ pub async fn sync_round(state: &Arc, config: &SyncConfig) { state.poke_dirty.notify_one(); } state.stats.record_sync_success(&url); - state.peers.lock().await.mark_alive(&url); + state.peers.lock().await.mark_synced(&url); } Err(e) => { crate::stats::bump(&state.stats.sync_errors); @@ -282,7 +282,7 @@ pub async fn run_poke_sync_loop(state: Arc, config: SyncConfig) { state.poke_dirty.notify_one(); } state.stats.record_sync_success(&url); - state.peers.lock().await.mark_alive(&url); + state.peers.lock().await.mark_synced(&url); } Err(e) => { crate::stats::bump(&state.stats.sync_errors); From 4bf637e34e06b63abb13f9e0ab02ad1cb1d0dd63 Mon Sep 17 00:00:00 2001 From: Buffrr Date: Sat, 1 Aug 2026 13:44:28 +0200 Subject: [PATCH 6/6] fix: coerce source_ip in bootstrap_from's returned peer list too --- relay/src/http.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/relay/src/http.rs b/relay/src/http.rs index 286acb5..4d02311 100644 --- a/relay/src/http.rs +++ b/relay/src/http.rs @@ -1063,19 +1063,20 @@ pub async fn bootstrap_from( p.url.len() <= 256 && crate::peer::validate_peer_url(&p.url, state.allow_private_peers).is_ok() }); + // The claimed source_ip in a peers list is remote-controlled and + // unverifiable — coerce it to the unspecified address before ANY use + // (including the returned vec) so it can never claim an IP slot or + // displace entries attributed to real client IPs (see + // PeerTable::announce). + for peer in &mut peers { + peer.source_ip = std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED); + } - // Add discovered peers to our table. The claimed source_ip in a peers - // list is remote-controlled and unverifiable — coerce it to the - // unspecified address so it can never claim an IP slot or displace - // entries attributed to real client IPs (see PeerTable::announce). + // Add discovered peers to our table { let mut peer_table = state.peers.lock().await; for peer in &peers { - peer_table.announce(&PeerInfo { - source_ip: std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), - url: peer.url.clone(), - capabilities: peer.capabilities, - }); + peer_table.announce(peer); } }