test: exercise engine snapshot updates under real parallelism

.github/workflows/nightly-tsan.yml

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Praxis Contributors
+
+name: Nightly ThreadSanitizer
+
+# The engine is shared across threads behind `Arc` and mutated while
+# requests are in flight. PR CI runs the seeded stress test on a
+# multi-threaded runtime; this job rebuilds it under ThreadSanitizer so
+# a data race is a red build rather than a wrong plugin count.
+#
+# Nightly only: TSan needs a nightly compiler, and the rebuild is too
+# slow for every pull request. `workflow_dispatch` is here so a race
+# report can be reproduced without waiting for the cron.
+
+on:
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  CARGO_INCREMENTAL: 0
+  TSAN_OPTIONS: halt_on_error=1
+
+jobs:
+  tsan-engine-stress:
+    name: engine concurrency under TSan
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
+          key: ${{ runner.os }}-cargo-tsan-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-tsan-
+      - run: rustup toolchain install nightly --component rust-src --profile minimal
+      - run: rustup target add x86_64-unknown-linux-gnu --toolchain nightly
+      - run: make test-tsan

CONTRIBUTING.md

@@ -141,3 +141,33 @@ Enforcing one of the allowed groups is welcome as a focused change, one lint at
 time, separate from feature work. `docs/lints.md` is worth reading first: it
 records which lints clippy reports as machine-fixable but cannot actually fix, and
 where a lint's suggested rewrite is worse than the code it replaces.
+
+## Multi-threaded Tokio tests
+
+`#[tokio::test]` defaults to `current_thread`. Tasks yield at `.await` but never
+run on two OS threads at the same time, so a load and a store that have no await
+between them cannot overlap.
+
+Tests that exercise registration, unregister, hot reload (`load_config` /
+`from_config`), or route-cache fill and invalidation use
+`#[tokio::test(flavor = "multi_thread")]`, including when the body is a single
+task. The flavor is set by the surface under test so a later spawn starts from
+the runtime a host uses. A single-task test of those APIs does not itself
+create overlap.
+
+Overlap is asserted in `crates/ppe-core/tests/engine_concurrency.rs`.
+
+```rust
+#[tokio::test(flavor = "multi_thread")]
+async fn register_while_other_tasks_invoke() { /* ... */ }
+```
+
+Sequential tests that do not touch those surfaces stay on `current_thread`.
+
+A seeded stress test lives in `crates/ppe-core/tests/engine_concurrency.rs`.
+Replay a failure with `PPE_STRESS_SEED`. Nightly CI runs that test under
+ThreadSanitizer (`make test-tsan`; Linux only, the target is hardcoded). The
+`Release` / `Acquire` pairing `mutate_runtime` describes is documented by the
+extracted loom model in `crates/ppe-core/tests/loom_generation_snapshot.rs`.
+That model is not wired to `engine.rs`. Default `cargo test` does not compile
+it; run `RUSTFLAGS='--cfg loom' cargo test -p praxis-policy-core --test loom_generation_snapshot`.

Cargo.toml

@@ -108,6 +108,9 @@ paste = "1"
 futures = "0.3"
 hashbrown = "0.17"
 arc-swap = "1.9"
+# Exhaustive scheduler for a tiny model of the generation / snapshot
+# pairing. Dev-only, and only compiled when a crate enables `--cfg loom`.
+loom = "0.7"
 wildmatch = "2"
 rmp-serde = "1"
 serde_bytes = "0.11"
@@ -189,7 +192,7 @@ redundant_lifetimes = "deny"
 # A stale feature name in a `cfg` predicate is otherwise only a warning, which
 # is how a rename can silently disable a gated export. Denying it turns that
 # into a build error at the moment the rename happens.
-unexpected_cfgs = "deny"
+unexpected_cfgs = { level = "deny", check-cfg = ['cfg(loom)'] }
 # Closed from the import backlog: measured at zero violations across
 # --workspace --all-targets --all-features, so enforcing costs no code change.
 unused_extern_crates = "deny"

Makefile

@@ -40,6 +40,7 @@ help:
 	@echo ""
 	@echo "Test:"
 	@echo "  test              Run all workspace tests"
+	@echo "  test-tsan         Engine concurrency stress under ThreadSanitizer (nightly)"
 	@echo ""
 	@echo "Supply chain & coverage:"
 	@echo "  audit             cargo deny check (advisories, licenses, bans, sources)"
@@ -159,6 +160,19 @@ test:
 	@$(CARGO) test --workspace
 	@$(CARGO) test --workspace --all-features
 
+# ThreadSanitizer on the engine concurrency stress test. Needs nightly, a
+# Linux target, and an instrumented libstd (`-Zbuild-std`). The sanitizer
+# does not run on the pinned stable toolchain or on macOS. `--test-threads=1`
+# keeps TSan's own reports from overlapping.
+.PHONY: test-tsan
+test-tsan:
+	@echo "ThreadSanitizer: praxis-policy-core engine concurrency ..."
+	@RUSTFLAGS="-Zsanitizer=thread" CARGO_INCREMENTAL=0 \
+		$(CARGO) +$(NIGHTLY) test -Zbuild-std=std,panic_abort \
+		-p praxis-policy-core --test engine_concurrency \
+		--target x86_64-unknown-linux-gnu -- --test-threads=1
+	@echo "test-tsan passed"
+
 # =============================================================================
 # Supply chain & coverage
 # =============================================================================

crates/ppe-core/Cargo.toml

@@ -77,5 +77,11 @@ praxis-policy-core = { path = ".", features = ["test-util"] }
 tokio = { workspace = true, features = ["test-util"] }
 tokio-util = { workspace = true }
 
+# loom is pulled only under `--cfg loom` so default `cargo test` does not
+# compile `generator` / `cc`. Run the model with
+# `RUSTFLAGS='--cfg loom' cargo test -p praxis-policy-core --test loom_generation_snapshot`.
+[target.'cfg(loom)'.dev-dependencies]
+loom = { workspace = true }
+
 [lints]
 workspace = true