From bbc0e3ab40e66e471d616f55642058acac15ea71 Mon Sep 17 00:00:00 2001 From: Crane-Liu Date: Wed, 29 Jul 2026 15:27:38 +0800 Subject: [PATCH] Add: progressable native-run lifecycle Split native execution into prepare, launch, poll/wait, and finalize phases while preserving the blocking simpler_run and ChipWorker.run contract. Use a sticky wakeup for launch acceptance so executor bootstrap does not busy-spin a host core. Carry host-build dep-gen capture state with the native run and adopt it on the executor before execution. Keep a single unfinished native run per runner, reject stale slot, lease, and epoch tokens, preserve STRACE identity across phases, and harden cleanup diagnostics and teardown against repeated interruption and cross-worker contamination. Add lifecycle and cross-thread handoff coverage on simulation and A2/A3 hardware. --- docs/chip-level-arch.md | 17 +- docs/dfx/dep-gen.md | 7 +- docs/dfx/host-trace.md | 17 + docs/dynamic-linking.md | 54 +- docs/l3-l2-orch-comm.md | 12 +- docs/task-flow.md | 11 + python/bindings/task_interface.cpp | 205 ++++++- python/simpler/task_interface.py | 31 +- python/simpler/worker.py | 143 +++-- .../platform/onboard/host/device_runner.cpp | 33 +- .../platform/onboard/host/device_runner.h | 3 + src/a2a3/platform/sim/host/device_runner.cpp | 31 +- src/a2a3/platform/sim/host/device_runner.h | 3 + .../host/dep_gen_host_graph.cpp | 34 +- .../runtime/dep_gen_host_graph.h | 24 +- src/a5/platform/sim/host/device_runner.cpp | 4 + src/common/log/include/common/strace.h | 72 ++- .../platform/onboard/host/c_api_shared.cpp | 333 ++++++++--- .../onboard/host/device_runner_base.cpp | 39 +- .../onboard/host/device_runner_base.h | 20 + src/common/platform/sim/host/c_api_shared.cpp | 324 ++++++++--- .../platform/sim/host/device_runner_base.cpp | 43 ++ .../platform/sim/host/device_runner_base.h | 24 + src/common/worker/chip_worker.cpp | 257 ++++++++- src/common/worker/chip_worker.h | 96 +++- src/common/worker/native_run_launch_signal.h | 46 ++ src/common/worker/native_run_state.h | 85 +++ src/common/worker/pto_runtime_c_api.h | 84 ++- .../native_run_lifecycle/conftest.py | 33 ++ .../orchestration/long_vector_orch.cpp | 66 +++ .../test_native_run_lifecycle.py | 194 +++++++ tests/ut/cpp/CMakeLists.txt | 15 + tests/ut/cpp/a2a3/test_dep_gen_host_graph.cpp | 92 +++ .../common/test_native_run_launch_signal.cpp | 55 ++ tests/ut/py/test_worker/test_host_worker.py | 47 ++ .../test_worker/test_l3_l2_message_queue.py | 2 +- .../ut/py/test_worker/test_l3_l2_orch_comm.py | 537 +++++++++++++++++- 37 files changed, 2769 insertions(+), 324 deletions(-) create mode 100644 src/common/worker/native_run_launch_signal.h create mode 100644 src/common/worker/native_run_state.h create mode 100644 tests/st/a2a3/host_build_graph/native_run_lifecycle/conftest.py create mode 100644 tests/st/a2a3/host_build_graph/native_run_lifecycle/kernels/orchestration/long_vector_orch.cpp create mode 100644 tests/st/a2a3/host_build_graph/native_run_lifecycle/test_native_run_lifecycle.py create mode 100644 tests/ut/cpp/a2a3/test_dep_gen_host_graph.cpp create mode 100644 tests/ut/cpp/common/test_native_run_launch_signal.cpp diff --git a/docs/chip-level-arch.md b/docs/chip-level-arch.md index 7175e332e8..ecd75e46bc 100644 --- a/docs/chip-level-arch.md +++ b/docs/chip-level-arch.md @@ -122,13 +122,20 @@ simpler_init(ctx, device_id, // attach + binary takeove aicpu_binary, aicpu_size, aicore_binary, aicore_size); size_t size = get_runtime_size(); -register_callable(ctx, cid, callable); // one-time per callable -simpler_run(ctx, runtime, cid, args, config); // per-launch — no binaries; config - // carries aicpu_thread_num, - // diagnostics + ring overrides -unregister_callable(ctx, cid); +size_t alignment = get_runtime_alignment(); +void *runtime = allocate_zeroed_aligned(size, alignment); // stable until finalize +simpler_register_callable(ctx, cid, callable); // one-time per callable + +// Progressable form; simpler_run(...) composes these phases synchronously. +simpler_prepare_run(ctx, runtime, cid, args, config); // bind, no device launch +simpler_launch_run(ctx, runtime); // returns after launch fence +simpler_wait_run(ctx, runtime); // or poll until complete +simpler_finalize_run(ctx, runtime); // validate, copy back, destroy + +simpler_unregister_callable(ctx, cid); finalize_device(ctx); destroy_device_context(ctx); +free(runtime); ``` ### Layer 3: Python API (`python/bindings/task_interface.cpp` via nanobind) diff --git a/docs/dfx/dep-gen.md b/docs/dfx/dep-gen.md index 0de6739ca0..b310072399 100644 --- a/docs/dfx/dep-gen.md +++ b/docs/dfx/dep-gen.md @@ -90,8 +90,9 @@ nothing to capture-then-reconstruct. shared-memory ring, and the drain thread are all skipped (`dep_gen_host_graph_active()` tells the runner). Nothing is dropped under back-pressure because nothing is streamed. -- **Output.** The same `deps.json`, written at run teardown from the graph that - run's orchestration built. +- **Output.** The same `deps.json`, written at run teardown. After prepare's + host orchestration builds the graph, the phased runtime moves it into + run-owned storage and the executor adopts it before execution. --- @@ -394,7 +395,7 @@ list; only the dep_gen replay graph loses the tail. | Capture call site (device-orch) | `src/{a2a3,a5}/runtime/tensormap_and_ringbuffer/runtime/pto_orchestrator.cpp` `submit_task_common` | One conditional block that snapshots inputs into the ring when `is_dep_gen_enabled()`; fires for both `submit_task` and `submit_dummy_task`. The schema carries `kernel_ids[3] = {aic, aiv0, aiv1}` so the swimlane post-processor can resolve `task_id → kernel` from `deps.json` at level=1 where the AICore record is the sole device-side identity source. Inactive subslots stay at `INVALID_KERNEL_ID = -1`. It also carries the SPMD logical block num (`block_num` on a2a3, `core_num` on a5's launch spec) as `tasks[].block_num`. | | Replay | `src/{a2a3,a5}/runtime/tensormap_and_ringbuffer/host/dep_gen_replay.{h,cpp}` | Pure CPU; runs dual-pass differential replay — `compute_task_fanin` (oracle) + inlined STEP A/B mirror (annotated) against two `PTO2TensorMap` instances. Emits `deps.json` when both passes agree per record. Platform-agnostic — a5 reuses the a2a3 source verbatim. | | Host-direct capture (host-orch) | `src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h`, `src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp` | Task / tensor / edge tables filled from `submit_task_common` + `compute_task_fanin`'s `Annotate` hooks (`src/a2a3/runtime/host_build_graph/runtime/pto_dep_compute.h`), reset per orchestration by `run_host_orchestration`, serialized by the same `deps.json` writer. The runtime translation unit carries weak no-op fallbacks so the AICPU build links without it. | -| Device-runner hookup | `src/{a2a3,a5}/platform/{onboard,sim}/host/device_runner.cpp` | `dep_gen_host_graph_active()` picks the shape: host-orch calls `dep_gen_host_graph_emit(deps_path)` and skips collector init/start/reconcile entirely; device-orch calls `dep_gen_replay_emit_deps_json(records.data(), records.size(), deps_path)` post-`reconcile_counters`. The c_api latches the CallConfig before the bind so host capture is armed before the orchestration it records. | +| Device-runner hookup | `src/{a2a3,a5}/platform/{onboard,sim}/host/device_runner.cpp` | `dep_gen_host_graph_active()` picks the shape: host-orch moves the prepare thread's completed capture into run-owned storage, adopts it on the executor, calls `dep_gen_host_graph_emit(deps_path)` at teardown, and skips collector init/start/reconcile entirely; device-orch calls `dep_gen_replay_emit_deps_json(records.data(), records.size(), deps_path)` post-`reconcile_counters`. The c_api latches the CallConfig before the bind so host capture is armed before the orchestration it records. | | Viewer | `simpler_setup/tools/deps_viewer.py` | `deps.json` → text (default) or pan/zoom HTML | | Test | `tests/st/{a2a3,a5}/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py` + `test_dep_gen_chain.py`, `tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py` | Smoke test + 6-edge validation against `vector_example` orchestration (both platforms share byte-identical orchestration code). The host_build_graph case runs the *same* orchestration through host-direct capture and asserts the same 6 edges, so a divergence between the two shapes fails a test. | diff --git a/docs/dfx/host-trace.md b/docs/dfx/host-trace.md index cc01b6b095..76287561d5 100644 --- a/docs/dfx/host-trace.md +++ b/docs/dfx/host-trace.md @@ -63,6 +63,23 @@ device-log lines. A phase that was never stamped (0 ns) is skipped — e.g. `so_load` is ~0 on a cached-callable run. See [device-phases.md](device-phases.md) for the device-side mechanism. +The phased native-run interface preserves this same marker contract. Prepare +allocates one `inv` and records the host-wall start; prepare, the blocking +executor thread, and finalize temporarily bind that `(inv, hid)` while emitting +their spans. Finalize releases the runner claim, destroys the per-run state, and +then emits the stored `simpler_run` wall, so the root includes that cleanup tail. +No trace scope or synthetic nesting remains active between C API calls. For +direct phased use the host wall is the full prepare-to-finalize lifetime, +including time the caller spends polling or doing other host work; blocking +`simpler_run` is the same phases composed back-to-back. + +| Depth | Span names | +| ----- | ---------- | +| 0 | `simpler_run` | +| 1 | `simpler_run.bind`, `simpler_run.runner_run`, `simpler_run.validate` | +| 2 | `simpler_run.bind.args`, `simpler_run.bind.prebuilt`, `simpler_run.runner_run.device_wall` | +| 3 | `simpler_run.runner_run.device_wall.{preamble,so_load,graph_build,config_validate,arena_wire,sm_reset,post_orch,orch,sched,task_slot_*}` | + ## Reading the markers — `strace_timing.py` ```bash diff --git a/docs/dynamic-linking.md b/docs/dynamic-linking.md index 06ea8f3e74..cfc2e96178 100644 --- a/docs/dynamic-linking.md +++ b/docs/dynamic-linking.md @@ -292,9 +292,12 @@ ChipWorker.init(device_id, bins) # Python wrapper _ChipWorker.init(host_path, aicpu_path, aicore_path, device_id) # C++ dlopen(host_runtime.so, RTLD_LOCAL) dlsym: create_device_context, destroy_device_context, simpler_init, - get_runtime_size, register_callable, simpler_run, unregister_callable, - finalize_device + get_runtime_size, get_runtime_alignment, simpler_register_callable, + simpler_prepare_run, simpler_launch_run, simpler_poll_run, + simpler_wait_run, simpler_finalize_run, simpler_run, + simpler_unregister_callable, finalize_device create_device_context() → DeviceContextHandle + allocate zeroed, aligned, stable native-run storage per pipeline slot simpler_init(ctx, device_id, aicpu*, aicpu_size, aicore*, aicore_size) DeviceRunner::attach_current_thread(device_id) pto_cpu_sim_bind_device(device_id) @@ -303,16 +306,20 @@ ChipWorker.init(device_id, bins) # Python wrapper ChipWorker.run(handle, args, config) # public wrapper path simpler_run(ctx, buf, internal callable entry, args, config) - new (buf) Runtime() - DeviceRunner::bind_callable_to_runtime(r, cid, api, args, rings) # replay + per-run bind - DeviceRunner::run(r, config) # applies config; width already resolved pre-bind - clear_cpu_sim_shared_storage() - ensure_binaries_loaded() dlopen aicpu/aicore SOs once - launch AICPU + AICore threads - join all threads - unload_executor_binaries() dlclose aicpu/aicore SOs - validate_runtime_impl(r) copy results, remove kernels - r->~Runtime() + simpler_prepare_run(...) + new (buf) NativeRunState() owns Runtime + executor state + DeviceRunner::bind_callable_to_runtime(r, cid, api, args, rings) + simpler_launch_run(...) + executor thread: DeviceRunner::run(r, config) + clear_cpu_sim_shared_storage() + ensure_binaries_loaded() dlopen aicpu/aicore SOs once + launch AICPU + AICore threads + join all threads + unload_executor_binaries() dlclose aicpu/aicore SOs + simpler_wait_run(...) + simpler_finalize_run(...) + validate_runtime_impl(r) copy results, remove kernels + state->~NativeRunState() destroys Runtime ChipWorker.finalize() finalize_device(ctx) @@ -351,20 +358,23 @@ device_worker_main(device_id) for each callable: ChipWorker.register_callable(callable) # returns opaque handle - register_callable(ctx, internal callable entry, callable) + simpler_register_callable(ctx, internal callable entry, callable) upload child kernels, copy orch SO to device buffer for each launch with that handle: ChipWorker.run(handle, args, config) simpler_run(ctx, buf, internal callable entry, args, config) - new (buf) Runtime() - bind_callable_to_runtime() replay + rtMalloc, rtMemcpy to device - DeviceRunner::run() - ensure_binaries_loaded() already done by init - launch_aicore_kernel() cached rtRegisterAllKernel handle - + rtKernelLaunchWithHandleV2 - launch_aicpu_kernel(Run) rtsLaunchCpuKernel, cached rtFuncHandle - aclrtSynchronizeStreamWithTimeout() wait on both streams - validate_runtime_impl() rtMemcpy results back to host + simpler_prepare_run(...) + new (buf) NativeRunState() owns Runtime + executor state + bind_callable_to_runtime() replay + rtMalloc, rtMemcpy to device + simpler_launch_run(...) + executor thread: DeviceRunner::run() + ensure_binaries_loaded() already done by init + launch_aicore_kernel() cached rtRegisterAllKernel handle + + rtKernelLaunchWithHandleV2 + launch_aicpu_kernel(Run) rtsLaunchCpuKernel, cached rtFuncHandle + aclrtSynchronizeStreamWithTimeout() wait on both streams + simpler_wait_run(...) + simpler_finalize_run(...) rtMemcpy results back; destroy state ChipWorker.finalize() finalize_device(ctx) rtDeviceReset() diff --git a/docs/l3-l2-orch-comm.md b/docs/l3-l2-orch-comm.md index e476404635..51d8f42fba 100644 --- a/docs/l3-l2-orch-comm.md +++ b/docs/l3-l2-orch-comm.md @@ -162,11 +162,13 @@ therefore release the child allocation only after no parent operation can still touch it. If an unadopted native owner reports a cleanup failure while an import unwinds, -the diagnostic stays on the importing thread until the Worker consumes it and -poisons itself. It cannot be consumed by a Worker importing concurrently on a -different thread. If the diagnostic survives to a later create on the same -thread, its original Worker is no longer identifiable, so that Worker is -conservatively stopped with an explicit attribution message. +the owner records the diagnostic under its Worker's stable owner token. The +owning Worker consumes that diagnostic at its next admission, create rollback, +or close boundary and poisons itself. Transfer is two-phase: native storage is +read non-destructively and acknowledged only after the Python poison is +published, so an interruption at that boundary cannot lose the diagnostic. +Another Worker cannot consume or be poisoned by that error, even when both +Workers run on the same thread. ## 4. Signal Counters diff --git a/docs/task-flow.md b/docs/task-flow.md index f6d2c0a92b..0c07c4fcf9 100644 --- a/docs/task-flow.md +++ b/docs/task-flow.md @@ -340,6 +340,17 @@ second device execution, or cross-run publication overlap: the endpoint remains a single synchronous round trip, and the scheduler dispatches only the run that holds the FIFO head and still owns its lease. +The L2 host-runtime boundary is nevertheless progressable. Its opaque native +run storage supports `prepare -> launch -> poll/wait -> finalize`, and the +existing `simpler_run` / `ChipWorker.run` surface is the blocking composition +of those phases. `prepare` constructs and binds the per-run `Runtime` without +crossing the device launch fence; `launch` returns after the backend has +actually submitted its execution; `finalize` owns validation, copy-back, DFX, +and Runtime destruction. Until execution-only `DeviceRunner` state is made +per-run, the backend admits only one prepared or launched native run at a time. +The single-frame child intentionally continues to use the blocking composition; +the phase split is the B3a ownership seam, not a claim of mailbox overlap. + Simulation implements the same depth, so the contract means the same thing on both platforms: its runner owns one arena bank and one retained temporary buffer per slot, and its single-entry prebuilt-arena cache stays owned by diff --git a/python/bindings/task_interface.cpp b/python/bindings/task_interface.cpp index 1e2507d713..bb0dfac049 100644 --- a/python/bindings/task_interface.cpp +++ b/python/bindings/task_interface.cpp @@ -305,6 +305,9 @@ class AclRuntimeApi { AclRuntimeApi &acl_api() { static std::once_flag once; + // Intentionally process-lifetime: late Python finalizers may still need + // the initialized ACL dispatch table after ordinary static destruction + // begins, so deleting it would reintroduce a destruction-order use-after-free. static AclRuntimeApi *api{nullptr}; std::call_once(once, []() { auto candidate = std::make_unique(); @@ -317,24 +320,59 @@ AclRuntimeApi &acl_api() { class L3HostMappedRegionCleanupErrors { public: - void record(const std::string &message) noexcept { + void record(const std::string &owner_token, const std::string &message) noexcept { try { - append_cleanup_error(error_, message); + std::lock_guard lk(mu_); + append_cleanup_error(errors_[owner_token], message); } catch (...) {} } - std::string take() { return std::exchange(error_, {}); } + std::string take(const std::string &owner_token) { + std::lock_guard lk(mu_); + auto it = errors_.find(owner_token); + if (it == errors_.end()) { + return {}; + } + std::string error = std::move(it->second); + errors_.erase(it); + return error; + } + + std::string peek(const std::string &owner_token) const { + std::lock_guard lk(mu_); + auto it = errors_.find(owner_token); + return it == errors_.end() ? std::string{} : it->second; + } + + void acknowledge(const std::string &owner_token, const std::string &observed) { + if (observed.empty()) { + return; + } + std::lock_guard lk(mu_); + auto it = errors_.find(owner_token); + if (it == errors_.end()) { + return; + } + if (it->second == observed) { + errors_.erase(it); + return; + } + if (it->second.size() > observed.size() + 2 && it->second.compare(0, observed.size(), observed) == 0 && + it->second.compare(observed.size(), 2, "; ") == 0) { + it->second.erase(0, observed.size() + 2); + } + } private: - std::string error_; + mutable std::mutex mu_; + std::unordered_map errors_; }; L3HostMappedRegionCleanupErrors &l3_host_mapped_region_cleanup_errors() { - // A return-boundary owner destroyed by Python while unwinding an import is - // finalized on the importing thread. Keep its diagnostic thread-local so - // concurrent Workers cannot consume and misattribute one another's error. - // Leak the tiny sink to remain usable during late Python finalization. - static thread_local auto *errors = new L3HostMappedRegionCleanupErrors(); + // Return-boundary owners can be finalized after their importing call has + // unwound. The process-lifetime registry preserves Worker-keyed diagnostics + // until that same Worker reaches an admission or close boundary. + static auto *errors = new L3HostMappedRegionCleanupErrors(); return *errors; } @@ -349,13 +387,16 @@ class L3HostMappedRegion { std::string cleanup_error; close_collecting(cleanup_error); if (!cleanup_error.empty()) { - l3_host_mapped_region_cleanup_errors().record(cleanup_error); + l3_host_mapped_region_cleanup_errors().record(owner_token, cleanup_error); } } catch (...) { - l3_host_mapped_region_cleanup_errors().record("L3-L2 mapped-region cleanup failed with an unknown error"); + l3_host_mapped_region_cleanup_errors().record( + owner_token, "L3-L2 mapped-region cleanup failed with an unknown error" + ); } } + std::string owner_token; L3L2RegionAccessProfile profile{L3L2RegionAccessProfile::SIM_POSIX_SHM}; int fd{-1}; uint64_t device_addr{0}; @@ -590,6 +631,8 @@ class L3HostMappedRegionEntry { return; } state_ = State::CLOSING; + // A counter_wait lease may remain held for the rest of its timeout; + // the mapping stays valid until every such waiter has returned. idle_.wait(lk, [this]() { return active_leases_ == 0; }); @@ -741,12 +784,14 @@ void close_l3_host_mapped_region(uint64_t handle) { l3_host_mapped_region_regist class L3HostMappedRegionHandle { public: - explicit L3HostMappedRegionHandle(uint64_t handle) : - handle_(handle) {} + explicit L3HostMappedRegionHandle(uint64_t handle, std::string owner_token) : + handle_(handle), + owner_token_(std::move(owner_token)) {} L3HostMappedRegionHandle(const L3HostMappedRegionHandle &) = delete; L3HostMappedRegionHandle &operator=(const L3HostMappedRegionHandle &) = delete; L3HostMappedRegionHandle(L3HostMappedRegionHandle &&other) noexcept : - handle_(std::exchange(other.handle_, 0)) {} + handle_(std::exchange(other.handle_, 0)), + owner_token_(std::move(other.owner_token_)) {} L3HostMappedRegionHandle &operator=(L3HostMappedRegionHandle &&) = delete; ~L3HostMappedRegionHandle() noexcept { @@ -756,10 +801,10 @@ class L3HostMappedRegionHandle { try { close_l3_host_mapped_region(handle_); } catch (const std::exception &exc) { - l3_host_mapped_region_cleanup_errors().record(exc.what()); + l3_host_mapped_region_cleanup_errors().record(owner_token_, exc.what()); } catch (...) { l3_host_mapped_region_cleanup_errors().record( - "L3-L2 mapped-region owner cleanup failed with an unknown error" + owner_token_, "L3-L2 mapped-region owner cleanup failed with an unknown error" ); } } @@ -768,6 +813,7 @@ class L3HostMappedRegionHandle { private: uint64_t handle_{0}; + std::string owner_token_; }; class L2ChildOnboardRegionRegistry { @@ -1642,6 +1688,11 @@ NB_MODULE(_task_interface, m) { // breakdown) is no longer returned from run(); the platform emits it as // `[STRACE]` log markers — parse with simpler_setup.tools.strace_timing. + nb::class_(m, "_ChipWorkerNativeRun") + .def_ro("slot_id", &ChipWorkerNativeRun::slot_id) + .def_ro("generation", &ChipWorkerNativeRun::generation) + .def_ro("run_epoch", &ChipWorkerNativeRun::run_epoch); + // --- ChipWorker --- nb::class_(m, "_ChipWorker") .def(nb::init<>()) @@ -1734,6 +1785,61 @@ NB_MODULE(_task_interface, m) { nb::arg("callable_id"), nb::arg("args"), nb::arg("config"), nb::arg("slot_id"), nb::arg("generation"), "Internal generation-safe pipeline-slot launch for pre-encoded task args." ) + .def( + "_prepare_native_run_with_pipeline_lease", + [](ChipWorker &self, int32_t callable_id, TaskArgs &args, const CallConfig &config, uint32_t slot_id, + uint64_t generation) { + return self.prepare_native_run( + callable_id, make_view(args), config, PipelineSlotLease{slot_id, 0, generation} + ); + }, + nb::arg("callable_id"), nb::arg("args"), nb::arg("config"), nb::arg("slot_id"), nb::arg("generation"), + nb::call_guard(), + "Prepare a generation-bound native run without crossing its device launch fence." + ) + .def( + "_prepare_native_run_with_pipeline_lease", + [](ChipWorker &self, int32_t callable_id, ChipStorageTaskArgs &args, const CallConfig &config, + uint32_t slot_id, uint64_t generation) { + return self.prepare_native_run(callable_id, &args, config, PipelineSlotLease{slot_id, 0, generation}); + }, + nb::arg("callable_id"), nb::arg("args"), nb::arg("config"), nb::arg("slot_id"), nb::arg("generation"), + nb::call_guard(), + "Prepare a generation-bound native run from pre-encoded task args." + ) + .def( + "_prepare_native_run_from_blob", + [](ChipWorker &self, int32_t callable_id, uint64_t args_blob_ptr, size_t blob_capacity, + const CallConfig &config, uint32_t slot_id, uint64_t generation) { + TaskArgsView view = read_blob(reinterpret_cast(args_blob_ptr), blob_capacity); + return self.prepare_native_run(callable_id, view, config, PipelineSlotLease{slot_id, 0, generation}); + }, + nb::arg("callable_id"), nb::arg("args_blob_ptr"), nb::arg("blob_capacity"), nb::arg("config"), + nb::arg("slot_id"), nb::arg("generation"), nb::call_guard(), + "Prepare a generation-bound native run from a raw mailbox TaskArgs blob." + ) + .def( + "_launch_native_run", + [](ChipWorker &self, const ChipWorkerNativeRun &run, uint64_t accepted_state_addr, int32_t accepted_value) { + self.launch_native_run(run, reinterpret_cast(accepted_state_addr), accepted_value); + }, + nb::arg("run"), nb::arg("accepted_state_addr") = 0, nb::arg("accepted_value") = 0, + nb::call_guard(), + "Launch a prepared native run and return after its real device launch fence." + ) + .def( + "_poll_native_run", &ChipWorker::poll_native_run, nb::arg("run"), + "Return whether a launched native run has reached its completion fence." + ) + .def( + "_wait_native_run", &ChipWorker::wait_native_run, nb::arg("run"), nb::call_guard(), + "Wait for a launched native run's completion fence." + ) + .def( + "_finalize_native_run", &ChipWorker::finalize_native_run, nb::arg("run"), + nb::call_guard(), + "Validate, copy back, emit diagnostics, and destroy a prepared native run." + ) .def( "run_from_blob", [](ChipWorker &self, int32_t callable_id, uint64_t args_blob_ptr, size_t blob_capacity, @@ -1912,12 +2018,18 @@ NB_MODULE(_task_interface, m) { m.def( "_l3_host_mapped_region_import_sim", - [](const std::string &token, uint64_t mapping_bytes) -> L3HostMappedRegionHandle { + [](const std::string &token, uint64_t mapping_bytes, + const std::string &owner_token) -> L3HostMappedRegionHandle { if (mapping_bytes == 0 || mapping_bytes > static_cast(std::numeric_limits::max())) { throw std::invalid_argument("L3-L2 sim L3 Host mapped-region import requires a positive mapping size"); } + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 mapped-region import requires a non-empty Worker owner token"); + } + std::string handle_owner_token = owner_token; std::string name = shm_name_for_open(token); auto mapping = std::make_unique(); + mapping->owner_token = owner_token; mapping->fd = shm_open(name.c_str(), O_RDWR, 0); if (mapping->fd < 0) { throw std::runtime_error("L3-L2 sim L3 Host mapped-region import shm_open failed"); @@ -1933,21 +2045,28 @@ NB_MODULE(_task_interface, m) { mapping->profile = L3L2RegionAccessProfile::SIM_POSIX_SHM; mapping->device_addr = reinterpret_cast(base); mapping->mapping_bytes = mapping_bytes; - return L3HostMappedRegionHandle(l3_host_mapped_region_registry().emplace(std::move(mapping))); + uint64_t handle = l3_host_mapped_region_registry().emplace(std::move(mapping)); + return L3HostMappedRegionHandle(handle, std::move(handle_owner_token)); }, - nb::arg("token"), nb::arg("mapping_bytes"), nb::call_guard(), + nb::arg("token"), nb::arg("mapping_bytes"), nb::arg("owner_token"), nb::call_guard(), "Import a sim L3-L2 POSIX shm region for L3 Host mapped-region access." ); m.def( "_l3_host_mapped_region_import_onboard", - [](int device_id, uint64_t shareable_handle, uint64_t mapping_bytes) -> L3HostMappedRegionHandle { + [](int device_id, uint64_t shareable_handle, uint64_t mapping_bytes, + const std::string &owner_token) -> L3HostMappedRegionHandle { if (device_id < 0) { throw std::invalid_argument("L3-L2 onboard mapped-region import requires a non-negative device id"); } if (mapping_bytes == 0 || mapping_bytes > static_cast(std::numeric_limits::max())) { throw std::invalid_argument("L3-L2 onboard mapped-region import requires a positive mapping size"); } + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 mapped-region import requires a non-empty Worker owner token"); + } + std::string handle_owner_token = owner_token; auto mapping = std::make_unique(); + mapping->owner_token = owner_token; mapping->profile = L3L2RegionAccessProfile::ONBOARD_VMM; mapping->device_id = device_id; mapping->mapping_bytes = mapping_bytes; @@ -1959,9 +2078,10 @@ NB_MODULE(_task_interface, m) { mapping->device_addr = reinterpret_cast(mapped_addr); api.vmm_map_with_check(mapped_addr, mapping_bytes, mapping->vmm_handle); api.vmm_set_access_with_check(mapped_addr, mapping_bytes, device_id); - return L3HostMappedRegionHandle(l3_host_mapped_region_registry().emplace(std::move(mapping))); + uint64_t handle = l3_host_mapped_region_registry().emplace(std::move(mapping)); + return L3HostMappedRegionHandle(handle, std::move(handle_owner_token)); }, - nb::arg("device_id"), nb::arg("shareable_handle"), nb::arg("mapping_bytes"), + nb::arg("device_id"), nb::arg("shareable_handle"), nb::arg("mapping_bytes"), nb::arg("owner_token"), nb::call_guard(), "Import an onboard VMM L3-L2 region for L3 Host mapped-region access." ); m.def( @@ -1980,10 +2100,45 @@ NB_MODULE(_task_interface, m) { ); m.def( "_l3_host_mapped_region_take_cleanup_error", - []() { - return l3_host_mapped_region_cleanup_errors().take(); + [](const std::string &owner_token) { + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 cleanup-error lookup requires a non-empty Worker owner token"); + } + return l3_host_mapped_region_cleanup_errors().take(owner_token); + }, + nb::arg("owner_token"), + "Take a cleanup error recorded by an unadopted native mapped-region owner for one Worker." + ); + m.def( + "_l3_host_mapped_region_peek_cleanup_error", + [](const std::string &owner_token) { + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 cleanup-error lookup requires a non-empty Worker owner token"); + } + return l3_host_mapped_region_cleanup_errors().peek(owner_token); + }, + nb::arg("owner_token"), "Read one Worker's mapped-region cleanup error without consuming it." + ); + m.def( + "_l3_host_mapped_region_ack_cleanup_error", + [](const std::string &owner_token, const std::string &observed) { + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 cleanup-error acknowledgement requires a Worker owner token"); + } + l3_host_mapped_region_cleanup_errors().acknowledge(owner_token, observed); + }, + nb::arg("owner_token"), nb::arg("observed"), + "Acknowledge the mapped-region cleanup error already published by one Worker." + ); + m.def( + "_l3_host_mapped_region_record_cleanup_error_for_test", + [](const std::string &owner_token, const std::string &message) { + if (owner_token.empty()) { + throw std::invalid_argument("L3-L2 cleanup-error injection requires a non-empty Worker owner token"); + } + l3_host_mapped_region_cleanup_errors().record(owner_token, message); }, - "Take a cleanup error recorded by an unadopted native mapped-region owner on this thread." + nb::arg("owner_token"), nb::arg("message"), "Inject one Worker-owned mapped-region cleanup error." ); m.def( "_l3_host_mapped_region_fail_next_registry_insert_for_test", diff --git a/python/simpler/task_interface.py b/python/simpler/task_interface.py index 68e02f2767..4a82e67cb1 100644 --- a/python/simpler/task_interface.py +++ b/python/simpler/task_interface.py @@ -1363,6 +1363,35 @@ def _run_slot_with_pipeline_lease(self, callable_id, args, slot_id, generation, setattr(config, k, v) self._impl._run_with_pipeline_lease(int(callable_id), args, config, int(slot_id), int(generation)) + def _prepare_native_run_with_pipeline_lease(self, callable_id, args, slot_id, generation, config=None, **kwargs): + """Prepare one native run without crossing its device launch fence. + + Private B3a seam for the hierarchical endpoint. The returned token is + bound to both the pipeline lease generation and a unique prepare epoch; + it must be passed back to launch/poll/wait/finalize on this ChipWorker. + Keep every tensor backing buffer referenced by ``args`` alive until + finalize returns. + """ + if config is None: + config = CallConfig() + for k, v in kwargs.items(): + setattr(config, k, v) + return self._impl._prepare_native_run_with_pipeline_lease( + int(callable_id), args, config, int(slot_id), int(generation) + ) + + def _launch_native_run(self, run): + self._impl._launch_native_run(run) + + def _poll_native_run(self, run): + return bool(self._impl._poll_native_run(run)) + + def _wait_native_run(self, run): + self._impl._wait_native_run(run) + + def _finalize_native_run(self, run): + self._impl._finalize_native_run(run) + def _unregister_slot(self, callable_id): self._impl.unregister_callable(int(callable_id)) @@ -1391,7 +1420,7 @@ def runtime_slot_count(self): @property def runtime_buffer_addrs(self): - """Address of each host Runtime staging buffer, in slot order.""" + """Address of each opaque host native-run storage buffer, in slot order.""" return list(self._impl.runtime_buffer_addrs) def arena_bank_gm_heap_base(self, bank_id): diff --git a/python/simpler/worker.py b/python/simpler/worker.py index c7817ba3cd..202f0362f4 100644 --- a/python/simpler/worker.py +++ b/python/simpler/worker.py @@ -89,10 +89,11 @@ def my_l4_orch(orch, args, config): WorkerType, _l3_child_onboard_region_close, _l3_child_onboard_region_create, + _l3_host_mapped_region_ack_cleanup_error, _l3_host_mapped_region_close, _l3_host_mapped_region_import_onboard, _l3_host_mapped_region_import_sim, - _l3_host_mapped_region_take_cleanup_error, + _l3_host_mapped_region_peek_cleanup_error, _mailbox_load_i32, _mailbox_store_i32, read_args_from_blob, @@ -764,12 +765,16 @@ def _join_candidates(self) -> None: self._record_error(exc) def _drain(self, cursor: _ThreadFanoutDrainCursor) -> None: - try: - while not cursor.exhausted: + # Every launched candidate may still reference caller-owned state, so + # this boundary cannot abandon the cursor after an interruption. Keep + # retrying on a constant stack until ownership is fully drained. + while True: + try: + if cursor.exhausted: + return cursor.advance() - except BaseException as exc: # noqa: BLE001 - self._record_error(exc) - self._drain(cursor) + except BaseException as exc: # noqa: BLE001 + self._record_error(exc) def run(self) -> None: cursor = _ThreadFanoutDrainCursor( @@ -2768,15 +2773,22 @@ def __init__(self, handles: list[RunHandle]) -> None: self.first_error: BaseException | None = None def drain(self, pending_error: BaseException | None = None) -> None: - try: - if pending_error is not None and self.first_error is None: - self.first_error = pending_error - while self._handles: + if pending_error is not None and self.first_error is None: + self.first_error = pending_error + # These references become safe to drop only after native teardown and + # no later close attempt re-enters this terminal phase. Retry on a + # constant stack so repeated interruptions neither leak stack frames + # nor report cleanup complete early. + while True: + try: + if not self._handles: + return handle = self._handles[-1] handle._keepalive = None self._handles.pop() - except BaseException as exc: # noqa: BLE001 - self.drain(exc) + except BaseException as exc: # noqa: BLE001 + if self.first_error is None: + self.first_error = exc class RunHandle: @@ -3311,6 +3323,16 @@ def __init__( # work would build on it. Sticky — there is no recovery short of close. # Guarded by _hierarchical_start_cv. self._ordered_cleanup_error: BaseException | None = None + # Sticky copy of this Worker's native return-boundary mapping cleanup + # diagnostic. Native storage is acknowledged only after this field and + # the admission poison above have been published. + self._l3_host_mapped_cleanup_error: RuntimeError | None = None + # (snapshot awaiting acknowledgement, retained distinct detail + # fragments). This is an ordered diagnostic set, not a resource-count + # ledger: an identical repeated failure adds no actionable information. + # Keep it immutable so one attribute assignment publishes both pieces + # atomically with respect to asynchronous Python exceptions. + self._l3_host_mapped_cleanup_state: tuple[str | None, tuple[str, ...]] = (None, ()) # submit graph construction is serialized by _submit_mu. Resource # creation helpers use this pointer to bind new objects to the handle # being built; the pointer is cleared before submit() returns. @@ -4609,6 +4631,7 @@ def _operation_lease(self, api: str): host-buffer / remote-memory).""" tid = threading.get_ident() with self._hierarchical_start_cv: + self._consume_l3_host_mapped_cleanup_error_locked(api) if self._lifecycle is not _Lifecycle.READY: raise RuntimeError(f"Worker.{api}: requires an initialized (READY) worker") from self._startup_error if self._ordered_cleanup_error is not None: @@ -6322,20 +6345,61 @@ def _validate_l3_l2_orch_comm_host_buffer(self, tensor) -> None: f"L3-L2 payload Tensor size {nbytes} exceeds registered shared storage {registered_nbytes}" ) + def _consume_l3_host_mapped_cleanup_error_locked(self, api: str) -> RuntimeError | None: + """Publish and then acknowledge this Worker's native cleanup debt. + + Must hold ``_hierarchical_start_cv``. The native read is intentionally + non-destructive: an asynchronous exception before both sticky Python + fields are assigned leaves the diagnostic available for the next + boundary. Acknowledgement may be interrupted only after admission is + already poisoned. + """ + cleanup_error = _l3_host_mapped_region_peek_cleanup_error(self._owner_id) + pending_snapshot, details = self._l3_host_mapped_cleanup_state + if not cleanup_error: + if pending_snapshot is not None: + self._l3_host_mapped_cleanup_state = (None, details) + return self._l3_host_mapped_cleanup_error + + if cleanup_error != pending_snapshot: + detail = cleanup_error + if pending_snapshot is not None and cleanup_error.startswith(f"{pending_snapshot}; "): + # Native acknowledgement removes an observed prefix. If a + # finalizer appends another diagnostic before that ack, retain + # only the newly appended suffix in Python's stable copy. + detail = cleanup_error[len(pending_snapshot) + 2 :] + if detail and detail not in details: + details = (*details, detail) + self._l3_host_mapped_cleanup_state = (cleanup_error, details) + + leaked = self._l3_host_mapped_cleanup_error + if leaked is None: + leaked = RuntimeError( + f"Worker.{api}: a native L3 Host mapping owned by this Worker finalized without explicit close " + "and could not be reclaimed; no further work is admitted" + ) + self._l3_host_mapped_cleanup_error = leaked + leaked.__cause__ = RuntimeError("; ".join(details)) + if self._ordered_cleanup_error is None: + self._ordered_cleanup_error = leaked + + _l3_host_mapped_region_ack_cleanup_error(self._owner_id, cleanup_error) + self._l3_host_mapped_cleanup_state = (None, details) + return self._l3_host_mapped_cleanup_error + + def _consume_l3_host_mapped_cleanup_error(self, api: str) -> RuntimeError | None: + with self._hierarchical_start_cv: + return self._consume_l3_host_mapped_cleanup_error_locked(api) + def _create_l3_l2_region(self, worker_id: int, payload_bytes: int, counter_bytes: int): # noqa: PLR0912 if payload_bytes <= 0: raise ValueError("create_l3_l2_region: payload_bytes must be positive") if counter_bytes <= 0 or counter_bytes % 4 != 0: raise ValueError("create_l3_l2_region: counter_bytes must be positive and a multiple of 4") self._validate_l3_l2_worker_id(int(worker_id)) - prior_native_cleanup_error = _l3_host_mapped_region_take_cleanup_error() - if prior_native_cleanup_error: - raise self._record_unreclaimable( - "create_l3_l2_region: a native L3 Host mapping finalized without explicit close on this " - "thread and could not be reclaimed; its owning Worker is no longer identifiable, so this " - "Worker is conservatively stopped and no further work is admitted", - RuntimeError(prior_native_cleanup_error), - ) + prior_native_cleanup_error = self._consume_l3_host_mapped_cleanup_error("create_l3_l2_region") + if prior_native_cleanup_error is not None: + raise prior_native_cleanup_error resources = self._building_run_resources req_shm = SharedMemory(create=True, size=_REGION_CREATE_REQUEST_BYTES) reply_shm = SharedMemory(create=True, size=_REGION_CREATE_REPLY_BYTES) @@ -6375,12 +6439,15 @@ def _create_l3_l2_region(self, worker_id: int, payload_bytes: int, counter_bytes ) counter_offset, total_bytes = validate_region_create_reply(reply, expected_access_profile) if platform.endswith("sim"): - native_mapping_handle = _l3_host_mapped_region_import_sim(reply.backing_shm, int(reply.mapping_bytes)) + native_mapping_handle = _l3_host_mapped_region_import_sim( + reply.backing_shm, int(reply.mapping_bytes), self._owner_id + ) else: native_mapping_handle = _l3_host_mapped_region_import_onboard( int(reply.device_id), int(reply.shareable_handle), int(reply.mapping_bytes), + self._owner_id, ) l3_host_mapping = L3HostRegionMapping( worker_id=int(worker_id), @@ -6425,17 +6492,6 @@ def _create_l3_l2_region(self, worker_id: int, payload_bytes: int, counter_bytes _l3_host_mapped_region_close(int(native_mapping_handle)) except BaseException as mapping_exc: # noqa: BLE001 mapping_cleanup_error = mapping_exc - deferred_native_cleanup_error = _l3_host_mapped_region_take_cleanup_error() - if deferred_native_cleanup_error: - deferred_exc = RuntimeError(deferred_native_cleanup_error) - if mapping_cleanup_error is not None: - combined_exc = RuntimeError( - f"{mapping_cleanup_error}; deferred native cleanup also failed: {deferred_native_cleanup_error}" - ) - combined_exc.__cause__ = mapping_cleanup_error - mapping_cleanup_error = combined_exc - else: - mapping_cleanup_error = deferred_exc if not region_id: # The failure may have landed after the child created its # region. The reply shm is parent-owned and zero-filled and the @@ -6472,6 +6528,17 @@ def _create_l3_l2_region(self, worker_id: int, payload_bytes: int, counter_bytes f"{int(worker_id)}; it is leaked and no further work is admitted", release_exc, ) + deferred_native_cleanup_error = self._consume_l3_host_mapped_cleanup_error("create_l3_l2_region rollback") + if deferred_native_cleanup_error is not None: + deferred_exc = deferred_native_cleanup_error.__cause__ or deferred_native_cleanup_error + if mapping_cleanup_error is not None: + combined_exc = RuntimeError( + f"{mapping_cleanup_error}; deferred native cleanup also failed: {deferred_exc}" + ) + combined_exc.__cause__ = mapping_cleanup_error + mapping_cleanup_error = combined_exc + else: + mapping_cleanup_error = deferred_exc if mapping_cleanup_error is not None: raise self._record_unreclaimable( f"create_l3_l2_region: rollback could not close the L3 Host mapping for region " @@ -8126,6 +8193,7 @@ def close(self) -> None: # noqa: PLR0912, PLR0915 -- lifecycle linearization: r # cannot strand a joiner — the joiner's bounded re-check recovers a # skipped notify. A pre-publication interruption becomes that immutable # outcome's error; an interruption after publication cannot retract it. + deferred_native_cleanup_error: RuntimeError | None = None attempt: _CloseAttempt | None = None result: BaseException | None = None teardown_tree = False @@ -8167,8 +8235,11 @@ def close(self) -> None: # noqa: PLR0912, PLR0915 -- lifecycle linearization: r # intact — may be retried by this call. prior = self._close_completion if prior is not None and prior.done and (self._teardown_attempted or prior.error is None): + deferred_native_cleanup_error = self._consume_l3_host_mapped_cleanup_error_locked("close") if prior.error is not None: raise prior.error + if deferred_native_cleanup_error is not None: + raise deferred_native_cleanup_error return # A device-bound native object must be finalized on the init-owner # thread — always, even after that thread has exited (affinity @@ -8179,6 +8250,7 @@ def close(self) -> None: # noqa: PLR0912, PLR0915 -- lifecycle linearization: r "Worker.close(): a worker with a live native tree must be closed on the thread that " "init()'d it (native teardown is thread-bound)" ) + deferred_native_cleanup_error = self._consume_l3_host_mapped_cleanup_error_locked("close") # Claim: publish CLOSED (permanent admission fence) and install a # fresh teardown attempt. self._lifecycle = _Lifecycle.CLOSED @@ -8243,7 +8315,9 @@ def close(self) -> None: # noqa: PLR0912, PLR0915 -- lifecycle linearization: r # Fence drain makes this close terminal even when the # run error is the only remaining outcome and no native # resource needs teardown. - self._teardown_attempted = teardown_tree or result is not None + self._teardown_attempted = ( + teardown_tree or result is not None or deferred_native_cleanup_error is not None + ) if teardown_tree: self._teardown_ready_tree() teardown_completed = True @@ -8255,6 +8329,9 @@ def close(self) -> None: # noqa: PLR0912, PLR0915 -- lifecycle linearization: r had_live = True # conservative default if a read below is interrupted detached_registry: tuple[dict, dict, dict] | None = None try: + deferred_native_cleanup_error = self._consume_l3_host_mapped_cleanup_error("close") + if result is None and deferred_native_cleanup_error is not None: + result = deferred_native_cleanup_error # A successful tree teardown is the reclamation boundary # for abandoned run ownership. Drain it before the residual # inventory: that diagnostic probe is interruptible and an diff --git a/src/a2a3/platform/onboard/host/device_runner.cpp b/src/a2a3/platform/onboard/host/device_runner.cpp index 981a213582..21a110a6fc 100644 --- a/src/a2a3/platform/onboard/host/device_runner.cpp +++ b/src/a2a3/platform/onboard/host/device_runner.cpp @@ -69,6 +69,13 @@ extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_replay_emit_d extern "C" __attribute__((weak, visibility("hidden"))) bool dep_gen_host_graph_active() { return false; } extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_set_enabled(bool /*enable*/) {} +extern "C" __attribute__((weak, visibility("hidden"))) void *dep_gen_host_graph_take_capture() { return nullptr; } +extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_adopt_capture( + void * /*capture*/ +) noexcept {} +extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_destroy_capture( + void * /*capture*/ +) noexcept {} extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_host_graph_emit(const char * /*deps_json_path*/) { LOG_DEBUG("dep_gen host graph not implemented for this runtime — deps.json skipped"); return -1; @@ -208,12 +215,24 @@ int DeviceRunner::destroy_comm_stream(void *stream) { void DeviceRunner::set_dep_gen_enabled(bool enable) { enable_dep_gen_ = enable; // Arms host-side capture for a host-orch runtime (no-op weak stub for the - // device-orch one). Enabling clears the previous run's graph, so this must - // stay ahead of the orchestration it captures — the c_api latches the - // CallConfig before the bind for exactly that reason. + // device-orch one). The c_api latches the CallConfig before bind, and the + // orchestration entry resets the graph before recording it. dep_gen_host_graph_set_enabled(enable); } +void *DeviceRunner::take_native_run_thread_state() { + if (!enable_dep_gen_ || !dep_gen_host_graph_active()) return nullptr; + return dep_gen_host_graph_take_capture(); +} + +void DeviceRunner::adopt_native_run_thread_state(void *snapshot) noexcept { + dep_gen_host_graph_adopt_capture(snapshot); +} + +void DeviceRunner::destroy_native_run_thread_state(void *snapshot) noexcept { + dep_gen_host_graph_destroy_capture(snapshot); +} + int DeviceRunner::run(Runtime &runtime, const CallConfig &config) { const unsigned selected_pipeline_slot = pipeline_slot(); // Latch this run's diagnostic enables onto the runner before the collector @@ -584,10 +603,6 @@ int DeviceRunner::launch_run(Runtime &runtime, int num_aicore, int launch_aicpu_ return rc; } - // Both kernels are enqueued. Publish before reap_run synchronizes either - // stream; the parent retains mailbox ownership until TASK_DONE. - publish_task_accepted(); - return 0; } @@ -622,8 +637,8 @@ int DeviceRunner::reap_run(unsigned slot) { // order (mgmt's final-drain pass into L2 has poll as its consumer). teardown_shared_collectors_after_run(); - // a2a3-only dep_gen teardown: host-orch emits the graph it captured during - // this run's orchestration; device-orch stops the collector, reconciles the + // a2a3-only dep_gen teardown: host-orch emits the graph snapshot adopted + // from the prepare thread; device-orch stops the collector, reconciles the // ring, and replays the records. if (enable_dep_gen_) { const std::string deps = make_deps_json_path(output_prefix_); diff --git a/src/a2a3/platform/onboard/host/device_runner.h b/src/a2a3/platform/onboard/host/device_runner.h index 62f8183754..37276deacf 100644 --- a/src/a2a3/platform/onboard/host/device_runner.h +++ b/src/a2a3/platform/onboard/host/device_runner.h @@ -133,6 +133,9 @@ class DeviceRunner : public DeviceRunnerBase { * `DeviceRunnerBase`. */ void set_dep_gen_enabled(bool enable) override; + void *take_native_run_thread_state() override; + void adopt_native_run_thread_state(void *snapshot) noexcept override; + void destroy_native_run_thread_state(void *snapshot) noexcept override; /** * Cleanup all resources diff --git a/src/a2a3/platform/sim/host/device_runner.cpp b/src/a2a3/platform/sim/host/device_runner.cpp index 912f793249..7269700704 100644 --- a/src/a2a3/platform/sim/host/device_runner.cpp +++ b/src/a2a3/platform/sim/host/device_runner.cpp @@ -67,6 +67,13 @@ extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_replay_emit_d extern "C" __attribute__((weak, visibility("hidden"))) bool dep_gen_host_graph_active() { return false; } extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_set_enabled(bool /*enable*/) {} +extern "C" __attribute__((weak, visibility("hidden"))) void *dep_gen_host_graph_take_capture() { return nullptr; } +extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_adopt_capture( + void * /*capture*/ +) noexcept {} +extern "C" __attribute__((weak, visibility("hidden"))) void dep_gen_host_graph_destroy_capture( + void * /*capture*/ +) noexcept {} extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_host_graph_emit(const char * /*deps_json_path*/) { LOG_DEBUG("dep_gen host graph not implemented for this runtime — deps.json skipped"); return -1; @@ -224,12 +231,24 @@ int DeviceRunner::invoke_device_register(const RegisterCallableArgs ®_args) { void DeviceRunner::set_dep_gen_enabled(bool enable) { enable_dep_gen_ = enable; // Arms host-side capture for a host-orch runtime (no-op weak stub for the - // device-orch one). Enabling clears the previous run's graph, so this must - // stay ahead of the orchestration it captures — the c_api latches the - // CallConfig before the bind for exactly that reason. + // device-orch one). The c_api latches the CallConfig before bind, and the + // orchestration entry resets the graph before recording it. dep_gen_host_graph_set_enabled(enable); } +void *DeviceRunner::take_native_run_thread_state() { + if (!enable_dep_gen_ || !dep_gen_host_graph_active()) return nullptr; + return dep_gen_host_graph_take_capture(); +} + +void DeviceRunner::adopt_native_run_thread_state(void *snapshot) noexcept { + dep_gen_host_graph_adopt_capture(snapshot); +} + +void DeviceRunner::destroy_native_run_thread_state(void *snapshot) noexcept { + dep_gen_host_graph_destroy_capture(snapshot); +} + int DeviceRunner::run(Runtime &runtime, const CallConfig &config) { apply_call_config(config); // prepare_launch_shape() resolved block_dim before the graph was built, so @@ -529,6 +548,10 @@ int DeviceRunner::run(Runtime &runtime, const CallConfig &config) { })); } + // Both simulated kernel thread groups now exist. This is the sim's real + // launch boundary: publish before joining either group. + publish_task_accepted(); + for (auto &t : aicpu_threads) { t.join(); } @@ -600,7 +623,7 @@ int DeviceRunner::run(Runtime &runtime, const CallConfig &config) { pmu_collector_.reconcile_counters(); } - // Host-orch emits the graph it captured during this run's orchestration; + // Host-orch emits the graph snapshot adopted from the prepare thread; // device-orch stops the collector, reconciles the ring, and replays. if (enable_dep_gen_) { const std::string deps = make_deps_json_path(output_prefix_); diff --git a/src/a2a3/platform/sim/host/device_runner.h b/src/a2a3/platform/sim/host/device_runner.h index a57d34715a..90735852ab 100644 --- a/src/a2a3/platform/sim/host/device_runner.h +++ b/src/a2a3/platform/sim/host/device_runner.h @@ -39,6 +39,9 @@ class DeviceRunner : public SimDeviceRunnerBase { // runtime uses instead of the device collector. Defined in the .cpp so this // header stays free of the runtime-provided capture symbols. void set_dep_gen_enabled(bool enable) override; + void *take_native_run_thread_state() override; + void adopt_native_run_thread_state(void *snapshot) noexcept override; + void destroy_native_run_thread_state(void *snapshot) noexcept override; private: int ensure_binaries_loaded() override; diff --git a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp index 9f266c1992..86b635c28b 100644 --- a/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp +++ b/src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp @@ -195,14 +195,13 @@ void fill_producer(EdgeAnnot &e, const PTO2TensorMapEntry &entry) { } // --------------------------------------------------------------------------- -// Capture state — one graph per run, owned by the thread that issues the run +// Capture state — thread-local while built, then moved with its native run // --------------------------------------------------------------------------- struct HostGraphState { bool enabled = false; - // A graph was captured on this thread since the last enable. Distinguishes - // "this orchestration submitted nothing" from "capture was never armed on - // the thread that emits" — the second is a wiring bug and says so. + // A graph was captured since the last reset. Distinguishes "this + // orchestration submitted nothing" from a missing capture/adoption handoff. bool captured = false; std::vector tasks; std::vector tensors; @@ -496,6 +495,25 @@ extern "C" void dep_gen_host_graph_set_enabled(bool enable) { state().enabled = extern "C" bool dep_gen_host_graph_active() { return true; } +extern "C" void *dep_gen_host_graph_take_capture() { + HostGraphState ¤t = state(); + if (!current.enabled) return nullptr; + auto *capture = new HostGraphState(std::move(current)); + current = HostGraphState{}; + return capture; +} + +extern "C" void dep_gen_host_graph_adopt_capture(void *capture) noexcept { + if (capture == nullptr) return; + auto *captured_state = static_cast(capture); + state() = std::move(*captured_state); + delete captured_state; +} + +extern "C" void dep_gen_host_graph_destroy_capture(void *capture) noexcept { + delete static_cast(capture); +} + extern "C" int dep_gen_host_graph_emit(const char *deps_json_path) { if (deps_json_path == nullptr) { LOG_ERROR("dep_gen host graph: null deps_json_path"); @@ -505,12 +523,10 @@ extern "C" int dep_gen_host_graph_emit(const char *deps_json_path) { if (!s.captured) { // An empty graph here is not "the orchestration submitted nothing" — // begin_task() would have set captured even for a graph of one task. - // It means capture was never armed on this thread, i.e. the runner - // emitted from a thread that did not run the orchestration. + // It means capture was never armed or the run-owned snapshot was not + // adopted onto this executor thread before teardown. LOG_ERROR( - "dep_gen host graph: no capture ran on this thread — deps.json not written to %s " - "(capture is armed per-thread by set_enabled + begin_capture)", - deps_json_path + "dep_gen host graph: no capture was adopted on this thread — deps.json not written to %s", deps_json_path ); return -3; } diff --git a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h index 90c49e942f..c4d286267f 100644 --- a/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h +++ b/src/a2a3/runtime/host_build_graph/runtime/dep_gen_host_graph.h @@ -31,18 +31,17 @@ * end_task() — closes the task, after its last dependency step * * Control surface, called from the device runner (same host_runtime.so): - * set_enabled() / active() / emit() + * set_enabled() / active() / take_capture() / adopt_capture() / emit() * * The runtime translation unit links weak no-op fallbacks (pto_orchestrator.cpp) * so the AICPU build, which has no host graph, resolves without this .cpp. * - * The graph is per-thread state. A runner is bound to the thread that issues - * its run (both c_api_shared.cpp files key it off a `pthread_key_t`), and every - * call above lands on that same thread — enable and emit from the runner's - * `simpler_run`, capture from the orchestration the bind runs inline. Two - * runners on two threads therefore build two independent graphs instead of - * racing one, which is the same per-runner isolation the device-orch shape gets - * from `DeviceRunner::dep_gen_collector_` being a member. + * The graph is per-thread state while it is being built. After bind, prepare + * moves the completed graph into run-owned storage; launch adopts that snapshot + * into the executor's thread-local state before DeviceRunner::run emits it. + * This keeps capture lock-free while allowing serialized lifecycle calls to + * use different host threads and preventing two prepared contexts on one + * thread from overwriting one another. * * Per-task producer dedup mirrors PTO2FaninBuilder, which keys on (ring, slot); * this keys on producer task id. The two agree only because host_build_graph is @@ -125,6 +124,15 @@ void dep_gen_host_graph_set_enabled(bool enable); */ bool dep_gen_host_graph_active(); +/** Move the current thread's capture into an opaque, caller-owned snapshot. */ +void *dep_gen_host_graph_take_capture(); + +/** Adopt and consume a snapshot on the current execution thread. */ +void dep_gen_host_graph_adopt_capture(void *capture) noexcept; + +/** Destroy a snapshot that will not be launched. */ +void dep_gen_host_graph_destroy_capture(void *capture) noexcept; + /** * Write the captured graph to `deps_json_path`. Returns 0 on success, non-zero * if capture was off/empty or the file could not be written. diff --git a/src/a5/platform/sim/host/device_runner.cpp b/src/a5/platform/sim/host/device_runner.cpp index 2acdb05f26..1aa9d795ba 100644 --- a/src/a5/platform/sim/host/device_runner.cpp +++ b/src/a5/platform/sim/host/device_runner.cpp @@ -473,6 +473,10 @@ int DeviceRunner::run(Runtime &runtime, const CallConfig &config) { })); } + // Both simulated kernel thread groups now exist. This is the sim's real + // launch boundary: publish before joining either group. + publish_task_accepted(); + for (auto &t : aicpu_threads) { t.join(); } diff --git a/src/common/log/include/common/strace.h b/src/common/log/include/common/strace.h index a21a8482e4..9ce4d359d0 100644 --- a/src/common/log/include/common/strace.h +++ b/src/common/log/include/common/strace.h @@ -30,8 +30,9 @@ * tid thread id (multi-threaded orch stays attributable) * inv process-wide simpler_run() invocation id (atomic-allocated, so * (pid, inv) is unique even across concurrent calls) — grouping key - * ONLY (gathers one call's spans together); not a token index. Set - * once per call via StraceScope::next_inv(). + * ONLY (gathers one call's spans together); not a token index. A + * lexical call sets it via StraceScope::next_inv(); a phased call + * allocates once and binds that id in each phase. * hid content-derived callable hash (ELF Build-ID 64); stable across slot * reuse / processes / runs. Parser buckets by hid; the most-frequent * bucket is decode, a once-seen bucket is prefill, etc. @@ -122,7 +123,9 @@ class StraceScope { ~StraceScope() { const auto t1 = std::chrono::steady_clock::now(); - const long long ts = static_cast(t0_.time_since_epoch().count()); + const long long ts = static_cast( + std::chrono::duration_cast(t0_.time_since_epoch()).count() + ); const long long dur = static_cast(std::chrono::duration_cast(t1 - t0_).count()); // depth printed is the scope's own level (post-decrement so the @@ -138,8 +141,8 @@ class StraceScope { StraceScope(const StraceScope &) = delete; StraceScope &operator=(const StraceScope &) = delete; - /** Begin a new invocation: allocate a process-wide unique id and make it the - * active id for this thread. Call once at simpler_run entry. + /** Begin a lexical invocation: allocate a process-wide unique id and make + * it the active id for this thread. Call once at simpler_run entry. * * The id generator is a process-wide atomic, not the per-thread counter, so * `(pid, inv)` uniquely identifies one invocation even when several threads @@ -147,9 +150,12 @@ class StraceScope { * would start at 1 and the parser would merge their spans. The resolved id * is stored in the per-thread slot (`inv()`) so nested scopes / emit_span_at * on this thread read the right value. */ - static unsigned next_inv() { + static unsigned allocate_inv() { static std::atomic global_inv{0}; - const unsigned id = global_inv.fetch_add(1, std::memory_order_acq_rel) + 1; + return global_inv.fetch_add(1, std::memory_order_acq_rel) + 1; + } + static unsigned next_inv() { + const unsigned id = allocate_inv(); inv() = id; return id; } @@ -169,6 +175,39 @@ class StraceScope { std::chrono::steady_clock::time_point t0_; }; +/** + * Temporarily bind one invocation to the current thread at a known parent + * depth. The previous state is restored on exit, so phased callers do not + * leave invocation identity or synthetic nesting active between API calls. + */ +class StraceContextScope { +public: + StraceContextScope(unsigned inv, uint64_t hid, int base_depth) : + state_(strace_state()), + saved_(*state_) { + state_->inv = inv; + state_->hid = hid; + state_->depth = base_depth; + } + + ~StraceContextScope() { *state_ = saved_; } + + StraceContextScope(const StraceContextScope &) = delete; + StraceContextScope &operator=(const StraceContextScope &) = delete; + +private: + ThreadState *state_; + ThreadState saved_; +}; + +/** Current steady-clock timestamp in the marker grammar's nanosecond unit. */ +inline long long strace_now_ns() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count() + ); +} + /** * Emit a marker for a span whose duration was measured elsewhere (e.g. a device * phase: AICPU cycles → ns). Shares the current thread's inv/hid grouping so the @@ -188,6 +227,11 @@ emit_span_at(const char *name, long long ts_ns, long long dur_ns, int depth, con ); } +/** Emit an explicitly timed host-domain span in the active invocation. */ +inline void emit_host_span_at(const char *name, long long ts_ns, long long dur_ns, int depth) { + emit_span_at(name, ts_ns, dur_ns, depth, ""); +} + } // namespace simpler::strace // Concatenation helpers so each scope gets a unique variable name per line. @@ -200,8 +244,18 @@ emit_span_at(const char *name, long long ts_ns, long long dur_ns, int depth, con #define STRACE_A(name, attrs) ::simpler::strace::StraceScope STRACE_CAT(_strace_, __LINE__)(name, attrs) /** Begin a new invocation group (call once per simpler_run); returns inv id. */ #define STRACE_NEW_INV() ::simpler::strace::StraceScope::next_inv() +/** Allocate an invocation id without changing the current thread's context. */ +#define STRACE_ALLOC_INV() ::simpler::strace::StraceScope::allocate_inv() /** Set the callable hash for subsequent spans on this thread. */ #define STRACE_SET_HID(h) ::simpler::strace::StraceScope::set_hid(h) +/** Bind an existing invocation + its synthetic parent depth for this scope. */ +#define STRACE_CONTEXT(inv, hid, depth) \ + ::simpler::strace::StraceContextScope STRACE_CAT(_strace_context_, __LINE__)((inv), (hid), (depth)) +/** Read the current host monotonic clock in nanoseconds. */ +#define STRACE_NOW_NS() ::simpler::strace::strace_now_ns() +/** Emit a host-domain span measured across disjoint API calls. */ +#define STRACE_HOST_SPAN_AT(name, ts_ns, dur_ns, depth) \ + ::simpler::strace::emit_host_span_at((name), (ts_ns), (dur_ns), (depth)) /** Emit a device-domain span (device-clock start `ts_ns` + measured `dur_ns`). */ #define STRACE_DEV_SPAN_AT(name, ts_ns, dur_ns, depth) \ ::simpler::strace::emit_span_at((name), (ts_ns), (dur_ns), (depth)) @@ -211,7 +265,11 @@ emit_span_at(const char *name, long long ts_ns, long long dur_ns, int depth, con #define STRACE(name) ((void)0) #define STRACE_A(name, attrs) ((void)0) #define STRACE_NEW_INV() ((void)0) +#define STRACE_ALLOC_INV() 0U #define STRACE_SET_HID(h) ((void)0) +#define STRACE_CONTEXT(inv, hid, depth) ((void)0) +#define STRACE_NOW_NS() 0LL +#define STRACE_HOST_SPAN_AT(name, ts_ns, dur_ns, depth) ((void)0) #define STRACE_DEV_SPAN_AT(name, ts_ns, dur_ns, depth) ((void)0) #endif // SIMPLER_HOST_STRACE diff --git a/src/common/platform/onboard/host/c_api_shared.cpp b/src/common/platform/onboard/host/c_api_shared.cpp index 73df184464..e19c80b52d 100644 --- a/src/common/platform/onboard/host/c_api_shared.cpp +++ b/src/common/platform/onboard/host/c_api_shared.cpp @@ -30,12 +30,15 @@ #include "prepare_callable_common.h" #include "pto_runtime_c_api.h" #include "task_args.h" +#include "native_run_state.h" #include #include #include #include +#include +#include #include #include @@ -51,6 +54,11 @@ // time against `libunified_dlog.so` / `libascendalog.so`. extern "C" int dlog_setlevel(int moduleId, int level, int enableEvent); +using OnboardNativeRunState = NativeRunState; +// Phase entry points validate raw caller storage before beginning object +// lifetime, so the on-storage magic must remain the leading bytes. +static_assert(__builtin_offsetof(OnboardNativeRunState, magic) == 0, "native-run magic must lead runtime storage"); + extern "C" { /* =========================================================================== @@ -248,9 +256,18 @@ static const HostApi g_host_api = { * `DeviceRunnerBase *`. * =========================================================================== */ -void destroy_device_context(DeviceContextHandle ctx) { delete static_cast(ctx); } +void destroy_device_context(DeviceContextHandle ctx) { + DeviceRunnerBase *runner = static_cast(ctx); + if (runner != nullptr && runner->native_run_active()) { + LOG_ERROR("destroy_device_context: refusing to destroy a context with an unfinalized native run"); + return; + } + delete runner; +} + +size_t get_runtime_size(void) { return sizeof(OnboardNativeRunState); } -size_t get_runtime_size(void) { return sizeof(Runtime); } +size_t get_runtime_alignment(void) { return alignof(OnboardNativeRunState); } void *device_malloc_ctx(DeviceContextHandle ctx, size_t size) { if (ctx == NULL) return NULL; @@ -290,6 +307,10 @@ int finalize_device(DeviceContextHandle ctx) { if (ctx == NULL) return -1; try { DeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR("finalize_device: native run must be finalized first"); + return -1; + } return runner->finalize(); } catch (...) { return -1; @@ -388,6 +409,10 @@ int simpler_init( int simpler_register_callable(DeviceContextHandle ctx, int32_t callable_id, const void *callable) { if (ctx == NULL || callable == NULL) return -1; DeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR("simpler_register_callable: native run must be finalized before mutating the callable registry"); + return -1; + } pthread_once(&g_runner_key_once, create_runner_key); pthread_setspecific(g_runner_key, ctx); @@ -543,22 +568,73 @@ static void emit_device_phase_markers(DeviceRunnerBase *runner) { } } -int simpler_run( +static OnboardNativeRunState *native_run_state(DeviceContextHandle ctx, RuntimeHandle runtime, const char *operation) { + if (ctx == nullptr || runtime == nullptr) return nullptr; + uint64_t magic = 0; + std::memcpy(&magic, runtime, sizeof(magic)); + if (magic != OnboardNativeRunState::kMagic) { + LOG_ERROR("%s: runtime does not contain a prepared native run", operation); + return nullptr; + } + auto *state = static_cast(runtime); + if (state->runner != static_cast(ctx)) { + LOG_ERROR("%s: prepared run belongs to a different device context", operation); + return nullptr; + } + return state; +} + +static void emit_native_run_host_wall(unsigned trace_inv, uint64_t trace_hid, long long trace_start_ns) { + const long long end_ns = STRACE_NOW_NS(); + STRACE_CONTEXT(trace_inv, trace_hid, 0); + STRACE_HOST_SPAN_AT("simpler_run", trace_start_ns, end_ns - trace_start_ns, 0); +} + +static int cleanup_failed_prepare(OnboardNativeRunState *state, int execution_rc, bool clear_gm_sm) { + const unsigned trace_inv = state->trace_inv; + const uint64_t trace_hid = state->trace_hid; + const long long trace_start_ns = state->trace_start_ns; + if (clear_gm_sm) state->runtime.set_gm_sm_ptr(nullptr); + int validation_rc = -1; + try { + validation_rc = validate_runtime_impl(&state->runtime, &g_host_api, execution_rc); + } catch (...) { + validation_rc = -1; + } + if (state->runner_claimed) { + state->runner->release_native_run(state); + state->runner_claimed = false; + } + destroy_native_run_state(state); + emit_native_run_host_wall(trace_inv, trace_hid, trace_start_ns); + return validation_rc != 0 ? validation_rc : execution_rc; +} + +int simpler_prepare_run( DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config ) { - if (ctx == NULL || runtime == NULL || config == NULL) return -1; + if (ctx == nullptr || runtime == nullptr || config == nullptr) return -1; + if (reinterpret_cast(runtime) % alignof(OnboardNativeRunState) != 0) { + LOG_ERROR("simpler_prepare_run: runtime storage does not satisfy get_runtime_alignment()"); + return -1; + } DeviceRunnerBase *runner = static_cast(ctx); - if (!runner->has_callable(callable_id)) { - LOG_ERROR("simpler_run: callable_id=%d not registered", callable_id); + LOG_ERROR("simpler_prepare_run: callable_id=%d not registered", callable_id); return -1; } if (!runner->can_accept_run()) { - LOG_ERROR( - "simpler_run: runner is unusable after a prior device failure; refusing callable_id=%d before resource " - "provisioning", - callable_id - ); + LOG_ERROR("simpler_prepare_run: runner is unusable after a prior device failure"); + return -1; + } + uint64_t magic = 0; + std::memcpy(&magic, runtime, sizeof(magic)); + if (magic == OnboardNativeRunState::kMagic) { + LOG_ERROR("simpler_prepare_run: runtime already contains a prepared run; finalize it before reuse"); + return -1; + } + if (magic != 0) { + LOG_ERROR("simpler_prepare_run: runtime storage was not zero-initialized before its first use"); return -1; } @@ -568,84 +644,176 @@ int simpler_run( pthread_setspecific(g_runner_key, nullptr); }); - STRACE_NEW_INV(); - STRACE_SET_HID(runner->callable_hash(callable_id)); - STRACE("simpler_run"); - + OnboardNativeRunState *state = nullptr; + const uint64_t trace_hid = runner->callable_hash(callable_id); + const unsigned trace_inv = STRACE_ALLOC_INV(); + const long long trace_start_ns = STRACE_NOW_NS(); try { + state = new (runtime) OnboardNativeRunState(runner, *config, trace_hid); + if (!runner->try_acquire_native_run(state, &state->launch_signal)) { + LOG_ERROR("simpler_prepare_run: another native run is active on this device context"); + destroy_native_run_state(state); + return -1; + } + state->runner_claimed = true; + state->trace_inv = trace_inv; + state->trace_start_ns = trace_start_ns; + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + int rc = runner->attach_current_thread(runner->device_id()); - if (rc != 0) return rc; + if (rc != 0) return cleanup_failed_prepare(state, rc, true); - Runtime *r = new (runtime) Runtime(); - // RAII the placement-new'd Runtime so its dtor fires on every exit - // (normal returns, the rc-check early-returns below, AND the catch(...) - // path). The prior manual `r->~Runtime()` on each return leaked the - // Runtime on any exception thrown inside the try block. - auto runtime_guard = RAIIScopeGuard([r]() { - r->~Runtime(); - }); - // Platform device-memory hooks. host_api is a platform capability, not - // runtime state — the shared g_host_api table (built once at load time) - // is passed explicitly into the runtime impls rather than stored on - // `Runtime` or reassembled per run. - // Core geometry first: a host-side orchestrator runs to completion - // inside the bind below, and reads worker_count to size its cluster - // spreading. Resolving after the bind would hand it the zeros a fresh - // Runtime carries. - rc = runner->prepare_launch_shape(*r, *config); - if (rc != 0) { - r->set_gm_sm_ptr(nullptr); - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); - return validation_rc != 0 ? validation_rc : rc; - } + rc = runner->prepare_launch_shape(state->runtime, state->config); + if (rc != 0) return cleanup_failed_prepare(state, rc, true); - // Latch the diagnostic enables before the bind: a host-orch runtime - // builds its whole task graph inside it, so a diagnostic that hooks the - // orchestrator (dep_gen) has to be armed by now. run() latches again at - // its entry — the call is idempotent. - runner->apply_call_config(*config); + runner->apply_call_config(state->config); { STRACE("simpler_run.bind"); - // One-step bind: restore kernel addrs + active_callable_id and run - // the per-run binding (tensor args, GM heap, SM alloc). The - // CallableState-derived host_orch_func_ptr + signature stay inside - // the runner — no longer returned across this boundary. rc = runner->bind_callable_to_runtime( - *r, callable_id, &g_host_api, args, config->runtime_env.ring_task_window, config->runtime_env.ring_heap, - config->runtime_env.ring_dep_pool + state->runtime, callable_id, &g_host_api, args, state->config.runtime_env.ring_task_window, + state->config.runtime_env.ring_heap, state->config.runtime_env.ring_dep_pool ); } - if (rc != 0) { - r->set_gm_sm_ptr(nullptr); - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); - return validation_rc != 0 ? validation_rc : rc; - } + if (rc != 0) return cleanup_failed_prepare(state, rc, true); + state->host_thread_state = runner->take_native_run_thread_state(); + return 0; + } catch (...) { + if (state != nullptr) return cleanup_failed_prepare(state, -1, true); + return -1; + } +} - { - STRACE("simpler_run.runner_run"); - // run() latches the diagnostic enables from config via - // apply_call_config() and consumes block_dim / aicpu_thread_num. - rc = runner->run(*r, *config); - } - if (rc != 0) { - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); - return validation_rc != 0 ? validation_rc : rc; - } +int simpler_launch_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + OnboardNativeRunState *state = native_run_state(ctx, runtime, "simpler_launch_run"); + if (state == nullptr || state->phase.load(std::memory_order_acquire) != NativeRunPhase::Prepared) return -1; + if (!state->runner->can_accept_run()) return -1; + if (!state->runner_claimed || !state->runner->native_run_owned_by(state)) return -1; + + state->phase.store(NativeRunPhase::Launching, std::memory_order_release); + + try { + // The compatibility backend uses one blocking executor per run. The + // prepare-through-finalize runner claim limits it to one per context. + state->executor = state->runner->create_thread([state, ctx]() { + pthread_once(&g_runner_key_once, create_runner_key); + pthread_setspecific(g_runner_key, ctx); + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + int rc = -1; + try { + int attach_rc = state->runner->attach_current_thread(state->runner->device_id()); + if (attach_rc == 0) { + state->adopt_host_thread_state(); + { + STRACE("simpler_run.runner_run"); + rc = state->runner->run(state->runtime, state->config); + } + } else { + rc = attach_rc; + } + } catch (...) { + rc = -1; + } + pthread_setspecific(g_runner_key, nullptr); + state->execution_rc.store(rc, std::memory_order_relaxed); + state->execution_done.store(true, std::memory_order_release); + state->launch_signal.notify(); + }); + } catch (...) { + state->phase.store(NativeRunPhase::Prepared, std::memory_order_release); + return -1; + } - { - STRACE("simpler_run.validate"); - rc = validate_runtime_impl(r, &g_host_api, 0); + state->launch_signal.wait(); + if (state->execution_done.load(std::memory_order_acquire)) { + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return state->execution_rc.load(std::memory_order_relaxed); + } + state->phase.store(NativeRunPhase::Running, std::memory_order_release); + return 0; +} + +int simpler_poll_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + OnboardNativeRunState *state = native_run_state(ctx, runtime, "simpler_poll_run"); + if (state == nullptr) return SIMPLER_NATIVE_RUN_POLL_ERROR; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Prepared) return SIMPLER_NATIVE_RUN_POLL_ERROR; + if (phase == NativeRunPhase::Complete || state->execution_done.load(std::memory_order_acquire)) { + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return SIMPLER_NATIVE_RUN_POLL_COMPLETE; + } + return SIMPLER_NATIVE_RUN_POLL_NOT_READY; +} + +int simpler_wait_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + OnboardNativeRunState *state = native_run_state(ctx, runtime, "simpler_wait_run"); + if (state == nullptr) return -1; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Prepared || phase == NativeRunPhase::Launching) return -1; + if (state->executor.joinable()) state->executor.join(); + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return state->execution_rc.load(std::memory_order_relaxed); +} + +int simpler_finalize_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + OnboardNativeRunState *state = native_run_state(ctx, runtime, "simpler_finalize_run"); + if (state == nullptr) return -1; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Launching) return -1; + const unsigned trace_inv = state->trace_inv; + const uint64_t trace_hid = state->trace_hid; + const long long trace_start_ns = state->trace_start_ns; + + pthread_once(&g_runner_key_once, create_runner_key); + pthread_setspecific(g_runner_key, ctx); + auto tsd_guard = RAIIScopeGuard([]() { + pthread_setspecific(g_runner_key, nullptr); + }); + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + + int execution_rc = -1; + const bool launched = phase != NativeRunPhase::Prepared; + if (launched) { + if (state->executor.joinable()) state->executor.join(); + execution_rc = state->execution_rc.load(std::memory_order_relaxed); + } + + int validation_rc = -1; + try { + if (!launched) state->runtime.set_gm_sm_ptr(nullptr); + int attach_rc = state->runner->attach_current_thread(state->runner->device_id()); + if (attach_rc == 0) { + { + STRACE("simpler_run.validate"); + validation_rc = validate_runtime_impl(&state->runtime, &g_host_api, launched ? execution_rc : -1); + } + if (launched && execution_rc == 0) emit_device_phase_markers(state->runner); + } else { + validation_rc = attach_rc; } - // Device-domain phase markers: the AICPU subdivision of the on-NPU wall - // (device_wall + preamble/so_load/graph_build/post_orch/orch/sched). - // host_wall is the simpler_run STRACE span; both flow via the log, not a - // return value. - emit_device_phase_markers(runner); - return rc; } catch (...) { - return -1; + validation_rc = -1; } + + if (state->runner_claimed) { + state->runner->release_native_run(state); + state->runner_claimed = false; + } + destroy_native_run_state(state); + emit_native_run_host_wall(trace_inv, trace_hid, trace_start_ns); + if (validation_rc != 0) return validation_rc; + return launched ? execution_rc : 0; +} + +int simpler_run( + DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config +) { + int rc = simpler_prepare_run(ctx, runtime, callable_id, args, config); + if (rc != 0) return rc; + rc = simpler_launch_run(ctx, runtime); + if (rc == 0) rc = simpler_wait_run(ctx, runtime); + int finalize_rc = simpler_finalize_run(ctx, runtime); + return finalize_rc != 0 ? finalize_rc : rc; } int set_task_accepted_state_ctx(DeviceContextHandle ctx, volatile int32_t *state, int32_t accepted_value) { @@ -660,7 +828,9 @@ int set_task_accepted_state_ctx(DeviceContextHandle ctx, volatile int32_t *state int select_pipeline_slot_ctx(DeviceContextHandle ctx, uint32_t slot_id) { if (ctx == NULL) return -1; try { - return static_cast(ctx)->select_pipeline_slot(slot_id); + DeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) return -1; + return runner->select_pipeline_slot(slot_id); } catch (...) { return -1; } @@ -669,7 +839,9 @@ int select_pipeline_slot_ctx(DeviceContextHandle ctx, uint32_t slot_id) { int select_arena_bank_ctx(DeviceContextHandle ctx, uint32_t bank_id) { if (ctx == NULL) return -1; try { - return static_cast(ctx)->select_arena_bank(bank_id); + DeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) return -1; + return runner->select_arena_bank(bank_id); } catch (...) { return -1; } @@ -696,7 +868,14 @@ uint64_t get_retained_temp_addr_ctx(DeviceContextHandle ctx, uint32_t slot_id) { int simpler_unregister_callable(DeviceContextHandle ctx, int32_t callable_id) { if (ctx == NULL) return -1; try { - return static_cast(ctx)->unregister_callable(callable_id); + DeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR( + "simpler_unregister_callable: native run must be finalized before mutating the callable registry" + ); + return -1; + } + return runner->unregister_callable(callable_id); } catch (...) { return -1; } diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index 1f92a8984e..fb71abeb84 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -46,6 +46,7 @@ #include "host/acl_error_log.h" #include "host/raii_scope_guard.h" #include "host_log.h" +#include "native_run_launch_signal.h" #include "platform_comm/comm.h" #include "pto_runtime_c_api.h" #include "task_args.h" @@ -1012,7 +1013,13 @@ int DeviceRunnerBase::launch_aicpu_kernel( // exported symbol (simpler_aicpu_exec). LaunchBuiltInOp dispatches via // rtsLaunchCpuKernel on the cached rtFuncHandle resolved by // LoadAicpuOp::Init at first-time bootstrap. - return load_aicpu_op_.LaunchBuiltInOp(stream, k_args, sizeof(KernelArgs), aicpu_num, kernel_name); + int rc = load_aicpu_op_.LaunchBuiltInOp(stream, k_args, sizeof(KernelArgs), aicpu_num, kernel_name); + if (rc == 0) { + // Both onboard arches enqueue AICore before this Run launch. A + // successful return is therefore the common post-enqueue boundary. + publish_task_accepted(); + } + return rc; } int DeviceRunnerBase::launch_aicpu_payload( @@ -1487,8 +1494,38 @@ int DeviceRunnerBase::set_task_accepted_state(volatile int32_t *state, int32_t a return 0; } +bool DeviceRunnerBase::try_acquire_native_run(const void *owner, NativeRunLaunchSignal *launch_signal) { + if (owner == nullptr || launch_signal == nullptr) return false; + const void *expected = nullptr; + if (!active_native_run_.compare_exchange_strong( + expected, owner, std::memory_order_acq_rel, std::memory_order_acquire + )) { + return false; + } + native_launch_signal_ = launch_signal; + return true; +} + +void DeviceRunnerBase::release_native_run(const void *owner) { + if (active_native_run_.load(std::memory_order_acquire) != owner) return; + native_launch_signal_ = nullptr; + const void *expected = owner; + (void)active_native_run_.compare_exchange_strong( + expected, nullptr, std::memory_order_release, std::memory_order_relaxed + ); +} + +bool DeviceRunnerBase::native_run_active() const { + return active_native_run_.load(std::memory_order_acquire) != nullptr; +} + +bool DeviceRunnerBase::native_run_owned_by(const void *owner) const { + return owner != nullptr && active_native_run_.load(std::memory_order_acquire) == owner; +} + void DeviceRunnerBase::publish_task_accepted() const { if (task_accepted_state_ != nullptr) { __atomic_store_n(task_accepted_state_, task_accepted_value_, __ATOMIC_RELEASE); } + if (native_launch_signal_ != nullptr) native_launch_signal_->notify(); } diff --git a/src/common/platform/onboard/host/device_runner_base.h b/src/common/platform/onboard/host/device_runner_base.h index 65ceb006e0..2b74cba3a4 100644 --- a/src/common/platform/onboard/host/device_runner_base.h +++ b/src/common/platform/onboard/host/device_runner_base.h @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -69,6 +70,7 @@ struct HostApi; // common/host_api.h — fwd-declared to keep task_interface headers out struct CallConfig; // task_interface/call_config.h — per-run config threaded into run() +class NativeRunLaunchSignal; /** * Common base class for both a2a3 and a5 onboard `DeviceRunner`s. @@ -93,6 +95,16 @@ class DeviceRunnerBase { /** Bind this runner's launch-acceptance publication target. */ int set_task_accepted_state(volatile int32_t *state, int32_t accepted_value); + /** + * Reserve the runner for one native prepared-run execution. The opaque + * owner and runner-owned timing and diagnostic state remain exclusive + * through validation/finalize. + */ + bool try_acquire_native_run(const void *owner, NativeRunLaunchSignal *launch_signal); + void release_native_run(const void *owner); + bool native_run_active() const; + bool native_run_owned_by(const void *owner) const; + /** Carry an already-validated run lease into resource selection. */ int select_pipeline_slot(uint32_t slot_id); int select_arena_bank(uint32_t bank_id); @@ -517,6 +529,12 @@ class DeviceRunnerBase { */ virtual void set_dep_gen_enabled(bool /*enable*/) {} + // Transfer any runtime-specific TLS captured during prepare to the run's + // executor. A non-null snapshot stays caller-owned until adopt or destroy. + virtual void *take_native_run_thread_state() { return nullptr; } + virtual void adopt_native_run_thread_state(void * /*snapshot*/) noexcept {} + virtual void destroy_native_run_thread_state(void * /*snapshot*/) noexcept {} + /** * Launch an AICPU kernel. Internal helper used by the subclass's * `run()`; thin wrapper that dispatches through `load_aicpu_op_`'s @@ -875,6 +893,8 @@ class DeviceRunnerBase { size_t host_dlopen_total_{0}; volatile int32_t *task_accepted_state_{nullptr}; int32_t task_accepted_value_{0}; + std::atomic active_native_run_{nullptr}; + NativeRunLaunchSignal *native_launch_signal_{nullptr}; // ---- State shared by both a2a3 and a5 --------------------------------- // diff --git a/src/common/platform/sim/host/c_api_shared.cpp b/src/common/platform/sim/host/c_api_shared.cpp index 87cb8ab18c..5a95715378 100644 --- a/src/common/platform/sim/host/c_api_shared.cpp +++ b/src/common/platform/sim/host/c_api_shared.cpp @@ -28,6 +28,7 @@ #include "device_runner_base.h" #include "prepare_callable_common.h" #include "task_args.h" +#include "native_run_state.h" #include #include @@ -35,6 +36,7 @@ #include #include #include +#include #include #include @@ -45,6 +47,11 @@ #include "host/raii_scope_guard.h" #include "runtime.h" +using SimNativeRunState = NativeRunState; +// Phase entry points validate raw caller storage before beginning object +// lifetime, so the on-storage magic must remain the leading bytes. +static_assert(__builtin_offsetof(SimNativeRunState, magic) == 0, "native-run magic must lead runtime storage"); + extern "C" { /* =========================================================================== @@ -241,9 +248,18 @@ static const HostApi g_host_api = { * Public C API (resolved by ChipWorker via dlsym) * =========================================================================== */ -void destroy_device_context(DeviceContextHandle ctx) { delete static_cast(ctx); } +void destroy_device_context(DeviceContextHandle ctx) { + SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner != nullptr && runner->native_run_active()) { + LOG_ERROR("destroy_device_context: refusing to destroy a context with an unfinalized native run"); + return; + } + delete runner; +} -size_t get_runtime_size(void) { return sizeof(Runtime); } +size_t get_runtime_size(void) { return sizeof(SimNativeRunState); } + +size_t get_runtime_alignment(void) { return alignof(SimNativeRunState); } void *device_malloc_ctx(DeviceContextHandle ctx, size_t size) { if (ctx == NULL) return NULL; @@ -283,6 +299,10 @@ int finalize_device(DeviceContextHandle ctx) { if (ctx == NULL) return -1; try { SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR("finalize_device: native run must be finalized first"); + return -1; + } int rc = runner->finalize(); int dev = pto_cpu_sim_get_bound_device(); if (dev >= 0) { @@ -364,6 +384,10 @@ int simpler_init( int simpler_register_callable(DeviceContextHandle ctx, int32_t callable_id, const void *callable) { if (ctx == NULL || callable == NULL) return -1; SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR("simpler_register_callable: native run must be finalized before mutating the callable registry"); + return -1; + } pthread_once(&g_runner_key_once, create_runner_key); pthread_setspecific(g_runner_key, ctx); @@ -507,104 +531,261 @@ static void emit_device_phase_markers(SimDeviceRunnerBase *runner) { } } -int simpler_run( +static SimNativeRunState *native_run_state(DeviceContextHandle ctx, RuntimeHandle runtime, const char *operation) { + if (ctx == nullptr || runtime == nullptr) return nullptr; + uint64_t magic = 0; + std::memcpy(&magic, runtime, sizeof(magic)); + if (magic != SimNativeRunState::kMagic) { + LOG_ERROR("%s: runtime does not contain a prepared native run", operation); + return nullptr; + } + auto *state = static_cast(runtime); + if (state->runner != static_cast(ctx)) { + LOG_ERROR("%s: prepared run belongs to a different device context", operation); + return nullptr; + } + return state; +} + +static void emit_native_run_host_wall(unsigned trace_inv, uint64_t trace_hid, long long trace_start_ns) { + const long long end_ns = STRACE_NOW_NS(); + STRACE_CONTEXT(trace_inv, trace_hid, 0); + STRACE_HOST_SPAN_AT("simpler_run", trace_start_ns, end_ns - trace_start_ns, 0); +} + +static int cleanup_failed_prepare(SimNativeRunState *state, int execution_rc, bool clear_gm_sm) { + const unsigned trace_inv = state->trace_inv; + const uint64_t trace_hid = state->trace_hid; + const long long trace_start_ns = state->trace_start_ns; + if (clear_gm_sm) state->runtime.set_gm_sm_ptr(nullptr); + int validation_rc = -1; + try { + validation_rc = validate_runtime_impl(&state->runtime, &g_host_api, execution_rc); + } catch (...) { + validation_rc = -1; + } + if (state->runner_claimed) { + state->runner->release_native_run(state); + state->runner_claimed = false; + } + destroy_native_run_state(state); + emit_native_run_host_wall(trace_inv, trace_hid, trace_start_ns); + return validation_rc != 0 ? validation_rc : execution_rc; +} + +int simpler_prepare_run( DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config ) { - if (ctx == NULL || runtime == NULL || config == NULL) return -1; + if (ctx == nullptr || runtime == nullptr || config == nullptr) return -1; + if (reinterpret_cast(runtime) % alignof(SimNativeRunState) != 0) { + LOG_ERROR("simpler_prepare_run: runtime storage does not satisfy get_runtime_alignment()"); + return -1; + } SimDeviceRunnerBase *runner = static_cast(ctx); - if (!runner->has_callable(callable_id)) { - LOG_ERROR("simpler_run: callable_id=%d not registered", callable_id); + LOG_ERROR("simpler_prepare_run: callable_id=%d not registered", callable_id); + return -1; + } + uint64_t magic = 0; + std::memcpy(&magic, runtime, sizeof(magic)); + if (magic == SimNativeRunState::kMagic) { + LOG_ERROR("simpler_prepare_run: runtime already contains a prepared run; finalize it before reuse"); + return -1; + } + if (magic != 0) { + LOG_ERROR("simpler_prepare_run: runtime storage was not zero-initialized before its first use"); return -1; } pthread_once(&g_runner_key_once, create_runner_key); pthread_setspecific(g_runner_key, ctx); + auto tsd_guard = RAIIScopeGuard([]() { + pthread_setspecific(g_runner_key, nullptr); + }); - STRACE_NEW_INV(); - STRACE_SET_HID(static_cast(callable_id)); - STRACE("simpler_run"); - + SimNativeRunState *state = nullptr; + const uint64_t trace_hid = static_cast(callable_id); + const unsigned trace_inv = STRACE_ALLOC_INV(); + const long long trace_start_ns = STRACE_NOW_NS(); try { - Runtime *r = new (runtime) Runtime(); - // RAII the placement-new'd Runtime so its dtor fires on every exit - // (normal returns, the rc-check early-returns below, AND the catch(...) - // path). Mirrors the onboard c_api_shared fix from PR #928. - auto runtime_guard = RAIIScopeGuard([r]() { - r->~Runtime(); - }); - - // Platform device-memory hooks. host_api is a platform capability, not - // runtime state — the shared g_host_api table (built once at load time) - // is passed explicitly into the runtime impls rather than stored on - // `Runtime` or reassembled per run. - // Core geometry first: a host-side orchestrator runs to completion - // inside the bind below, and reads worker_count to size its cluster - // spreading. Resolving after the bind would hand it the zeros a fresh - // Runtime carries. - int rc = runner->prepare_launch_shape(*r, *config); - if (rc != 0) { - r->set_gm_sm_ptr(nullptr); - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); - pthread_setspecific(g_runner_key, nullptr); - return validation_rc != 0 ? validation_rc : rc; + state = new (runtime) SimNativeRunState(runner, *config, trace_hid); + if (!runner->try_acquire_native_run(state, &state->launch_signal)) { + LOG_ERROR("simpler_prepare_run: another native run is active on this device context"); + destroy_native_run_state(state); + return -1; } + state->runner_claimed = true; + state->trace_inv = trace_inv; + state->trace_start_ns = trace_start_ns; + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + + int rc = runner->attach_current_thread(runner->device_id()); + if (rc != 0) return cleanup_failed_prepare(state, rc, true); + + rc = runner->prepare_launch_shape(state->runtime, state->config); + if (rc != 0) return cleanup_failed_prepare(state, rc, true); - // Latch the diagnostic enables before the bind: a host-orch runtime - // builds its whole task graph inside it, so a diagnostic that hooks the - // orchestrator (dep_gen) has to be armed by now. run() latches again at - // its entry — the call is idempotent. - runner->apply_call_config(*config); + runner->apply_call_config(state->config); { STRACE("simpler_run.bind"); - // One-step bind: replay CallableState + run the per-run binding. The - // host_orch_func_ptr + signature stay inside the runner. rc = runner->bind_callable_to_runtime( - *r, callable_id, &g_host_api, args, config->runtime_env.ring_task_window, config->runtime_env.ring_heap, - config->runtime_env.ring_dep_pool + state->runtime, callable_id, &g_host_api, args, state->config.runtime_env.ring_task_window, + state->config.runtime_env.ring_heap, state->config.runtime_env.ring_dep_pool ); } - if (rc != 0) { - r->set_gm_sm_ptr(nullptr); - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); - pthread_setspecific(g_runner_key, nullptr); - return validation_rc != 0 ? validation_rc : rc; - } + if (rc != 0) return cleanup_failed_prepare(state, rc, true); + state->host_thread_state = runner->take_native_run_thread_state(); + return 0; + } catch (...) { + if (state != nullptr) return cleanup_failed_prepare(state, -1, true); + return -1; + } +} - { - STRACE("simpler_run.runner_run"); - // run() latches the diagnostic enables from config via - // apply_call_config() and consumes block_dim / aicpu_thread_num. - rc = runner->run(*r, *config); - } - if (rc != 0) { - int validation_rc = validate_runtime_impl(r, &g_host_api, rc); +int simpler_launch_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + SimNativeRunState *state = native_run_state(ctx, runtime, "simpler_launch_run"); + if (state == nullptr || state->phase.load(std::memory_order_acquire) != NativeRunPhase::Prepared) return -1; + if (!state->runner_claimed || !state->runner->native_run_owned_by(state)) return -1; + + state->phase.store(NativeRunPhase::Launching, std::memory_order_release); + + try { + // The compatibility backend uses one blocking executor per run. The + // prepare-through-finalize runner claim limits it to one per context. + state->executor = state->runner->create_thread([state, ctx]() { + pthread_once(&g_runner_key_once, create_runner_key); + pthread_setspecific(g_runner_key, ctx); + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + int rc = -1; + try { + int attach_rc = state->runner->attach_current_thread(state->runner->device_id()); + if (attach_rc == 0) { + state->adopt_host_thread_state(); + { + STRACE("simpler_run.runner_run"); + rc = state->runner->run(state->runtime, state->config); + } + } else { + rc = attach_rc; + } + } catch (...) { + rc = -1; + } pthread_setspecific(g_runner_key, nullptr); - return validation_rc != 0 ? validation_rc : rc; - } + state->execution_rc.store(rc, std::memory_order_relaxed); + state->execution_done.store(true, std::memory_order_release); + state->launch_signal.notify(); + }); + } catch (...) { + state->phase.store(NativeRunPhase::Prepared, std::memory_order_release); + return -1; + } - { - STRACE("simpler_run.validate"); - rc = validate_runtime_impl(r, &g_host_api, 0); - } + state->launch_signal.wait(); + if (state->execution_done.load(std::memory_order_acquire)) { + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return state->execution_rc.load(std::memory_order_relaxed); + } + state->phase.store(NativeRunPhase::Running, std::memory_order_release); + return 0; +} + +int simpler_poll_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + SimNativeRunState *state = native_run_state(ctx, runtime, "simpler_poll_run"); + if (state == nullptr) return SIMPLER_NATIVE_RUN_POLL_ERROR; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Prepared) return SIMPLER_NATIVE_RUN_POLL_ERROR; + if (phase == NativeRunPhase::Complete || state->execution_done.load(std::memory_order_acquire)) { + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return SIMPLER_NATIVE_RUN_POLL_COMPLETE; + } + return SIMPLER_NATIVE_RUN_POLL_NOT_READY; +} + +int simpler_wait_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + SimNativeRunState *state = native_run_state(ctx, runtime, "simpler_wait_run"); + if (state == nullptr) return -1; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Prepared || phase == NativeRunPhase::Launching) return -1; + if (state->executor.joinable()) state->executor.join(); + state->phase.store(NativeRunPhase::Complete, std::memory_order_release); + return state->execution_rc.load(std::memory_order_relaxed); +} + +int simpler_finalize_run(DeviceContextHandle ctx, RuntimeHandle runtime) { + SimNativeRunState *state = native_run_state(ctx, runtime, "simpler_finalize_run"); + if (state == nullptr) return -1; + NativeRunPhase phase = state->phase.load(std::memory_order_acquire); + if (phase == NativeRunPhase::Launching) return -1; + const unsigned trace_inv = state->trace_inv; + const uint64_t trace_hid = state->trace_hid; + const long long trace_start_ns = state->trace_start_ns; + + pthread_once(&g_runner_key_once, create_runner_key); + pthread_setspecific(g_runner_key, ctx); + auto tsd_guard = RAIIScopeGuard([]() { pthread_setspecific(g_runner_key, nullptr); - emit_device_phase_markers(runner); - return rc; + }); + STRACE_CONTEXT(state->trace_inv, state->trace_hid, 1); + + int execution_rc = -1; + const bool launched = phase != NativeRunPhase::Prepared; + if (launched) { + if (state->executor.joinable()) state->executor.join(); + execution_rc = state->execution_rc.load(std::memory_order_relaxed); + } + + int validation_rc = -1; + try { + if (!launched) state->runtime.set_gm_sm_ptr(nullptr); + int attach_rc = state->runner->attach_current_thread(state->runner->device_id()); + if (attach_rc == 0) { + { + STRACE("simpler_run.validate"); + validation_rc = validate_runtime_impl(&state->runtime, &g_host_api, launched ? execution_rc : -1); + } + if (launched && execution_rc == 0) emit_device_phase_markers(state->runner); + } else { + validation_rc = attach_rc; + } } catch (...) { - pthread_setspecific(g_runner_key, nullptr); - return -1; + validation_rc = -1; + } + + if (state->runner_claimed) { + state->runner->release_native_run(state); + state->runner_claimed = false; } + destroy_native_run_state(state); + emit_native_run_host_wall(trace_inv, trace_hid, trace_start_ns); + if (validation_rc != 0) return validation_rc; + return launched ? execution_rc : 0; +} + +int simpler_run( + DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config +) { + int rc = simpler_prepare_run(ctx, runtime, callable_id, args, config); + if (rc != 0) return rc; + rc = simpler_launch_run(ctx, runtime); + if (rc == 0) rc = simpler_wait_run(ctx, runtime); + int finalize_rc = simpler_finalize_run(ctx, runtime); + return finalize_rc != 0 ? finalize_rc : rc; } int select_pipeline_slot_ctx(DeviceContextHandle ctx, uint32_t slot_id) { if (ctx == NULL) return -1; - return static_cast(ctx)->select_pipeline_slot(slot_id); + SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) return -1; + return runner->select_pipeline_slot(slot_id); } int select_arena_bank_ctx(DeviceContextHandle ctx, uint32_t bank_id) { if (ctx == NULL) return -1; - return static_cast(ctx)->select_arena_bank(bank_id); + SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) return -1; + return runner->select_arena_bank(bank_id); } uint64_t get_arena_bank_gm_heap_base_ctx(DeviceContextHandle ctx, uint32_t bank_id) { @@ -624,7 +805,14 @@ uint64_t get_retained_temp_addr_ctx(DeviceContextHandle ctx, uint32_t slot_id) { int simpler_unregister_callable(DeviceContextHandle ctx, int32_t callable_id) { if (ctx == NULL) return -1; try { - return static_cast(ctx)->unregister_callable(callable_id); + SimDeviceRunnerBase *runner = static_cast(ctx); + if (runner->native_run_active()) { + LOG_ERROR( + "simpler_unregister_callable: native run must be finalized before mutating the callable registry" + ); + return -1; + } + return runner->unregister_callable(callable_id); } catch (...) { return -1; } diff --git a/src/common/platform/sim/host/device_runner_base.cpp b/src/common/platform/sim/host/device_runner_base.cpp index 217971929f..b0b77f2ae7 100644 --- a/src/common/platform/sim/host/device_runner_base.cpp +++ b/src/common/platform/sim/host/device_runner_base.cpp @@ -28,6 +28,7 @@ #include "common/host_api.h" #include "cpu_sim_context.h" #include "host/raii_scope_guard.h" +#include "native_run_launch_signal.h" #include "task_args.h" #include "utils/elf_build_id.h" @@ -82,6 +83,48 @@ bool create_temp_so_file(const std::string &path_template, const uint8_t *data, // SimDeviceRunnerBase Implementation // ============================================================================= +int SimDeviceRunnerBase::set_task_accepted_state(volatile int32_t *state, int32_t accepted_value) { + task_accepted_state_ = state; + task_accepted_value_ = accepted_value; + return 0; +} + +bool SimDeviceRunnerBase::try_acquire_native_run(const void *owner, NativeRunLaunchSignal *launch_signal) { + if (owner == nullptr || launch_signal == nullptr) return false; + const void *expected = nullptr; + if (!active_native_run_.compare_exchange_strong( + expected, owner, std::memory_order_acq_rel, std::memory_order_acquire + )) { + return false; + } + native_launch_signal_ = launch_signal; + return true; +} + +void SimDeviceRunnerBase::release_native_run(const void *owner) { + if (active_native_run_.load(std::memory_order_acquire) != owner) return; + native_launch_signal_ = nullptr; + const void *expected = owner; + (void)active_native_run_.compare_exchange_strong( + expected, nullptr, std::memory_order_release, std::memory_order_relaxed + ); +} + +bool SimDeviceRunnerBase::native_run_active() const { + return active_native_run_.load(std::memory_order_acquire) != nullptr; +} + +bool SimDeviceRunnerBase::native_run_owned_by(const void *owner) const { + return owner != nullptr && active_native_run_.load(std::memory_order_acquire) == owner; +} + +void SimDeviceRunnerBase::publish_task_accepted() const { + if (task_accepted_state_ != nullptr) { + __atomic_store_n(task_accepted_state_, task_accepted_value_, __ATOMIC_RELEASE); + } + if (native_launch_signal_ != nullptr) native_launch_signal_->notify(); +} + int SimDeviceRunnerBase::select_pipeline_slot(uint32_t slot_id) { if (slot_id >= PTO_PIPELINE_MAX_DEPTH) return -1; pipeline_slot_ = slot_id; diff --git a/src/common/platform/sim/host/device_runner_base.h b/src/common/platform/sim/host/device_runner_base.h index 0dcc307a7b..1225fd5eff 100644 --- a/src/common/platform/sim/host/device_runner_base.h +++ b/src/common/platform/sim/host/device_runner_base.h @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,7 @@ struct HostApi; // common/host_api.h — fwd-declared to keep task_interface headers out struct CallConfig; // task_interface/call_config.h — per-run config threaded into run() +class NativeRunLaunchSignal; // Width sim resolves the CallConfig "auto" sentinel to, deliberately below // PLATFORM_MAX_BLOCKDIM (24 on a2a3, 36 on a5). The simulator runs one OS @@ -99,6 +101,21 @@ class SimDeviceRunnerBase { // a2a3 and a5 both override; an arch without dep_gen leaves the no-op. virtual void set_dep_gen_enabled(bool /*enable*/) {} + // Transfer any runtime-specific TLS captured during prepare to the run's + // executor. A non-null snapshot stays caller-owned until adopt or destroy. + virtual void *take_native_run_thread_state() { return nullptr; } + virtual void adopt_native_run_thread_state(void * /*snapshot*/) noexcept {} + virtual void destroy_native_run_thread_state(void * /*snapshot*/) noexcept {} + + /** Bind an optional external launch-acceptance target. */ + int set_task_accepted_state(volatile int32_t *state, int32_t accepted_value); + + /** Reserve the runner's single active native execution through finalize. */ + bool try_acquire_native_run(const void *owner, NativeRunLaunchSignal *launch_signal); + void release_native_run(const void *owner); + bool native_run_active() const; + bool native_run_owned_by(const void *owner) const; + // --- Shared methods -------------------------------------------------- int setup_static_arena(size_t gm_heap_size, size_t gm_sm_size, size_t runtime_arena_size); @@ -224,6 +241,8 @@ class SimDeviceRunnerBase { protected: // --- Helpers usable by subclass run() / finalize() ------------------- + /** Publish after both simulated kernel thread groups have been created. */ + void publish_task_accepted() const; int ensure_device_initialized(); virtual int ensure_binaries_loaded() = 0; // Hand the orch-SO descriptor to the sim AICPU register entry. Built @@ -363,6 +382,11 @@ class SimDeviceRunnerBase { Runtime *last_runtime_{nullptr}; + volatile int32_t *task_accepted_state_{nullptr}; + int32_t task_accepted_value_{0}; + std::atomic active_native_run_{nullptr}; + NativeRunLaunchSignal *native_launch_signal_{nullptr}; + // Dynamically loaded executor libraries (shared infra; the dlsym'd function- // pointer table itself lives on the subclass since signatures diverge // per-arch — a2a3 vs a5 differ on aicore_execute and several setters). diff --git a/src/common/worker/chip_worker.cpp b/src/common/worker/chip_worker.cpp index 8cc111168e..4430489012 100644 --- a/src/common/worker/chip_worker.cpp +++ b/src/common/worker/chip_worker.cpp @@ -15,8 +15,12 @@ #include +#include +#include #include #include +#include +#include #include #include #include @@ -47,6 +51,17 @@ T load_optional_symbol(void *handle, const char *name) { return reinterpret_cast(sym); } +uint64_t next_native_run_epoch() { + static std::atomic epoch{0}; + uint64_t current = epoch.load(std::memory_order_relaxed); + while (current != std::numeric_limits::max()) { + if (epoch.compare_exchange_weak(current, current + 1, std::memory_order_relaxed)) { + return current + 1; + } + } + throw std::overflow_error("native-run epoch space is exhausted"); +} + std::vector read_binary_file(const std::string &path) { std::ifstream f(path, std::ios::binary | std::ios::ate); if (!f) { @@ -66,6 +81,30 @@ std::vector read_binary_file(const std::string &path) { } // namespace +ChipWorker::RuntimeStorage::RuntimeStorage(size_t size, size_t alignment) { + void *storage = nullptr; + if (posix_memalign(&storage, alignment, size) != 0) { + throw std::bad_alloc(); + } + std::memset(storage, 0, size); + data_ = storage; +} + +ChipWorker::RuntimeStorage::~RuntimeStorage() { std::free(data_); } + +ChipWorker::RuntimeStorage::RuntimeStorage(RuntimeStorage &&other) noexcept : + data_(other.data_) { + other.data_ = nullptr; +} + +ChipWorker::RuntimeStorage &ChipWorker::RuntimeStorage::operator=(RuntimeStorage &&other) noexcept { + if (this == &other) return *this; + std::free(data_); + data_ = other.data_; + other.data_ = nullptr; + return *this; +} + ChipWorker::~ChipWorker() { finalize(); } void ChipWorker::init( @@ -115,9 +154,15 @@ void ChipWorker::init( copy_to_device_ctx_fn_ = load_symbol(handle, "copy_to_device_ctx"); copy_from_device_ctx_fn_ = load_symbol(handle, "copy_from_device_ctx"); get_runtime_size_fn_ = load_symbol(handle, "get_runtime_size"); + get_runtime_alignment_fn_ = load_symbol(handle, "get_runtime_alignment"); simpler_init_fn_ = load_symbol(handle, "simpler_init"); register_callable_fn_ = load_symbol(handle, "simpler_register_callable"); run_fn_ = load_symbol(handle, "simpler_run"); + prepare_run_fn_ = load_symbol(handle, "simpler_prepare_run"); + launch_run_fn_ = load_symbol(handle, "simpler_launch_run"); + poll_run_fn_ = load_symbol(handle, "simpler_poll_run"); + wait_run_fn_ = load_symbol(handle, "simpler_wait_run"); + finalize_run_fn_ = load_symbol(handle, "simpler_finalize_run"); select_pipeline_slot_fn_ = load_symbol(handle, "select_pipeline_slot_ctx"); select_arena_bank_fn_ = load_symbol(handle, "select_arena_bank_ctx"); get_arena_bank_gm_heap_base_fn_ = @@ -177,16 +222,23 @@ void ChipWorker::init( } try { - // One host Runtime per slot, always. This buffer is not the - // RUNTIME_IMAGE resource: the contract classifies the device-resident - // image, while this holds the host-side Runtime object a run - // constructs in place — its tensor leases, launch arguments, and - // validate/finalize state. That is per-run whatever the device image - // sharing is, so a runtime whose image is DEVICE_SCRATCH still needs - // its own buffer per slot or preparing N+1 overwrites live state of N. - std::vector> runtime_bufs( - resolved_contract.pipeline_depth, std::vector(get_runtime_size_fn_()) - ); + // One opaque native-run storage buffer per slot, always. The host + // runtime constructs its per-run Runtime + phase state behind this + // ABI boundary. This storage is not the RUNTIME_IMAGE resource: the + // contract classifies the device-resident image, while this owns the + // host-side tensor leases, launch arguments, and validation/finalize + // state. It is per-run even when the device image is DEVICE_SCRATCH. + const size_t runtime_size = get_runtime_size_fn_(); + const size_t runtime_alignment = get_runtime_alignment_fn_(); + if (runtime_size < sizeof(uint64_t) || runtime_alignment < sizeof(void *) || + (runtime_alignment & (runtime_alignment - 1)) != 0) { + throw std::runtime_error("host runtime returned unsupported native-run storage size/alignment"); + } + std::vector runtime_bufs; + runtime_bufs.reserve(resolved_contract.pipeline_depth); + for (uint32_t slot = 0; slot < resolved_contract.pipeline_depth; ++slot) { + runtime_bufs.emplace_back(runtime_size, runtime_alignment); + } runtime_bufs_.swap(runtime_bufs); } catch (...) { destroy_device_context_fn_(device_ctx_); @@ -240,9 +292,15 @@ void ChipWorker::init( copy_to_device_ctx_fn_ = nullptr; copy_from_device_ctx_fn_ = nullptr; get_runtime_size_fn_ = nullptr; + get_runtime_alignment_fn_ = nullptr; simpler_init_fn_ = nullptr; register_callable_fn_ = nullptr; run_fn_ = nullptr; + prepare_run_fn_ = nullptr; + launch_run_fn_ = nullptr; + poll_run_fn_ = nullptr; + wait_run_fn_ = nullptr; + finalize_run_fn_ = nullptr; select_pipeline_slot_fn_ = nullptr; select_arena_bank_fn_ = nullptr; get_arena_bank_gm_heap_base_fn_ = nullptr; @@ -286,9 +344,15 @@ void ChipWorker::init( copy_to_device_ctx_fn_ = nullptr; copy_from_device_ctx_fn_ = nullptr; get_runtime_size_fn_ = nullptr; + get_runtime_alignment_fn_ = nullptr; simpler_init_fn_ = nullptr; register_callable_fn_ = nullptr; run_fn_ = nullptr; + prepare_run_fn_ = nullptr; + launch_run_fn_ = nullptr; + poll_run_fn_ = nullptr; + wait_run_fn_ = nullptr; + finalize_run_fn_ = nullptr; select_pipeline_slot_fn_ = nullptr; select_arena_bank_fn_ = nullptr; get_arena_bank_gm_heap_base_fn_ = nullptr; @@ -340,6 +404,7 @@ void ChipWorker::init( } void ChipWorker::finalize() { + cleanup_native_runs_noexcept(); // Defensive: if the user never called comm_destroy, reclaim all owned // communicator handles and streams before tearing down the device context. clear_comm_sessions(); @@ -363,8 +428,14 @@ void ChipWorker::finalize() { copy_to_device_ctx_fn_ = nullptr; copy_from_device_ctx_fn_ = nullptr; get_runtime_size_fn_ = nullptr; + get_runtime_alignment_fn_ = nullptr; register_callable_fn_ = nullptr; run_fn_ = nullptr; + prepare_run_fn_ = nullptr; + launch_run_fn_ = nullptr; + poll_run_fn_ = nullptr; + wait_run_fn_ = nullptr; + finalize_run_fn_ = nullptr; select_pipeline_slot_fn_ = nullptr; select_arena_bank_fn_ = nullptr; get_arena_bank_gm_heap_base_fn_ = nullptr; @@ -530,6 +601,172 @@ void ChipWorker::run_on_slot( } } +ChipWorkerNativeRun ChipWorker::prepare_native_run( + int32_t callable_id, TaskArgsView args, const CallConfig &config, const PipelineSlotLease &lease +) { + ChipStorageTaskArgs chip_storage = view_to_chip_storage(args); + return prepare_native_run(callable_id, &chip_storage, config, lease); +} + +ChipWorkerNativeRun ChipWorker::prepare_native_run( + int32_t callable_id, const ChipStorageTaskArgs *args, const CallConfig &config, const PipelineSlotLease &lease +) { + if (lease.reserved != 0 || lease.generation == 0 || lease.slot_id >= pipeline_contract_.pipeline_depth) { + throw std::runtime_error("native-run pipeline lease is outside the runtime PipelineContract"); + } + if (!pipeline_generations_.admit(lease)) { + throw std::runtime_error("native-run pipeline lease generation is stale"); + } + return prepare_native_run_on_slot(callable_id, args, config, lease.slot_id, lease.generation); +} + +ChipWorkerNativeRun ChipWorker::prepare_native_run_on_slot( + int32_t callable_id, const ChipStorageTaskArgs *args, const CallConfig &config, uint32_t slot_id, + uint64_t generation +) { + config.validate(); + if (!initialized_) { + throw std::runtime_error("ChipWorker not initialized; call init() first"); + } + if (args == nullptr) { + throw std::runtime_error("prepare_native_run requires task args"); + } + if (slot_id >= runtime_bufs_.size()) { + throw std::runtime_error("prepare_native_run slot is outside the runtime PipelineContract"); + } + NativeRunSlotState &state = native_run_states_[slot_id]; + if (state.phase != NativeRunPhase::EMPTY) { + throw std::runtime_error("prepare_native_run slot already owns an unfinished native run"); + } + for (const NativeRunSlotState &candidate : native_run_states_) { + if (candidate.phase != NativeRunPhase::EMPTY) { + throw std::runtime_error( + "prepare_native_run cannot select another slot while an unfinished native run owns the runner" + ); + } + } + + (void)select_slot_resources(slot_id); + const uint64_t run_epoch = next_native_run_epoch(); + int rc = prepare_run_fn_(device_ctx_, runtime_bufs_[slot_id].data(), callable_id, args, &config); + if (rc != 0) { + throw std::runtime_error("prepare_native_run failed with code " + std::to_string(rc)); + } + state.lease_generation = generation; + state.run_epoch = run_epoch; + state.phase = NativeRunPhase::PREPARED; + state.wait_rc = 0; + return ChipWorkerNativeRun{slot_id, generation, run_epoch}; +} + +ChipWorker::NativeRunSlotState & +ChipWorker::require_native_run(const ChipWorkerNativeRun &run, NativeRunPhase first, NativeRunPhase second) { + if (run.slot_id >= runtime_bufs_.size()) { + throw std::runtime_error("native-run token slot is outside the runtime PipelineContract"); + } + NativeRunSlotState &state = native_run_states_[run.slot_id]; + if (state.lease_generation != run.generation || state.run_epoch != run.run_epoch || + (state.phase != first && state.phase != second)) { + throw std::runtime_error("native-run token is stale or used in the wrong phase"); + } + return state; +} + +void ChipWorker::launch_native_run( + const ChipWorkerNativeRun &run, volatile int32_t *accepted_state, int32_t accepted_value +) { + NativeRunSlotState &state = require_native_run(run, NativeRunPhase::PREPARED, NativeRunPhase::PREPARED); + if (accepted_state != nullptr && set_task_accepted_state_fn_ != nullptr) { + int bind_rc = set_task_accepted_state_fn_(device_ctx_, accepted_state, accepted_value); + if (bind_rc != 0) { + throw std::runtime_error("set_task_accepted_state_ctx failed with code " + std::to_string(bind_rc)); + } + } + auto clear_accepted_state = [&]() { + if (accepted_state != nullptr && set_task_accepted_state_fn_ != nullptr) { + (void)set_task_accepted_state_fn_(device_ctx_, nullptr, 0); + } + }; + + int rc = -1; + try { + rc = launch_run_fn_(device_ctx_, runtime_bufs_[run.slot_id].data()); + } catch (...) { + clear_accepted_state(); + throw; + } + clear_accepted_state(); + if (rc != 0) { + state.phase = NativeRunPhase::REAPED; + state.wait_rc = rc; + throw std::runtime_error("launch_native_run failed with code " + std::to_string(rc)); + } + state.phase = NativeRunPhase::LAUNCHED; +} + +bool ChipWorker::poll_native_run(const ChipWorkerNativeRun &run) { + NativeRunSlotState &state = require_native_run(run, NativeRunPhase::LAUNCHED, NativeRunPhase::REAPED); + if (state.phase == NativeRunPhase::REAPED) { + return true; + } + int rc = poll_run_fn_(device_ctx_, runtime_bufs_[run.slot_id].data()); + if (rc == SIMPLER_NATIVE_RUN_POLL_NOT_READY) { + return false; + } + if (rc == SIMPLER_NATIVE_RUN_POLL_COMPLETE) { + state.phase = NativeRunPhase::REAPED; + return true; + } + throw std::runtime_error("poll_native_run failed with code " + std::to_string(rc)); +} + +void ChipWorker::wait_native_run(const ChipWorkerNativeRun &run) { + NativeRunSlotState &state = require_native_run(run, NativeRunPhase::LAUNCHED, NativeRunPhase::REAPED); + if (state.phase == NativeRunPhase::REAPED) { + return; + } + state.wait_rc = wait_run_fn_(device_ctx_, runtime_bufs_[run.slot_id].data()); + state.phase = NativeRunPhase::REAPED; +} + +void ChipWorker::finalize_native_run(const ChipWorkerNativeRun &run) { + if (run.slot_id >= runtime_bufs_.size()) { + throw std::runtime_error("native-run token slot is outside the runtime PipelineContract"); + } + NativeRunSlotState &state = native_run_states_[run.slot_id]; + if (state.lease_generation != run.generation || state.run_epoch != run.run_epoch || + state.phase == NativeRunPhase::EMPTY) { + throw std::runtime_error("native-run token is stale or already finalized"); + } + if (state.phase == NativeRunPhase::LAUNCHED) { + wait_native_run(run); + } + int wait_rc = state.wait_rc; + int finalize_rc = finalize_run_fn_(device_ctx_, runtime_bufs_[run.slot_id].data()); + state = NativeRunSlotState{}; + int rc = finalize_rc != 0 ? finalize_rc : wait_rc; + if (rc != 0) { + throw std::runtime_error("finalize_native_run failed with code " + std::to_string(rc)); + } +} + +void ChipWorker::cleanup_native_runs_noexcept() noexcept { + if (device_ctx_ == nullptr || finalize_run_fn_ == nullptr) { + return; + } + for (size_t slot_id = 0; slot_id < runtime_bufs_.size(); ++slot_id) { + NativeRunSlotState &state = native_run_states_[slot_id]; + if (state.phase == NativeRunPhase::EMPTY) { + continue; + } + if (state.phase == NativeRunPhase::LAUNCHED && wait_run_fn_ != nullptr) { + (void)wait_run_fn_(device_ctx_, runtime_bufs_[slot_id].data()); + } + (void)finalize_run_fn_(device_ctx_, runtime_bufs_[slot_id].data()); + state = NativeRunSlotState{}; + } +} + void ChipWorker::unregister_callable(int32_t callable_id) { if (!initialized_) { throw std::runtime_error("ChipWorker not initialized; call init() first"); diff --git a/src/common/worker/chip_worker.h b/src/common/worker/chip_worker.h index 2a31935641..5582237ce0 100644 --- a/src/common/worker/chip_worker.h +++ b/src/common/worker/chip_worker.h @@ -12,6 +12,8 @@ #ifndef SRC_COMMON_WORKER_CHIP_WORKER_H_ #define SRC_COMMON_WORKER_CHIP_WORKER_H_ +#include +#include #include #include #include @@ -23,6 +25,16 @@ #include "pto_runtime_c_api.h" #include "types.h" +/** Opaque identity for one prepared native run owned by a ChipWorker. */ +struct ChipWorkerNativeRun { + uint32_t slot_id{0}; + // Generation of the externally minted pipeline lease. One lease may + // dispatch several runs, so this is not sufficient run identity alone. + uint64_t generation{0}; + // Process-unique identity for exactly one prepare attempt. + uint64_t run_epoch{0}; +}; + class ChipWorker { public: ChipWorker() = default; @@ -89,6 +101,38 @@ class ChipWorker { volatile int32_t *accepted_state = nullptr, int32_t accepted_value = 0 ); + /** + * Progressable native-run lifecycle. + * + * prepare_native_run performs per-run Runtime construction and binding but + * does not launch device work. launch_native_run returns only after the + * backend crosses its launch fence (or terminates with an error), while + * poll/wait observe completion and finalize owns validation, copy-back, + * diagnostics, and Runtime destruction. The blocking run() overloads are + * the compatibility composition of these phases. A successful prepare + * transfers cleanup ownership to the caller: launch failure does not consume + * the token, and the caller must still finalize it. The blocking composition + * performs that cleanup internally on every exit. + * + * The current backend keeps execution-only state on DeviceRunner, so it + * admits one prepared/active native run at a time. The + * slot/lease-generation/process-unique-run-epoch token prevents a delayed + * phase call from touching reused storage, including another run under the + * same pipeline lease or on another ChipWorker. + */ + ChipWorkerNativeRun prepare_native_run( + int32_t callable_id, const ChipStorageTaskArgs *args, const CallConfig &config, const PipelineSlotLease &lease + ); + ChipWorkerNativeRun prepare_native_run( + int32_t callable_id, TaskArgsView args, const CallConfig &config, const PipelineSlotLease &lease + ); + void launch_native_run( + const ChipWorkerNativeRun &run, volatile int32_t *accepted_state = nullptr, int32_t accepted_value = 0 + ); + bool poll_native_run(const ChipWorkerNativeRun &run); + void wait_native_run(const ChipWorkerNativeRun &run); + void finalize_native_run(const ChipWorkerNativeRun &run); + // Per-callable_id preparation. Requires init() first and a callable_id // in [0, MAX_REGISTERED_CALLABLE_IDS) (cap 64). void register_callable(int32_t callable_id, const void *callable); @@ -164,9 +208,9 @@ class ChipWorker { unsigned pipeline_depth() const { return pipeline_contract_.pipeline_depth; } size_t runtime_slot_count() const { return runtime_bufs_.size(); } - /// Host Runtime staging buffer address of every copy the contract asked - /// for, in slot order. Two copies hold distinct storage; tests read this to - /// prove per-run buffers are not one buffer under two slot ids. + /// Opaque host native-run storage address for every slot the contract + /// asked for. Two slots hold distinct storage; tests read this to prove + /// per-run state is not one buffer under two slot ids. std::vector runtime_buffer_addrs() const; /// Committed GM heap base of one arena bank on the bound runner, or 0 when @@ -186,6 +230,7 @@ class ChipWorker { using CopyToDeviceCtxFn = int (*)(void *, void *, const void *, size_t); using CopyFromDeviceCtxFn = int (*)(void *, void *, const void *, size_t); using GetRuntimeSizeFn = size_t (*)(); + using GetRuntimeAlignmentFn = size_t (*)(); using GetCommittedDeviceMemoryFn = size_t (*)(void *); // From host_runtime.so. Single platform-side init that does (a) thread // attach + device-id record, (b) executor binary takeover, (c) onboard @@ -195,6 +240,8 @@ class ChipWorker { ); using SimplerRegisterCallableFn = int (*)(void *, int32_t, const void *); using SimplerRunFn = int (*)(void *, void *, int32_t, const void *, const CallConfig *); + using SimplerPrepareRunFn = int (*)(void *, void *, int32_t, const void *, const CallConfig *); + using SimplerNativeRunFn = int (*)(void *, void *); using SetTaskAcceptedStateFn = int (*)(void *, volatile int32_t *, int32_t); using SelectPipelineSlotFn = int (*)(void *, uint32_t); using SelectArenaBankFn = int (*)(void *, uint32_t); @@ -244,10 +291,16 @@ class ChipWorker { CopyToDeviceCtxFn copy_to_device_ctx_fn_ = nullptr; CopyFromDeviceCtxFn copy_from_device_ctx_fn_ = nullptr; GetRuntimeSizeFn get_runtime_size_fn_ = nullptr; + GetRuntimeAlignmentFn get_runtime_alignment_fn_ = nullptr; GetCommittedDeviceMemoryFn device_committed_memory_fn_ = nullptr; SimplerInitFn simpler_init_fn_ = nullptr; SimplerRegisterCallableFn register_callable_fn_ = nullptr; SimplerRunFn run_fn_ = nullptr; + SimplerPrepareRunFn prepare_run_fn_ = nullptr; + SimplerNativeRunFn launch_run_fn_ = nullptr; + SimplerNativeRunFn poll_run_fn_ = nullptr; + SimplerNativeRunFn wait_run_fn_ = nullptr; + SimplerNativeRunFn finalize_run_fn_ = nullptr; SetTaskAcceptedStateFn set_task_accepted_state_fn_ = nullptr; SelectPipelineSlotFn select_pipeline_slot_fn_ = nullptr; SelectArenaBankFn select_arena_bank_fn_ = nullptr; @@ -286,7 +339,42 @@ class ChipWorker { ); uint32_t select_slot_resources(uint32_t slot_id); - std::vector> runtime_bufs_; + enum class NativeRunPhase : uint8_t { EMPTY, PREPARED, LAUNCHED, REAPED }; + struct NativeRunSlotState { + uint64_t lease_generation{0}; + uint64_t run_epoch{0}; + NativeRunPhase phase{NativeRunPhase::EMPTY}; + int wait_rc{0}; + }; + ChipWorkerNativeRun prepare_native_run_on_slot( + int32_t callable_id, const ChipStorageTaskArgs *args, const CallConfig &config, uint32_t slot_id, + uint64_t generation + ); + NativeRunSlotState &require_native_run(const ChipWorkerNativeRun &run, NativeRunPhase first, NativeRunPhase second); + void cleanup_native_runs_noexcept() noexcept; + + class RuntimeStorage { + public: + RuntimeStorage() = default; + RuntimeStorage(size_t size, size_t alignment); + ~RuntimeStorage(); + RuntimeStorage(RuntimeStorage &&other) noexcept; + RuntimeStorage &operator=(RuntimeStorage &&other) noexcept; + RuntimeStorage(const RuntimeStorage &) = delete; + RuntimeStorage &operator=(const RuntimeStorage &) = delete; + + void *data() { return data_; } + const void *data() const { return data_; } + + private: + void *data_{nullptr}; + }; + + // Allocated once during init and never resized. Each allocation honors the + // runtime-reported alignment and keeps a stable C ABI address through + // prepare/finalize even if the owning vector itself is moved. + std::vector runtime_bufs_; + std::array native_run_states_{}; PipelineSlotGenerationFilter pipeline_generations_; PipelineContract pipeline_contract_{PTO_PIPELINE_CONTRACT_ABI_VERSION, 0, 1, {}}; // device_id_ is set once in init() and never modified afterward. All diff --git a/src/common/worker/native_run_launch_signal.h b/src/common/worker/native_run_launch_signal.h new file mode 100644 index 0000000000..ef17a46348 --- /dev/null +++ b/src/common/worker/native_run_launch_signal.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef SRC_COMMON_WORKER_NATIVE_RUN_LAUNCH_SIGNAL_H_ +#define SRC_COMMON_WORKER_NATIVE_RUN_LAUNCH_SIGNAL_H_ + +#include +#include + +/** Sticky one-shot wakeup for the host thread waiting on launch readiness. */ +class NativeRunLaunchSignal { +public: + NativeRunLaunchSignal() = default; + NativeRunLaunchSignal(const NativeRunLaunchSignal &) = delete; + NativeRunLaunchSignal &operator=(const NativeRunLaunchSignal &) = delete; + + void wait() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this]() { + return notified_; + }); + } + + void notify() { + { + std::lock_guard lock(mutex_); + notified_ = true; + } + cv_.notify_one(); + } + +private: + std::mutex mutex_; + std::condition_variable cv_; + bool notified_{false}; +}; + +#endif // SRC_COMMON_WORKER_NATIVE_RUN_LAUNCH_SIGNAL_H_ diff --git a/src/common/worker/native_run_state.h b/src/common/worker/native_run_state.h new file mode 100644 index 0000000000..f21283dd9f --- /dev/null +++ b/src/common/worker/native_run_state.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef SRC_COMMON_WORKER_NATIVE_RUN_STATE_H_ +#define SRC_COMMON_WORKER_NATIVE_RUN_STATE_H_ + +#include +#include +#include +#include + +#include "call_config.h" +#include "native_run_launch_signal.h" +#include "runtime.h" + +/** Internal phase of the caller-owned opaque native-run storage. */ +enum class NativeRunPhase : uint8_t { + Prepared, + Launching, + Running, + Complete, +}; + +/** + * Caller-owned state for one progressable native lifecycle. Runtime and + * CallConfig are per-run; Runner-owned streams, diagnostics, and timing require + * exclusive ownership from prepare through finalize. + */ +template +struct NativeRunState { + static constexpr uint64_t kMagic = UINT64_C(0x534d504c52554e31); // "SMPLRUN1" + + NativeRunState(Runner *runner_in, const CallConfig &config_in, uint64_t trace_hid_in) : + runner(runner_in), + config(config_in), + trace_hid(trace_hid_in) {} + + ~NativeRunState() { + if (executor.joinable()) executor.join(); + if (host_thread_state != nullptr) { + runner->destroy_native_run_thread_state(host_thread_state); + } + } + + /** Move prepare-thread state into the executor before runner->run(). */ + void adopt_host_thread_state() noexcept { + void *snapshot = host_thread_state; + host_thread_state = nullptr; + if (snapshot != nullptr) runner->adopt_native_run_thread_state(snapshot); + } + + uint64_t magic{kMagic}; + Runner *runner{nullptr}; + CallConfig config{}; + Runtime runtime{}; + uint64_t trace_hid{0}; + unsigned trace_inv{0}; + long long trace_start_ns{0}; + std::thread executor{}; + std::atomic execution_rc{-1}; + std::atomic execution_done{false}; + std::atomic phase{NativeRunPhase::Prepared}; + NativeRunLaunchSignal launch_signal{}; + void *host_thread_state{nullptr}; + bool runner_claimed{false}; +}; + +/** End object lifetime, then mark the caller-owned storage reusable. */ +template +void destroy_native_run_state(NativeRunState *state) { + void *storage = state; + state->~NativeRunState(); + constexpr uint64_t kEmpty = 0; + std::memcpy(storage, &kEmpty, sizeof(kEmpty)); +} + +#endif // SRC_COMMON_WORKER_NATIVE_RUN_STATE_H_ diff --git a/src/common/worker/pto_runtime_c_api.h b/src/common/worker/pto_runtime_c_api.h index 5b0a35902f..1e6757618d 100644 --- a/src/common/worker/pto_runtime_c_api.h +++ b/src/common/worker/pto_runtime_c_api.h @@ -9,7 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ /** - * PTO Runtime C API — canonical header + * simpler host-runtime C API — canonical header * * Declares all C-linkage functions exported by the host runtime .so. * Both the ChipWorker (consumer, resolves public symbols via dlsym) and the @@ -20,11 +20,14 @@ * stubs rather than omitting symbols): * - lifecycle: create_device_context, destroy_device_context, * simpler_init, finalize_device - * - sizing: get_runtime_size + * - sizing: get_runtime_size, get_runtime_alignment * - device-mem: device_malloc_ctx, device_free_ctx, * committed_device_memory_ctx, * copy_to_device_ctx, copy_from_device_ctx - * - prepared run: simpler_register_callable, simpler_run, unregister_callable, + * - prepared run: simpler_register_callable, simpler_prepare_run, + * simpler_launch_run, simpler_poll_run, simpler_wait_run, + * simpler_finalize_run, simpler_run, + * simpler_unregister_callable, * get_aicpu_dlopen_count, get_host_dlopen_count, * get_run_stream_set_create_count, * simpler_provision_dma_workspace @@ -36,8 +39,14 @@ * Optional metadata: * - pipeline: get_pipeline_contract * - * Memory management: caller allocates a buffer of get_runtime_size() bytes - * and passes it to simpler_run(). Error codes: 0 = success, negative = error. + * Native-run storage: caller allocates at least get_runtime_size() bytes with + * get_runtime_alignment() alignment, zero-initializes it before first use, and + * keeps its address stable from prepare through finalize. After finalize the + * same storage may be reused without re-zeroing. The storage must not be moved, + * copied, or used for overlapping runs while it contains a prepared run. The + * caller must serialize phase functions for a given context/storage pair; + * poll/wait/finalize are not concurrent operations on the same run. + * Error codes: 0 = success, negative = error. */ #ifndef SRC_COMMON_WORKER_PTO_RUNTIME_C_API_H_ @@ -65,6 +74,13 @@ enum { PTO_RUNTIME_ERR_UNSUPPORTED = -2, }; +/** Return values from simpler_poll_run(). */ +enum { + SIMPLER_NATIVE_RUN_POLL_ERROR = -1, + SIMPLER_NATIVE_RUN_POLL_NOT_READY = 0, + SIMPLER_NATIVE_RUN_POLL_COMPLETE = 1, +}; + enum { PTO_PIPELINE_CONTRACT_ABI_VERSION = 1, PTO_PIPELINE_MAX_RESOURCES = 8, @@ -159,13 +175,18 @@ DeviceContextHandle create_device_context(void); /** * Destroy a device context created by create_device_context(). - * Calls finalize internally, then frees the underlying object. + * The caller must finalize every prepared native run and call + * finalize_device() first. An active native run makes this operation log an + * error and leave the context alive; otherwise it frees the underlying object. */ void destroy_device_context(DeviceContextHandle ctx); -/** Return sizeof(Runtime) for caller buffer allocation. */ +/** Return the byte size of the opaque prepared-run storage. */ size_t get_runtime_size(void); +/** Return the required byte alignment of the opaque prepared-run storage. */ +size_t get_runtime_alignment(void); + /** Allocate device memory in the given device context. */ void *device_malloc_ctx(DeviceContextHandle ctx, size_t size); @@ -226,7 +247,8 @@ int simpler_init( /** * Release all device resources held by the context. - * Must be called before destroy_device_context() / dlclose(). + * Must be called before destroy_device_context() / dlclose(). Returns an error + * without teardown while a prepared native run remains unfinalized. */ int finalize_device(DeviceContextHandle ctx); @@ -241,7 +263,9 @@ int finalize_device(DeviceContextHandle ctx); * MAX_REGISTERED_CALLABLE_IDS in the AICPU executor) and rejects ids outside * `[0, 64)`. Lifetime: caller must `unregister_callable` before * `finalize_device` to release the device-side orch SO buffer; kernels stay - * resident until finalize regardless. + * resident until finalize regardless. Register and unregister mutate state + * referenced by a prepared run, so both are rejected from successful prepare + * until its matching finalize. * =========================================================================== */ /** @@ -256,6 +280,8 @@ int finalize_device(DeviceContextHandle ctx); * * `device_id` and the executor binaries are not threaded through this entry * — they were captured by `simpler_init` and live on the DeviceRunner. + * Callable-registry mutation is rejected while a native run is prepared or + * executing on this context. * * @return 0 on success, negative on error (NULL ctx, callable_id out of * range, upload/copy failure, or AICPU prewarm failure). @@ -285,8 +311,8 @@ int simpler_register_callable(DeviceContextHandle ctx, int32_t callable_id, cons * per-scope-depth-ring array of RUNTIME_ENV_RING_COUNT entries; 0 = unset, * precedence per ring: per-ring entry > PTO2_RING_* env var > compile-time * default). Ring overrides are consumed by tensormap_and_ringbuffer only; other - * runtime variants accept and ignore them. Wire-compatible POD; the platform - * reads it by pointer without copying. + * runtime variants accept and ignore them. Wire-compatible POD; prepare copies + * it into the native-run state before returning. * * @return 0 on success, negative on error (no prep state, NULL ctx/config, etc.). */ @@ -294,6 +320,41 @@ int simpler_run( DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config ); +/** + * Build and bind one run into caller-owned opaque storage without launching it. + * A successful prepare must be paired with simpler_finalize_run(), even when + * launch is abandoned or fails. The `args` container itself is consumed during + * prepare and need not remain alive afterward, but every tensor backing buffer + * referenced by it must remain valid through finalize (which may copy results + * back to those addresses). See the storage size/alignment/lifetime contract at + * the top of this header. + */ +int simpler_prepare_run( + DeviceContextHandle ctx, RuntimeHandle runtime, int32_t callable_id, const void *args, const CallConfig *config +); + +/** + * Launch a prepared run. Returns only after the platform has published its + * real kernel-launch marker, or after execution terminates before that marker. + */ +int simpler_launch_run(DeviceContextHandle ctx, RuntimeHandle runtime); + +/** + * Non-blocking completion query. Returns SIMPLER_NATIVE_RUN_POLL_NOT_READY, + * SIMPLER_NATIVE_RUN_POLL_COMPLETE, or a negative validation/phase error. Call + * only after simpler_launch_run() returns. + */ +int simpler_poll_run(DeviceContextHandle ctx, RuntimeHandle runtime); + +/** Wait for device execution to terminate. Does not release prepared resources. */ +int simpler_wait_run(DeviceContextHandle ctx, RuntimeHandle runtime); + +/** + * Wait if needed, validate/copy results, and release the opaque prepared run. + * Also safely aborts a run that was prepared but never launched. + */ +int simpler_finalize_run(DeviceContextHandle ctx, RuntimeHandle runtime); + /** Select the per-run/exec-handle slot used by the next synchronous run. */ int select_pipeline_slot_ctx(DeviceContextHandle ctx, uint32_t slot_id); @@ -335,6 +396,7 @@ int set_task_accepted_state_ctx(DeviceContextHandle ctx, volatile int32_t *state * `launch_device_register` triggers `dlclose` + reload), or at process * exit. Long-running processes that register / unregister cids without ever * reusing them will hold the AICPU SO handle until shutdown. + * Rejected while a native run is prepared or executing on this context. * * @return 0 on success or if callable_id was not registered, negative on error. */ diff --git a/tests/st/a2a3/host_build_graph/native_run_lifecycle/conftest.py b/tests/st/a2a3/host_build_graph/native_run_lifecycle/conftest.py new file mode 100644 index 0000000000..87959e1a8f --- /dev/null +++ b/tests/st/a2a3/host_build_graph/native_run_lifecycle/conftest.py @@ -0,0 +1,33 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Give the private native-run lifecycle test an empty L2 callable table.""" + +import pytest + + +@pytest.fixture(scope="class") +def st_worker(request, st_platform, device_pool): + cls = request.node.cls + if cls is None or not hasattr(cls, "_st_runtime"): + pytest.skip("isolated st_worker requires a SceneTestCase subclass") + + ids = device_pool.allocate(1) + if not ids: + pytest.fail("no devices available for isolated L2 worker") + try: + from simpler.worker import Worker # noqa: PLC0415 + + worker = Worker(level=2, device_id=ids[0], platform=st_platform, runtime=cls._st_runtime) + worker.init() + try: + yield worker + finally: + worker.close() + finally: + device_pool.release(ids) diff --git a/tests/st/a2a3/host_build_graph/native_run_lifecycle/kernels/orchestration/long_vector_orch.cpp b/tests/st/a2a3/host_build_graph/native_run_lifecycle/kernels/orchestration/long_vector_orch.cpp new file mode 100644 index 0000000000..d788c355b6 --- /dev/null +++ b/tests/st/a2a3/host_build_graph/native_run_lifecycle/kernels/orchestration/long_vector_orch.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include "pto_orchestration_api.h" // NOLINT(build/include_subdir) + +namespace { + +constexpr uint64_t kAdd = 0; +constexpr uint64_t kAddScalar = 1; +constexpr int kChainLength = 64; + +} // namespace + +extern "C" { + +__attribute__((visibility("default"))) PTO2OrchestrationConfig aicpu_orchestration_config(const L2TaskArgs &args) { + (void)args; + return PTO2OrchestrationConfig{.expected_arg_count = 3}; +} + +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const L2TaskArgs &args) { + const Tensor &a = args.tensor(0).ref(); + const Tensor &b = args.tensor(1).ref(); + const Tensor &out = args.tensor(2).ref(); + uint32_t shape[1] = {a.shapes[0]}; + TensorCreateInfo temporary(shape, 1, DataType::FLOAT32); + + L0TaskArgs add_args; + add_args.add_input(a); + add_args.add_input(b); + add_args.add_output(temporary); + TaskOutputTensors add_outputs = rt_submit_aiv_task(kAdd, add_args); + Tensor current = add_outputs.get_ref(0); + + union { + float f32; + uint64_t u64; + } scalar{}; + scalar.f32 = 1.0F; + for (int i = 0; i < kChainLength; ++i) { + L0TaskArgs step_args; + step_args.add_input(current); + if (i + 1 == kChainLength) { + step_args.add_output(out); + } else { + step_args.add_output(temporary); + } + step_args.add_scalar(scalar.u64); + TaskOutputTensors step_outputs = rt_submit_aiv_task(kAddScalar, step_args); + if (i + 1 != kChainLength) { + current = step_outputs.get_ref(0); + } + } +} + +} // extern "C" diff --git a/tests/st/a2a3/host_build_graph/native_run_lifecycle/test_native_run_lifecycle.py b/tests/st/a2a3/host_build_graph/native_run_lifecycle/test_native_run_lifecycle.py new file mode 100644 index 0000000000..ea2a35fc9f --- /dev/null +++ b/tests/st/a2a3/host_build_graph/native_run_lifecycle/test_native_run_lifecycle.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""End-to-end validation of the B3a prepare/launch/poll/wait/finalize seam.""" + +import pytest +import torch +from simpler.task_interface import ArgDirection as D + +from simpler_setup import SceneTestCase, TaskArgsBuilder, Tensor, scene_test +from simpler_setup.scene_test import _build_chip_task_args, _compare_outputs +from simpler_setup.tools.strace_timing import group_invocations, parse_spans + +_VECTOR_KERNELS = "../vector_example/kernels/aiv" +_SLOT = 0 +_GENERATION = 1 +_SIZE = 128 * 128 +_CHAIN_LENGTH = 64 + + +@scene_test(level=2, runtime="host_build_graph") +class TestNativeRunLifecycle(SceneTestCase): + CALLABLE = { + "orchestration": { + "source": "kernels/orchestration/long_vector_orch.cpp", + "function_name": "aicpu_orchestration_entry", + "signature": [D.IN, D.IN, D.OUT], + }, + "incores": [ + { + "func_id": 0, + "source": f"{_VECTOR_KERNELS}/kernel_add.cpp", + "core_type": "aiv", + "signature": [D.IN, D.IN, D.OUT], + }, + { + "func_id": 1, + "source": f"{_VECTOR_KERNELS}/kernel_add_scalar.cpp", + "core_type": "aiv", + "signature": [D.IN, D.OUT], + }, + ], + } + + CASES = [ + { + "name": "phase_split_preserves_blocking_compatibility", + "platforms": ["a2a3sim", "a2a3"], + "config": {"aicpu_thread_num": 4}, + "params": {"a": 2.0, "b": 3.0}, + } + ] + + def generate_args(self, params): + return TaskArgsBuilder( + Tensor("a", torch.full((_SIZE,), params["a"], dtype=torch.float32)), + Tensor("b", torch.full((_SIZE,), params["b"], dtype=torch.float32)), + Tensor("out", torch.zeros(_SIZE, dtype=torch.float32)), + ) + + def compute_golden(self, args, params): + args.out[:] = args.a + args.b + _CHAIN_LENGTH + + def test_run(self, st_platform, st_worker, request, capfd): + super().test_run(st_platform, st_worker, request) + + spans = list(parse_spans(capfd.readouterr().err.splitlines())) + invocations = [inv for inv in group_invocations(spans) if "simpler_run" in inv.by_name()] + assert len(invocations) == 3, "abandoned, direct, and blocking runs must each emit one trace invocation" + + common_depths = { + "simpler_run": 0, + "simpler_run.bind": 1, + "simpler_run.validate": 1, + } + launched_depths = { + **common_depths, + "simpler_run.runner_run": 1, + "simpler_run.runner_run.device_wall": 2, + } + launched_count = 0 + for invocation in invocations: + by_name = invocation.by_name() + expected_depths = launched_depths if "simpler_run.runner_run" in by_name else common_depths + launched_count += "simpler_run.runner_run" in by_name + assert expected_depths.keys() <= by_name.keys() + assert len({span.hid for span in invocation.spans}) == 1 + assert sum(span.name == "simpler_run" for span in invocation.spans) == 1 + for name, depth in expected_depths.items(): + assert by_name[name].depth == depth + + root = by_name["simpler_run"] + root_end = root.ts + root.dur + for name in expected_depths.keys() - {"simpler_run", "simpler_run.runner_run.device_wall"}: + stage = by_name[name] + assert root.ts <= stage.ts <= stage.ts + stage.dur <= root_end + assert launched_count == 2 + + def _run_and_validate_l2( # noqa: PLR0913 + self, + worker, + callable_obj, + case, + rounds=1, + skip_golden=False, + enable_l2_swimlane=False, + enable_dump_args=False, + enable_pmu=0, + enable_dep_gen=False, + enable_scope_stats=False, + output_prefix="", + ): + del rounds, skip_golden, enable_l2_swimlane, enable_dump_args + del enable_pmu, enable_dep_gen, enable_scope_stats, output_prefix + config = self._build_config(case["config"]) + chip_worker = worker._chip_worker + assert chip_worker is not None + chip_worker._register_callable_at_slot(_SLOT, callable_obj) + native_run = None + try: + test_args = self.generate_args(case["params"]) + chip_args, output_names = _build_chip_task_args(test_args, self.CALLABLE["orchestration"]["signature"]) + golden_args = test_args.clone() + self.compute_golden(golden_args, case["params"]) + + stream_count_before_prepare = chip_worker.run_stream_set_create_count + native_run = chip_worker._prepare_native_run_with_pipeline_lease( + _SLOT, chip_args, _SLOT, _GENERATION, config=config + ) + first_run = native_run + assert chip_worker.run_stream_set_create_count == stream_count_before_prepare + assert torch.count_nonzero(test_args.out) == 0, "prepare crossed the device launch fence" + with pytest.raises(RuntimeError, match="unfinished native run|owns the runner"): + chip_worker._prepare_native_run_with_pipeline_lease(_SLOT, chip_args, 1, _GENERATION, config=config) + with pytest.raises(RuntimeError, match="unregister_callable failed"): + chip_worker._unregister_slot(_SLOT) + with pytest.raises(RuntimeError, match="register_callable failed"): + chip_worker._register_callable_at_slot(1, callable_obj) + + # A prepared run can be abandoned explicitly. Finalize releases its + # claim and registry dependencies without launching or copying back. + chip_worker._finalize_native_run(native_run) + native_run = None + assert torch.count_nonzero(test_args.out) == 0 + chip_worker._unregister_slot(_SLOT) + chip_worker._register_callable_at_slot(_SLOT, callable_obj) + + test_args = self.generate_args(case["params"]) + chip_args, output_names = _build_chip_task_args(test_args, self.CALLABLE["orchestration"]["signature"]) + golden_args = test_args.clone() + self.compute_golden(golden_args, case["params"]) + native_run = chip_worker._prepare_native_run_with_pipeline_lease( + _SLOT, chip_args, _SLOT, _GENERATION, config=config + ) + assert first_run.generation == native_run.generation + assert first_run.run_epoch != native_run.run_epoch + with pytest.raises(RuntimeError, match="stale|wrong phase"): + chip_worker._launch_native_run(first_run) + chip_worker._launch_native_run(native_run) + chip_worker._wait_native_run(native_run) + assert chip_worker._poll_native_run(native_run) + chip_worker._finalize_native_run(native_run) + _compare_outputs(test_args, golden_args, output_names, self.RTOL, self.ATOL) + + with pytest.raises(RuntimeError, match="stale|finalized|wrong phase"): + chip_worker._poll_native_run(first_run) + native_run = None + + # The existing blocking surface remains the compatibility path. + second_args = self.generate_args(case["params"]) + second_chip_args, second_output_names = _build_chip_task_args( + second_args, self.CALLABLE["orchestration"]["signature"] + ) + second_golden = second_args.clone() + self.compute_golden(second_golden, case["params"]) + chip_worker._run_slot(_SLOT, second_chip_args, config=config) + _compare_outputs(second_args, second_golden, second_output_names, self.RTOL, self.ATOL) + finally: + if native_run is not None: + try: + chip_worker._finalize_native_run(native_run) + except Exception: + pass + chip_worker._unregister_slot(_SLOT) + + +if __name__ == "__main__": + SceneTestCase.run_module(__name__) diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 851a05f818..2e918ad29c 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -381,6 +381,17 @@ target_link_libraries(test_run_stream_slots PRIVATE ${GTEST_MAIN_LIB} ${GTEST_LI add_test(NAME test_run_stream_slots COMMAND test_run_stream_slots) set_tests_properties(test_run_stream_slots PROPERTIES LABELS "no_hardware") +# Native launch readiness is a sticky blocking handoff: notification may race +# either side of wait(), and the waiting host thread must sleep until signaled. +add_executable(test_native_run_launch_signal common/test_native_run_launch_signal.cpp) +target_include_directories(test_native_run_launch_signal PRIVATE + ${GTEST_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/../../../src/common/worker +) +target_link_libraries(test_native_run_launch_signal PRIVATE ${GTEST_MAIN_LIB} ${GTEST_LIB} pthread) +add_test(NAME test_native_run_launch_signal COMMAND test_native_run_launch_signal) +set_tests_properties(test_native_run_launch_signal PROPERTIES LABELS "no_hardware") + # --------------------------------------------------------------------------- # Types / task_interface tests (src/common/task_interface/) # --------------------------------------------------------------------------- @@ -669,11 +680,15 @@ add_a2a3_runtime_test(test_task_allocator a2a3/test_task_allocator.cpp) add_a2a3_runtime_test(test_scope_deadlock_detection common/test_scope_deadlock_detection.cpp) add_a2a3_hbg_runtime_test(test_hbg_task_allocator a2a3/test_task_allocator.cpp) add_a2a3_hbg_runtime_test(test_hbg_tensormap a2a3/test_hbg_tensormap.cpp) +add_a2a3_hbg_runtime_test(test_hbg_dep_gen_host_graph a2a3/test_dep_gen_host_graph.cpp) # PTO2TensorMap's out-of-line members (reserve_layout / init / valid_count) live # in this .cpp; no other add_a2a3_hbg_runtime_test target needs them. target_sources(test_hbg_tensormap PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/runtime/shared/pto_tensormap.cpp ) +target_sources(test_hbg_dep_gen_host_graph PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a2a3/runtime/host_build_graph/host/dep_gen_host_graph.cpp +) add_a2a3_runtime_test(test_dep_list_pool a2a3/test_dep_list_pool.cpp) add_a2a3_runtime_test(test_scheduler_state a2a3/test_scheduler_state.cpp) add_a2a3_runtime_test(test_task_state a2a3/test_task_state.cpp) diff --git a/tests/ut/cpp/a2a3/test_dep_gen_host_graph.cpp b/tests/ut/cpp/a2a3/test_dep_gen_host_graph.cpp new file mode 100644 index 0000000000..b97dd331bb --- /dev/null +++ b/tests/ut/cpp/a2a3/test_dep_gen_host_graph.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include + +#include + +#include "dep_gen_host_graph.h" + +namespace { + +std::filesystem::path output_path(const char *name) { + return std::filesystem::temp_directory_path() / + (std::string("simpler_dep_gen_") + name + "_" + std::to_string(::getpid()) + ".json"); +} + +void capture_task(uint64_t task_id, uint64_t predecessor = 0) { + const int32_t kernel_ids[3] = {1, -1, -1}; + dep_gen_host_graph_begin_task(task_id, false, false, kernel_ids, 1, 0, nullptr, nullptr); + if (predecessor != 0) dep_gen_host_graph_add_explicit_edge(predecessor); + dep_gen_host_graph_end_task(); +} + +std::string read_file(const std::filesystem::path &path) { + std::ifstream input(path); + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +} // namespace + +TEST(DepGenHostGraphTest, CaptureMovesFromPrepareThreadToExecutorThread) { + const std::filesystem::path path = output_path("handoff"); + std::filesystem::remove(path); + dep_gen_host_graph_set_enabled(true); + dep_gen_host_graph_begin_capture(); + capture_task(11); + capture_task(12, 11); + + void *capture = dep_gen_host_graph_take_capture(); + ASSERT_NE(capture, nullptr); + EXPECT_EQ(dep_gen_host_graph_emit(path.c_str()), -3); + + int emit_rc = -1; + std::thread executor([&]() { + dep_gen_host_graph_adopt_capture(capture); + emit_rc = dep_gen_host_graph_emit(path.c_str()); + }); + executor.join(); + + ASSERT_EQ(emit_rc, 0); + const std::string json = read_file(path); + EXPECT_NE(json.find("\"task_id\":\"11\""), std::string::npos); + EXPECT_NE(json.find("\"task_id\":\"12\""), std::string::npos); + EXPECT_NE(json.find("\"pred\":\"11\",\"succ\":\"12\""), std::string::npos); + std::filesystem::remove(path); +} + +TEST(DepGenHostGraphTest, DestroyedCaptureDoesNotContaminateNextRun) { + const std::filesystem::path path = output_path("abandoned"); + std::filesystem::remove(path); + dep_gen_host_graph_set_enabled(true); + dep_gen_host_graph_begin_capture(); + capture_task(101); + dep_gen_host_graph_destroy_capture(dep_gen_host_graph_take_capture()); + + dep_gen_host_graph_set_enabled(true); + dep_gen_host_graph_begin_capture(); + capture_task(202); + void *capture = dep_gen_host_graph_take_capture(); + ASSERT_NE(capture, nullptr); + dep_gen_host_graph_adopt_capture(capture); + ASSERT_EQ(dep_gen_host_graph_emit(path.c_str()), 0); + + const std::string json = read_file(path); + EXPECT_EQ(json.find("\"task_id\":\"101\""), std::string::npos); + EXPECT_NE(json.find("\"task_id\":\"202\""), std::string::npos); + std::filesystem::remove(path); +} diff --git a/tests/ut/cpp/common/test_native_run_launch_signal.cpp b/tests/ut/cpp/common/test_native_run_launch_signal.cpp new file mode 100644 index 0000000000..16e207af11 --- /dev/null +++ b/tests/ut/cpp/common/test_native_run_launch_signal.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include + +#include "native_run_launch_signal.h" + +TEST(NativeRunLaunchSignalTest, WaitBlocksWithoutConsumingCpu) { + NativeRunLaunchSignal signal; + std::promise waiter_started; + std::promise waiter_returned; + auto started = waiter_started.get_future(); + auto returned = waiter_returned.get_future(); + + std::thread waiter([&]() { + waiter_started.set_value(); + signal.wait(); + waiter_returned.set_value(); + }); + + started.wait(); + const std::clock_t cpu_start = std::clock(); + const std::future_status wait_status = returned.wait_for(std::chrono::milliseconds(200)); + const double cpu_seconds = static_cast(std::clock() - cpu_start) / CLOCKS_PER_SEC; + signal.notify(); + const std::future_status return_status = returned.wait_for(std::chrono::seconds(1)); + waiter.join(); + + EXPECT_EQ(wait_status, std::future_status::timeout); + EXPECT_EQ(return_status, std::future_status::ready); + EXPECT_LT(cpu_seconds, 0.05); +} + +TEST(NativeRunLaunchSignalTest, NotificationBeforeWaitIsRemembered) { + NativeRunLaunchSignal signal; + signal.notify(); + + auto waiter = std::async(std::launch::async, [&]() { + signal.wait(); + }); + EXPECT_EQ(waiter.wait_for(std::chrono::seconds(1)), std::future_status::ready); +} diff --git a/tests/ut/py/test_worker/test_host_worker.py b/tests/ut/py/test_worker/test_host_worker.py index 3dcc144ec9..7208526ada 100644 --- a/tests/ut/py/test_worker/test_host_worker.py +++ b/tests/ut/py/test_worker/test_host_worker.py @@ -4340,6 +4340,53 @@ def run_fanout() -> None: allow_rank_zero.set() runner.join(5.0) + def test_fanout_drain_uses_constant_stack_through_many_interruptions(self): + first_interrupt = KeyboardInterrupt("first phase interruption") + interruptions_remaining = 1_250 + + def interrupted_phase() -> None: + nonlocal interruptions_remaining + if interruptions_remaining <= 0: + return + interruptions_remaining -= 1 + if interruptions_remaining == 1_249: + raise first_interrupt + raise KeyboardInterrupt(f"phase interruption {interruptions_remaining}") + + cursor = worker_mod._ThreadFanoutDrainCursor(phases=(interrupted_phase,), after_phase=None) + fanout = worker_mod._ThreadFanout((), lambda _item: None, "test_constant_stack_", None, None) + + fanout._drain(cursor) + + assert cursor.exhausted + assert fanout._first_error is first_interrupt + + def test_abandoned_keepalive_drain_uses_constant_stack_through_many_interruptions(self): + first_interrupt = KeyboardInterrupt("first keepalive interruption") + + class InterruptingHandleList(list): + def __init__(self, *items): + super().__init__(items) + self.interruptions_remaining = 1_250 + + def __bool__(self): + if self.interruptions_remaining <= 0: + return len(self) != 0 + self.interruptions_remaining -= 1 + if self.interruptions_remaining == 1_249: + raise first_interrupt + raise KeyboardInterrupt(f"keepalive interruption {self.interruptions_remaining}") + + retained = SimpleNamespace(_keepalive=object()) + handles = InterruptingHandleList(retained) + cursor = worker_mod._AbandonedRunKeepaliveCursor(cast(Any, handles)) + + cursor.drain() + + assert cursor.first_error is first_interrupt + assert retained._keepalive is None + assert not handles + def test_domain_fanout_cancels_and_retries_an_ambiguously_launched_thread(self, monkeypatch): worker = self._worker() rank_zero_entered = threading.Event() diff --git a/tests/ut/py/test_worker/test_l3_l2_message_queue.py b/tests/ut/py/test_worker/test_l3_l2_message_queue.py index fbcd7a95c4..5e5302cceb 100644 --- a/tests/ut/py/test_worker/test_l3_l2_message_queue.py +++ b/tests/ut/py/test_worker/test_l3_l2_message_queue.py @@ -128,7 +128,7 @@ def __init__(self): self.fail_next_cmd: Optional[str] = None self.original_helpers: list[tuple[object, str, object]] = [] - def import_region(self, _token: str, mapping_bytes: int) -> int: + def import_region(self, _token: str, mapping_bytes: int, _owner_token: str) -> int: self.payload = bytearray(int(mapping_bytes)) self.counters = {} self.counter_mapping_offset = int(mapping_bytes) - L3L2_QUEUE_COUNTER_BYTES diff --git a/tests/ut/py/test_worker/test_l3_l2_orch_comm.py b/tests/ut/py/test_worker/test_l3_l2_orch_comm.py index c25c57a60e..7291c70b03 100644 --- a/tests/ut/py/test_worker/test_l3_l2_orch_comm.py +++ b/tests/ut/py/test_worker/test_l3_l2_orch_comm.py @@ -198,7 +198,7 @@ def test_sim_direct_region_uses_lifecycle_control_and_l3_host_metadata(monkeypat monkeypatch.setattr( worker_module, "_l3_host_mapped_region_import_sim", - lambda token, mapping_bytes: calls.append(("import", token, mapping_bytes)) or 99, + lambda token, mapping_bytes, owner_token: calls.append(("import", token, mapping_bytes, owner_token)) or 99, ) monkeypatch.setattr( l3_l2_orch_comm, @@ -235,7 +235,7 @@ def test_sim_direct_region_uses_lifecycle_control_and_l3_host_metadata(monkeypat assert l3_host_mapping is not None assert l3_host_mapping.handle != region.descriptor.payload_base assert l3_host_mapping.counter_offset == 64 - assert calls[0] == ("import", "sim-direct-1", 192) + assert calls[0] == ("import", "sim-direct-1", 192, worker._owner_id) assert calls[1][0:3] == ("write", 99, 0) assert calls[2][0:3] == ("read", 99, 8) assert calls[3] == ("test", 99, 128, 7, int(WaitCmp.EQ)) @@ -254,8 +254,8 @@ def test_onboard_direct_region_imports_vmm_shareable_handle_and_uses_l3_host_met monkeypatch.setattr( worker_module, "_l3_host_mapped_region_import_onboard", - lambda device_id, shareable_handle, mapping_bytes: calls.append( - ("import_onboard", device_id, shareable_handle, mapping_bytes) + lambda device_id, shareable_handle, mapping_bytes, owner_token: calls.append( + ("import_onboard", device_id, shareable_handle, mapping_bytes, owner_token) ) or 123, ) @@ -275,7 +275,7 @@ def test_onboard_direct_region_imports_vmm_shareable_handle_and_uses_l3_host_met assert l3_host_mapping is not None assert l3_host_mapping.access_profile == l3_l2_orch_comm.L3L2RegionAccessProfile.ONBOARD_VMM assert l3_host_mapping.counter_offset == 64 - assert calls[0] == ("import_onboard", 2, 0xABCDEF, 192) + assert calls[0] == ("import_onboard", 2, 0xABCDEF, 192, worker._owner_id) assert calls[1] == ("notify", 123, 128, 9, int(NotifyOp.Set)) finally: worker._close_l3_l2_orch_comm() @@ -289,7 +289,7 @@ def test_sim_direct_create_import_failure_rolls_back_l2_host_region(monkeypatch) monkeypatch.setattr( worker_module, "_l3_host_mapped_region_import_sim", - lambda _token, _mapping_bytes: (_ for _ in ()).throw(RuntimeError("import failed")), + lambda _token, _mapping_bytes, _owner_token: (_ for _ in ()).throw(RuntimeError("import failed")), ) with pytest.raises(RuntimeError, match="import failed"): @@ -383,7 +383,7 @@ def test_onboard_direct_mapping_allows_granularity_aligned_mapping(monkeypatch): region = worker._create_l3_l2_region(0, 64, 128) - assert calls == [(2, 0xABCDEF, 65536)] + assert calls == [(2, 0xABCDEF, 65536, worker._owner_id)] assert region._l3_host_mapping is not None assert region._l3_host_mapping.total_bytes == 192 finally: @@ -403,7 +403,7 @@ def append(self, item) -> None: resources = worker_module._RunResources() worker._building_run_resources = resources close_calls: list[int] = [] - monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size: 55) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size, _owner_token: 55) monkeypatch.setattr( l3_l2_orch_comm, "_l3_host_mapped_region_close", @@ -440,7 +440,7 @@ def append(self, item) -> None: worker, shm, fake_c_worker = _make_started_sim_worker() worker._live_l3_l2_regions = _AppendThenInterrupt() - monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size: 55) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size, _owner_token: 55) monkeypatch.setattr( l3_l2_orch_comm, "_l3_host_mapped_region_close", @@ -466,11 +466,23 @@ def append(self, item) -> None: def test_unadopted_native_mapping_cleanup_failure_poisons_worker(monkeypatch): worker, shm, fake_c_worker = _make_started_sim_worker() cleanup_errors = iter(("", "native owner cleanup failed")) - monkeypatch.setattr(worker_module, "_l3_host_mapped_region_take_cleanup_error", lambda: next(cleanup_errors)) + consumed_owner_tokens: list[str] = [] + acknowledgements: list[tuple[str, str]] = [] + + def peek_cleanup_error(owner_token: str) -> str: + consumed_owner_tokens.append(owner_token) + return next(cleanup_errors) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_peek_cleanup_error", peek_cleanup_error) + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_ack_cleanup_error", + lambda owner_token, observed: acknowledgements.append((owner_token, observed)), + ) monkeypatch.setattr( worker_module, "_l3_host_mapped_region_import_sim", - lambda _token, _size: (_ for _ in ()).throw(KeyboardInterrupt("interrupted native adoption")), + lambda _token, _size, _owner_token: (_ for _ in ()).throw(KeyboardInterrupt("interrupted native adoption")), ) try: @@ -479,6 +491,8 @@ def test_unadopted_native_mapping_cleanup_failure_poisons_worker(monkeypatch): assert isinstance(excinfo.value.__cause__, RuntimeError) assert "native owner cleanup failed" in str(excinfo.value.__cause__) + assert consumed_owner_tokens == [worker._owner_id, worker._owner_id] + assert acknowledgements == [(worker._owner_id, "native owner cleanup failed")] assert fake_c_worker.release_calls == [(0, 1)] with pytest.raises(RuntimeError, match="no further work is admitted"): worker._require_no_ordered_cleanup_failure("submit") @@ -488,6 +502,486 @@ def test_unadopted_native_mapping_cleanup_failure_poisons_worker(monkeypatch): shm.unlink() +def test_interrupted_cleanup_ack_happens_after_region_rollback(monkeypatch): + worker, shm, fake_c_worker = _make_started_sim_worker() + cleanup_errors = iter(("", "native owner cleanup failed")) + ack_interrupt = KeyboardInterrupt("interrupted cleanup acknowledgement") + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda _owner_token: next(cleanup_errors), + ) + + def interrupt_ack(_owner_token: str, _observed: str) -> None: + assert fake_c_worker.release_calls == [(0, 1)] + raise ack_interrupt + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", interrupt_ack) + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_import_sim", + lambda _token, _size, _owner_token: (_ for _ in ()).throw(KeyboardInterrupt("interrupted native adoption")), + ) + + try: + with pytest.raises(KeyboardInterrupt) as caught: + worker._create_l3_l2_region(0, 64, 128) + + assert caught.value is ack_interrupt + assert fake_c_worker.release_calls == [(0, 1)] + assert worker._ordered_cleanup_error is worker._l3_host_mapped_cleanup_error + finally: + worker._close_l3_l2_orch_comm() + shm.close() + shm.unlink() + + +def test_deferred_native_cleanup_error_only_poisons_owning_worker_on_admission(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + peer = Worker(level=3, num_sub_workers=0) + owner._lifecycle = worker_module._Lifecycle.READY + peer._lifecycle = worker_module._Lifecycle.READY + errors = {owner._owner_id: "owner mapping cleanup failed"} + consumed_owner_tokens: list[str] = [] + + def peek_cleanup_error(owner_token: str) -> str: + consumed_owner_tokens.append(owner_token) + return errors.get(owner_token, "") + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_peek_cleanup_error", peek_cleanup_error) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + with peer._operation_lease("submit"): + pass + + assert peer._ordered_cleanup_error is None + with pytest.raises(RuntimeError, match="no further work is admitted") as excinfo: + with owner._operation_lease("submit"): + pass + + assert isinstance(excinfo.value.__cause__, RuntimeError) + assert "owner mapping cleanup failed" in str(excinfo.value.__cause__.__cause__) + assert consumed_owner_tokens == [peer._owner_id, owner._owner_id] + + +def test_close_consumes_only_its_deferred_native_cleanup_error(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + peer = Worker(level=3, num_sub_workers=0) + errors = {owner._owner_id: "owner mapping cleanup failed"} + consumed_owner_tokens: list[str] = [] + + def peek_cleanup_error(owner_token: str) -> str: + consumed_owner_tokens.append(owner_token) + return errors.get(owner_token, "") + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_peek_cleanup_error", peek_cleanup_error) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + peer.close() + with pytest.raises(RuntimeError, match="native L3 Host mapping") as excinfo: + owner.close() + + assert "owner mapping cleanup failed" in str(excinfo.value.__cause__) + assert consumed_owner_tokens == [peer._owner_id, peer._owner_id, owner._owner_id, owner._owner_id] + + +def test_cleanup_error_survives_interrupted_peek_boundary(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + peer = Worker(level=3, num_sub_workers=0) + owner._lifecycle = worker_module._Lifecycle.READY + peer._lifecycle = worker_module._Lifecycle.READY + errors = {owner._owner_id: "owner mapping cleanup failed"} + interrupt = KeyboardInterrupt("interrupted native cleanup-error lookup") + interrupt_owner_once = True + + def peek_cleanup_error(owner_token: str) -> str: + nonlocal interrupt_owner_once + if owner_token == owner._owner_id and interrupt_owner_once: + interrupt_owner_once = False + raise interrupt + return errors.get(owner_token, "") + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_peek_cleanup_error", peek_cleanup_error) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + with pytest.raises(KeyboardInterrupt) as caught: + with owner._operation_lease("submit"): + pass + + assert caught.value is interrupt + assert owner._ordered_cleanup_error is None + assert errors == {owner._owner_id: "owner mapping cleanup failed"} + with peer._operation_lease("submit"): + pass + with pytest.raises(RuntimeError, match="no further work is admitted"): + with owner._operation_lease("submit"): + pass + assert owner._ordered_cleanup_error is owner._l3_host_mapped_cleanup_error + assert errors == {} + + +def test_cleanup_error_ack_interrupt_happens_after_poison_publication(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + owner._lifecycle = worker_module._Lifecycle.READY + errors = {owner._owner_id: "owner mapping cleanup failed"} + interrupt = KeyboardInterrupt("interrupted cleanup-error acknowledgement") + interrupt_ack_once = True + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + nonlocal interrupt_ack_once + if errors.get(owner_token) == observed: + errors.pop(owner_token) + if interrupt_ack_once: + interrupt_ack_once = False + raise interrupt + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + with pytest.raises(KeyboardInterrupt) as caught: + with owner._operation_lease("submit"): + pass + + assert caught.value is interrupt + assert owner._ordered_cleanup_error is owner._l3_host_mapped_cleanup_error + assert errors == {} + with pytest.raises(RuntimeError, match="no further work is admitted"): + with owner._operation_lease("submit"): + pass + + +def test_later_native_cleanup_error_is_retained_in_sticky_poison(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + owner._lifecycle = worker_module._Lifecycle.READY + errors: dict[str, str] = {owner._owner_id: "cleanup A"} + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + with pytest.raises(RuntimeError, match="no further work is admitted"): + with owner._operation_lease("submit"): + pass + + errors[owner._owner_id] = "cleanup B" + with pytest.raises(RuntimeError, match="no further work is admitted"): + with owner._operation_lease("submit"): + pass + + assert owner._l3_host_mapped_cleanup_error is not None + assert owner._l3_host_mapped_cleanup_error.__cause__ is not None + assert str(owner._l3_host_mapped_cleanup_error.__cause__) == "cleanup A; cleanup B" + assert errors == {} + + +def test_interrupted_ack_replay_does_not_duplicate_cleanup_detail(monkeypatch): + owner = Worker(level=3, num_sub_workers=0) + owner._lifecycle = worker_module._Lifecycle.READY + errors: dict[str, str] = {owner._owner_id: "cleanup A"} + interrupt = KeyboardInterrupt("interrupted before native acknowledgement") + interrupt_once = True + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + nonlocal interrupt_once + if interrupt_once: + interrupt_once = False + raise interrupt + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + with pytest.raises(KeyboardInterrupt) as caught: + with owner._operation_lease("submit"): + pass + assert caught.value is interrupt + + with pytest.raises(RuntimeError, match="no further work is admitted"): + with owner._operation_lease("submit"): + pass + + assert owner._l3_host_mapped_cleanup_error is not None + assert owner._l3_host_mapped_cleanup_error.__cause__ is not None + assert str(owner._l3_host_mapped_cleanup_error.__cause__) == "cleanup A" + assert errors == {} + + +def test_late_cleanup_error_after_successful_close_replays_stably(monkeypatch): + worker = Worker(level=3, num_sub_workers=0) + errors: dict[str, str] = {} + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + worker.close() + errors[worker._owner_id] = "late owner mapping cleanup failed" + + with pytest.raises(RuntimeError, match="native L3 Host mapping") as first: + worker.close() + with pytest.raises(RuntimeError) as replayed: + worker.close() + + assert replayed.value is first.value + assert errors == {} + + +def test_concurrent_close_publishes_joiner_cleanup_error_to_every_caller(monkeypatch): + worker = Worker(level=3, num_sub_workers=0) + worker._lifecycle = worker_module._Lifecycle.READY + worker._worker = cast(Any, object()) + worker._init_owner_thread = threading.current_thread() + errors: dict[str, str] = {} + teardown_started = threading.Event() + joiner_errors: list[BaseException] = [] + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + + def teardown_tree() -> None: + errors[worker._owner_id] = "joiner observed mapping cleanup failed" + teardown_started.set() + worker._worker = None + + monkeypatch.setattr(worker, "_teardown_ready_tree", teardown_tree) + + def join_close() -> None: + assert teardown_started.wait(5.0) + try: + worker.close() + except BaseException as exc: # noqa: BLE001 + joiner_errors.append(exc) + + joiner = threading.Thread(target=join_close) + joiner.start() + try: + with pytest.raises(RuntimeError, match="native L3 Host mapping") as owner_error: + worker.close() + finally: + joiner.join(5.0) + + assert not joiner.is_alive() + assert joiner_errors == [owner_error.value] + + +def test_cleanup_error_after_final_consume_waits_for_next_close_attempt(monkeypatch): + worker = Worker(level=3, num_sub_workers=0) + worker._lifecycle = worker_module._Lifecycle.READY + worker._worker = cast(Any, object()) + worker._init_owner_thread = threading.current_thread() + errors: dict[str, str] = {} + late_recorded = threading.Event() + joiner_waiting = threading.Event() + joiner_errors: list[BaseException] = [] + has_live_calls = 0 + + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + monkeypatch.setattr(worker, "_teardown_ready_tree", lambda: setattr(worker, "_worker", None)) + + def has_live_resources() -> bool: + nonlocal has_live_calls + has_live_calls += 1 + if has_live_calls == 1: + return True + if has_live_calls == 2: + errors[worker._owner_id] = "late owner mapping cleanup failed" + late_recorded.set() + assert joiner_waiting.wait(5.0) + return False + + monkeypatch.setattr(worker, "_has_live_resources", has_live_resources) + real_close_wait = worker._hierarchical_start_cv.wait + joiner: threading.Thread + + def close_wait(timeout=None): + if threading.current_thread() is joiner: + joiner_waiting.set() + return real_close_wait(timeout=timeout) + + monkeypatch.setattr(worker._hierarchical_start_cv, "wait", close_wait) + + def join_close() -> None: + assert late_recorded.wait(5.0) + try: + worker.close() + except BaseException as exc: # noqa: BLE001 + joiner_errors.append(exc) + + joiner = threading.Thread(target=join_close) + joiner.start() + try: + worker.close() + finally: + joiner.join(5.0) + + assert not joiner.is_alive() + assert joiner_errors == [] + assert worker._close_completion is not None + assert worker._close_completion.error is None + assert errors == {worker._owner_id: "late owner mapping cleanup failed"} + + with pytest.raises(RuntimeError, match="native L3 Host mapping") as first: + worker.close() + with pytest.raises(RuntimeError) as replayed: + worker.close() + assert replayed.value is first.value + + +def test_wrong_thread_close_does_not_consume_owner_cleanup_error(monkeypatch): + worker = Worker(level=3, num_sub_workers=0) + worker._lifecycle = worker_module._Lifecycle.READY + worker._worker = cast(Any, object()) + worker._init_owner_thread = threading.current_thread() + errors = {worker._owner_id: "owner mapping cleanup failed"} + peeked: list[str] = [] + foreign_errors: list[BaseException] = [] + + def peek_cleanup_error(owner_token: str) -> str: + peeked.append(owner_token) + return errors.get(owner_token, "") + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_peek_cleanup_error", peek_cleanup_error) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + monkeypatch.setattr(worker, "_teardown_ready_tree", lambda: setattr(worker, "_worker", None)) + + def close_from_foreign_thread() -> None: + try: + worker.close() + except BaseException as exc: # noqa: BLE001 + foreign_errors.append(exc) + + foreign = threading.Thread(target=close_from_foreign_thread) + foreign.start() + foreign.join(5.0) + + assert len(foreign_errors) == 1 + assert "thread that init()'d it" in str(foreign_errors[0]) + assert peeked == [] + assert errors == {worker._owner_id: "owner mapping cleanup failed"} + with pytest.raises(RuntimeError, match="native L3 Host mapping"): + worker.close() + assert errors == {} + + +def test_cleanup_error_survives_close_drain_timeout_retry(monkeypatch): + worker = Worker(level=3, num_sub_workers=0) + worker._lifecycle = worker_module._Lifecycle.READY + worker._worker = cast(Any, object()) + worker._init_owner_thread = threading.current_thread() + worker._active_ops = 1 + errors = {worker._owner_id: "owner mapping cleanup failed"} + + monkeypatch.setattr(worker_module, "_ROLLBACK_GRACEFUL_TIMEOUT_S", 0.001) + monkeypatch.setattr( + worker_module, + "_l3_host_mapped_region_peek_cleanup_error", + lambda owner_token: errors.get(owner_token, ""), + ) + + def acknowledge_cleanup_error(owner_token: str, observed: str) -> None: + if errors.get(owner_token) == observed: + errors.pop(owner_token) + + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_ack_cleanup_error", acknowledge_cleanup_error) + monkeypatch.setattr(worker, "_teardown_ready_tree", lambda: setattr(worker, "_worker", None)) + + with pytest.raises(TimeoutError): + worker.close() + assert worker._l3_host_mapped_cleanup_error is not None + assert errors == {} + + worker._active_ops = 0 + with pytest.raises(RuntimeError, match="native L3 Host mapping") as retry: + worker.close() + with pytest.raises(RuntimeError) as replayed: + worker.close() + + assert replayed.value is retry.value + + +def test_native_mapping_cleanup_errors_are_keyed_by_owner_token(): + owner_token = "owner-a" + peer_token = "owner-b" + _task_interface_ext._l3_host_mapped_region_take_cleanup_error(owner_token) + _task_interface_ext._l3_host_mapped_region_take_cleanup_error(peer_token) + + _task_interface_ext._l3_host_mapped_region_record_cleanup_error_for_test( + owner_token, "owner mapping cleanup failed" + ) + + assert _task_interface_ext._l3_host_mapped_region_peek_cleanup_error(peer_token) == "" + observed = _task_interface_ext._l3_host_mapped_region_peek_cleanup_error(owner_token) + assert observed == "owner mapping cleanup failed" + _task_interface_ext._l3_host_mapped_region_record_cleanup_error_for_test(owner_token, "later cleanup failed") + _task_interface_ext._l3_host_mapped_region_ack_cleanup_error(owner_token, observed) + assert _task_interface_ext._l3_host_mapped_region_peek_cleanup_error(owner_token) == "later cleanup failed" + _task_interface_ext._l3_host_mapped_region_ack_cleanup_error(owner_token, "later cleanup failed") + assert _task_interface_ext._l3_host_mapped_region_take_cleanup_error(owner_token) == "" + + def test_onboard_region_create_handler_uses_named_export_fields(monkeypatch): req_shm = SharedMemory(create=True, size=l3_l2_orch_comm._REGION_CREATE_REQUEST_BYTES) reply_shm = SharedMemory(create=True, size=l3_l2_orch_comm._REGION_CREATE_REPLY_BYTES) @@ -586,7 +1080,7 @@ def test_l3_host_mapped_counter_wait_releases_gil_for_python_notifier(): shm = SharedMemory(create=True, size=64) handle = 0 try: - owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64) + owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64, "counter-wait-test") handle = int(owner) def notify() -> None: @@ -612,7 +1106,7 @@ def test_l3_host_mapped_sim_payload_and_counter_helpers_roundtrip(): shm = SharedMemory(create=True, size=128) handle = 0 try: - owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 128) + owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 128, "roundtrip-test") handle = int(owner) src_t = ctypes.c_uint8 * 8 src = src_t(*range(10, 18)) @@ -648,7 +1142,7 @@ def test_l3_host_mapped_region_close_makes_sim_handle_unusable(): shm = SharedMemory(create=True, size=64) handle = 0 try: - owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64) + owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64, "closed-handle-test") handle = int(owner) _task_interface_ext._l3_host_mapped_region_close(handle) @@ -665,7 +1159,7 @@ def test_l3_host_mapped_import_owner_closes_unadopted_mapping(): shm = SharedMemory(create=True, size=64) raw_handle = 0 try: - owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64) + owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64, "unadopted-owner-test") raw_handle = int(owner) del owner gc.collect() @@ -685,6 +1179,7 @@ def test_sim_import_registry_failure_releases_pre_registry_mapping(): shm = SharedMemory(create=True, size=64) shm_token = shm.name.lstrip("/") + owner_token = "registry-failure-test" def mapped_resource_counts() -> tuple[int, int]: fd_count = 0 @@ -700,15 +1195,15 @@ def mapped_resource_counts() -> tuple[int, int]: try: baseline = mapped_resource_counts() - _task_interface_ext._l3_host_mapped_region_take_cleanup_error() + _task_interface_ext._l3_host_mapped_region_take_cleanup_error(owner_token) _task_interface_ext._l3_host_mapped_region_fail_next_registry_insert_for_test() with pytest.raises(RuntimeError, match="injected mapped-region registry insertion failure"): - _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64) + _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64, owner_token) gc.collect() assert mapped_resource_counts() == baseline - assert _task_interface_ext._l3_host_mapped_region_take_cleanup_error() == "" + assert _task_interface_ext._l3_host_mapped_region_take_cleanup_error(owner_token) == "" finally: shm.close() shm.unlink() @@ -718,7 +1213,7 @@ def test_l3_host_mapped_concurrent_closes_wait_for_in_flight_counter_wait(): shm = SharedMemory(create=True, size=64) handle = 0 try: - owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64) + owner = _task_interface_ext._l3_host_mapped_region_import_sim(shm.name, 64, "concurrent-close-test") handle = int(owner) close_entered = [threading.Event(), threading.Event()] close_done = [threading.Event(), threading.Event()] @@ -761,7 +1256,7 @@ def close_mapping(index: int) -> None: def test_sim_direct_transfer_failure_poisons_only_region(monkeypatch): worker, shm, _fake_c_worker = _make_started_sim_worker() try: - monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size: 55) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size, _owner_token: 55) monkeypatch.setattr( l3_l2_orch_comm, "_l3_host_mapped_payload_write", @@ -791,7 +1286,7 @@ def release(worker_id: int, region_id: int) -> None: try: fake_c_worker.control_l3_l2_region_release = release - monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size: 77) + monkeypatch.setattr(worker_module, "_l3_host_mapped_region_import_sim", lambda _token, _size, _owner_token: 77) monkeypatch.setattr( l3_l2_orch_comm, "_l3_host_mapped_region_close",