diff --git a/docs/investigations/2026-07-a2a3-sdma-fault-teardown.md b/docs/investigations/2026-07-a2a3-sdma-fault-teardown.md index adada44d93..a9f4a27a3d 100644 --- a/docs/investigations/2026-07-a2a3-sdma-fault-teardown.md +++ b/docs/investigations/2026-07-a2a3-sdma-fault-teardown.md @@ -6,6 +6,63 @@ risk and ordinary Workers are unaffected; full recovery inside an SDMA-enabled Worker after a fault is deferred pending a CANN runtime-and-driver fix +## 2026-08 package-specific follow-up + +PR #1664 adds a narrower containment for the installed CANN 9.0.0 and driver +26.0.rc1 combination. Only run streams belonging to a Worker that actually +provisioned the SDMA workspace use stop-on-failure mode. Fatal close on such a +Worker then waits for `max(10 seconds, configured op timeout + 5 seconds)`, +attempts one force reset, and forgets the failed generation's host handles +without issuing per-resource destroy/free calls. + +This is an empirically validated workaround, not an SDMA retirement contract. +CANN still exposes neither a completion fence nor a portable upper bound for +the final CP-process stream release. Ordinary a2a3 Workers and all a5 Workers +therefore keep normal stream mode, their existing error/diagnostic behavior, +and no added handoff delay. The hardware regression test keeps the +package-specific workaround visible by covering real SDMA provisioning followed +by an AICore fault and a bounded `Worker.close()`. + +### Reset attempts are budgeted per stream population, not globally + +`force_reset_device()` drains the card before resetting it and returns 0 only +when its post-reset probe confirms a usable generation, so a second call runs +against a settled card and can recover a poison the first could not. Ordinary +poison therefore keeps a bounded three-attempt budget on both a2a3 and a5. + +A Worker holding the 48 CP-process SDMA streams is the exception and gets a +single attempt: there a reset that does not confirm has already blocked on the +driver's 150/300-second remote-event timeout, so a retry multiplies that wait +without adding a completion condition. + +Collapsing both populations onto a single attempt regresses ordinary +fault-injection recovery — the poisoned card stays poisoned for the next +process that lands on it, which surfaces as unrelated tests failing with +507018/507046 on the devices a fault-injection test just used. + +### Emergency shutdown broadcasts in parallel but still joins the cores + +The AICore exit acknowledgement is what leaves a card usable for the next +process: only after a core confirms it stopped does the AICPU quiesce that +core's register block, putting the dispatch register back to idle and closing +the fast path. Returning from emergency shutdown while cores are still running +leaves the card poisoned past the host's device reset, and the next process on +that device fails at launch. + +So the fatal path signals every handshake'd core first and joins them second, +rather than signalling and waiting one core at a time. Cores drain +concurrently and the per-core quiesce is preserved. + +The join takes **one deadline for the whole group**, not one timeout per core. +That distinction is load-bearing on the fatal path, where every core is +typically dead: the onboard deinit timeout is 1 second, so a per-core deadline +costs a second per unresponsive core and pushes an AICore-timeout run past a +10-second budget, while a shared deadline caps the whole group at one second. + +The wait is an on-device poll of the core's `COND` register, so it costs no +host or remote operation and is unrelated to the CANN SDMA teardown problem +above. + ## Question Can simpler keep PTO-ISA async SDMA available while avoiding the roughly diff --git a/src/a2a3/platform/include/aicpu/platform_regs.h b/src/a2a3/platform/include/aicpu/platform_regs.h index 00cc51c45c..f2ab3f493f 100644 --- a/src/a2a3/platform/include/aicpu/platform_regs.h +++ b/src/a2a3/platform/include/aicpu/platform_regs.h @@ -139,6 +139,30 @@ inline void write_reg(uint64_t reg_base_addr, RegId reg, uint64_t value) { */ void platform_init_aicore_regs(uint64_t reg_addr); +/** Send the AICore exit signal without waiting for an acknowledgement. */ +void platform_signal_aicore_exit(uint64_t reg_addr); + +/** + * Absolute sys-cnt value one deinit timeout from now. Share a single deadline + * across a group of cores so the whole group costs one timeout rather than one + * per unresponsive core. + */ +uint64_t platform_aicore_exit_deadline(); + +/** + * Wait for a signalled AICore to acknowledge exit, then quiesce its register + * block (dispatch register back to idle, fast path closed). + * + * Pairs with platform_signal_aicore_exit when stopping several cores: signal + * them all, take one deadline from platform_aicore_exit_deadline(), then finish + * each core against that shared deadline. + * + * @param deadline absolute sys-cnt value to give up at. + * @return 0 once the core acknowledges, -1 on timeout — an unresponsive core is + * left for the host's device reset to clear. + */ +int32_t platform_finish_aicore_exit(uint64_t reg_addr, uint64_t deadline); + /** * Deinitialize AICore registers before termination * diff --git a/src/a2a3/platform/onboard/host/device_runner.cpp b/src/a2a3/platform/onboard/host/device_runner.cpp index 3c07181174..609aace237 100644 --- a/src/a2a3/platform/onboard/host/device_runner.cpp +++ b/src/a2a3/platform/onboard/host/device_runner.cpp @@ -22,11 +22,14 @@ #include +#include #include +#include #include #include #include #include +#include #include #include "acl/acl.h" #include "host/acl_error_log.h" @@ -43,6 +46,7 @@ #include "utils/elf_build_id.h" #include "host/host_regs.h" // Register address retrieval #include "host/raii_scope_guard.h" +#include "utils/fatal_shutdown_latch.h" // dep_gen has two shapes, one per orchestration site, and each runtime provides // the strong symbols for the one it uses: @@ -525,26 +529,70 @@ int DeviceRunner::drain_run(uint32_t pipeline_slot) { void DeviceRunner::cleanup_active_run(bool retire_aicore) noexcept { if (!active_run_.owns_resources) return; + // A poisoned card cannot retire per-resource frees or stream destroys, and + // issuing them can block in the driver. Drop host-side ownership instead; + // finalize()'s force reset invalidates the whole device generation. + const bool abandon = device_unusable_.load(std::memory_order_acquire); + // Collectors must stop before their backing arguments are released; the // per-run stream retires last. Each cleanup operation is idempotent. - finalize_collectors(); - (void)kernel_args_.finalize_device_kernel_args(); - (void)kernel_args_.finalize_runtime_args(); + finalize_collectors(abandon); + if (abandon) { + kernel_args_.abandon_after_device_failure(); + } else { + (void)kernel_args_.finalize_device_kernel_args(); + (void)kernel_args_.finalize_runtime_args(); + } if (kernel_args_.args.pmu_reg_addrs != 0) { - (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.pmu_reg_addrs)); + if (!abandon) { + (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.pmu_reg_addrs)); + } kernel_args_.args.pmu_reg_addrs = 0; } if (kernel_args_.args.regs != 0) { - (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.regs)); + if (!abandon) { + (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.regs)); + } kernel_args_.args.regs = 0; } - if (retire_aicore && !active_run_.aicore_retirement_attempted) { + if (retire_aicore && !abandon && !active_run_.aicore_retirement_attempted) { active_run_.aicore_retirement_attempted = true; (void)retire_run_aicore_stream(active_run_.slot, RunStreamSlots::CompletionStatus::Unproven); } active_run_.owns_resources = false; } +int DeviceRunner::create_run_stream(void **out) { + if (out == nullptr) return -1; + *out = nullptr; + + rtStream_t stream = nullptr; + int rc = rtStreamCreate(&stream, 0); + if (rc != 0) { + LOG_ERROR("rtStreamCreate (run stream) failed: %d", rc); + ACL_LOG_ERROR_DETAIL(rc); + return rc; + } + + // CANN 9.0.0 + driver 26.0.rc1 needs the host to observe an SDMA-generation + // AICore fault before the device reaches DEV_RUNNING_DOWN; otherwise reset + // walks the 48 CP-process streams and blocks in a 300-second remote event. + // Keep this workaround strictly on Workers that actually provisioned SDMA. + // Ordinary a2a3 and every a5 stream retain their normal error/diagnostic + // contract (for example 507046 rather than the early 507015). + if (dma_workspace_handle_ != nullptr) { + aclError acl_rc = aclrtSetStreamFailureMode(stream, ACL_STOP_ON_FAILURE); + if (acl_rc != ACL_SUCCESS) { + LOG_ERROR("aclrtSetStreamFailureMode (SDMA run stream) failed: %d", static_cast(acl_rc)); + (void)rtStreamDestroy(stream); + return static_cast(acl_rc); + } + } + + *out = stream; + return 0; +} + int DeviceRunner::ensure_run_stream_set(unsigned slot) { int rc = run_stream_slots_.acquire(slot); if (rc != 0) { @@ -948,6 +996,86 @@ int DeviceRunner::finalize() { return 0; } + // Fatal path: the ordinary stream completion/error boundary has already + // reaped the submitted run. Stop host collector threads locally, drain and + // force-reset the card, then forget old handles without per-resource + // RTS/HAL calls. + if (device_unusable_.load(std::memory_order_acquire)) { + // A Worker that provisioned SDMA holds 48 CP-process streams, which is + // what makes both the handoff delay and the single reset attempt below + // necessary. Read before abandon_common_after_device_failure() clears + // the handle. + const bool sdma_provisioned = dma_workspace_handle_ != nullptr; + + // CANN exposes no retirement fence for its CP-process SDMA streams. + // On the package named above, repeated hardware A/B runs established + // this handoff delay as a containment workaround, not a portable + // runtime guarantee. Do not delay ordinary (non-SDMA) failures. + if (sdma_provisioned) { + constexpr uint64_t kCANN900FatalHandoffMs = 10000; + constexpr uint64_t kDeviceDownSettleMarginMs = 5000; + const uint64_t op_timeout_ms = (timeout_config_.op_execute_timeout_us + 999) / 1000; + const uint64_t configured_handoff_ms = op_timeout_ms + kDeviceDownSettleMarginMs; + const uint64_t reset_delay_ms = std::max(kCANN900FatalHandoffMs, configured_handoff_ms); + LOG_WARN( + "SDMA fatal teardown workaround (CANN 9.0.0/driver 26.0.rc1): " + "waiting %llu ms before force reset; no runtime retirement fence is available", + static_cast(reset_delay_ms) + ); + std::this_thread::sleep_for(std::chrono::milliseconds(reset_delay_ms)); + } + + finalize_collectors(true); + + // force_reset_device() drains before it resets and returns 0 only when + // its post-reset probe confirms the card, so a second pass runs against + // a settled card and can recover a poison the first pass could not + // (verified on a2a3). An SDMA-provisioned card gets a single attempt: + // there a non-confirming reset already blocks on the driver's + // remote-event timeout, which a retry only multiplies. + constexpr int kFatalResetAttempts = 3; + int reset_rc = attempt_fatal_reset( + [this]() { + return force_reset_device(); + }, + sdma_provisioned ? 1 : kFatalResetAttempts + ); + const bool reset_confirmed = reset_rc == 0; + if (!reset_confirmed) { + LOG_ERROR( + "Fatal teardown: force reset of device %d did not confirm clean (rc=%d); " + "quarantining old handles without per-resource RTS calls", + device_id_, reset_rc + ); + } + + run_stream_slots_.abandon_all(); + int abandon_rc = abandon_common_after_device_failure(); + + // Only finalize the ACL owner after force reset established a clean + // generation. On reset failure, aclFinalize may itself walk poisoned + // stream resources and enter the same remote-event timeout. + if (acl_ready_) { + if (reset_confirmed) { + int finalize_rc = aclFinalize(); + if (finalize_rc != 0) { + LOG_ERROR("aclFinalize failed during fatal finalize: %d", finalize_rc); + if (abandon_rc == 0) abandon_rc = finalize_rc; + } + } else { + LOG_WARN("Fatal teardown: skipping aclFinalize because device reset was not confirmed"); + } + acl_ready_ = false; + } + + device_id_ = -1; + if (reset_confirmed) { + device_unusable_.store(false, std::memory_order_release); + } + LOG_WARN("DeviceRunner finalized after fatal device failure"); + return abandon_rc != 0 ? abandon_rc : reset_rc; + } + int rc = attach_current_thread(device_id_); if (rc != 0) { LOG_ERROR("Failed to attach finalize thread to device %d: %d", device_id_, rc); @@ -1007,57 +1135,11 @@ int DeviceRunner::finalize() { } } - // On the poison path the soft reset above does NOT clear the op-timeout - // sticky-error — a fresh in-process Worker.init then fails at rtStreamCreate - // 507899. A FORCE reset clears it, so the next Worker on this card inits - // clean in the SAME process and the remaining tests run instead of cascading - // / being skipped. Only reached on the (rare) device-poison path; onboard - // work always holds an exclusive task-submit lock on the card (enforced by - // .claude/rules/running-onboard.md), and the reset scopes to this card alone, - // so it cannot disturb other devices/users. - int reset_rc = 0; - if (device_unusable_.load(std::memory_order_acquire)) { - // Bounded retry: a single force reset normally clears the op-timeout - // sticky-error (verified 5/5 on a2a3), but the poison occasionally needs - // a drain-then-reset cycle, so retry up to kMaxResetAttempts. - // force_reset_device() drains (best-effort) and resets inside its own - // ACL/bound scope, and returns 0 only when its post-reset probe confirms - // the card is actually clean. - constexpr int kMaxResetAttempts = 3; - for (int attempt = 1; attempt <= kMaxResetAttempts; ++attempt) { - reset_rc = force_reset_device(); - if (reset_rc == 0) { - if (attempt > 1) { - LOG_WARN( - "DeviceRunner finalize: device %d recovered on force-reset attempt %d/%d", device_id_, attempt, - kMaxResetAttempts - ); - } - break; - } - LOG_ERROR( - "DeviceRunner finalize: force-reset attempt %d/%d of device %d did not confirm clean (rc=%d)", attempt, - kMaxResetAttempts, device_id_, reset_rc - ); - } - if (reset_rc != 0) { - LOG_ERROR( - "DeviceRunner finalize: device %d still poisoned after %d force-reset attempts; leaving it marked " - "unusable so the layer above (st_worker poison-skip + dispatcher retry) recovers it.", - device_id_, kMaxResetAttempts - ); - } - } - + // Only the healthy path reaches here: a poisoned card returned from the + // fatal branch at the top of finalize(), which owns the force reset. device_id_ = -1; - // Clear the poison flag only if the force reset actually recovered the card, - // so a still-poisoned card stays flagged: a reused DeviceRunner then fails - // admission instead of being treated as clean. On the normal (not - // unusable) path reset_rc stays 0 and the flag is already false. - if (reset_rc == 0) { - device_unusable_.store(false, std::memory_order_release); - } - return rc != 0 ? rc : reset_rc; + device_unusable_.store(false, std::memory_order_release); + return rc; } // `launch_aicpu_kernel` and `launch_aicore_kernel` live on `DeviceRunnerBase`. @@ -1230,15 +1312,20 @@ int DeviceRunner::init_scope_stats(int num_threads, int device_id) { return 0; } -void DeviceRunner::finalize_collectors() { - auto unregister_cb = [](void *dev_ptr, int device_id) -> int { +void DeviceRunner::finalize_collectors(bool abandon_device_resources) { + auto healthy_unregister_cb = [](void *dev_ptr, int device_id) -> int { HalHostUnregisterFn fn = get_halHostUnregister(); if (fn != nullptr) { return fn(dev_ptr, device_id); } return 0; }; - auto free_cb = [this](void *dev_ptr) -> int { + auto no_op_unregister_cb = [](void *, int) -> int { + return 0; + }; + auto unregister_cb = abandon_device_resources ? no_op_unregister_cb : healthy_unregister_cb; + auto free_cb = [this, abandon_device_resources](void *dev_ptr) -> int { + if (abandon_device_resources) return 0; return mem_alloc_.free(dev_ptr); }; diff --git a/src/a2a3/platform/onboard/host/device_runner.h b/src/a2a3/platform/onboard/host/device_runner.h index f8152cf2b4..3bdd66c243 100644 --- a/src/a2a3/platform/onboard/host/device_runner.h +++ b/src/a2a3/platform/onboard/host/device_runner.h @@ -232,13 +232,14 @@ class DeviceRunner : public DeviceRunnerBase { // fresh AICore stream per run, handle kept when a destroy fails — is // testable without a device. RunStreamSlots run_stream_slots_{ - [](void **out) { - return rtStreamCreate(reinterpret_cast(out), 0); + [this](void **out) { + return create_run_stream(out); }, [](void *stream) { return rtStreamDestroy(static_cast(stream)); } }; + int create_run_stream(void **out); int ensure_run_stream_set(unsigned slot); // Destroys this run's AICore stream. Returns the driver's error and KEEPS // the handle when the destroy fails: the stream may still hold the previous @@ -359,7 +360,7 @@ class DeviceRunner : public DeviceRunnerBase { * collectors in a pristine, re-initializable state) and from finalize() * as a backstop before mem_alloc_.finalize(). */ - void finalize_collectors(); + void finalize_collectors(bool abandon_device_resources = false); // Shared enable flags (`enable_chip_swimlane_`, `enable_dump_args_`, // `enable_pmu_`, `enable_scope_stats_`, `chip_swimlane_level_`, // `pmu_event_type_`, `output_prefix_`) live on `DeviceRunnerBase`. diff --git a/src/a2a3/platform/shared/aicpu/platform_regs.cpp b/src/a2a3/platform/shared/aicpu/platform_regs.cpp index 2f68a8b8c2..517425a3e0 100644 --- a/src/a2a3/platform/shared/aicpu/platform_regs.cpp +++ b/src/a2a3/platform/shared/aicpu/platform_regs.cpp @@ -62,19 +62,16 @@ void platform_init_aicore_regs(uint64_t reg_addr) { write_reg(reg_addr, RegId::DATA_MAIN_BASE, AICPU_IDLE_TASK_ID); } -int32_t platform_deinit_aicore_regs(uint64_t reg_addr) { - // Send exit signal to AICore - write_reg(reg_addr, RegId::DATA_MAIN_BASE, AICORE_EXIT_SIGNAL); +void platform_signal_aicore_exit(uint64_t reg_addr) { write_reg(reg_addr, RegId::DATA_MAIN_BASE, AICORE_EXIT_SIGNAL); } + +uint64_t platform_aicore_exit_deadline() { return get_sys_cnt_aicpu() + inner_get_deinit_timeout_ticks(); } - // Wait for AICore to acknowledge exit, with timeout. Timeout is - // variant-specific (sim wider than onboard) — see - // inner_get_deinit_timeout_ticks declaration in platform_regs.h. - // On timeout, skip register cleanup (AICore is unresponsive; host will +int32_t platform_finish_aicore_exit(uint64_t reg_addr, uint64_t deadline) { + // Wait for AICore to acknowledge exit, until the caller's deadline. On + // timeout, skip register cleanup (AICore is unresponsive; host will // aclrtResetDevice to clear all hardware state). - const uint64_t deinit_timeout_ticks = inner_get_deinit_timeout_ticks(); - uint64_t t0 = get_sys_cnt_aicpu(); while (read_reg(reg_addr, RegId::COND) != AICORE_EXITED_VALUE) { - if (get_sys_cnt_aicpu() - t0 > deinit_timeout_ticks) { + if (get_sys_cnt_aicpu() > deadline) { return -1; } } @@ -86,6 +83,11 @@ int32_t platform_deinit_aicore_regs(uint64_t reg_addr) { return 0; } +int32_t platform_deinit_aicore_regs(uint64_t reg_addr) { + platform_signal_aicore_exit(reg_addr); + return platform_finish_aicore_exit(reg_addr, platform_aicore_exit_deadline()); +} + uint32_t platform_get_physical_cores_count() { return DAV_2201::PLATFORM_MAX_PHYSICAL_CORES * PLATFORM_CORES_PER_BLOCKDIM; } diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp index bddbe7fedc..6f9f0236fe 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/aicpu/aicpu_executor.cpp @@ -857,9 +857,11 @@ int32_t AicpuExecutor::run(Runtime *runtime) { } } - // Always shutdown AICore — even if sched_ctx_.completed_ was already true. - // platform_deinit_aicore_regs is idempotent; orchestrator threads have + // Shutdown AICore even when sched_ctx_.completed_ was already true: + // platform_deinit_aicore_regs is idempotent, and orchestrator threads have // core_trackers_[thread_idx].core_num() == 0 so they skip the loop harmlessly. + // A fatal run is the exception — shutdown() returns immediately there, + // because emergency_shutdown() has already quiesced every core. int32_t shutdown_rc = sched_ctx_.shutdown(thread_idx); if (shutdown_rc != 0 && run_rc == 0) { run_rc = shutdown_rc; diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md index 5d62329bb7..090b51a1b1 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/RUNTIME_LOGIC.md @@ -603,7 +603,7 @@ Public surface (called from `AicpuExecutor::init/run/deinit`): | `init(runtime, aicpu_thread_num, sched_thread_num, regs_base)` | once per run | Handshake + assign cores, reset counters, latch `regs_base`, bind `func_id_to_addr_` | | `bind_runtime(rt)` | device-orch only | Wire `sched_` to `rt->scheduler` once the orchestrator thread creates `rt` | | `resolve_and_dispatch(runtime, thread_idx)` | per scheduler thread | Main dispatch loop | -| `shutdown(thread_idx)` | per thread on exit | `platform_deinit_aicore_regs` for this thread's cores; PMU finalize when enabled | +| `shutdown(thread_idx)` | per thread on exit | `platform_deinit_aicore_regs` for this thread's cores; PMU finalize when enabled. No-op on a fatal run — `emergency_shutdown` has already quiesced every core, and PMU finalize is skipped with it | | `on_orchestration_done(runtime, rt, thread_idx, total_tasks)` | orchestrator thread | Publish core assignments, latch task count, fold inline-completed tasks, flip `orchestrator_done_` (or `emergency_shutdown` on fatal) | | `deinit()` | once per run | Reset every scheduler-owned field to its post-construction default | | Read-only accessors | various | `aic_count()` / `aiv_count()` / `is_completed()` / `completed_tasks_count()` | diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp index a19872b753..4f442617cb 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_cold_path.cpp @@ -10,6 +10,8 @@ */ #include "scheduler_context.h" +#include "utils/fatal_shutdown_latch.h" + #include #include @@ -65,17 +67,13 @@ LoopAction SchedulerContext::handle_orchestrator_exit( "completed_tasks=%d, total_tasks=%d", thread_idx, orch_err, completed_tasks_.load(std::memory_order_relaxed), total_tasks_ ); - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } + emergency_shutdown(runtime); return LoopAction::BREAK_LOOP; } int32_t sched_err = header->sched_error_code.load(std::memory_order_acquire); if (sched_err != PTO2_ERROR_NONE) { LOG_ERROR("Thread %d: Scheduler fatal error detected (code=%d)", thread_idx, sched_err); - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } + emergency_shutdown(runtime); return LoopAction::BREAK_LOOP; } @@ -102,17 +100,13 @@ SchedulerContext::check_idle_fatal_error(int32_t thread_idx, PTO2SharedMemoryHea int32_t orch_err = header->orch_error_code.load(std::memory_order_acquire); if (orch_err != PTO2_ERROR_NONE) { LOG_ERROR("Thread %d: Fatal error detected (code=%d), sending EXIT_SIGNAL to all cores", thread_idx, orch_err); - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } + emergency_shutdown(runtime); return LoopAction::BREAK_LOOP; } int32_t sched_err = header->sched_error_code.load(std::memory_order_acquire); if (sched_err != PTO2_ERROR_NONE) { LOG_ERROR("Thread %d: Scheduler fatal error detected (code=%d)", thread_idx, sched_err); - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } + emergency_shutdown(runtime); return LoopAction::BREAK_LOOP; } return LoopAction::NONE; @@ -452,7 +446,7 @@ int32_t SchedulerContext::handle_timeout_exit( // sees the locators above already settled. header->sched_stall_detail.store(cls.detail, std::memory_order_release); } - if (!completed_.exchange(true, std::memory_order_acq_rel)) { + if (begin_emergency_shutdown()) { log_shutdown_stall_snapshot(thread_idx, idle_iterations, last_progress_count); #if SIMPLER_DFX // Capture the in-flight kernels' partial output before signalling the @@ -472,7 +466,7 @@ int32_t SchedulerContext::handle_timeout_exit( ); } #endif - emergency_shutdown(runtime); + signal_emergency_shutdown(runtime); } #if SIMPLER_DFX uint64_t sched_timeout_ts = get_sys_cnt_aicpu(); @@ -631,8 +625,18 @@ void SchedulerContext::log_chip_swimlane_summary(int32_t thread_idx, [[maybe_unu // Shutdown: deinit AICore regs for this thread's cores (and PMU finalize if enabled). // Orchestrator threads have core_trackers_[thread_idx].core_num() == 0 -> no-op. // platform_deinit_aicore_regs is idempotent; safe to call after early completion. +// +// A fatal run returns before any of that: emergency_shutdown() has already +// broadcast exit to every core and quiesced its register block, so re-running +// the per-thread path would only re-poll cores that have already stopped. PMU +// finalize is skipped with it — the host force-resets the card after a fatal +// run, so counters read here would not survive into the next generation. // ============================================================================= int32_t SchedulerContext::shutdown(int32_t thread_idx) { + if (fatal_shutdown_started_.load(std::memory_order_acquire)) { + return 0; + } + const int32_t *cores = core_trackers_[thread_idx].core_ids(); int32_t core_num = core_trackers_[thread_idx].core_num(); if (core_num == 0) return 0; @@ -963,11 +967,7 @@ void SchedulerContext::assign_own_clusters(int32_t tidx) { // Abort the run on a handshake failure discovered without the all-thread barrier // (non-DFX path): latch completion so every scheduler thread exits its dispatch // loop, and broadcast exit to whatever cores did come up. Idempotent. -void SchedulerContext::abort_and_shutdown(Runtime *runtime) { - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } -} +void SchedulerContext::abort_and_shutdown(Runtime *runtime) { emergency_shutdown(runtime); } // Profiling-subsystem init (leader-only). pmu_aicpu_init needs every core's // physical_core_id, so the barrier-free init path calls this behind an @@ -1046,20 +1046,38 @@ bool SchedulerContext::assign_cores_to_threads() { } // ============================================================================= -// Emergency shutdown: broadcast exit signal to every handshake'd core and -// deinit their AICore register blocks. Idempotent. +// Emergency shutdown: elect one thread, broadcast exit to every handshake'd +// core, then join them. Idempotent — the per-thread shutdown() path no-ops once +// fatal shutdown has started, so cores are quiesced exactly once. // ============================================================================= -void SchedulerContext::emergency_shutdown(Runtime *runtime) { +bool SchedulerContext::begin_emergency_shutdown() { + return publish_fatal_shutdown(fatal_shutdown_started_, completed_); +} + +void SchedulerContext::signal_emergency_shutdown(Runtime *runtime) { (void)runtime; // exit is now delivered via each core's register block, not GM LOG_WARN("Emergency shutdown: sending exit signal to all initialized cores"); + // Broadcast to every core before joining any of them, so the cores drain + // concurrently and a dead core's timeout does not serialize behind the + // cores ahead of it. Cores never opened (reg_addr==0) are reaped by the + // host device reset that follows. + for (int32_t i = 0; i < cores_total_num_; i++) { + if (core_exec_states_[i].reg_addr != 0) { + platform_signal_aicore_exit(core_exec_states_[i].reg_addr); + } + } + // The join is what leaves the card usable for the next process: it quiesces + // each core's register block (dispatch idle, fast path closed) once the core + // confirms it stopped. Returning while cores still run leaves the card + // poisoned past the host's device reset. One deadline covers the whole + // group, so a fatal run where every core is dead costs a single deinit + // timeout rather than one per core. The wait is an on-device register poll, + // so it adds no host or remote operation. + const uint64_t exit_deadline = platform_aicore_exit_deadline(); int32_t timeout_count = 0; for (int32_t i = 0; i < cores_total_num_; i++) { - // platform_deinit_aicore_regs writes DATA_MAIN_BASE=EXIT, which both - // releases a core still polling for its window to open and signals it to - // exit. Cores never opened (reg_addr==0) are reaped by the host device - // reset that follows a handshake failure. if (core_exec_states_[i].reg_addr != 0) { - if (platform_deinit_aicore_regs(core_exec_states_[i].reg_addr) != 0) { + if (platform_finish_aicore_exit(core_exec_states_[i].reg_addr, exit_deadline) != 0) { timeout_count++; } } @@ -1069,6 +1087,12 @@ void SchedulerContext::emergency_shutdown(Runtime *runtime) { } } +void SchedulerContext::emergency_shutdown(Runtime *runtime) { + if (begin_emergency_shutdown()) { + signal_emergency_shutdown(runtime); + } +} + // ============================================================================= // Lifecycle: init / deinit // ============================================================================= @@ -1320,6 +1344,7 @@ void SchedulerContext::deinit() { total_tasks_ = 0; orchestrator_done_.store(false, std::memory_order_release); completed_.store(false, std::memory_order_release); + fatal_shutdown_started_.store(false, std::memory_order_release); // Reset core discovery and assignment state aic_count_ = 0; @@ -1388,9 +1413,7 @@ void SchedulerContext::on_orchestration_done( orch_err = sched_->sm_header->orch_error_code.load(std::memory_order_relaxed); } if (orch_err != PTO2_ERROR_NONE) { - if (!completed_.exchange(true, std::memory_order_acq_rel)) { - emergency_shutdown(runtime); - } + emergency_shutdown(runtime); } #if SIMPLER_DFX diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h index f7388094ae..a1d5f8f02b 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/runtime/scheduler/scheduler_context.h @@ -179,6 +179,7 @@ class SchedulerContext { // Device orchestration: set by last orchestrator when graph is built; schedulers poll it. std::atomic orchestrator_done_{false}; std::atomic completed_{false}; + std::atomic fatal_shutdown_started_{false}; uint64_t *func_id_to_addr_{nullptr}; // --- Thread/core configuration --- @@ -220,8 +221,10 @@ class SchedulerContext { // Assign discovered cores (cluster = 1 AIC + 2 AIV) round-robin across scheduler threads. bool assign_cores_to_threads(); - // Emergency shutdown: broadcast exit signal to every handshake'd core and - // deinit their AICore register blocks. Idempotent. + // Publish fatal state before completion, then elect one thread to broadcast + // exit to every handshake'd core. Idempotent. + bool begin_emergency_shutdown(); + void signal_emergency_shutdown(Runtime *runtime); void emergency_shutdown(Runtime *runtime); // ========================================================================= diff --git a/src/a5/platform/onboard/host/device_runner.cpp b/src/a5/platform/onboard/host/device_runner.cpp index b5027a6127..a1c100dfdb 100644 --- a/src/a5/platform/onboard/host/device_runner.cpp +++ b/src/a5/platform/onboard/host/device_runner.cpp @@ -42,6 +42,7 @@ #include "utils/fnv1a_64.h" #include "host/host_regs.h" // Register address retrieval #include "host/raii_scope_guard.h" +#include "utils/fatal_shutdown_latch.h" // dep_gen_replay_emit_deps_json: strong symbol provided by // runtime/tensormap_and_ringbuffer/host/dep_gen_replay.cpp when that runtime is @@ -477,12 +478,23 @@ int DeviceRunner::drain_run(uint32_t pipeline_slot) { void DeviceRunner::cleanup_active_run() noexcept { if (!run_resources_owned_) return; + // A poisoned card cannot retire per-resource frees, and issuing them can + // block in the driver. Drop host-side ownership instead; finalize()'s force + // reset invalidates the whole device generation. + const bool abandon = device_unusable_.load(std::memory_order_acquire); + // Collectors stop before device/runtime arguments and register buffers. - finalize_collectors(); - (void)kernel_args_.finalize_device_kernel_args(); - (void)kernel_args_.finalize_runtime_args(); + finalize_collectors(abandon); + if (abandon) { + kernel_args_.abandon_after_device_failure(); + } else { + (void)kernel_args_.finalize_device_kernel_args(); + (void)kernel_args_.finalize_runtime_args(); + } if (kernel_args_.args.regs != 0) { - (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.regs)); + if (!abandon) { + (void)mem_alloc_.free(reinterpret_cast(kernel_args_.args.regs)); + } kernel_args_.args.regs = 0; } run_resources_owned_ = false; @@ -683,6 +695,58 @@ int DeviceRunner::finalize() { return 0; } + // Fatal cleanup must not walk poisoned streams, mappings, or allocations. + // Stop collector threads locally, drain and force-reset the card, then + // forget the old generation's handles. + if (device_unusable_.load(std::memory_order_acquire)) { + finalize_collectors(true); + + // force_reset_device() drains before it resets and returns 0 only when + // its post-reset probe confirms the card, so a second pass runs against + // a settled card and can recover a poison the first pass could not. A + // Worker holding CP-process SDMA streams gets a single attempt: there a + // non-confirming reset already blocks on the driver's remote-event + // timeout, which a retry only multiplies. Read dma_workspace_handle_ + // before abandon_common_after_device_failure() clears it. + constexpr int kFatalResetAttempts = 3; + const bool sdma_provisioned = dma_workspace_handle_ != nullptr; + int reset_rc = attempt_fatal_reset( + [this]() { + return force_reset_device(); + }, + sdma_provisioned ? 1 : kFatalResetAttempts + ); + const bool reset_confirmed = reset_rc == 0; + if (!reset_confirmed) { + LOG_ERROR( + "Fatal teardown: force reset of device %d did not confirm clean (rc=%d); " + "quarantining old handles without per-resource RTS calls", + device_id_, reset_rc + ); + } + + int abandon_rc = abandon_common_after_device_failure(); + if (acl_ready_) { + if (reset_confirmed) { + int finalize_rc = aclFinalize(); + if (finalize_rc != 0) { + LOG_ERROR("aclFinalize failed during fatal finalize: %d", finalize_rc); + if (abandon_rc == 0) abandon_rc = finalize_rc; + } + } else { + LOG_WARN("Fatal teardown: skipping aclFinalize because device reset was not confirmed"); + } + acl_ready_ = false; + } + + device_id_ = -1; + if (reset_confirmed) { + device_unusable_.store(false, std::memory_order_release); + } + LOG_WARN("DeviceRunner finalized after fatal device failure"); + return abandon_rc != 0 ? abandon_rc : reset_rc; + } + int rc = attach_current_thread(device_id_); if (rc != 0) { LOG_ERROR("Failed to attach finalize thread to device %d: %d", device_id_, rc); @@ -725,67 +789,23 @@ int DeviceRunner::finalize() { } } - // On the poison path the soft reset above does NOT clear the op-timeout - // sticky-error — a fresh in-process Worker.init then fails at rtStreamCreate - // 507899. A FORCE reset clears it, so the next Worker on this card inits - // clean in the SAME process and the remaining tests run instead of cascading - // / being skipped. Only reached on the (rare) device-poison path; onboard - // work always holds an exclusive task-submit lock on the card (enforced by - // .claude/rules/running-onboard.md), and the reset is verified to scope to - // this card alone, so it cannot disturb other devices/users. - int reset_rc = 0; - if (device_unusable_.load(std::memory_order_acquire)) { - // Bounded retry: a single force reset normally clears the op-timeout - // sticky-error, but the poison occasionally needs a drain-then-reset - // cycle, so retry up to kMaxResetAttempts. force_reset_device() drains - // (best-effort) and resets inside its own ACL/bound scope, and returns 0 - // only when its post-reset probe confirms the card is actually clean. - constexpr int kMaxResetAttempts = 3; - for (int attempt = 1; attempt <= kMaxResetAttempts; ++attempt) { - reset_rc = force_reset_device(); - if (reset_rc == 0) { - if (attempt > 1) { - LOG_WARN( - "DeviceRunner finalize: device %d recovered on force-reset attempt %d/%d", device_id_, attempt, - kMaxResetAttempts - ); - } - break; - } - LOG_ERROR( - "DeviceRunner finalize: force-reset attempt %d/%d of device %d did not confirm clean (rc=%d)", attempt, - kMaxResetAttempts, device_id_, reset_rc - ); - } - if (reset_rc != 0) { - LOG_ERROR( - "DeviceRunner finalize: device %d still poisoned after %d force-reset attempts; leaving it marked " - "unusable so the layer above (st_worker poison-skip + dispatcher retry) recovers it.", - device_id_, kMaxResetAttempts - ); - } - } - + // Only the healthy path reaches here: a poisoned card returned from the + // fatal branch at the top of finalize(), which owns the force reset. device_id_ = -1; - // Clear the poison flag only if the force reset actually recovered the card, - // so a still-poisoned card stays flagged: a reused DeviceRunner then fails - // admission instead of being treated as clean. On the normal (not - // unusable) path reset_rc stays 0 and the flag is already false. - if (reset_rc == 0) { - device_unusable_.store(false, std::memory_order_release); - } - return rc != 0 ? rc : reset_rc; + device_unusable_.store(false, std::memory_order_release); + return rc; } // `launch_aicpu_kernel` and `launch_aicore_kernel` live on `DeviceRunnerBase`. -void DeviceRunner::finalize_collectors() { +void DeviceRunner::finalize_collectors(bool abandon_device_resources) { // On drain or enqueue rollback, release the diagnostics collectors' shared // memory. They are only re-initialized per run, so a // Worker reused across runs (e.g. a pytest session-scoped worker pool) would // otherwise re-enter init_chip_swimlane() with stale state still allocated. // Matches a2a3's finalize_collectors(). - auto free_cb = [this](void *dev_ptr) -> int { + auto free_cb = [this, abandon_device_resources](void *dev_ptr) -> int { + if (abandon_device_resources) return 0; return mem_alloc_.free(dev_ptr); }; if (chip_swimlane_collector_.is_initialized()) { diff --git a/src/a5/platform/onboard/host/device_runner.h b/src/a5/platform/onboard/host/device_runner.h index 4a55780908..185b01e8cc 100644 --- a/src/a5/platform/onboard/host/device_runner.h +++ b/src/a5/platform/onboard/host/device_runner.h @@ -285,5 +285,5 @@ class DeviceRunner : public DeviceRunnerBase { // whose init succeeded, in the only safe order (stop() joins mgmt before // poll). Idempotent — collectors that never initialized are skipped. // Does not release device memory; full release happens in finalize(). - void finalize_collectors(); + void finalize_collectors(bool abandon_device_resources = false); }; diff --git a/src/common/aicpu_loader/host/load_aicpu_op.cpp b/src/common/aicpu_loader/host/load_aicpu_op.cpp index 9ab3dc90cd..218770b355 100644 --- a/src/common/aicpu_loader/host/load_aicpu_op.cpp +++ b/src/common/aicpu_loader/host/load_aicpu_op.cpp @@ -238,6 +238,17 @@ void LoadAicpuOp::Finalize() { } } +void LoadAicpuOp::AbandonAfterDeviceFailure() { + binary_handle_ = nullptr; + func_handles_.clear(); + inner_fp_ = 0; + inner_so_basename_.clear(); + if (!json_file_path_.empty()) { + std::remove(json_file_path_.c_str()); + json_file_path_.clear(); + } +} + LoadAicpuOp::~LoadAicpuOp() { Finalize(); } bool LoadAicpuOp::GenerateAicpuOpJson(const std::string &json_path, const std::string &kernel_so) { diff --git a/src/common/aicpu_loader/host/load_aicpu_op.h b/src/common/aicpu_loader/host/load_aicpu_op.h index d8de4ce249..13fbb377b5 100644 --- a/src/common/aicpu_loader/host/load_aicpu_op.h +++ b/src/common/aicpu_loader/host/load_aicpu_op.h @@ -122,6 +122,14 @@ class LoadAicpuOp { /** @brief Release binary handle + function handles + temporary JSON. */ void Finalize(); + /** + * @brief Forget runtime handles without calling rtsBinaryUnload. + * + * Used after a force reset, or when the device is already unusable and + * another runtime teardown request could block waiting for device service. + */ + void AbandonAfterDeviceFailure(); + /** * @brief Launch a runtime SO entry point via rtsLaunchCpuKernel. * diff --git a/src/common/platform/include/host/memory_allocator.h b/src/common/platform/include/host/memory_allocator.h index 175b840cef..3eebb6a761 100644 --- a/src/common/platform/include/host/memory_allocator.h +++ b/src/common/platform/include/host/memory_allocator.h @@ -91,6 +91,20 @@ class MemoryAllocator { */ int finalize(); + /** + * Forget tracked device pointers without calling the platform free API. + * + * Use only after a device reset has already invalidated every allocation, + * or when a fatal device state makes further runtime calls unsafe. This + * makes the RAII destructor a no-op while leaving healthy teardown on the + * normal finalize() path. + */ + void abandon_after_device_failure() { + std::scoped_lock lk(mu_); + ptr_size_map_.clear(); + committed_bytes_ = 0; + } + /** * Get number of tracked allocations * diff --git a/src/common/platform/include/host/run_stream_slots.h b/src/common/platform/include/host/run_stream_slots.h index 1b45c8e7d6..bd9424aef7 100644 --- a/src/common/platform/include/host/run_stream_slots.h +++ b/src/common/platform/include/host/run_stream_slots.h @@ -157,6 +157,14 @@ class RunStreamSlots { return first_error; } + /** Forget every handle after device reset without invoking destroy_. */ + void abandon_all() { + for (Slot &s : slots_) { + s.aicpu = nullptr; + s.aicore = nullptr; + } + } + void *aicpu(unsigned slot) const { return slot < slots_.size() ? slots_[slot].aicpu : nullptr; } void *aicore(unsigned slot) const { return slot < slots_.size() ? slots_[slot].aicore : nullptr; } bool ready(unsigned slot) const { return aicpu(slot) != nullptr && aicore(slot) != nullptr; } diff --git a/src/common/platform/onboard/host/device_runner_base.cpp b/src/common/platform/onboard/host/device_runner_base.cpp index c72b60db56..6262aea9ac 100644 --- a/src/common/platform/onboard/host/device_runner_base.cpp +++ b/src/common/platform/onboard/host/device_runner_base.cpp @@ -218,6 +218,12 @@ void DeviceRunnerBase::release_graph_execution_buffers() { } } +void DeviceRunnerBase::abandon_graph_execution_buffers() { + for (GraphExecutionBufferMap &by_key : graph_execution_buffers_) { + by_key.clear(); + } +} + void DeviceRunnerBase::clear_temporary_buffer() { for (size_t slot = 0; slot < retained_temp_addrs_.size(); ++slot) { if (retained_temp_addrs_[slot] == nullptr) continue; @@ -1069,7 +1075,11 @@ int DeviceRunnerBase::launch_aicpu_payload( return load_aicpu_op_.LaunchBuiltInOp(stream, args, args_size, aicpu_num, kernel_name); } -int DeviceRunnerBase::finalize_common() { +int DeviceRunnerBase::finalize_common() { return finalize_common_impl(false); } + +int DeviceRunnerBase::abandon_common_after_device_failure() { return finalize_common_impl(true); } + +int DeviceRunnerBase::finalize_common_impl(bool abandon_device_resources) { int rc = 0; auto capture = [&rc](int err) { if (err != 0 && rc == 0) rc = err; @@ -1095,27 +1105,48 @@ int DeviceRunnerBase::finalize_common() { // error-state stream at finalize wedges subsequent tests (observed: 507018 // / 507899 / 507901 cascade across the whole st-onboard-a2a3 suite). // rtStreamDestroy on an error-state stream is the supported teardown path. + if (abandon_device_resources) { + LOG_WARN("Fatal teardown: force reset/quarantine finished; skipping per-resource RTS destroy/free calls"); + } if (stream_aicpu_ != nullptr) { - capture(rtStreamDestroy(stream_aicpu_)); + if (!abandon_device_resources) { + capture(rtStreamDestroy(stream_aicpu_)); + } stream_aicpu_ = nullptr; } if (stream_aicore_ != nullptr) { - capture(rtStreamDestroy(stream_aicore_)); + if (!abandon_device_resources) { + capture(rtStreamDestroy(stream_aicore_)); + } stream_aicore_ = nullptr; } - // Release the async-DMA provider (SDMA STARS streams + workspace) while RTS - // is live, before the subclass device reset. Null unless the Worker was - // created with SDMA enabled; idempotent so a reused runner re-provisions. + // Release the async-DMA provider (SDMA STARS streams + workspace) only on + // healthy teardown. A fatal reset invalidates its device resources as a + // group, so running its per-stream destructor afterwards is unsafe. if (dma_workspace_handle_ != nullptr) { - dma_workspace_release(dma_workspace_handle_); + if (!abandon_device_resources) { + dma_workspace_release(dma_workspace_handle_); + } dma_workspace_handle_ = nullptr; } + for (int kind = 0; kind < DMA_WORKSPACE_KIND_COUNT; ++kind) + dma_workspace_addr_[kind] = 0; // LoadAicpuOp holds a binary_handle_ from rtsBinaryLoadFromFile; unload it // here while RTS is live so ~LoadAicpuOp's idempotent Finalize() no-ops // instead of unloading after aclFinalize (see the invariant above). - load_aicpu_op_.Finalize(); + if (abandon_device_resources) { + load_aicpu_op_.AbandonAfterDeviceFailure(); + // A force reset invalidates every device allocation at once. If the + // reset failed, the device is quarantined and per-allocation rtFree is + // still unsafe. Forget allocator ownership before the shared host-side + // cleanup below, so arena/free backstops become local no-ops. + mem_alloc_.abandon_after_device_failure(); + kernel_args_.abandon_after_device_failure(); + } else { + load_aicpu_op_.Finalize(); + } // aicore_bin_handle_ was registered once via rtRegisterAllKernel; CANN // releases its device-side state when the device context tears down. @@ -1127,12 +1158,14 @@ int DeviceRunnerBase::finalize_common() { aicpu_init_launched_ = false; // Release any chip callable buffers callers forgot to unregister. - for (auto &kv : chip_callable_buffers_) { - mem_alloc_.free(reinterpret_cast(kv.second.chip_dev)); - LOG_DEBUG( - "Freed chip callable buffer: chip_dev=0x%lx, size=%zu, hash=0x%lx", kv.second.chip_dev, - kv.second.total_size, kv.first - ); + if (!abandon_device_resources) { + for (auto &kv : chip_callable_buffers_) { + mem_alloc_.free(reinterpret_cast(kv.second.chip_dev)); + LOG_DEBUG( + "Freed chip callable buffer: chip_dev=0x%lx, size=%zu, hash=0x%lx", kv.second.chip_dev, + kv.second.total_size, kv.first + ); + } } chip_callable_buffers_.clear(); @@ -1155,9 +1188,15 @@ int DeviceRunnerBase::finalize_common() { // mem_alloc_.finalize() so the arenas free through the still-live // allocator, not after it. for (auto &bank : arena_banks_) { - bank->gm_heap.release(); - bank->gm_sm.release(); - bank->runtime_pool.release(); + if (abandon_device_resources) { + bank->gm_heap.abandon_after_device_failure(); + bank->gm_sm.abandon_after_device_failure(); + bank->runtime_pool.abandon_after_device_failure(); + } else { + bank->gm_heap.release(); + bank->gm_sm.release(); + bank->runtime_pool.release(); + } } prebuilt_runtime_arena_cache_valid_ = false; prebuilt_runtime_arena_cache_key_.clear(); @@ -1166,20 +1205,30 @@ int DeviceRunnerBase::finalize_common() { prebuilt_runtime_arena_cache_runtime_arena_base_ = nullptr; prebuilt_runtime_arena_cache_image_.clear(); - release_graph_execution_buffers(); - clear_temporary_buffer(); + if (abandon_device_resources) { + abandon_graph_execution_buffers(); + retained_temp_addrs_.fill(nullptr); + retained_temp_sizes_.fill(0); + } else { + release_graph_execution_buffers(); + clear_temporary_buffer(); + } // Free the device-phase/task-timing buffer (allocated lazily in run()) while // mem_alloc_ and the device context are still live. free_tensor() routes // through mem_alloc_.free(), so it must run before mem_alloc_.finalize() // and before the subclass's `rtDeviceReset()` tears down the device runtime. if (device_wall_dev_ptr_ != nullptr) { - free_tensor(device_wall_dev_ptr_); + if (!abandon_device_resources) { + free_tensor(device_wall_dev_ptr_); + } device_wall_dev_ptr_ = nullptr; } // Free all remaining allocations (including handshake buffer and binGmAddr) - mem_alloc_.finalize(); + if (!abandon_device_resources) { + mem_alloc_.finalize(); + } block_dim_ = 0; worker_count_ = 0; @@ -1194,6 +1243,9 @@ int DeviceRunnerBase::finalize_common() { bank->cached_gm_sm_size = 0; bank->cached_runtime_arena_size = 0; } + if (abandon_device_resources) { + LOG_WARN("Fatal teardown: host-side ownership cleared without further device calls"); + } return rc; } diff --git a/src/common/platform/onboard/host/device_runner_base.h b/src/common/platform/onboard/host/device_runner_base.h index 1d65f28d52..0eb23b1c4a 100644 --- a/src/common/platform/onboard/host/device_runner_base.h +++ b/src/common/platform/onboard/host/device_runner_base.h @@ -856,6 +856,23 @@ class DeviceRunnerBase { int finalize_common(); void release_graph_execution_buffers(); + /** + * Drop the retained graph-execution buffers without freeing them. + * + * The fatal counterpart of release_graph_execution_buffers(): a force reset + * has already invalidated every allocation, so only the host-side map is + * cleared. + */ + void abandon_graph_execution_buffers(); + + /** + * Clear host-side ownership after a fatal device failure without issuing + * per-resource RTS calls. The caller must first attempt a force reset. + */ + int abandon_common_after_device_failure(); + + int finalize_common_impl(bool abandon_device_resources); + /** * Stamp the active callable_id onto a Runtime so the AICPU knows which * orch_so_table_ slot to dispatch. The orch SO itself was already delivered diff --git a/src/common/platform/onboard/host/device_runner_helpers.h b/src/common/platform/onboard/host/device_runner_helpers.h index 8fe4bcae73..fa1ac3df52 100644 --- a/src/common/platform/onboard/host/device_runner_helpers.h +++ b/src/common/platform/onboard/host/device_runner_helpers.h @@ -88,6 +88,17 @@ struct KernelArgsHelper { /** Free device memory allocated for the device-resident `KernelArgs` copy. */ int finalize_device_kernel_args(); + /** + * Clear device-pointer bookkeeping without calling the allocator. + * + * Used only by fatal teardown after reset/quarantine. + */ + void abandon_after_device_failure() { + args.runtime_args = nullptr; + device_k_args_ = nullptr; + allocator_ = nullptr; + } + /** * Implicit conversion operators for seamless use with runtime APIs. * diff --git a/src/common/utils/device_arena.h b/src/common/utils/device_arena.h index ffe34c479e..c2cc8436be 100644 --- a/src/common/utils/device_arena.h +++ b/src/common/utils/device_arena.h @@ -134,6 +134,11 @@ class DeviceArena { // a fresh reserve+commit cycle can run. void release() noexcept; + // Forget the backing buffer without invoking the injected free callback. + // Use only after device reset, or when a fatal device state makes any + // further per-allocation runtime call unsafe. + void abandon_after_device_failure() noexcept; + bool is_committed() const noexcept { return committed_; } void *base() const noexcept { return base_; } @@ -254,3 +259,13 @@ inline void DeviceArena::release() noexcept { committed_ = false; attached_ = false; } + +inline void DeviceArena::abandon_after_device_failure() noexcept { + raw_base_ = nullptr; + base_ = nullptr; + raw_size_ = 0; + cursor_ = 0; + region_count_ = 0; + committed_ = false; + attached_ = false; +} diff --git a/src/common/utils/fatal_shutdown_latch.h b/src/common/utils/fatal_shutdown_latch.h new file mode 100644 index 0000000000..2d8b06e60c --- /dev/null +++ b/src/common/utils/fatal_shutdown_latch.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +/** + * Publish fatal teardown before publishing run completion. + * + * A thread that observes `completed` with acquire ordering is guaranteed to + * observe `fatal_started` too, so it cannot enter the healthy per-thread + * shutdown path for a fatal run and race the emergency broadcast for the same + * cores. The return value elects exactly one caller to run that broadcast. + */ +inline bool publish_fatal_shutdown(std::atomic &fatal_started, std::atomic &completed) noexcept { + const bool first = !fatal_started.exchange(true, std::memory_order_acq_rel); + completed.store(true, std::memory_order_release); + return first; +} + +/** + * Drive the fatal-path device reset until an attempt confirms the card clean. + * + * Each attempt drains the device before resetting it, so a later attempt runs + * against a settled card and can confirm clean where the first did not. + * + * `max_attempts` must be 1 whenever the device still holds CP-process SDMA + * streams: there a reset that does not confirm blocks on the driver's + * 150/300-second remote-event timeout, and a further attempt only multiplies + * that wait without a new completion condition. Values below 1 are treated + * as 1. + * + * Returns 0 on the first confirming attempt, otherwise the last attempt's + * error; the caller quarantines host-side handles on a non-zero return. + */ +template +inline int attempt_fatal_reset(ResetFn &&reset, int max_attempts) { + const int attempts = max_attempts < 1 ? 1 : max_attempts; + int rc = 0; + for (int attempt = 0; attempt < attempts; ++attempt) { + rc = reset(); + if (rc == 0) break; + } + return rc; +} diff --git a/tests/st/aicore_op_timeout/test_aicore_op_timeout.py b/tests/st/aicore_op_timeout/test_aicore_op_timeout.py index f13b31a375..6ae3e53d59 100644 --- a/tests/st/aicore_op_timeout/test_aicore_op_timeout.py +++ b/tests/st/aicore_op_timeout/test_aicore_op_timeout.py @@ -56,25 +56,33 @@ def _build_chip_callable(platform: str) -> ChipCallable: ) -@pytest.mark.platforms(["a2a3", "a5"]) -@pytest.mark.device_count(1) -@pytest.mark.runtime(RUNTIME) -@pytest.mark.timeout(60) -def test_aicore_op_timeout_surfaces_as_runtime_error(st_platform, st_device_ids, monkeypatch): +def _exercise_aicore_timeout(st_platform, st_device_ids, monkeypatch, tmp_path, *, enable_sdma: bool): configure_logging("error") monkeypatch.setenv("SIMPLER_SCHEDULER_TIMEOUT_MS", "2000") monkeypatch.setenv("SIMPLER_OP_EXECUTE_TIMEOUT_US", "3000000") monkeypatch.setenv("SIMPLER_STREAM_SYNC_TIMEOUT_MS", "4000") chip_callable = _build_chip_callable(st_platform) - worker = Worker(level=2, platform=st_platform, runtime=RUNTIME, device_id=int(st_device_ids[0])) + worker = Worker( + level=2, + platform=st_platform, + runtime=RUNTIME, + device_id=int(st_device_ids[0]), + enable_sdma=enable_sdma, + ) handle = worker.register(chip_callable) worker.init() + close_elapsed = None try: config = CallConfig() # >=2 so the orchestration thread and the scheduler thread don't fight # for a single AICPU; smaller configs may not dispatch the AIC task. config.aicpu_thread_num = 2 + # Keep device-backed DFX buffers alive through the injected failure. + # Fatal cleanup must stop the host threads and forget those mappings + # without unregistering/freeing buffers on the poisoned card. + config.enable_chip_swimlane = 1 + config.output_prefix = str(tmp_path) t0 = time.monotonic() # Acceptable error codes for the STARS-killed AICore op. Device status @@ -94,7 +102,12 @@ def test_aicore_op_timeout_surfaces_as_runtime_error(st_platform, st_device_ids, # regression we care about is that the timeout chain reaps the hang in # single-digit seconds and surfaces either the device classification or # a valid host fallback rather than deadlocking. - with pytest.raises(RuntimeError, match=r"run failed with code (-100|507(046|018|000))"): + error_codes = r"(-100|507(046|018|000))" + if enable_sdma: + # CANN 9.0.0/driver 26.0.rc1 containment deliberately stops the + # SDMA run stream early so reset precedes DEV_RUNNING_DOWN. + error_codes = r"(-100|507(046|018|015|000))" + with pytest.raises(RuntimeError, match=rf"run failed with code {error_codes}"): worker.run(handle, ChipStorageTaskArgs(), config) elapsed = time.monotonic() - t0 @@ -103,4 +116,42 @@ def test_aicore_op_timeout_surfaces_as_runtime_error(st_platform, st_device_ids, # If this fires, the timeout chain is broken (or absent). assert elapsed < 10, f"run() took {elapsed:.1f}s — timeout chain did not fire" finally: + close_t0 = time.monotonic() worker.close() + close_elapsed = time.monotonic() - close_t0 + + # The SDMA case is issue #1425: one reset attempt only, because a failed + # reset there already blocks on the driver event a retry would multiply. + # The ordinary case budgets the full kFatalResetAttempts=3, since each + # attempt drains before resetting and can recover what the previous one + # could not. One attempt costs the stream-sync budget (4 s here) plus the + # driver reset (~11 s on the a5 CI package) plus a small probe, so the + # ceiling has to hold three of those — otherwise a retry that works + # correctly, just slowly, is reported as an unbounded teardown. + # Every limit stays far below the 150/300 s driver-event stalls this + # regression exists to catch. + close_limit = 30 if enable_sdma else (60 if st_platform == "a5" else 45) + assert close_elapsed < close_limit, ( + f"Worker.close() took {close_elapsed:.1f}s with enable_sdma={enable_sdma}; fatal teardown did not stay bounded" + ) + + +@pytest.mark.platforms(["a2a3", "a5"]) +@pytest.mark.device_count(1) +@pytest.mark.runtime(RUNTIME) +# Sits above the in-test close ceiling (60 s on a5) plus worker setup and the +# 10 s run budget, so the assertions report first; this mark is only the +# backstop for a genuine hang. +@pytest.mark.timeout(180) +def test_aicore_op_timeout_surfaces_as_runtime_error(st_platform, st_device_ids, monkeypatch, tmp_path): + _exercise_aicore_timeout(st_platform, st_device_ids, monkeypatch, tmp_path, enable_sdma=False) + + +@pytest.mark.sdma +@pytest.mark.platforms(["a2a3"]) +@pytest.mark.device_count(1) +@pytest.mark.runtime(RUNTIME) +@pytest.mark.timeout(90) +def test_sdma_worker_aicore_fault_teardown_is_bounded(st_platform, st_device_ids, monkeypatch, tmp_path): + """Real SDMA provisioning must not turn one AICore fault into a five-minute close.""" + _exercise_aicore_timeout(st_platform, st_device_ids, monkeypatch, tmp_path, enable_sdma=True) diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 64d2bbbc5e..e6686db492 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -476,6 +476,7 @@ target_include_directories(test_runtime_orch_so PRIVATE ) add_common_utils_test(test_device_arena common/test_device_arena.cpp) add_common_utils_test(test_thread_completion_gate common/test_thread_completion_gate.cpp) +add_common_utils_test(test_fatal_shutdown_latch common/test_fatal_shutdown_latch.cpp) add_common_utils_test(test_buffer_pool_manager common/test_buffer_pool_manager.cpp) target_sources(test_buffer_pool_manager PRIVATE ${CMAKE_SOURCE_DIR}/../../../src/common/log/host_log.cpp diff --git a/tests/ut/cpp/common/test_device_arena.cpp b/tests/ut/cpp/common/test_device_arena.cpp index af4e1d975b..8211ac2765 100644 --- a/tests/ut/cpp/common/test_device_arena.cpp +++ b/tests/ut/cpp/common/test_device_arena.cpp @@ -222,6 +222,32 @@ TEST(DeviceArenaTest, ReleaseFreesAndAllowsReuse) { EXPECT_EQ(m.alloc_count, 2); } +TEST(DeviceArenaTest, AbandonClearsOwnershipWithoutCallingBackendFree) { + MockBackend m; + DeviceArena arena(mock_alloc, mock_free, &m); + + arena.reserve(128, 64); + ASSERT_NE(arena.commit(1024), nullptr); + EXPECT_EQ(m.alloc_count, 1); + + arena.abandon_after_device_failure(); + EXPECT_EQ(m.free_count, 0); + EXPECT_FALSE(arena.is_committed()); + EXPECT_EQ(arena.base(), nullptr); + EXPECT_EQ(arena.total_size(), 0u); + + // Destruction/release after abandonment must remain a local no-op. + arena.release(); + EXPECT_EQ(m.free_count, 0); + + // The mock has no device reset to reclaim abandoned storage, so release + // its host allocation directly after the behavior assertions. + for (void *ptr : m.live) { + std::free(ptr); + } + m.live.clear(); +} + TEST(DeviceArenaTest, ZeroSizedRegionDoesNotAdvanceCursor) { MockBackend m; DeviceArena arena(mock_alloc, mock_free, &m); diff --git a/tests/ut/cpp/common/test_fatal_shutdown_latch.cpp b/tests/ut/cpp/common/test_fatal_shutdown_latch.cpp new file mode 100644 index 0000000000..52564bd943 --- /dev/null +++ b/tests/ut/cpp/common/test_fatal_shutdown_latch.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include + +#include "utils/fatal_shutdown_latch.h" + +namespace { + +TEST(FatalShutdownLatchTest, CompletionNeverBecomesVisibleBeforeFatalState) { + for (int iteration = 0; iteration < 1000; ++iteration) { + std::atomic fatal_started{false}; + std::atomic completed{false}; + std::atomic observed_fatal{false}; + + std::thread observer([&]() { + while (!completed.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + observed_fatal.store(fatal_started.load(std::memory_order_acquire), std::memory_order_relaxed); + }); + EXPECT_TRUE(publish_fatal_shutdown(fatal_started, completed)); + observer.join(); + + EXPECT_TRUE(observed_fatal.load(std::memory_order_relaxed)); + } +} + +TEST(FatalShutdownLatchTest, ExactlyOneCallerOwnsTheEmergencySignalBroadcast) { + std::atomic fatal_started{false}; + std::atomic completed{false}; + std::atomic leaders{0}; + std::vector publishers; + + for (int i = 0; i < 16; ++i) { + publishers.emplace_back([&]() { + if (publish_fatal_shutdown(fatal_started, completed)) { + leaders.fetch_add(1, std::memory_order_relaxed); + } + }); + } + for (auto &publisher : publishers) { + publisher.join(); + } + + EXPECT_TRUE(fatal_started.load(std::memory_order_acquire)); + EXPECT_TRUE(completed.load(std::memory_order_acquire)); + EXPECT_EQ(leaders.load(std::memory_order_relaxed), 1); +} + +TEST(FatalShutdownLatchTest, SdmaProvisionedCardGetsASingleResetAttempt) { + int attempts = 0; + int rc = attempt_fatal_reset( + [&]() { + ++attempts; + return 507007; + }, + 1 + ); + + EXPECT_EQ(rc, 507007); + EXPECT_EQ(attempts, 1); +} + +TEST(FatalShutdownLatchTest, OrdinaryPoisonRetriesUpToTheAttemptBudget) { + int attempts = 0; + int rc = attempt_fatal_reset( + [&]() { + ++attempts; + return 507007; + }, + 3 + ); + + EXPECT_EQ(rc, 507007); + EXPECT_EQ(attempts, 3); +} + +TEST(FatalShutdownLatchTest, RetryStopsOnTheFirstAttemptThatConfirmsTheCard) { + int attempts = 0; + int rc = attempt_fatal_reset( + [&]() { + return ++attempts == 2 ? 0 : 507007; + }, + 3 + ); + + EXPECT_EQ(rc, 0); + EXPECT_EQ(attempts, 2); +} + +TEST(FatalShutdownLatchTest, AttemptBudgetBelowOneStillResetsOnce) { + int attempts = 0; + int rc = attempt_fatal_reset( + [&]() { + ++attempts; + return 0; + }, + 0 + ); + + EXPECT_EQ(rc, 0); + EXPECT_EQ(attempts, 1); +} + +} // namespace diff --git a/tests/ut/cpp/common/test_memory_allocator.cpp b/tests/ut/cpp/common/test_memory_allocator.cpp index 6e74652b02..7a42adf83e 100644 --- a/tests/ut/cpp/common/test_memory_allocator.cpp +++ b/tests/ut/cpp/common/test_memory_allocator.cpp @@ -10,6 +10,7 @@ */ #include +#include #include @@ -75,6 +76,23 @@ TEST(MemoryAllocatorTest, FinalizeReleasesAllAndZeroesCounter) { EXPECT_EQ(a.committed_bytes(), 0u); } +TEST(MemoryAllocatorTest, AbandonClearsTrackingWithoutCallingPlatformFree) { + MemoryAllocator a; + void *p = a.alloc(128); + void *q = a.alloc(256); + ASSERT_NE(p, nullptr); + ASSERT_NE(q, nullptr); + + a.abandon_after_device_failure(); + EXPECT_EQ(a.committed_bytes(), 0u); + EXPECT_EQ(a.get_allocation_count(), 0u); + EXPECT_EQ(a.finalize(), 0); + + // The sim test has no device reset to reclaim abandoned allocations. + std::free(p); + std::free(q); +} + TEST(MemoryAllocatorTest, DestructorFreesLiveAllocationsWithoutLeak) { { MemoryAllocator a; diff --git a/tests/ut/cpp/hierarchical/test_run_stream_slots.cpp b/tests/ut/cpp/hierarchical/test_run_stream_slots.cpp index 8a5d38fa30..61a1017cc2 100644 --- a/tests/ut/cpp/hierarchical/test_run_stream_slots.cpp +++ b/tests/ut/cpp/hierarchical/test_run_stream_slots.cpp @@ -364,6 +364,20 @@ TEST(RunStreamSlots, DestroyAllReportsFailureAndRetriesWhatSurvived) { EXPECT_EQ(fake.live_count(), 0u); } +TEST(RunStreamSlots, AbandonAllClearsHandlesWithoutDestroyingThem) { + FakeStreams fake; + RunStreamSlots slots = make_slots(fake); + ASSERT_EQ(slots.acquire(0), 0); + + slots.abandon_all(); + + EXPECT_EQ(slots.aicpu(0), nullptr); + EXPECT_EQ(slots.aicore(0), nullptr); + EXPECT_EQ(fake.live_count(), 2u); + EXPECT_EQ(slots.destroy_all(), 0); + EXPECT_EQ(fake.live_count(), 2u); +} + // A failed create leaves nothing half-owned behind. TEST(RunStreamSlots, AFailedCreateLeavesTheSlotEmpty) { FakeStreams fake;