diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db2a66ca80..ef2b4d7b94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -595,8 +595,11 @@ jobs: # device fault/sync domain, so a fault on that device pays a multi-minute # teardown (issue #1425). Keep EVERY SDMA test out of this general sweep # — which also runs the aicore_op_timeout fault-injection test — and run - # them on their own dedicated device(s) in the SDMA step below, so an - # SDMA fault's slow teardown can never mix with ordinary fault recovery. + # them in the SDMA step below, after this sweep, so an SDMA fault's slow + # teardown can never mix with ordinary fault recovery. Device + # disjointness holds only on aarch64 (`task-submit --device auto`); the + # x86_64 branch has no task-submit and shares ${DEVICE_RANGE}, so there + # ordering is the separation. # # Selected by marker, not by path. `pytest --ignore=` exits # 0 and says nothing, so a rename used to drop the quarantine silently @@ -609,8 +612,8 @@ jobs: --run "python -m pytest examples tests/st -m 'not sdma' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600" fi - # SDMA pytest — every SDMA test runs here on its own dedicated device(s), - # isolated from the general sweep's aicore_op_timeout fault-injection test + # SDMA pytest — every SDMA test runs here, after the sweep above, isolated + # from the general sweep's aicore_op_timeout fault-injection test # (issue #1425). Both demos create their Worker with enable_sdma=True so the # runtime provisions the SDMA workspace: prefetch_async_demo on 1 device (L2), # sdma_async_completion_demo on 2 devices (L3, workspace provisioned per chip diff --git a/README.md b/README.md index 00c796910e..eadc3af3ee 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,8 @@ See runtime docs per arch: [a2a3](src/a2a3/docs/runtimes.md), [a5](src/a5/docs/r pytest examples tests/st --platform a2a3sim # Hardware scene tests (requires Ascend device) -pytest examples tests/st --platform a2a3 --device 4-7 +# SDMA cases are quarantined by marker, as in CI; run them separately with -m sdma +pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 # Python unit tests pytest tests/ut -m "not requires_hardware" -v diff --git a/docs/testing.md b/docs/testing.md index c7aad4b633..769739a827 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,8 +46,13 @@ ctest --test-dir tests/ut/cpp/build -L "^requires_hardware(_a2a3)?$" --output-on # Scene tests (pytest, @scene_test classes) pytest examples tests/st # all sim platforms (auto-parametrized) pytest examples tests/st --platform a2a3sim # specific sim -pytest examples tests/st --platform a2a3 # hardware -pytest examples tests/st --platform a2a3 --device 4-7 # hardware with device pool +pytest examples tests/st -m "not sdma" --platform a2a3 # hardware +pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 # hardware with device pool + +# SDMA cases run separately, as they do in CI: they are quarantined by +# @pytest.mark.sdma so no fault-injection case shares a device with a +# provisioned SDMA workspace (issue #1425) +pytest examples tests/st -m sdma --platform a2a3 --device 4-5 # Single scene test (standalone) python examples/a2a3/tensormap_and_ringbuffer/vector_example/test_vector_example.py -p a2a3sim @@ -625,8 +630,8 @@ pytest examples tests/st --platform a2a3sim # Standalone (single case) python test_my_kernel.py -p a2a3sim -# On hardware -pytest examples tests/st --platform a2a3 +# On hardware (SDMA cases quarantined by marker; run them with -m sdma) +pytest examples tests/st -m "not sdma" --platform a2a3 ``` Key fields: diff --git a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md index af3c36d640..92a3865898 100644 --- a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md +++ b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md @@ -63,7 +63,7 @@ exception you must surface the CANN device slog, which is otherwise hidden: ```bash ASCEND_SLOG_PRINT_TO_STDOUT=1 ASCEND_GLOBAL_LOG_LEVEL=1 \ - python -m pytest examples tests/st --platform a2a3 --device -v \ + python -m pytest examples tests/st -m "not sdma" --platform a2a3 --device -v \ --pto-session-timeout 600 ``` diff --git a/docs/user/reference/cli.md b/docs/user/reference/cli.md index 704380e9ee..16ab6c5748 100644 --- a/docs/user/reference/cli.md +++ b/docs/user/reference/cli.md @@ -6,7 +6,7 @@ How you select what runs, turn diagnostics on, and read the artifacts back. ```bash pytest examples tests/st --platform a2a3sim # simulation, no device -pytest examples tests/st --platform a2a3 --device 4-7 # hardware +pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 # hardware (SDMA quarantined by marker) python examples/my_example/test_my_example.py -p a2a3sim # standalone, no pytest ``` diff --git a/examples/README.md b/examples/README.md index acd0c309c3..6798586931 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,7 +17,7 @@ on both simulators: ```bash pytest examples --platform a2a3sim -pytest examples --platform a2a3 --device 0-1 # hardware +pytest examples -m "not sdma" --platform a2a3 --device 0-1 # hardware (SDMA quarantined by marker) ``` A single example: diff --git a/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md index 7782020b4d..23cabc3922 100644 --- a/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/host_build_graph/docs/SUBMIT_BY_CLUSTER.md @@ -210,7 +210,7 @@ python tests/st/a2a3/tensormap_and_ringbuffer/batch_paged_attention/test_batch_p Final validation: ```bash -pytest examples tests/st --platform a2a3 +pytest examples tests/st -m "not sdma" --platform a2a3 # SDMA cases quarantined by marker; run them with -m sdma ``` ## 14. Resolved Decisions diff --git a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 1d8ee744be..63eb2cde91 100644 --- a/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a2a3/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -208,7 +208,7 @@ python tests/st/a2a3/tensormap_and_ringbuffer/batch_paged_attention/test_batch_p Final validation: ```bash -pytest examples tests/st --platform a2a3 +pytest examples tests/st -m "not sdma" --platform a2a3 # SDMA cases quarantined by marker; run them with -m sdma ``` ## 14. Resolved Decisions diff --git a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md index 1d8ee744be..63eb2cde91 100644 --- a/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md +++ b/src/a5/runtime/tensormap_and_ringbuffer/docs/SUBMIT_BY_CLUSTER.md @@ -208,7 +208,7 @@ python tests/st/a2a3/tensormap_and_ringbuffer/batch_paged_attention/test_batch_p Final validation: ```bash -pytest examples tests/st --platform a2a3 +pytest examples tests/st -m "not sdma" --platform a2a3 # SDMA cases quarantined by marker; run them with -m sdma ``` ## 14. Resolved Decisions