diff --git a/.github/actions/pod-run-example/action.yml b/.github/actions/pod-run-example/action.yml deleted file mode 100644 index 9f93d8bb5a..0000000000 --- a/.github/actions/pod-run-example/action.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Run one pod example across the pair -description: >- - Start the peer's L3 daemon, run one example's parent against it, then stop the - daemon and pull its logs back. Call once per example; pod-stage has already - put the tree and the venv on the peer, which every example shares. - -inputs: - example: - description: Example directory name. Names this example's log directory, so it must be unique within a run. - required: true - env-prefix: - description: >- - Environment-variable prefix the example's parent script reads, e.g. - SIMPLER_VECTOR_ADD_MIXED_L3. Taken as an input rather than derived from - `example`, so an example is free to name its variables as it likes. The - daemon side needs no prefix — it is the same generic session server for - every example and is started directly below. - required: true - parent-script: - description: Repo-relative script that runs the parent side. - required: true - local-devices: - description: Device ids the local L3 owns. Defaults to the pod config's. - required: false - default: '' - remote-devices: - description: Device ids the peer's L3 owns. Defaults to the pod config's. - required: false - default: '' - daemon-port: - description: Port the peer's daemon listens on. Defaults to the pod config's. - required: false - default: '' - -runs: - using: composite - steps: - # The daemon must outlive this step, so its ssh is backgrounded with its - # output redirected to a file. The pid goes to a file because the collect - # step below cannot inherit a shell variable. python3, not the venv's - # python: this poll only needs the stdlib and the venv is not activated. - - name: Start the peer's L3 daemon (${{ inputs.example }}) - shell: bash - working-directory: ${{ github.workspace }} - env: - EXAMPLE: ${{ inputs.example }} - ENV_PREFIX: ${{ inputs.env-prefix }} - IN_DAEMON_PORT: ${{ inputs.daemon-port }} - run: | - set -euo pipefail - source "$POD_SSH_HELPER" - DAEMON_PORT="${IN_DAEMON_PORT:-$POD_L3_DAEMON_PORT}" - LOCAL_LOGS="$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}" - REMOTE_LOGS="output/pod-ci/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}" - mkdir -p "$LOCAL_LOGS" "$RUN_DIR/$EXAMPLE/parent-${POD_MACHINE}/ascend" - { - echo "POD_EXAMPLE=$EXAMPLE" - echo "POD_EXAMPLE_ENV_PREFIX=$ENV_PREFIX" - echo "POD_EXAMPLE_DAEMON_PORT=$DAEMON_PORT" - echo "POD_EXAMPLE_REMOTE_LOGS=$REMOTE_LOGS" - } >> "$GITHUB_ENV" - - pod_ssh " - pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true - " || true - - pod_ssh " - set -eo pipefail - cd '$REMOTE_WORKDIR' - export PYTHONPATH="\${PYTHONPATH:-}" - export CMAKE_PREFIX_PATH="\${CMAKE_PREFIX_PATH:-}" - source '$POD_REMOTE_CANN_ENV' - set -u - source .venv/bin/activate - # The job-level timeouts are the parent's environment; ssh carries no - # environment, so the peer would otherwise schedule against the - # built-in defaults and the two halves of one run would disagree on - # how long a stall may last. - export SIMPLER_SCHEDULER_TIMEOUT_MS='$SIMPLER_SCHEDULER_TIMEOUT_MS' - export SIMPLER_OP_EXECUTE_TIMEOUT_US='$SIMPLER_OP_EXECUTE_TIMEOUT_US' - export SIMPLER_STREAM_SYNC_TIMEOUT_MS='$SIMPLER_STREAM_SYNC_TIMEOUT_MS' - mkdir -p '$REMOTE_LOGS' - export ASCEND_PROCESS_LOG_PATH="\$PWD/$REMOTE_LOGS" - echo '[pod-daemon] machine${POD_REMOTE_MACHINE} listening on ${REMOTE_DAEMON_HOST}:${DAEMON_PORT}' - python -m simpler.remote_l3_worker --host '${REMOTE_DAEMON_HOST}' --port '${DAEMON_PORT}' - " > "$LOCAL_LOGS/daemon.ssh.log" 2>&1 & - echo $! > "$RUNNER_TEMP/pod-daemon-$EXAMPLE.pid" - - # A daemon that dies during import never opens the port, so on its own - # the connect probe only reports that after the full wait. The ssh - # process exits with it, so its absence is the earlier and more precise - # signal — and it is the one that can name the log holding the reason. - python3 - </dev/null - rm -f "$PID_FILE" - fi - # The daemon's chip children keep writing after the daemon itself is - # signalled, so wait for it to actually go before anything reads or - # removes that tree. - pod_ssh " - pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' || true - for _ in \$(seq 1 20); do - pgrep -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' >/dev/null || break - sleep 0.5 - done - " - rsync -a -e "$RSYNC_SSH" \ - "$REMOTE_TARGET:$REMOTE_WORKDIR/$POD_EXAMPLE_REMOTE_LOGS/" \ - "$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}/" 2>/dev/null - exit 0 diff --git a/.github/actions/pod-run-pytest/action.yml b/.github/actions/pod-run-pytest/action.yml new file mode 100644 index 0000000000..8f7738092d --- /dev/null +++ b/.github/actions/pod-run-pytest/action.yml @@ -0,0 +1,141 @@ +name: Run pod pytest across the pair +description: >- + Start the peer's L3 daemon, run pod-marked pytest against it, then stop the + daemon and pull its logs back. pod-stage has already put the tree and venv on + the peer. + +inputs: + pytest-args: + description: Arguments passed to `python -m pytest`. + required: true + +runs: + using: composite + steps: + # The daemon must outlive this step, so its ssh is backgrounded with its + # output redirected to a file. The pid goes to a file because the collect + # step below cannot inherit a shell variable. python3, not the venv's + # python: this poll only needs the stdlib and the venv is not activated. + - name: Start the peer's L3 daemon + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + source "$POD_SSH_HELPER" + DAEMON_PORT="$POD_L3_DAEMON_PORT" + LOCAL_LOGS="$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}" + REMOTE_LOGS="output/pod-ci/pytest/daemon-${POD_REMOTE_MACHINE}" + mkdir -p "$LOCAL_LOGS" "$RUN_DIR/pytest/parent-${POD_MACHINE}/ascend" + { + echo "POD_REMOTE_ENDPOINT=${REMOTE_DAEMON_HOST}:${DAEMON_PORT}" + echo "POD_PYTEST_REMOTE_LOGS=$REMOTE_LOGS" + } >> "$GITHUB_ENV" + + pod_ssh " + pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true + " || true + + pod_ssh " + set -eo pipefail + cd '$REMOTE_WORKDIR' + export PYTHONPATH=\"\${PYTHONPATH:-}\" + export CMAKE_PREFIX_PATH=\"\${CMAKE_PREFIX_PATH:-}\" + source '$POD_REMOTE_CANN_ENV' + set -u + source .venv/bin/activate + export SIMPLER_SCHEDULER_TIMEOUT_MS='$SIMPLER_SCHEDULER_TIMEOUT_MS' + export SIMPLER_OP_EXECUTE_TIMEOUT_US='$SIMPLER_OP_EXECUTE_TIMEOUT_US' + export SIMPLER_STREAM_SYNC_TIMEOUT_MS='$SIMPLER_STREAM_SYNC_TIMEOUT_MS' + mkdir -p '$REMOTE_LOGS' + export ASCEND_PROCESS_LOG_PATH=\"\$PWD/$REMOTE_LOGS\" + echo '[pod-daemon] machine${POD_REMOTE_MACHINE} listening on ${REMOTE_DAEMON_HOST}:${DAEMON_PORT}' + python -m simpler.remote_l3_worker --host '${REMOTE_DAEMON_HOST}' --port '${DAEMON_PORT}' + " > "$LOCAL_LOGS/daemon.ssh.log" 2>&1 & + echo $! > "$RUNNER_TEMP/pod-daemon-pytest.pid" + + # A daemon that dies during import never opens the port, so on its own + # the connect probe only reports that after the full wait. The ssh + # process exits with it, so its absence is the earlier and more precise + # signal, and it can name the log holding the reason. + python3 - </dev/null + rm -f "$PID_FILE" + fi + DAEMON_PORT="${POD_L3_DAEMON_PORT:-19073}" + pod_ssh " + pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true + for _ in \$(seq 1 20); do + pgrep -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' >/dev/null || break + sleep 0.5 + done + " + mkdir -p "$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}" + rsync -a -e "$RSYNC_SSH" \ + "$REMOTE_TARGET:$REMOTE_WORKDIR/${POD_PYTEST_REMOTE_LOGS:-output/pod-ci/pytest/daemon-${POD_REMOTE_MACHINE}}/" \ + "$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}/" 2>/dev/null + exit 0 diff --git a/.github/actions/pod-teardown/action.yml b/.github/actions/pod-teardown/action.yml index 5d3d3fdbdb..0b7c9ad28e 100644 --- a/.github/actions/pod-teardown/action.yml +++ b/.github/actions/pod-teardown/action.yml @@ -1,8 +1,8 @@ name: Clear the pod staging tree description: >- - Remove this run's tree from the peer. Call once, with `if: always()`. Each - pod-run-example already stopped its own daemon; the sweep here only catches an - example that died before reaching that step. + Remove this run's tree from the peer. Call once, with `if: always()`. The pod + pytest action already stopped its own daemon; the sweep here only catches a + run that died before reaching that step. runs: using: composite diff --git a/.github/workflows/_st-npu-a2a3.yml b/.github/workflows/_st-npu-a2a3.yml index 87f4a8c9ac..69f8eb1cfc 100644 --- a/.github/workflows/_st-npu-a2a3.yml +++ b/.github/workflows/_st-npu-a2a3.yml @@ -78,10 +78,10 @@ jobs: source /usr/local/Ascend/cann/set_env.sh source .venv/bin/activate if [ "$(uname -m)" = "x86_64" ]; then - python -m pytest examples tests/st -m "not sdma" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200 + python -m pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200 else task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \ - --run "python -m pytest examples tests/st -m 'not sdma' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200" + --run "python -m pytest examples tests/st -m 'not sdma and not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200" fi - name: Run pytest scene tests (a2a3 legacy SDMA paths) @@ -91,10 +91,10 @@ jobs: source .venv/bin/activate SDMA_IGNORE="--ignore=examples/a2a3/tensormap_and_ringbuffer/prefetch_async_demo --ignore=examples/a2a3/tensormap_and_ringbuffer/sdma_async_completion_demo" if [ "$(uname -m)" = "x86_64" ]; then - python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600 + python -m pytest examples tests/st $SDMA_IGNORE -m "not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600 else task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \ - --run "python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600" + --run "python -m pytest examples tests/st $SDMA_IGNORE -m 'not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600" fi - name: SDMA pytest (a2a3) diff --git a/.github/workflows/_st-npu-a5.yml b/.github/workflows/_st-npu-a5.yml index d3b250c91d..9ae0494005 100644 --- a/.github/workflows/_st-npu-a5.yml +++ b/.github/workflows/_st-npu-a5.yml @@ -59,9 +59,9 @@ jobs: python -m simpler_setup.tools.scene_test_compile examples tests/st \ --platform a5 --require-pto-isa --compile-workers 8 -q DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))") - PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa" + PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa -m 'not pod'" if [ "$(uname -m)" = "x86_64" ]; then - PYTEST="$PYTEST -m 'not sdma'" + PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa -m 'not sdma and not pod'" fi task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST --pto-session-timeout 1200" diff --git a/.github/workflows/_st-pod.yml b/.github/workflows/_st-pod.yml index 0421e5fadd..6527bbd771 100644 --- a/.github/workflows/_st-pod.yml +++ b/.github/workflows/_st-pod.yml @@ -86,7 +86,7 @@ jobs: set_default POD_REMOTE_STAGING_ROOT /data/workspace/ci-runner/pod-ci set_default POD_REMOTE_STAGING_TTL_DAYS 2 set_default POD_DAEMON_WAIT_S 120 - set_default POD_SMOKE_TIMEOUT_S 1800 + set_default POD_SMOKE_TIMEOUT_S 2400 set_default POD_CANN_ENV /usr/local/Ascend/cann/set_env.sh set_default POD_REMOTE_CANN_ENV "$POD_CANN_ENV" set_default POD_HTTP_PROXY "" @@ -170,63 +170,29 @@ jobs: https-proxy: ${{ env.POD_HTTPS_PROXY }} no-proxy: ${{ env.POD_NO_PROXY }} - # Staging and the peer-side build are per-run: every example below runs - # against the same tree and the same venv, so re-staging per example - # would repeat the whole cost of the job. + # Staging and the peer-side build are per-run: the pod pytest sweep runs + # against the same tree and the same venv, so re-staging per test would + # repeat the whole cost of the job. - name: Stage the checkout on the peer uses: ./.github/actions/pod-stage - # continue-on-error so a second example still runs when the first fails — - # one round of this job occupies two machines, and finding out about only - # the first failure wastes the second half of it. The summary step below - # is what makes the job red. - - name: vector_add_mixed_l3 - id: vector-add-mixed-l3 - continue-on-error: true - uses: ./.github/actions/pod-run-example + - name: Run pod pytest + uses: ./.github/actions/pod-run-pytest with: - example: vector_add_mixed_l3 - env-prefix: SIMPLER_VECTOR_ADD_MIXED_L3 - parent-script: examples/workers/l4/vector_add_mixed_l3/run_parent.sh - - - name: global_tload_mixed_l3 - id: global-tload-mixed-l3 - continue-on-error: true - uses: ./.github/actions/pod-run-example - with: - example: global_tload_mixed_l3 - env-prefix: SIMPLER_GLOBAL_TLOAD_MIXED_L3 - parent-script: examples/workers/l4/global_tload_mixed_l3/run_parent.sh - - - name: compute_then_tload_mixed_l3 - id: compute-then-tload-mixed-l3 - continue-on-error: true - uses: ./.github/actions/pod-run-example - with: - example: compute_then_tload_mixed_l3 - env-prefix: SIMPLER_COMPUTE_THEN_TLOAD_MIXED_L3 - parent-script: examples/workers/l4/compute_then_tload_mixed_l3/run_parent.sh + pytest-args: >- + examples tests/st + -m pod + --platform ${{ inputs.platform }} + --device ${{ env.POD_LOCAL_DEVICES }} + --max-parallel 1 + -v + --require-pto-isa + --pto-session-timeout 1200 - name: Clear the pod staging tree if: always() uses: ./.github/actions/pod-teardown - # continue-on-error rewrites a failed example's `conclusion` to success - # and leaves the truth in `outcome`, so this is the only step whose - # result means the examples passed. - - name: Report pod example results - if: always() - run: | - FAILED="" - [ "${{ steps.vector-add-mixed-l3.outcome }}" = "success" ] || FAILED="$FAILED vector_add_mixed_l3" - [ "${{ steps.global-tload-mixed-l3.outcome }}" = "success" ] || FAILED="$FAILED global_tload_mixed_l3" - [ "${{ steps.compute-then-tload-mixed-l3.outcome }}" = "success" ] || FAILED="$FAILED compute_then_tload_mixed_l3" - if [ -n "$FAILED" ]; then - echo "::error::pod examples failed:$FAILED" - exit 1 - fi - echo "all pod examples passed" - # Both sides' device logs and each daemon's output land under the run # directory and stay on the runner otherwise. A device-side failure — a # scheduler timeout on the peer, say — names its sub-class only there, so diff --git a/conftest.py b/conftest.py index 2e5e1a0c97..13bf5fd3d2 100644 --- a/conftest.py +++ b/conftest.py @@ -95,6 +95,13 @@ def release(self, ids: list[int]) -> None: _device_pool: DevicePool | None = None +class PodPeer(typing.NamedTuple): + endpoint: str + remote_device_ids: tuple[int, ...] + session_timeout_s: float + session_listen_host: str + + def pytest_addoption(parser): """Register CLI options.""" parser.addoption("--platform", action="store", default=None, help="Target platform (e.g., a2a3sim, a2a3)") @@ -430,6 +437,11 @@ def pytest_configure(config): config.addinivalue_line("markers", "platforms(list): supported platforms for standalone ST functions") config.addinivalue_line("markers", "requires_hardware: test needs Ascend toolchain and real device") config.addinivalue_line("markers", "device_count(n): number of NPU devices needed") + config.addinivalue_line("markers", "pod: test needs the pod runner and its peer machine") + config.addinivalue_line( + "markers", + "pod_remote_device_count(n): number of remote NPU devices needed on the peer machine", + ) config.addinivalue_line( "markers", "sdma: the test provisions the PTO-ISA async-SDMA workspace. " @@ -1245,6 +1257,62 @@ def st_platform(request): return p +@pytest.fixture(scope="session") +def st_pod_peer(): + """Pod endpoint and remote device pool from the pod runner environment.""" + endpoint = os.environ.get("POD_REMOTE_ENDPOINT") + if not endpoint: + pytest.skip("POD_REMOTE_ENDPOINT is required for pod tests") + remote_devices = os.environ.get("POD_REMOTE_DEVICES") + if not remote_devices: + pytest.skip("POD_REMOTE_DEVICES is required for pod tests") + try: + session_timeout_s = float(os.environ.get("POD_L3_SESSION_TIMEOUT_S", "120")) + except ValueError as e: + pytest.fail(f"POD_L3_SESSION_TIMEOUT_S must be a float: {e}") + # The remote peer connects back to the parent session runner. + return PodPeer( + endpoint=endpoint, + remote_device_ids=tuple(_parse_device_range(remote_devices)), + session_timeout_s=session_timeout_s, + session_listen_host=os.environ.get("POD_L3_SESSION_LISTEN_HOST", "0.0.0.0"), # noqa: S104 + ) + + +@pytest.fixture() +def st_pod_remote_device_ids(request, st_pod_peer): + """Allocate remote device IDs from the pod peer's default device pool. + + Every pod test gets the same leading slice, so this is collision-free only + while the pod job serializes the sweep with ``--max-parallel 1``. + """ + marker = request.node.get_closest_marker("pod_remote_device_count") + n = marker.args[0] if marker else 1 + if n > len(st_pod_peer.remote_device_ids): + pytest.fail( + f"need {n} remote devices but POD_REMOTE_DEVICES only has {len(st_pod_peer.remote_device_ids)} entries" + ) + return list(st_pod_peer.remote_device_ids[:n]) + + +@pytest.fixture() +def st_pod_logs(request, monkeypatch): + """Per-test parent log directory for pod scene tests.""" + if request.node.get_closest_marker("pod") is None: + pytest.fail("st_pod_logs requires @pytest.mark.pod") + run_dir = os.environ.get("RUN_DIR") + if not run_dir: + if not os.environ.get("POD_REMOTE_ENDPOINT") or not os.environ.get("POD_REMOTE_DEVICES"): + pytest.skip("pod runner environment is required for pod tests") + pytest.fail("RUN_DIR is required for pod tests") + machine = os.environ.get("POD_MACHINE", "parent") + nodeid = re.sub(r"[^A-Za-z0-9_.-]+", "_", request.node.nodeid) + log_path = os.path.join(run_dir, "pytest", f"parent-{machine}", "ascend", nodeid) + os.makedirs(log_path, exist_ok=True) + monkeypatch.setenv("ASCEND_PROCESS_LOG_PATH", log_path) + return log_path + + @pytest.fixture(scope="session") def _l2_worker_pool(request, st_platform): """Session-scoped L2 worker pool keyed by (runtime, device_id). diff --git a/docs/ci.md b/docs/ci.md index 01ce7576bd..9e389a05c2 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -58,10 +58,10 @@ PullRequest | `st-sim-a2a3` | `ubuntu-latest`, `macos-latest` | `pytest examples tests/st --platform a2a3sim` | | `st-sim-a5` | `ubuntu-latest`, `macos-latest` | `pytest examples tests/st --platform a5sim` | | `ut-a2a3` | a2a3 self-hosted | `pytest tests/ut --platform a2a3` + `ctest -L "^requires_hardware(_a2a3)?$" --resource-spec-file ...` + build `tools/cann-examples/query` and run `query version` (no device) + build `tools/cann-examples/aicpu-device-query` and `tools/cann-examples/aicpu-kernel-launch` (host + cross-compiled device SO, link smoke only) | -| `st-onboard-a2a3` | a2a3 self-hosted | `pytest examples tests/st -m "not sdma" --platform a2a3 --device ...`, then a separate `-m sdma` step, then adaptive-parallel DFX feature smokes | +| `st-onboard-a2a3` | a2a3 self-hosted | `pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device ...`, then a separate `-m sdma` step, then adaptive-parallel DFX feature smokes | | `ut-a5` | a5 self-hosted | `pytest tests/ut --platform a5` + `ctest -L "^requires_hardware(_a5)?$"` + build `tools/cann-examples/query` and run `query version` (no device) + build `tools/cann-examples/aicpu-device-query` and `tools/cann-examples/aicpu-kernel-launch` (link smoke only) | -| `st-onboard-a5` | a5 self-hosted | `pytest examples tests/st --platform a5 --device ...`, then adaptive-parallel DFX feature smokes; x86_64 runners add `-m "not sdma"` to the main sweep | -| `st-pod-onboard-a2a3` | a pair of `a2a3pod` machines | the L4 mixed local/remote examples, one L3 per machine | +| `st-onboard-a5` | a5 self-hosted | `pytest examples tests/st -m "not pod" --platform a5 --device ...`, then adaptive-parallel DFX feature smokes; x86_64 runners use `-m "not sdma and not pod"` | +| `st-pod-onboard-a2a3` | a pair of `a2a3pod` machines | `pytest examples tests/st -m pod --platform a2a3 --device ... --max-parallel 1`, one L3 daemon on the peer | ### Multi-machine pod jobs @@ -73,32 +73,28 @@ addresses, the device split, ports, the staging root, proxies — comes from a Adding or re-addressing a machine is an edit to that file. Only a machine hosting a runner needs one. -Its body splits by what is per-run and what is per-example: +Its body splits by what is per-run and what is per-pytest session: | Action | Called | What it does | | ------ | ------ | ------------ | | `pod-stage` | once | rsync this run's tree onto the peer and build it there | -| `pod-run-example` | once per example | start the peer's L3 daemon, run the example's parent, stop the daemon and pull its logs | +| `pod-run-pytest` | once | start the peer's L3 daemon, set the pod pytest environment, run `pytest examples tests/st -m pod`, stop the daemon and pull its logs | | `pod-teardown` | once, `if: always()` | remove the run's tree from the peer | -Staging and the peer-side build are the job's whole cost, and every example -runs against that same tree and venv, so they happen once; only the daemon and -the parent repeat. A job-level matrix over examples would instead repeat the -staging and both venvs per branch. +Staging and the peer-side build are the job's whole cost, and every pod test +runs against that same tree and venv, so they happen once. The pytest command +owns selection: adding an L4 pod example means adding a `test_*.py` wrapper with +`@pytest.mark.pod`, not editing `_st-pod.yml`. -Examples run with `continue-on-error` and a summary step decides the result: -one round holds two machines, so learning about only the first failure wastes -the second half of it. **A step reporting green there has not necessarily -passed** — `continue-on-error` rewrites a failed step's `conclusion` to success -and leaves the truth in `outcome`, which only the summary step reads. +Pod logs go to `output/pod-ci--/pytest/` and the whole directory +is uploaded as one artifact. Parent-side `ASCEND_PROCESS_LOG_PATH` is split per +pytest nodeid by `st_pod_logs`; peer-side daemon/device logs are grouped for the +pytest session. Reach for the artifact first on a device-side failure: the host +traceback only says the peer's scheduler gave up, and the sub-class saying why +is printed on the device. -Each example's logs go to `output/pod-ci--//` and the -whole directory is uploaded as one artifact. Reach for it first on a -device-side failure: the host traceback only says the peer's scheduler gave -up, and the sub-class saying why is printed on the device. - -Writing an example — the files, the entry module, the environment variables -`pod-run-example` sets — is covered in +Writing an example — the files, the entry module, the `run(...)` entry point, +the `test_*.py` pod wrapper, and the manual `run_parent.sh` — is covered in [`examples/workers/README.md`](../examples/workers/README.md). ### Nightly sanitizer sweep @@ -134,16 +130,16 @@ benefit — device bin-packing for L3, xdist fanout for L2, and a shared `ChipWorker` per `(runtime, device)`: ```bash -# Recommended CI invocation — a2a3 deselects the SDMA marker, as the job does, -# and runs it as a second pass afterwards -pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 -x +# Recommended CI invocation — a2a3 deselects SDMA and pod tests, as the job does, +# and runs SDMA as a second pass afterwards +pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 -x pytest examples tests/st -m sdma --platform a2a3 --device 4-5 -x -# A5 ARM64 runners run the full corpus -pytest examples tests/st --platform a5 --device 0-7 -x +# A5 ARM64 runners run the non-pod corpus +pytest examples tests/st -m "not pod" --platform a5 --device 0-7 -x -# A5 x86_64 runners deselect SDMA tests -pytest examples tests/st -m "not sdma" --platform a5 --device 0-7 -x +# A5 x86_64 runners deselect SDMA and pod tests +pytest examples tests/st -m "not sdma and not pod" --platform a5 --device 0-7 -x ``` `-x` (`--exitfirst`) is appropriate for CI, where aborting on first @@ -195,7 +191,7 @@ not need `--max-parallel` manually. The arch flags subtract `NON_CODE` before deciding, so a non-code-only change already makes both `false`. An arch-gated job therefore needs no separate non-code check. See [`.claude/rules/ci-change-detection.md`](../.claude/rules/ci-change-detection.md) for the invariants these gates must keep. -- **SDMA tests run as their own step inside `st-onboard-a2a3`.** The sweep deselects them with `-m "not sdma"` and a later step runs `-m sdma`. Ordering is what the two paths share: the SDMA step is always second, so no fault-injection case can land on a device that has already provisioned SDMA. Device acquisition differs by host arch — on aarch64 the SDMA step takes its own `task-submit --device auto --device-num 2`, so the two steps are disjoint in devices as well; on x86_64 there is no `task-submit` and both steps use the same `${DEVICE_RANGE}`, leaving ordering as the only separation. Provisioning the SDMA workspace creates device-only STARS streams that live in the device fault domain, so an AICore fault on a device that has provisioned SDMA costs minutes instead of milliseconds — the sweep's `aicore_op_timeout` fault injection must therefore never share a device with them ([#1425](https://github.com/hw-native-sys/simpler/issues/1425)). Selection is by marker on both sides, so the two cannot drift apart; the split can be dropped once #1425 is fixed. Nothing outside `st-onboard-a2a3` filters on the marker, so a local `pytest examples tests/st` still runs everything. +- **SDMA tests run as their own step inside `st-onboard-a2a3`.** The ordinary sweep deselects them with `-m "not sdma and not pod"` and a later step runs `-m sdma`. Ordering is what the two SDMA paths share: the SDMA step is always second, so no fault-injection case can land on a device that has already provisioned SDMA. Device acquisition differs by host arch — on aarch64 the SDMA step takes its own `task-submit --device auto --device-num 2`, so the two steps are disjoint in devices as well; on x86_64 there is no `task-submit` and both steps use the same `${DEVICE_RANGE}`, leaving ordering as the only separation. Provisioning the SDMA workspace creates device-only STARS streams that live in the device fault domain, so an AICore fault on a device that has provisioned SDMA costs minutes instead of milliseconds — the sweep's `aicore_op_timeout` fault injection must therefore never share a device with them ([#1425](https://github.com/hw-native-sys/simpler/issues/1425)). Selection is by marker on both sides, so the two cannot drift apart; the split can be dropped once #1425 is fixed. Pod tests are selected by `-m pod` in `st-pod-onboard-a2a3` and explicitly excluded from ordinary onboard ST lanes. ### CPU emergency lane (`ci-self-cpu.yml`) and the `/run-cpu` button @@ -245,7 +241,7 @@ runner pools, branched at run time on the host arch (`uname -m`): `--device ${DEVICE_RANGE}`. a5 runners always use `task-submit`. The x86_64 A5 scene-test sweep deselects -the `sdma` marker; ARM64 runs the full corpus, including SDMA tests. Steps that +the `sdma` marker; ARM64 runs the full non-pod corpus, including SDMA tests. Steps that only build (cmake, `RuntimeBuilder`, the `cann-examples` smokes) take no lock on either arch. The same device-lock rule applies to local onboard work — see diff --git a/docs/comm-domain.md b/docs/comm-domain.md index 7fc6a34563..1191ca4348 100644 --- a/docs/comm-domain.md +++ b/docs/comm-domain.md @@ -112,7 +112,8 @@ local/remote paths with the `sim` backend. It also exercises the real profile is `a3-fabric-v1` on real A3 devices. Those two examples are the in-repository harness for the Fabric path; a run that needs to know whether Fabric was covered should read that job rather than infer it from the -simulation checks. +simulation checks. The job now drives their `test_*.py` wrappers through +`pod-run-pytest` rather than calling `run_parent.sh` directly. --- diff --git a/docs/testing.md b/docs/testing.md index 732ee7be5b..d612408d84 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,21 +46,21 @@ ctest --test-dir tests/ut/cpp/build -L "^requires_hardware(_a2a3)?$" --output-on # Scene tests (pytest, @scene_test classes) pytest examples tests/st # all sim platforms (auto-parametrized) pytest examples tests/st --platform a2a3sim # specific sim -pytest examples tests/st -m "not sdma" --platform a2a3 # hardware -pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 # hardware with device pool +pytest examples tests/st -m "not sdma and not pod" --platform a2a3 # hardware +pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 # hardware with device pool # Compile the selected hardware batch without creating a Worker or using an NPU python -m simpler_setup.tools.scene_test_compile examples tests/st \ - -m "not sdma" --platform a2a3 --require-pto-isa --compile-workers 8 + -m "not sdma and not pod" --platform a2a3 --require-pto-isa --compile-workers 8 # SDMA cases run separately, as they do in CI: they are quarantined by # @pytest.mark.sdma so no fault-injection case shares a device with a # provisioned SDMA workspace (issue #1425) pytest examples tests/st -m sdma --platform a2a3 --device 4-5 -# A5 ARM64 runs the full corpus; A5 x86_64 deselects SDMA tests -pytest examples tests/st --platform a5 --device 0-7 -pytest examples tests/st -m "not sdma" --platform a5 --device 0-7 +# A5 ARM64 runs the non-pod corpus; A5 x86_64 also deselects SDMA tests +pytest examples tests/st -m "not pod" --platform a5 --device 0-7 +pytest examples tests/st -m "not sdma and not pod" --platform a5 --device 0-7 # Single scene test (standalone) python examples/a2a3/tensormap_and_ringbuffer/vector_example/test_vector_example.py -p a2a3sim @@ -665,7 +665,7 @@ pytest examples tests/st --platform a2a3sim python test_my_kernel.py -p a2a3sim # On hardware (SDMA cases quarantined by marker; run them with -m sdma) -pytest examples tests/st -m "not sdma" --platform a2a3 +pytest examples tests/st -m "not sdma and not pod" --platform a2a3 ``` Key fields: diff --git a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md index 92a3865898..c143d7017e 100644 --- a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md +++ b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md @@ -63,7 +63,7 @@ exception you must surface the CANN device slog, which is otherwise hidden: ```bash ASCEND_SLOG_PRINT_TO_STDOUT=1 ASCEND_GLOBAL_LOG_LEVEL=1 \ - python -m pytest examples tests/st -m "not sdma" --platform a2a3 --device -v \ + python -m pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device -v \ --pto-session-timeout 600 ``` diff --git a/docs/user/reference/cli.md b/docs/user/reference/cli.md index 83d361d166..1460f6937a 100644 --- a/docs/user/reference/cli.md +++ b/docs/user/reference/cli.md @@ -6,7 +6,7 @@ How you select what runs, turn diagnostics on, and read the artifacts back. ```bash pytest examples tests/st --platform a2a3sim # simulation, no device -pytest examples tests/st -m "not sdma" --platform a2a3 --device 4-7 # hardware (SDMA quarantined by marker) +pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) python examples/my_example/test_my_example.py -p a2a3sim # standalone, no pytest ``` diff --git a/examples/README.md b/examples/README.md index 6798586931..842d083c62 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,7 +17,7 @@ on both simulators: ```bash pytest examples --platform a2a3sim -pytest examples -m "not sdma" --platform a2a3 --device 0-1 # hardware (SDMA quarantined by marker) +pytest examples -m "not sdma and not pod" --platform a2a3 --device 0-1 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) ``` A single example: diff --git a/examples/workers/README.md b/examples/workers/README.md index c88ccc9254..f811dae79c 100644 --- a/examples/workers/README.md +++ b/examples/workers/README.md @@ -67,7 +67,8 @@ l4// README.md kernels/aiv/*.cpp kernels/orchestration/*.cpp - main.py # entry point: argparse + main() + main.py # entry point: argparse + main() delegating to run() + test_.py # @pytest.mark.pod wrapper collected by pod CI run_parent.sh # maps environment variables onto main.py's flags ``` @@ -88,9 +89,8 @@ it. Three things are load-bearing: ### The environment-variable contract -`run_parent.sh` exists to turn environment variables into `main.py`'s flags, -because that is the interface CI drives it through. Pick a prefix and read -these five; CI sets exactly them: +`run_parent.sh` exists to turn environment variables into `main.py`'s flags for +manual two-machine runs. Pick a prefix and read these five: | Variable | Meaning | | -------- | ------- | @@ -105,11 +105,10 @@ Anything else — platform, runtime — defaults inside `run_parent.sh`. ### Running it in CI The `st-pod-onboard-a2a3` job runs L4 examples across a pair of a2a3 machines. -Adding yours to it is one block plus one line, and the wiring, the log -artifact, and the failure semantics are described in -[`docs/ci.md`](../../docs/ci.md#multi-machine-pod-jobs). Read that before adding the -block — in particular why a step that reports green there may still have -failed. +The job runs one `pytest examples tests/st -m pod` sweep, so adding yours means +adding a `test_*.py` wrapper carrying `@pytest.mark.pod`. Do not edit +`_st-pod.yml`. The wiring and the log artifact are described in +[`docs/ci.md`](../../docs/ci.md#multi-machine-pod-jobs). ## Prerequisites @@ -149,9 +148,10 @@ require an Ascend NPU box with `ASCEND_HOME_PATH` set. L2 and L3 examples follow that uniform CLI. **L4 examples do not** — they need a peer's address and a device split on each side, so they take `--remote`, -`--local-devices` and `--remote-devices` instead of `-p`/`-d`, and are normally -launched through their `run_parent.sh`. See each L4 example's README for the -two-machine sequence. +`--local-devices` and `--remote-devices` instead of `-p`/`-d`. Each L4 example +now also ships a `test_*.py` wrapper for pytest collection; CI uses that wrapper +and `run_parent.sh` remains the manual entry point. See each L4 example's README +for the two-machine sequence. ## Related documentation diff --git a/examples/workers/l4/compute_then_tload_mixed_l3/README.md b/examples/workers/l4/compute_then_tload_mixed_l3/README.md index 1cfd8346cd..24972854fb 100644 --- a/examples/workers/l4/compute_then_tload_mixed_l3/README.md +++ b/examples/workers/l4/compute_then_tload_mixed_l3/README.md @@ -83,4 +83,8 @@ Any `max_diff` above its tolerance exits non-zero. The peer `TLOAD` needs real cross-device windows, so the default profile is `a3-fabric-v1` and requires real A3 devices on both machines. In CI the pod -job drives this script through `pod-run-example` with the same variables. +job runs the `test_compute_then_tload_mixed_l3.py` wrapper through +`pod-run-pytest`. CI supplies `POD_REMOTE_ENDPOINT`, `POD_REMOTE_DEVICES`, +`POD_L3_SESSION_TIMEOUT_S`, `POD_L3_SESSION_LISTEN_HOST`, and pytest's +`--platform` / `--device` options instead of the `SIMPLER_*` variables above; +`run_parent.sh` remains the manual entry point. diff --git a/examples/workers/l4/compute_then_tload_mixed_l3/main.py b/examples/workers/l4/compute_then_tload_mixed_l3/main.py index 9e5fa09f5d..01290f9e07 100644 --- a/examples/workers/l4/compute_then_tload_mixed_l3/main.py +++ b/examples/workers/l4/compute_then_tload_mixed_l3/main.py @@ -245,45 +245,58 @@ def _parse_args() -> argparse.Namespace: return parser.parse_args() -def main() -> int: # noqa: PLR0915 -- one linear two-phase scenario; splitting it would scatter the phase ordering this example exists to show +def run( # noqa: PLR0915 -- one linear two-phase scenario; splitting it would scatter the phase ordering this example exists to show + *, + remote: str, + local_devices: str, + remote_devices: str, + platform: str = "a2a3", + runtime: str = "tensormap_and_ringbuffer", + comm_profile: str = "a3-fabric-v1", + session_timeout: float = 120.0, + session_listen_host: str = "0.0.0.0", # noqa: S104 - Remote peer callbacks need a reachable listener. +) -> int: # The local L3 is a fork of this process, so its orchestration functions # reach the handles only through module state; locals would not survive # into the child. global _LOCAL_COMPUTE_HANDLE, _LOCAL_TLOAD_HANDLE # noqa: PLW0603 - args = _parse_args() - local_device = _parse_first_device(args.local_devices, label="local") - remote_device = _parse_first_device(args.remote_devices, label="remote") - - compute_callable = _build_compute_callable(args.platform, args.runtime) - tload_callable = _build_tload_callable(args.platform, args.runtime) - local_l3 = Worker( - level=3, - device_ids=[local_device], - num_sub_workers=0, - platform=args.platform, - runtime=args.runtime, - comm_profile=args.comm_profile, - global_device_ranks=(0,), - ) - _LOCAL_COMPUTE_HANDLE = local_l3.register(compute_callable) - _LOCAL_TLOAD_HANDLE = local_l3.register(tload_callable) + local_device = _parse_first_device(local_devices, label="local") + remote_device = _parse_first_device(remote_devices, label="remote") - worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=args.session_timeout) + local_l3: Worker | None = None + local_l3_attached = False + worker: Worker | None = None domain_handle: GlobalCommDomainHandle | None = None parent_keepalive: list[TaskArgs] = [] try: + compute_callable = _build_compute_callable(platform, runtime) + tload_callable = _build_tload_callable(platform, runtime) + local_l3 = Worker( + level=3, + device_ids=[local_device], + num_sub_workers=0, + platform=platform, + runtime=runtime, + comm_profile=comm_profile, + global_device_ranks=(0,), + ) + _LOCAL_COMPUTE_HANDLE = local_l3.register(compute_callable) + _LOCAL_TLOAD_HANDLE = local_l3.register(tload_callable) + + worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=session_timeout) local_node = worker.add_worker(local_l3) + local_l3_attached = True remote_node = worker.add_remote_worker( RemoteWorkerSpec( - endpoint=args.remote, - platform=args.platform, - runtime=args.runtime, + endpoint=remote, + platform=platform, + runtime=runtime, device_ids=(remote_device,), transport=HOST_TCP_TRANSPORT_PROFILE, - comm_profile=args.comm_profile, + comm_profile=comm_profile, global_device_ranks=(1,), - session_listen_host=args.session_listen_host, + session_listen_host=session_listen_host, allow_wildcard_session_bind=True, ) ) @@ -378,7 +391,27 @@ def verify_phase(orch, _args, _cfg): if domain_handle is not None and not domain_handle.freed: with contextlib.suppress(Exception): domain_handle.release() - worker.close() + try: + if worker is not None: + worker.close() + finally: + if local_l3 is not None and not local_l3_attached: + with contextlib.suppress(Exception): + local_l3.close() + + +def main() -> int: + args = _parse_args() + return run( + remote=args.remote, + local_devices=args.local_devices, + remote_devices=args.remote_devices, + platform=args.platform, + runtime=args.runtime, + comm_profile=args.comm_profile, + session_timeout=args.session_timeout, + session_listen_host=args.session_listen_host, + ) if __name__ == "__main__": diff --git a/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py b/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py new file mode 100644 index 0000000000..91bc43f3f6 --- /dev/null +++ b/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py @@ -0,0 +1,34 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Pod ST for examples/workers/l4/compute_then_tload_mixed_l3.""" + +import pytest + +from .main import run + + +def _device_spec(device_ids) -> str: + return ",".join(str(device_id) for device_id in device_ids) + + +@pytest.mark.pod +@pytest.mark.platforms(["a2a3"]) +@pytest.mark.runtime("tensormap_and_ringbuffer") +@pytest.mark.device_count(1) +@pytest.mark.pod_remote_device_count(1) +def test_compute_then_tload_mixed_l3(st_platform, st_device_ids, st_pod_peer, st_pod_remote_device_ids, st_pod_logs): + rc = run( + remote=st_pod_peer.endpoint, + local_devices=_device_spec(st_device_ids), + remote_devices=_device_spec(st_pod_remote_device_ids), + platform=st_platform, + session_timeout=st_pod_peer.session_timeout_s, + session_listen_host=st_pod_peer.session_listen_host, + ) + assert rc == 0 diff --git a/examples/workers/l4/global_tload_mixed_l3/README.md b/examples/workers/l4/global_tload_mixed_l3/README.md index 5f3e5bc87d..21cec4779c 100644 --- a/examples/workers/l4/global_tload_mixed_l3/README.md +++ b/examples/workers/l4/global_tload_mixed_l3/README.md @@ -81,4 +81,8 @@ Any `max_diff` above the tolerance exits non-zero. The peer `TLOAD` needs real cross-device windows, so the default profile is `a3-fabric-v1` and requires real A3 devices on both machines. In CI the pod -job drives this script through `pod-run-example` with the same variables. +job runs the `test_global_tload_mixed_l3.py` wrapper through `pod-run-pytest`. +CI supplies `POD_REMOTE_ENDPOINT`, `POD_REMOTE_DEVICES`, +`POD_L3_SESSION_TIMEOUT_S`, `POD_L3_SESSION_LISTEN_HOST`, and pytest's +`--platform` / `--device` options instead of the `SIMPLER_*` variables above; +`run_parent.sh` remains the manual entry point. diff --git a/examples/workers/l4/global_tload_mixed_l3/main.py b/examples/workers/l4/global_tload_mixed_l3/main.py index 1aa1c52331..d820aee6d3 100644 --- a/examples/workers/l4/global_tload_mixed_l3/main.py +++ b/examples/workers/l4/global_tload_mixed_l3/main.py @@ -151,43 +151,56 @@ def _parse_args() -> argparse.Namespace: return parser.parse_args() -def main() -> int: +def run( + *, + remote: str, + local_devices: str, + remote_devices: str, + platform: str = "a2a3", + runtime: str = "tensormap_and_ringbuffer", + comm_profile: str = "a3-fabric-v1", + session_timeout: float = 120.0, + session_listen_host: str = "0.0.0.0", # noqa: S104 - Remote peer callbacks need a reachable listener. +) -> int: # The local L3 is a fork of this process, so its orchestration function # reaches the handle only through module state; a local would not survive # into the child. global _LOCAL_CHIP_HANDLE # noqa: PLW0603 - args = _parse_args() - local_device = _parse_first_device(args.local_devices, label="local") - remote_device = _parse_first_device(args.remote_devices, label="remote") - - chip_callable = _build_tload_callable(args.platform, args.runtime) - local_l3 = Worker( - level=3, - device_ids=[local_device], - num_sub_workers=0, - platform=args.platform, - runtime=args.runtime, - comm_profile=args.comm_profile, - global_device_ranks=(0,), - ) - _LOCAL_CHIP_HANDLE = local_l3.register(chip_callable) + local_device = _parse_first_device(local_devices, label="local") + remote_device = _parse_first_device(remote_devices, label="remote") - worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=args.session_timeout) + local_l3: Worker | None = None + local_l3_attached = False + worker: Worker | None = None domain_handle: GlobalCommDomainHandle | None = None parent_keepalive: list[TaskArgs] = [] try: + chip_callable = _build_tload_callable(platform, runtime) + local_l3 = Worker( + level=3, + device_ids=[local_device], + num_sub_workers=0, + platform=platform, + runtime=runtime, + comm_profile=comm_profile, + global_device_ranks=(0,), + ) + _LOCAL_CHIP_HANDLE = local_l3.register(chip_callable) + + worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=session_timeout) local_node = worker.add_worker(local_l3) + local_l3_attached = True remote_node = worker.add_remote_worker( RemoteWorkerSpec( - endpoint=args.remote, - platform=args.platform, - runtime=args.runtime, + endpoint=remote, + platform=platform, + runtime=runtime, device_ids=(remote_device,), transport=HOST_TCP_TRANSPORT_PROFILE, - comm_profile=args.comm_profile, + comm_profile=comm_profile, global_device_ranks=(1,), - session_listen_host=args.session_listen_host, + session_listen_host=session_listen_host, allow_wildcard_session_bind=True, ) ) @@ -262,7 +275,27 @@ def read_and_release(orch, _args, _cfg): if domain_handle is not None and not domain_handle.freed: with contextlib.suppress(Exception): domain_handle.release() - worker.close() + try: + if worker is not None: + worker.close() + finally: + if local_l3 is not None and not local_l3_attached: + with contextlib.suppress(Exception): + local_l3.close() + + +def main() -> int: + args = _parse_args() + return run( + remote=args.remote, + local_devices=args.local_devices, + remote_devices=args.remote_devices, + platform=args.platform, + runtime=args.runtime, + comm_profile=args.comm_profile, + session_timeout=args.session_timeout, + session_listen_host=args.session_listen_host, + ) if __name__ == "__main__": diff --git a/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py b/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py new file mode 100644 index 0000000000..dbcf6fada3 --- /dev/null +++ b/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py @@ -0,0 +1,34 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Pod ST for examples/workers/l4/global_tload_mixed_l3.""" + +import pytest + +from .main import run + + +def _device_spec(device_ids) -> str: + return ",".join(str(device_id) for device_id in device_ids) + + +@pytest.mark.pod +@pytest.mark.platforms(["a2a3"]) +@pytest.mark.runtime("tensormap_and_ringbuffer") +@pytest.mark.device_count(1) +@pytest.mark.pod_remote_device_count(1) +def test_global_tload_mixed_l3(st_platform, st_device_ids, st_pod_peer, st_pod_remote_device_ids, st_pod_logs): + rc = run( + remote=st_pod_peer.endpoint, + local_devices=_device_spec(st_device_ids), + remote_devices=_device_spec(st_pod_remote_device_ids), + platform=st_platform, + session_timeout=st_pod_peer.session_timeout_s, + session_listen_host=st_pod_peer.session_listen_host, + ) + assert rc == 0 diff --git a/examples/workers/l4/vector_add_mixed_l3/main.py b/examples/workers/l4/vector_add_mixed_l3/main.py index 7853dd4933..b39d88fa08 100644 --- a/examples/workers/l4/vector_add_mixed_l3/main.py +++ b/examples/workers/l4/vector_add_mixed_l3/main.py @@ -12,6 +12,7 @@ from __future__ import annotations import argparse +import contextlib import ctypes from pathlib import Path from typing import Any @@ -243,35 +244,47 @@ def _parse_args() -> argparse.Namespace: return parser.parse_args() -def main() -> int: +def run( + *, + remote: str, + local_devices: str, + remote_devices: str, + platform: str = "a2a3", + runtime: str = "tensormap_and_ringbuffer", + session_timeout: float = 120.0, + session_listen_host: str = "0.0.0.0", # noqa: S104 - Remote peer callbacks need a reachable listener. +) -> int: # The local L3 is a fork of this process, so its orchestration function # reaches the handle only through module state; a local would not survive # into the child. global _LOCAL_CHIP_HANDLE # noqa: PLW0603 - args = _parse_args() - local_devices = _parse_device_ids(args.local_devices, label="local") - remote_devices = _parse_device_ids(args.remote_devices, label="remote") - - chip_callable = _build_vector_chip_callable(args.platform, args.runtime) - local_l3 = Worker(level=3, platform=args.platform, runtime=args.runtime, device_ids=local_devices) - _LOCAL_CHIP_HANDLE = local_l3.register(chip_callable) + local_device_ids = _parse_device_ids(local_devices, label="local") + remote_device_ids = _parse_device_ids(remote_devices, label="remote") - worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=args.session_timeout) + local_l3: Worker | None = None + local_l3_attached = False + worker: Worker | None = None remote_buffers: list[RemoteBufferHandle] = [] local_views: list[Any] = [] local_outputs: dict[str, tuple[Any, float]] = {} parent_keepalive: list[TaskArgs] = [] try: + chip_callable = _build_vector_chip_callable(platform, runtime) + local_l3 = Worker(level=3, platform=platform, runtime=runtime, device_ids=local_device_ids) + _LOCAL_CHIP_HANDLE = local_l3.register(chip_callable) + + worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=session_timeout) local_worker = worker.add_worker(local_l3) + local_l3_attached = True remote_worker = worker.add_remote_worker( RemoteWorkerSpec( - endpoint=args.remote, - platform=args.platform, - runtime=args.runtime, - device_ids=remote_devices, + endpoint=remote, + platform=platform, + runtime=runtime, + device_ids=remote_device_ids, transport=HOST_TCP_TRANSPORT_PROFILE, - session_listen_host=args.session_listen_host, + session_listen_host=session_listen_host, allow_wildcard_session_bind=True, ) ) @@ -325,7 +338,7 @@ def parent_orch(orch, _args, cfg): print( "vector_add_mixed_l3 passed: " - f"local[devices={args.local_devices}], remote={args.remote}[devices={args.remote_devices}], " + f"local[devices={local_devices}], remote={remote}[devices={remote_devices}], " f"elements={ELEMENTS}" ) return 0 @@ -333,17 +346,37 @@ def parent_orch(orch, _args, cfg): parent_keepalive.clear() _LOCAL_GROUP_KEEPALIVE.clear() _REMOTE_GROUP_KEEPALIVE.clear() - for handle in reversed(remote_buffers): - try: - worker.remote_free(handle) - except Exception as exc: # noqa: BLE001 - # The pod job diagnoses this example from stdout alone, so a - # leaked peer buffer has to name itself here or leave no trace. - print(f"[vector-add-mixed-l3] remote_free failed: {exc}") + if worker is not None: + for handle in reversed(remote_buffers): + try: + worker.remote_free(handle) + except Exception as exc: # noqa: BLE001 + # The pod job diagnoses this example from stdout alone, so a + # leaked peer buffer has to name itself here or leave no trace. + print(f"[vector-add-mixed-l3] remote_free failed: {exc}") # close() unlinks the owner Buffers, which fails while any view still aliases their shm. local_outputs.clear() local_views.clear() - worker.close() + try: + if worker is not None: + worker.close() + finally: + if local_l3 is not None and not local_l3_attached: + with contextlib.suppress(Exception): + local_l3.close() + + +def main() -> int: + args = _parse_args() + return run( + remote=args.remote, + local_devices=args.local_devices, + remote_devices=args.remote_devices, + platform=args.platform, + runtime=args.runtime, + session_timeout=args.session_timeout, + session_listen_host=args.session_listen_host, + ) if __name__ == "__main__": diff --git a/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py b/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py new file mode 100644 index 0000000000..a41aa2c150 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py @@ -0,0 +1,34 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Pod ST for examples/workers/l4/vector_add_mixed_l3.""" + +import pytest + +from .main import run + + +def _device_spec(device_ids) -> str: + return ",".join(str(device_id) for device_id in device_ids) + + +@pytest.mark.pod +@pytest.mark.platforms(["a2a3"]) +@pytest.mark.runtime("tensormap_and_ringbuffer") +@pytest.mark.device_count(2) +@pytest.mark.pod_remote_device_count(2) +def test_vector_add_mixed_l3(st_platform, st_device_ids, st_pod_peer, st_pod_remote_device_ids, st_pod_logs): + rc = run( + remote=st_pod_peer.endpoint, + local_devices=_device_spec(st_device_ids), + remote_devices=_device_spec(st_pod_remote_device_ids), + platform=st_platform, + session_timeout=st_pod_peer.session_timeout_s, + session_listen_host=st_pod_peer.session_listen_host, + ) + assert rc == 0 diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py b/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py new file mode 100644 index 0000000000..9911a78fbd --- /dev/null +++ b/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py @@ -0,0 +1,38 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Pod collection sentinel for tests/st. + +This intentionally re-runs the global_tload_mixed_l3 example path so pod CI +proves both `examples/` and `tests/st/` pod tests are collected by marker. +""" + +import pytest + +from examples.workers.l4.global_tload_mixed_l3.main import run + + +def _device_spec(device_ids) -> str: + return ",".join(str(device_id) for device_id in device_ids) + + +@pytest.mark.pod +@pytest.mark.platforms(["a2a3"]) +@pytest.mark.runtime("tensormap_and_ringbuffer") +@pytest.mark.device_count(1) +@pytest.mark.pod_remote_device_count(1) +def test_global_tload_mixed_l3_pod(st_platform, st_device_ids, st_pod_peer, st_pod_remote_device_ids, st_pod_logs): + rc = run( + remote=st_pod_peer.endpoint, + local_devices=_device_spec(st_device_ids), + remote_devices=_device_spec(st_pod_remote_device_ids), + platform=st_platform, + session_timeout=st_pod_peer.session_timeout_s, + session_listen_host=st_pod_peer.session_listen_host, + ) + assert rc == 0