CI: Collect L4 Pod Tests through pytest

.github/actions/pod-run-pytest/action.yml

@@ -0,0 +1,141 @@
+name: Run pod pytest across the pair
+description: >-
+  Start the peer's L3 daemon, run pod-marked pytest against it, then stop the
+  daemon and pull its logs back. pod-stage has already put the tree and venv on
+  the peer.
+
+inputs:
+  pytest-args:
+    description: Arguments passed to `python -m pytest`.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    # The daemon must outlive this step, so its ssh is backgrounded with its
+    # output redirected to a file. The pid goes to a file because the collect
+    # step below cannot inherit a shell variable. python3, not the venv's
+    # python: this poll only needs the stdlib and the venv is not activated.
+    - name: Start the peer's L3 daemon
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      run: |
+        set -euo pipefail
+        source "$POD_SSH_HELPER"
+        DAEMON_PORT="$POD_L3_DAEMON_PORT"
+        LOCAL_LOGS="$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}"
+        REMOTE_LOGS="output/pod-ci/pytest/daemon-${POD_REMOTE_MACHINE}"
+        mkdir -p "$LOCAL_LOGS" "$RUN_DIR/pytest/parent-${POD_MACHINE}/ascend"
+        {
+          echo "POD_REMOTE_ENDPOINT=${REMOTE_DAEMON_HOST}:${DAEMON_PORT}"
+          echo "POD_PYTEST_REMOTE_LOGS=$REMOTE_LOGS"
+        } >> "$GITHUB_ENV"
+
+        pod_ssh "
+          pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true
+        " || true
+
+        pod_ssh "
+          set -eo pipefail
+          cd '$REMOTE_WORKDIR'
+          export PYTHONPATH=\"\${PYTHONPATH:-}\"
+          export CMAKE_PREFIX_PATH=\"\${CMAKE_PREFIX_PATH:-}\"
+          source '$POD_REMOTE_CANN_ENV'
+          set -u
+          source .venv/bin/activate
+          export SIMPLER_SCHEDULER_TIMEOUT_MS='$SIMPLER_SCHEDULER_TIMEOUT_MS'
+          export SIMPLER_OP_EXECUTE_TIMEOUT_US='$SIMPLER_OP_EXECUTE_TIMEOUT_US'
+          export SIMPLER_STREAM_SYNC_TIMEOUT_MS='$SIMPLER_STREAM_SYNC_TIMEOUT_MS'
+          mkdir -p '$REMOTE_LOGS'
+          export ASCEND_PROCESS_LOG_PATH=\"\$PWD/$REMOTE_LOGS\"
+          echo '[pod-daemon] machine${POD_REMOTE_MACHINE} listening on ${REMOTE_DAEMON_HOST}:${DAEMON_PORT}'
+          python -m simpler.remote_l3_worker --host '${REMOTE_DAEMON_HOST}' --port '${DAEMON_PORT}'
+        " > "$LOCAL_LOGS/daemon.ssh.log" 2>&1 &
+        echo $! > "$RUNNER_TEMP/pod-daemon-pytest.pid"
+
+        # A daemon that dies during import never opens the port, so on its own
+        # the connect probe only reports that after the full wait. The ssh
+        # process exits with it, so its absence is the earlier and more precise
+        # signal, and it can name the log holding the reason.
+        python3 - <<PYWAIT
+        import os
+        import socket
+        import sys
+        import time
+
+        ssh_pid = int(open("$RUNNER_TEMP/pod-daemon-pytest.pid").read().strip())
+
+        def ssh_alive():
+            try:
+                os.kill(ssh_pid, 0)
+            except ProcessLookupError:
+                return False
+            try:
+                with open(f"/proc/{ssh_pid}/stat") as stat:
+                    return stat.read().rsplit(") ", 1)[1].split(" ", 1)[0] != "Z"
+            except OSError:
+                return False
+
+        deadline = time.time() + ${POD_DAEMON_WAIT_S}
+        while time.time() < deadline:
+            if not ssh_alive():
+                print(
+                    "remote daemon ssh exited before the port opened; "
+                    "see $LOCAL_LOGS/daemon.ssh.log",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            try:
+                with socket.create_connection(("${REMOTE_DAEMON_HOST}", ${DAEMON_PORT}), timeout=2):
+                    sys.exit(0)
+            except OSError:
+                time.sleep(1)
+        print("remote daemon did not become reachable", file=sys.stderr)
+        sys.exit(1)
+        PYWAIT
+
+    - name: Run pod pytest
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        PYTEST_ARGS: ${{ inputs.pytest-args }}
+      run: |
+        set -eo pipefail
+        export PYTHONPATH="${PYTHONPATH:-}"
+        export CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}"
+        source "$POD_CANN_ENV"
+        source .venv/bin/activate
+        set -u
+        mkdir -p "$RUN_DIR/pytest/parent-${POD_MACHINE}/ascend"
+        timeout "${POD_SMOKE_TIMEOUT_S}s" python -m pytest $PYTEST_ARGS
+
+    # Runs even when pytest failed, since a device-side failure names its reason
+    # only in the daemon or ASCEND logs. The staging tree itself stays for
+    # pod-teardown.
+    - name: Stop the daemon and collect peer logs
+      if: always()
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      run: |
+        set +e
+        [ -r "${POD_SSH_HELPER:-}" ] || exit 0
+        source "$POD_SSH_HELPER"
+
+        PID_FILE="$RUNNER_TEMP/pod-daemon-pytest.pid"
+        if [ -r "$PID_FILE" ]; then
+          kill "$(cat "$PID_FILE")" 2>/dev/null
+          rm -f "$PID_FILE"
+        fi
+        DAEMON_PORT="${POD_L3_DAEMON_PORT:-19073}"
+        pod_ssh "
+          pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true
+          for _ in \$(seq 1 20); do
+            pgrep -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' >/dev/null || break
+            sleep 0.5
+          done
+        "
+        mkdir -p "$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}"
+        rsync -a -e "$RSYNC_SSH" \
+          "$REMOTE_TARGET:$REMOTE_WORKDIR/${POD_PYTEST_REMOTE_LOGS:-output/pod-ci/pytest/daemon-${POD_REMOTE_MACHINE}}/" \
+          "$RUN_DIR/pytest/daemon-${POD_REMOTE_MACHINE}/" 2>/dev/null
+        exit 0

.github/actions/pod-teardown/action.yml

@@ -1,8 +1,8 @@
 name: Clear the pod staging tree
 description: >-
-  Remove this run's tree from the peer. Call once, with `if: always()`. Each
-  pod-run-example already stopped its own daemon; the sweep here only catches an
-  example that died before reaching that step.
+  Remove this run's tree from the peer. Call once, with `if: always()`. The pod
+  pytest action already stopped its own daemon; the sweep here only catches a
+  run that died before reaching that step.
 
 runs:
   using: composite

.github/workflows/_st-npu-a2a3.yml

@@ -78,10 +78,10 @@ jobs:
           source /usr/local/Ascend/cann/set_env.sh
           source .venv/bin/activate
           if [ "$(uname -m)" = "x86_64" ]; then
-            python -m pytest examples tests/st -m "not sdma" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200
+            python -m pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200
           else
             task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \
-              --run "python -m pytest examples tests/st -m 'not sdma' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200"
+              --run "python -m pytest examples tests/st -m 'not sdma and not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200"
           fi
 
       - name: Run pytest scene tests (a2a3 legacy SDMA paths)
@@ -91,10 +91,10 @@ jobs:
           source .venv/bin/activate
           SDMA_IGNORE="--ignore=examples/a2a3/tensormap_and_ringbuffer/prefetch_async_demo --ignore=examples/a2a3/tensormap_and_ringbuffer/sdma_async_completion_demo"
           if [ "$(uname -m)" = "x86_64" ]; then
-            python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600
+            python -m pytest examples tests/st $SDMA_IGNORE -m "not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600
           else
             task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \
-              --run "python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600"
+              --run "python -m pytest examples tests/st $SDMA_IGNORE -m 'not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600"
           fi
 
       - name: SDMA pytest (a2a3)

.github/workflows/_st-npu-a5.yml

@@ -59,9 +59,9 @@ jobs:
           python -m simpler_setup.tools.scene_test_compile examples tests/st \
             --platform a5 --require-pto-isa --compile-workers 8 -q
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
-          PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa"
+          PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa -m 'not pod'"
           if [ "$(uname -m)" = "x86_64" ]; then
-            PYTEST="$PYTEST -m 'not sdma'"
+            PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa -m 'not sdma and not pod'"
           fi
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST --pto-session-timeout 1200"