hw-native-sys · ChaoWao · Aug 5, 2026 · Aug 4, 2026
diff --git a/.github/actions/pod-run-example/action.yml b/.github/actions/pod-run-example/action.yml
@@ -0,0 +1,189 @@
+name: Run one pod example across the pair
+description: >-
+  Start the peer's L3 daemon, run one example's parent against it, then stop the
+  daemon and pull its logs back. Call once per example; pod-stage has already
+  put the tree and the venv on the peer, which every example shares.
+
+inputs:
+  example:
+    description: Example directory name. Names this example's log directory, so it must be unique within a run.
+    required: true
+  env-prefix:
+    description: >-
+      Environment-variable prefix the example's parent script reads, e.g.
+      SIMPLER_VECTOR_ADD_MIXED_L3. Taken as an input rather than derived from
+      `example`, so an example is free to name its variables as it likes. The
+      daemon side needs no prefix — it is the same generic session server for
+      every example and is started directly below.
+    required: true
+  parent-script:
+    description: Repo-relative script that runs the parent side.
+    required: true
+  local-devices:
+    description: Device ids the local L3 owns. Defaults to the pod config's.
+    required: false
+    default: ''
+  remote-devices:
+    description: Device ids the peer's L3 owns. Defaults to the pod config's.
+    required: false
+    default: ''
+  daemon-port:
+    description: Port the peer's daemon listens on. Defaults to the pod config's.
+    required: false
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    # The daemon must outlive this step, so its ssh is backgrounded with its
+    # output redirected to a file. The pid goes to a file because the collect
+    # step below cannot inherit a shell variable. python3, not the venv's
+    # python: this poll only needs the stdlib and the venv is not activated.
+    - name: Start the peer's L3 daemon (${{ inputs.example }})
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        EXAMPLE: ${{ inputs.example }}
+        ENV_PREFIX: ${{ inputs.env-prefix }}
+        IN_DAEMON_PORT: ${{ inputs.daemon-port }}
+      run: |
+        set -euo pipefail
+        source "$POD_SSH_HELPER"
+        DAEMON_PORT="${IN_DAEMON_PORT:-$POD_L3_DAEMON_PORT}"
+        LOCAL_LOGS="$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}"
+        REMOTE_LOGS="output/pod-ci/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}"
+        mkdir -p "$LOCAL_LOGS" "$RUN_DIR/$EXAMPLE/parent-${POD_MACHINE}/ascend"
+        {
+          echo "POD_EXAMPLE=$EXAMPLE"
+          echo "POD_EXAMPLE_ENV_PREFIX=$ENV_PREFIX"
+          echo "POD_EXAMPLE_DAEMON_PORT=$DAEMON_PORT"
+          echo "POD_EXAMPLE_REMOTE_LOGS=$REMOTE_LOGS"
+        } >> "$GITHUB_ENV"
+
+        pod_ssh "
+          pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true
+        " || true
+
+        pod_ssh "
+          set -eo pipefail
+          cd '$REMOTE_WORKDIR'
+          export PYTHONPATH="\${PYTHONPATH:-}"
+          export CMAKE_PREFIX_PATH="\${CMAKE_PREFIX_PATH:-}"
+          source '$POD_REMOTE_CANN_ENV'
+          set -u
+          source .venv/bin/activate
+          # The job-level timeouts are the parent's environment; ssh carries no
+          # environment, so the peer would otherwise schedule against the
+          # built-in defaults and the two halves of one run would disagree on
+          # how long a stall may last.
+          export SIMPLER_SCHEDULER_TIMEOUT_MS='$SIMPLER_SCHEDULER_TIMEOUT_MS'
+          export SIMPLER_OP_EXECUTE_TIMEOUT_US='$SIMPLER_OP_EXECUTE_TIMEOUT_US'
+          export SIMPLER_STREAM_SYNC_TIMEOUT_MS='$SIMPLER_STREAM_SYNC_TIMEOUT_MS'
+          mkdir -p '$REMOTE_LOGS'
+          export ASCEND_PROCESS_LOG_PATH="\$PWD/$REMOTE_LOGS"
+          echo '[pod-daemon] machine${POD_REMOTE_MACHINE} listening on ${REMOTE_DAEMON_HOST}:${DAEMON_PORT}'
+          python -m simpler.remote_l3_worker --host '${REMOTE_DAEMON_HOST}' --port '${DAEMON_PORT}'
+        " > "$LOCAL_LOGS/daemon.ssh.log" 2>&1 &
+        echo $! > "$RUNNER_TEMP/pod-daemon-$EXAMPLE.pid"
+
+        # A daemon that dies during import never opens the port, so on its own
+        # the connect probe only reports that after the full wait. The ssh
+        # process exits with it, so its absence is the earlier and more precise
+        # signal — and it is the one that can name the log holding the reason.
+        python3 - <<PYWAIT
+        import os
+        import socket
+        import sys
+        import time
+
+        ssh_pid = int(open("$RUNNER_TEMP/pod-daemon-$EXAMPLE.pid").read().strip())
+
+        def ssh_alive():
+            try:
+                os.kill(ssh_pid, 0)
+            except ProcessLookupError:
+                return False
+            # An exited child stays visible to kill(0) until bash reaps it, so
+            # the zombie state counts as gone.
+            try:
+                with open(f"/proc/{ssh_pid}/stat") as stat:
+                    return stat.read().rsplit(") ", 1)[1].split(" ", 1)[0] != "Z"
+            except OSError:
+                return False
+
+        deadline = time.time() + ${POD_DAEMON_WAIT_S}
+        while time.time() < deadline:
+            if not ssh_alive():
+                print(
+                    "remote daemon ssh exited before the port opened; "
+                    "see $LOCAL_LOGS/daemon.ssh.log",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            try:
+                with socket.create_connection(("${REMOTE_DAEMON_HOST}", ${DAEMON_PORT}), timeout=2):
+                    sys.exit(0)
+            except OSError:
+                time.sleep(1)
+        print("remote daemon did not become reachable", file=sys.stderr)
+        sys.exit(1)
+        PYWAIT
+
+    - name: Run the parent (${{ inputs.example }})
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        EXAMPLE: ${{ inputs.example }}
+        ENV_PREFIX: ${{ inputs.env-prefix }}
+        PARENT_SCRIPT: ${{ inputs.parent-script }}
+        IN_LOCAL_DEVICES: ${{ inputs.local-devices }}
+        IN_REMOTE_DEVICES: ${{ inputs.remote-devices }}
+      run: |
+        set -eo pipefail
+        export PYTHONPATH="${PYTHONPATH:-}"
+        export CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}"
+        source "$POD_CANN_ENV"
+        source .venv/bin/activate
+        set -u
+        export ASCEND_PROCESS_LOG_PATH="$RUN_DIR/$EXAMPLE/parent-${POD_MACHINE}/ascend"
+        export "${ENV_PREFIX}_REMOTE=${REMOTE_DAEMON_HOST}:${POD_EXAMPLE_DAEMON_PORT}"
+        export "${ENV_PREFIX}_LOCAL_DEVICES=${IN_LOCAL_DEVICES:-$POD_LOCAL_DEVICES}"
+        export "${ENV_PREFIX}_REMOTE_DEVICES=${IN_REMOTE_DEVICES:-$POD_REMOTE_DEVICES}"
+        export "${ENV_PREFIX}_SESSION_TIMEOUT=$POD_L3_SESSION_TIMEOUT_S"
+        export "${ENV_PREFIX}_SESSION_LISTEN_HOST=$POD_L3_SESSION_LISTEN_HOST"
+
+        timeout "${POD_SMOKE_TIMEOUT_S}s" bash "$PARENT_SCRIPT"
+
+    # Runs even when the parent failed — that is the case whose logs are worth
+    # having, since a device-side failure names its reason only there. The
+    # staging tree itself stays for the next example; pod-teardown clears it.
+    - name: Stop the daemon and collect its logs (${{ inputs.example }})
+      if: always()
+      shell: bash
+      working-directory: ${{ github.workspace }}
+      env:
+        EXAMPLE: ${{ inputs.example }}
+      run: |
+        set +e
+        [ -r "${POD_SSH_HELPER:-}" ] || exit 0
+        source "$POD_SSH_HELPER"
+
+        PID_FILE="$RUNNER_TEMP/pod-daemon-$EXAMPLE.pid"
+        if [ -r "$PID_FILE" ]; then
+          kill "$(cat "$PID_FILE")" 2>/dev/null
+          rm -f "$PID_FILE"
+        fi
+        # The daemon's chip children keep writing after the daemon itself is
+        # signalled, so wait for it to actually go before anything reads or
+        # removes that tree.
+        pod_ssh "
+          pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' || true
+          for _ in \$(seq 1 20); do
+            pgrep -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' >/dev/null || break
+            sleep 0.5
+          done
+        "
+        rsync -a -e "$RSYNC_SSH" \
+          "$REMOTE_TARGET:$REMOTE_WORKDIR/$POD_EXAMPLE_REMOTE_LOGS/" \
+          "$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}/" 2>/dev/null
+        exit 0
diff --git a/.github/actions/pod-stage/action.yml b/.github/actions/pod-stage/action.yml
@@ -0,0 +1,115 @@
+name: Stage the checkout on the pod peer
+description: >-
+  Put this run's source tree on the peer and build it there. Done once per run —
+  the tree and the venv it produces are what every pod example then runs
+  against, so re-staging per example would repeat the job's whole cost.
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      working-directory: ${{ github.workspace }}
+      run: |
+        set -euo pipefail
+        if ! command -v rsync >/dev/null 2>&1; then
+          echo "::error::rsync is required on the pod runner for remote checkout sync"
+          exit 1
+        fi
+
+        REMOTE_TARGET="${POD_REMOTE_USER:+$POD_REMOTE_USER@}$POD_REMOTE_HOST"
+        REMOTE_WORKDIR="$POD_REMOTE_STAGING_ROOT/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+        RUN_DIR="$GITHUB_WORKSPACE/output/pod-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+
+        # The peer's proxy is only reachable from the peer, so its liveness
+        # check has to run there; split the address here where the parsing is
+        # readable. pip below reaches PyPI over https, so an unreachable
+        # https proxy is the one that actually stops the staging build.
+        proxy_hostport() {
+          local hostport=${1#*://} host port
+          hostport=${hostport##*@}
+          hostport=${hostport%%/*}
+          host=${hostport%:*}
+          port=${hostport##*:}
+          [ "$host" != "$port" ] || port=$2
+          printf '%s %s' "$host" "$port"
+        }
+        REMOTE_PROXY_HOST=""
+        REMOTE_PROXY_PORT=""
+        REMOTE_PROXY_HTTPS_HOST=""
+        REMOTE_PROXY_HTTPS_PORT=""
+        if [ -n "$POD_REMOTE_HTTP_PROXY" ]; then
+          read -r REMOTE_PROXY_HOST REMOTE_PROXY_PORT <<<"$(proxy_hostport "$POD_REMOTE_HTTP_PROXY" 80)"
+        fi
+        if [ -n "$POD_REMOTE_HTTPS_PROXY" ]; then
+          read -r REMOTE_PROXY_HTTPS_HOST REMOTE_PROXY_HTTPS_PORT <<<"$(proxy_hostport "$POD_REMOTE_HTTPS_PROXY" 443)"
+        fi
+
+        {
+          echo "REMOTE_TARGET=$REMOTE_TARGET"
+          echo "REMOTE_WORKDIR=$REMOTE_WORKDIR"
+          echo "RUN_DIR=$RUN_DIR"
+          echo "REMOTE_DAEMON_HOST=$POD_REMOTE_HOST"
+          echo "POD_SSH_HELPER=$RUNNER_TEMP/pod-ssh.sh"
+        } >> "$GITHUB_ENV"
+
+        echo "pod parent=${POD_MACHINE} local=${POD_LOCAL_DEVICES} peer=${POD_REMOTE_MACHINE} remote=${POD_REMOTE_DEVICES}"
+
+        mkdir -p "$RUN_DIR"
+        mkdir -p "$HOME/.ssh"
+        chmod 700 "$HOME/.ssh"
+        touch "$HOME/.ssh/known_hosts"
+        chmod 600 "$HOME/.ssh/known_hosts"
+        ssh-keyscan -H -p "$POD_REMOTE_SSH_PORT" "$POD_REMOTE_HOST" >> "$HOME/.ssh/known_hosts" 2>/dev/null || true
+
+        # SSH_OPTS is a bash array and pod_ssh a function, neither of which any
+        # step or action boundary carries, so they live in a file the rest of
+        # the job sources.
+        cat > "$RUNNER_TEMP/pod-ssh.sh" <<'HELPER'
+        SSH_OPTS=(-p "$POD_REMOTE_SSH_PORT" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$HOME/.ssh/known_hosts")
+        RSYNC_SSH="ssh -p $POD_REMOTE_SSH_PORT -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HOME/.ssh/known_hosts"
+        pod_ssh() { ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" "$@"; }
+        HELPER
+        source "$RUNNER_TEMP/pod-ssh.sh"
+
+        # Each run stages a full source tree plus its own venv on the peer, so
+        # the staging root grows without a reclaim step. pod-teardown removes
+        # this run's directory, but a cancelled job is SIGKILLed and never
+        # reaches it — the age sweep is what reclaims those. The TTL must stay
+        # well above the job's timeout so a live run is never swept out from
+        # under itself.
+        pod_ssh "
+          mkdir -p '$POD_REMOTE_STAGING_ROOT'
+          find '$POD_REMOTE_STAGING_ROOT' -mindepth 1 -maxdepth 1 -type d -mtime +${POD_REMOTE_STAGING_TTL_DAYS} -exec rm -rf {} + || true
+          rm -rf '$REMOTE_WORKDIR' && mkdir -p '$REMOTE_WORKDIR'
+        "
+        rsync -a --delete -e "$RSYNC_SSH" \
+          --exclude .git --exclude .venv --exclude build --exclude output \
+          "$GITHUB_WORKSPACE/" "$REMOTE_TARGET:$REMOTE_WORKDIR/"
+
+        # The script goes to `bash -s` on stdin rather than to the peer's
+        # default shell: `pipefail` and `/dev/tcp` below are bash features, and
+        # nothing pins the peer's login shell to bash. Stdin also keeps the
+        # script one argument — ssh concatenates argv into a command string,
+        # which would re-split a multi-line `bash -c` payload on whitespace.
+        pod_ssh bash -s <<REMOTE_STAGE
+          set -eo pipefail
+          cd '$REMOTE_WORKDIR'
+          if [ -n '$REMOTE_PROXY_HOST' ] && ! (exec 3<>'/dev/tcp/$REMOTE_PROXY_HOST/$REMOTE_PROXY_PORT') 2>/dev/null; then
+            echo '::error::POD_REMOTE_HTTP_PROXY=$POD_REMOTE_HTTP_PROXY is not reachable from machine $POD_REMOTE_MACHINE'
+            exit 1
+          fi
+          if [ -n '$REMOTE_PROXY_HTTPS_HOST' ] && ! (exec 3<>'/dev/tcp/$REMOTE_PROXY_HTTPS_HOST/$REMOTE_PROXY_HTTPS_PORT') 2>/dev/null; then
+            echo '::error::POD_REMOTE_HTTPS_PROXY=$POD_REMOTE_HTTPS_PROXY is not reachable from machine $POD_REMOTE_MACHINE'
+            exit 1
+          fi
+          export PYTHONPATH="\${PYTHONPATH:-}"
+          export CMAKE_PREFIX_PATH="\${CMAKE_PREFIX_PATH:-}"
+          export http_proxy='$POD_REMOTE_HTTP_PROXY' https_proxy='$POD_REMOTE_HTTPS_PROXY' no_proxy='$POD_REMOTE_NO_PROXY'
+          export HTTP_PROXY='$POD_REMOTE_HTTP_PROXY' HTTPS_PROXY='$POD_REMOTE_HTTPS_PROXY' NO_PROXY='$POD_REMOTE_NO_PROXY'
+          source '$POD_REMOTE_CANN_ENV'
+          set -u
+          python3 -m venv --system-site-packages .venv
+          source .venv/bin/activate
+          pip install --upgrade pip
+          pip install '.[test]'
+        REMOTE_STAGE
diff --git a/.github/actions/pod-teardown/action.yml b/.github/actions/pod-teardown/action.yml
@@ -0,0 +1,23 @@
+name: Clear the pod staging tree
+description: >-
+  Remove this run's tree from the peer. Call once, with `if: always()`. Each
+  pod-run-example already stopped its own daemon; the sweep here only catches an
+  example that died before reaching that step.
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      working-directory: ${{ github.workspace }}
+      run: |
+        set +e
+        # pod-stage may never have run, so assume nothing.
+        [ -n "${REMOTE_WORKDIR:-}" ] || exit 0
+        [ -r "${POD_SSH_HELPER:-}" ] || exit 0
+        source "$POD_SSH_HELPER"
+
+        pod_ssh "pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST}' || true"
+        # rm -rf on a tree a surviving chip child is still writing fails with
+        # ENOTEMPTY, so retry once after giving it a moment.
+        pod_ssh "rm -rf '$REMOTE_WORKDIR' 2>/dev/null || { sleep 5; rm -rf '$REMOTE_WORKDIR'; }"
+        exit 0
diff --git a/.github/actions/setup-venv/action.yml b/.github/actions/setup-venv/action.yml
@@ -18,6 +18,29 @@ inputs:
     description: Source the CANN environment before creating the venv.
     required: false
     default: "false"
+  cann-env:
+    description: >-
+      Which CANN environment script source-cann sources. The pod runners carry
+      theirs in their own .env, so the path cannot be fixed here.
+    required: false
+    default: /usr/local/Ascend/cann/set_env.sh
+  http-proxy:
+    description: >-
+      Proxy for pip only, never exported job-wide — a runner that already
+      reaches its index directly must not be routed through an untested hop.
+      Empty keeps whatever the runner's own environment already says.
+    required: false
+    default: ""
+  https-proxy:
+    description: >-
+      Proxy for pip only. Empty keeps the runner's own setting.
+    required: false
+    default: ""
+  no-proxy:
+    description: >-
+      Hosts pip reaches without the proxy. Empty keeps the runner's own setting.
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -26,12 +49,24 @@ runs:
       shell: bash
       env:
         PACKAGES: ${{ inputs.packages }}
+        CANN_ENV: ${{ inputs.cann-env }}
+        IN_HTTP_PROXY: ${{ inputs.http-proxy }}
+        IN_HTTPS_PROXY: ${{ inputs.https-proxy }}
+        IN_NO_PROXY: ${{ inputs.no-proxy }}
       run: |
         set -f
         if [ "${{ inputs.source-cann }}" = "true" ]; then
-          source /usr/local/Ascend/cann/set_env.sh
+          source "$CANN_ENV"
         fi
 
+        # Only the pod workflow passes these; every other caller runs on a
+        # machine whose own environment already says how to reach an index.
+        # Exporting an empty input would erase that setting, so an unset input
+        # leaves the runner's environment untouched rather than forcing direct.
+        [ -z "$IN_HTTP_PROXY" ] || export http_proxy="$IN_HTTP_PROXY" HTTP_PROXY="$IN_HTTP_PROXY"
+        [ -z "$IN_HTTPS_PROXY" ] || export https_proxy="$IN_HTTPS_PROXY" HTTPS_PROXY="$IN_HTTPS_PROXY"
+        [ -z "$IN_NO_PROXY" ] || export no_proxy="$IN_NO_PROXY" NO_PROXY="$IN_NO_PROXY"
+
         VENV_ARGS=()
         if [ "${{ inputs.system-site-packages }}" = "true" ]; then
           VENV_ARGS+=(--system-site-packages)