diff --git a/.github/actions/pod-run-example/action.yml b/.github/actions/pod-run-example/action.yml new file mode 100644 index 0000000000..9f93d8bb5a --- /dev/null +++ b/.github/actions/pod-run-example/action.yml @@ -0,0 +1,189 @@ +name: Run one pod example across the pair +description: >- + Start the peer's L3 daemon, run one example's parent against it, then stop the + daemon and pull its logs back. Call once per example; pod-stage has already + put the tree and the venv on the peer, which every example shares. + +inputs: + example: + description: Example directory name. Names this example's log directory, so it must be unique within a run. + required: true + env-prefix: + description: >- + Environment-variable prefix the example's parent script reads, e.g. + SIMPLER_VECTOR_ADD_MIXED_L3. Taken as an input rather than derived from + `example`, so an example is free to name its variables as it likes. The + daemon side needs no prefix — it is the same generic session server for + every example and is started directly below. + required: true + parent-script: + description: Repo-relative script that runs the parent side. + required: true + local-devices: + description: Device ids the local L3 owns. Defaults to the pod config's. + required: false + default: '' + remote-devices: + description: Device ids the peer's L3 owns. Defaults to the pod config's. + required: false + default: '' + daemon-port: + description: Port the peer's daemon listens on. Defaults to the pod config's. + required: false + default: '' + +runs: + using: composite + steps: + # The daemon must outlive this step, so its ssh is backgrounded with its + # output redirected to a file. The pid goes to a file because the collect + # step below cannot inherit a shell variable. python3, not the venv's + # python: this poll only needs the stdlib and the venv is not activated. + - name: Start the peer's L3 daemon (${{ inputs.example }}) + shell: bash + working-directory: ${{ github.workspace }} + env: + EXAMPLE: ${{ inputs.example }} + ENV_PREFIX: ${{ inputs.env-prefix }} + IN_DAEMON_PORT: ${{ inputs.daemon-port }} + run: | + set -euo pipefail + source "$POD_SSH_HELPER" + DAEMON_PORT="${IN_DAEMON_PORT:-$POD_L3_DAEMON_PORT}" + LOCAL_LOGS="$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}" + REMOTE_LOGS="output/pod-ci/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}" + mkdir -p "$LOCAL_LOGS" "$RUN_DIR/$EXAMPLE/parent-${POD_MACHINE}/ascend" + { + echo "POD_EXAMPLE=$EXAMPLE" + echo "POD_EXAMPLE_ENV_PREFIX=$ENV_PREFIX" + echo "POD_EXAMPLE_DAEMON_PORT=$DAEMON_PORT" + echo "POD_EXAMPLE_REMOTE_LOGS=$REMOTE_LOGS" + } >> "$GITHUB_ENV" + + pod_ssh " + pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${DAEMON_PORT}' || true + " || true + + pod_ssh " + set -eo pipefail + cd '$REMOTE_WORKDIR' + export PYTHONPATH="\${PYTHONPATH:-}" + export CMAKE_PREFIX_PATH="\${CMAKE_PREFIX_PATH:-}" + source '$POD_REMOTE_CANN_ENV' + set -u + source .venv/bin/activate + # The job-level timeouts are the parent's environment; ssh carries no + # environment, so the peer would otherwise schedule against the + # built-in defaults and the two halves of one run would disagree on + # how long a stall may last. + export SIMPLER_SCHEDULER_TIMEOUT_MS='$SIMPLER_SCHEDULER_TIMEOUT_MS' + export SIMPLER_OP_EXECUTE_TIMEOUT_US='$SIMPLER_OP_EXECUTE_TIMEOUT_US' + export SIMPLER_STREAM_SYNC_TIMEOUT_MS='$SIMPLER_STREAM_SYNC_TIMEOUT_MS' + mkdir -p '$REMOTE_LOGS' + export ASCEND_PROCESS_LOG_PATH="\$PWD/$REMOTE_LOGS" + echo '[pod-daemon] machine${POD_REMOTE_MACHINE} listening on ${REMOTE_DAEMON_HOST}:${DAEMON_PORT}' + python -m simpler.remote_l3_worker --host '${REMOTE_DAEMON_HOST}' --port '${DAEMON_PORT}' + " > "$LOCAL_LOGS/daemon.ssh.log" 2>&1 & + echo $! > "$RUNNER_TEMP/pod-daemon-$EXAMPLE.pid" + + # A daemon that dies during import never opens the port, so on its own + # the connect probe only reports that after the full wait. The ssh + # process exits with it, so its absence is the earlier and more precise + # signal — and it is the one that can name the log holding the reason. + python3 - </dev/null + rm -f "$PID_FILE" + fi + # The daemon's chip children keep writing after the daemon itself is + # signalled, so wait for it to actually go before anything reads or + # removes that tree. + pod_ssh " + pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' || true + for _ in \$(seq 1 20); do + pgrep -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST} --port ${POD_EXAMPLE_DAEMON_PORT}' >/dev/null || break + sleep 0.5 + done + " + rsync -a -e "$RSYNC_SSH" \ + "$REMOTE_TARGET:$REMOTE_WORKDIR/$POD_EXAMPLE_REMOTE_LOGS/" \ + "$RUN_DIR/$EXAMPLE/daemon-${POD_REMOTE_MACHINE}/" 2>/dev/null + exit 0 diff --git a/.github/actions/pod-stage/action.yml b/.github/actions/pod-stage/action.yml new file mode 100644 index 0000000000..869d12aebb --- /dev/null +++ b/.github/actions/pod-stage/action.yml @@ -0,0 +1,115 @@ +name: Stage the checkout on the pod peer +description: >- + Put this run's source tree on the peer and build it there. Done once per run — + the tree and the venv it produces are what every pod example then runs + against, so re-staging per example would repeat the job's whole cost. + +runs: + using: composite + steps: + - shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + if ! command -v rsync >/dev/null 2>&1; then + echo "::error::rsync is required on the pod runner for remote checkout sync" + exit 1 + fi + + REMOTE_TARGET="${POD_REMOTE_USER:+$POD_REMOTE_USER@}$POD_REMOTE_HOST" + REMOTE_WORKDIR="$POD_REMOTE_STAGING_ROOT/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + RUN_DIR="$GITHUB_WORKSPACE/output/pod-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + + # The peer's proxy is only reachable from the peer, so its liveness + # check has to run there; split the address here where the parsing is + # readable. pip below reaches PyPI over https, so an unreachable + # https proxy is the one that actually stops the staging build. + proxy_hostport() { + local hostport=${1#*://} host port + hostport=${hostport##*@} + hostport=${hostport%%/*} + host=${hostport%:*} + port=${hostport##*:} + [ "$host" != "$port" ] || port=$2 + printf '%s %s' "$host" "$port" + } + REMOTE_PROXY_HOST="" + REMOTE_PROXY_PORT="" + REMOTE_PROXY_HTTPS_HOST="" + REMOTE_PROXY_HTTPS_PORT="" + if [ -n "$POD_REMOTE_HTTP_PROXY" ]; then + read -r REMOTE_PROXY_HOST REMOTE_PROXY_PORT <<<"$(proxy_hostport "$POD_REMOTE_HTTP_PROXY" 80)" + fi + if [ -n "$POD_REMOTE_HTTPS_PROXY" ]; then + read -r REMOTE_PROXY_HTTPS_HOST REMOTE_PROXY_HTTPS_PORT <<<"$(proxy_hostport "$POD_REMOTE_HTTPS_PROXY" 443)" + fi + + { + echo "REMOTE_TARGET=$REMOTE_TARGET" + echo "REMOTE_WORKDIR=$REMOTE_WORKDIR" + echo "RUN_DIR=$RUN_DIR" + echo "REMOTE_DAEMON_HOST=$POD_REMOTE_HOST" + echo "POD_SSH_HELPER=$RUNNER_TEMP/pod-ssh.sh" + } >> "$GITHUB_ENV" + + echo "pod parent=${POD_MACHINE} local=${POD_LOCAL_DEVICES} peer=${POD_REMOTE_MACHINE} remote=${POD_REMOTE_DEVICES}" + + mkdir -p "$RUN_DIR" + mkdir -p "$HOME/.ssh" + chmod 700 "$HOME/.ssh" + touch "$HOME/.ssh/known_hosts" + chmod 600 "$HOME/.ssh/known_hosts" + ssh-keyscan -H -p "$POD_REMOTE_SSH_PORT" "$POD_REMOTE_HOST" >> "$HOME/.ssh/known_hosts" 2>/dev/null || true + + # SSH_OPTS is a bash array and pod_ssh a function, neither of which any + # step or action boundary carries, so they live in a file the rest of + # the job sources. + cat > "$RUNNER_TEMP/pod-ssh.sh" <<'HELPER' + SSH_OPTS=(-p "$POD_REMOTE_SSH_PORT" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$HOME/.ssh/known_hosts") + RSYNC_SSH="ssh -p $POD_REMOTE_SSH_PORT -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HOME/.ssh/known_hosts" + pod_ssh() { ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" "$@"; } + HELPER + source "$RUNNER_TEMP/pod-ssh.sh" + + # Each run stages a full source tree plus its own venv on the peer, so + # the staging root grows without a reclaim step. pod-teardown removes + # this run's directory, but a cancelled job is SIGKILLed and never + # reaches it — the age sweep is what reclaims those. The TTL must stay + # well above the job's timeout so a live run is never swept out from + # under itself. + pod_ssh " + mkdir -p '$POD_REMOTE_STAGING_ROOT' + find '$POD_REMOTE_STAGING_ROOT' -mindepth 1 -maxdepth 1 -type d -mtime +${POD_REMOTE_STAGING_TTL_DAYS} -exec rm -rf {} + || true + rm -rf '$REMOTE_WORKDIR' && mkdir -p '$REMOTE_WORKDIR' + " + rsync -a --delete -e "$RSYNC_SSH" \ + --exclude .git --exclude .venv --exclude build --exclude output \ + "$GITHUB_WORKSPACE/" "$REMOTE_TARGET:$REMOTE_WORKDIR/" + + # The script goes to `bash -s` on stdin rather than to the peer's + # default shell: `pipefail` and `/dev/tcp` below are bash features, and + # nothing pins the peer's login shell to bash. Stdin also keeps the + # script one argument — ssh concatenates argv into a command string, + # which would re-split a multi-line `bash -c` payload on whitespace. + pod_ssh bash -s <'/dev/tcp/$REMOTE_PROXY_HOST/$REMOTE_PROXY_PORT') 2>/dev/null; then + echo '::error::POD_REMOTE_HTTP_PROXY=$POD_REMOTE_HTTP_PROXY is not reachable from machine $POD_REMOTE_MACHINE' + exit 1 + fi + if [ -n '$REMOTE_PROXY_HTTPS_HOST' ] && ! (exec 3<>'/dev/tcp/$REMOTE_PROXY_HTTPS_HOST/$REMOTE_PROXY_HTTPS_PORT') 2>/dev/null; then + echo '::error::POD_REMOTE_HTTPS_PROXY=$POD_REMOTE_HTTPS_PROXY is not reachable from machine $POD_REMOTE_MACHINE' + exit 1 + fi + export PYTHONPATH="\${PYTHONPATH:-}" + export CMAKE_PREFIX_PATH="\${CMAKE_PREFIX_PATH:-}" + export http_proxy='$POD_REMOTE_HTTP_PROXY' https_proxy='$POD_REMOTE_HTTPS_PROXY' no_proxy='$POD_REMOTE_NO_PROXY' + export HTTP_PROXY='$POD_REMOTE_HTTP_PROXY' HTTPS_PROXY='$POD_REMOTE_HTTPS_PROXY' NO_PROXY='$POD_REMOTE_NO_PROXY' + source '$POD_REMOTE_CANN_ENV' + set -u + python3 -m venv --system-site-packages .venv + source .venv/bin/activate + pip install --upgrade pip + pip install '.[test]' + REMOTE_STAGE diff --git a/.github/actions/pod-teardown/action.yml b/.github/actions/pod-teardown/action.yml new file mode 100644 index 0000000000..5d3d3fdbdb --- /dev/null +++ b/.github/actions/pod-teardown/action.yml @@ -0,0 +1,23 @@ +name: Clear the pod staging tree +description: >- + Remove this run's tree from the peer. Call once, with `if: always()`. Each + pod-run-example already stopped its own daemon; the sweep here only catches an + example that died before reaching that step. + +runs: + using: composite + steps: + - shell: bash + working-directory: ${{ github.workspace }} + run: | + set +e + # pod-stage may never have run, so assume nothing. + [ -n "${REMOTE_WORKDIR:-}" ] || exit 0 + [ -r "${POD_SSH_HELPER:-}" ] || exit 0 + source "$POD_SSH_HELPER" + + pod_ssh "pkill -f 'python -m simpler.remote_l3_worker --host ${REMOTE_DAEMON_HOST}' || true" + # rm -rf on a tree a surviving chip child is still writing fails with + # ENOTEMPTY, so retry once after giving it a moment. + pod_ssh "rm -rf '$REMOTE_WORKDIR' 2>/dev/null || { sleep 5; rm -rf '$REMOTE_WORKDIR'; }" + exit 0 diff --git a/.github/actions/setup-venv/action.yml b/.github/actions/setup-venv/action.yml index 0a17151507..c8fd176f02 100644 --- a/.github/actions/setup-venv/action.yml +++ b/.github/actions/setup-venv/action.yml @@ -18,6 +18,29 @@ inputs: description: Source the CANN environment before creating the venv. required: false default: "false" + cann-env: + description: >- + Which CANN environment script source-cann sources. The pod runners carry + theirs in their own .env, so the path cannot be fixed here. + required: false + default: /usr/local/Ascend/cann/set_env.sh + http-proxy: + description: >- + Proxy for pip only, never exported job-wide — a runner that already + reaches its index directly must not be routed through an untested hop. + Empty keeps whatever the runner's own environment already says. + required: false + default: "" + https-proxy: + description: >- + Proxy for pip only. Empty keeps the runner's own setting. + required: false + default: "" + no-proxy: + description: >- + Hosts pip reaches without the proxy. Empty keeps the runner's own setting. + required: false + default: "" runs: using: composite @@ -26,12 +49,24 @@ runs: shell: bash env: PACKAGES: ${{ inputs.packages }} + CANN_ENV: ${{ inputs.cann-env }} + IN_HTTP_PROXY: ${{ inputs.http-proxy }} + IN_HTTPS_PROXY: ${{ inputs.https-proxy }} + IN_NO_PROXY: ${{ inputs.no-proxy }} run: | set -f if [ "${{ inputs.source-cann }}" = "true" ]; then - source /usr/local/Ascend/cann/set_env.sh + source "$CANN_ENV" fi + # Only the pod workflow passes these; every other caller runs on a + # machine whose own environment already says how to reach an index. + # Exporting an empty input would erase that setting, so an unset input + # leaves the runner's environment untouched rather than forcing direct. + [ -z "$IN_HTTP_PROXY" ] || export http_proxy="$IN_HTTP_PROXY" HTTP_PROXY="$IN_HTTP_PROXY" + [ -z "$IN_HTTPS_PROXY" ] || export https_proxy="$IN_HTTPS_PROXY" HTTPS_PROXY="$IN_HTTPS_PROXY" + [ -z "$IN_NO_PROXY" ] || export no_proxy="$IN_NO_PROXY" NO_PROXY="$IN_NO_PROXY" + VENV_ARGS=() if [ "${{ inputs.system-site-packages }}" = "true" ]; then VENV_ARGS+=(--system-site-packages) diff --git a/.github/workflows/_st-pod.yml b/.github/workflows/_st-pod.yml new file mode 100644 index 0000000000..7ffe10d513 --- /dev/null +++ b/.github/workflows/_st-pod.yml @@ -0,0 +1,231 @@ +name: NPU Pod Scene Tests + +permissions: + contents: read + +# A pod job spans two machines: the runner it lands on drives its peer entirely +# over ssh, and the peer runs no workflow code. Every value that identifies +# either machine — addresses, device split, ports, staging root, proxy — comes +# from the runner's own .env, so nothing machine-specific lives in this file. +# See docs/ci.md for the action boundaries and examples/workers/README.md for +# how an example plugs into it. +on: + workflow_call: + inputs: + repository: + required: false + type: string + ref: + required: false + type: string + platform: + description: a2a3 or a5. + required: true + type: string + runs_on: + required: true + type: string + +jobs: + run: + name: st-pod-onboard-${{ inputs.platform }} + runs-on: ${{ fromJSON(inputs.runs_on) }} + timeout-minutes: 60 + # A runner claim covers only the machine the job landed on; the peer is + # driven over ssh and nothing else knows it is busy. Two runs of this job + # would then share the peer's devices and its POD_L3_DAEMON_PORT. The group + # is deliberately free of github.ref so runs from different PRs collide + # here, and cancel-in-progress stays false because a cancelled job is + # SIGKILLed before pod-teardown can clear the peer. + concurrency: + group: pod-${{ inputs.platform }} + cancel-in-progress: false + env: + SIMPLER_SCHEDULER_TIMEOUT_MS: "2000" + SIMPLER_OP_EXECUTE_TIMEOUT_US: "3000000" + SIMPLER_STREAM_SYNC_TIMEOUT_MS: "4000" + + steps: + # Each pod machine carries one .env describing itself and its peer, so + # this job never has to work out which machine it landed on. + - name: Load pod machine config + run: | + set -euo pipefail + + # The runner root holds .env and sits two levels above + # RUNNER_WORKSPACE (/_work/). POD_ENV_FILE overrides the + # derivation for a runner laid out differently. + ENV_FILE="${POD_ENV_FILE:-$(dirname "$(dirname "$RUNNER_WORKSPACE")")/.env}" + if [ ! -r "$ENV_FILE" ]; then + echo "::error::pod config not readable: $ENV_FILE (set POD_ENV_FILE to override)" + exit 1 + fi + echo "pod config: $ENV_FILE" + + # The runner service reads this same file for its own settings, so + # only POD_* is accepted here — an unrelated key in it must not reach + # the job environment through this step. + while IFS= read -r line || [ -n "$line" ]; do + line=${line%$'\r'} + line=${line#"${line%%[![:space:]]*}"} + case "$line" in ''|'#'*) continue ;; esac + [[ "$line" == *=* ]] || continue + key=${line%%=*} + value=${line#*=} + [[ "$key" =~ ^POD_[A-Z0-9_]+$ ]] || continue + value=${value%"${value##*[![:space:]]}"} + case "$value" in + \"*\") value=${value#\"}; value=${value%\"} ;; + \'*\') value=${value#\'}; value=${value%\'} ;; + esac + printf -v "$key" '%s' "$value" + done < "$ENV_FILE" + + set_default() { + local name=$1 + shift + [ -n "${!name-}" ] || printf -v "$name" '%s' "$*" + } + set_default POD_LOCAL_IP "" + set_default POD_REMOTE_USER "" + set_default POD_REMOTE_SSH_PORT 22 + set_default POD_L3_DAEMON_PORT 19073 + set_default POD_L3_SESSION_TIMEOUT_S 180 + set_default POD_L3_SESSION_LISTEN_HOST 0.0.0.0 + set_default POD_REMOTE_STAGING_ROOT /data/workspace/ci-runner/pod-ci + set_default POD_REMOTE_STAGING_TTL_DAYS 2 + set_default POD_DAEMON_WAIT_S 120 + set_default POD_SMOKE_TIMEOUT_S 1800 + set_default POD_CANN_ENV /usr/local/Ascend/cann/set_env.sh + set_default POD_REMOTE_CANN_ENV "$POD_CANN_ENV" + set_default POD_HTTP_PROXY "" + set_default POD_HTTPS_PROXY "" + set_default POD_NO_PROXY "" + # Each machine runs its own proxy on its own port, and the peer's + # values are evaluated on the peer — a loopback address here and + # there name two different proxies. Unset means "same as this + # machine's", which only holds for a shared LAN proxy. + set_default POD_REMOTE_HTTP_PROXY "$POD_HTTP_PROXY" + set_default POD_REMOTE_HTTPS_PROXY "$POD_HTTPS_PROXY" + set_default POD_REMOTE_NO_PROXY "$POD_NO_PROXY" + + # A proxy nothing listens on turns every pip call into five retries + # ending in an opaque ProxyError, several steps from the setting that + # caused it. Name the address here instead. + proxy_reachable() { + local hostport=${1#*://} host port + hostport=${hostport##*@} + hostport=${hostport%%/*} + host=${hostport%:*} + port=${hostport##*:} + [ "$host" != "$port" ] || port=80 + (exec 3<>"/dev/tcp/$host/$port") 2>/dev/null + } + for key in POD_HTTP_PROXY POD_HTTPS_PROXY; do + if [ -n "${!key}" ] && ! proxy_reachable "${!key}"; then + echo "::error::$key=${!key} is not reachable from this machine" + exit 1 + fi + done + + MISSING=0 + for key in POD_MACHINE POD_LOCAL_DEVICES POD_REMOTE_MACHINE POD_REMOTE_HOST POD_REMOTE_DEVICES; do + if [ -z "${!key-}" ]; then + echo "::error::$key is missing from $ENV_FILE" + MISSING=1 + fi + done + [ "$MISSING" = 0 ] || exit 1 + + # The file is the source of truth for identity; this only catches the + # wrong machine's .env having been copied onto this runner. + if [ -n "$POD_LOCAL_IP" ] && ! hostname -I | tr ' ' '\n' | grep -qx "$POD_LOCAL_IP"; then + echo "::error::POD_LOCAL_IP=$POD_LOCAL_IP is not an address of this host" + hostname -I || true + exit 1 + fi + + for key in POD_MACHINE POD_LOCAL_IP POD_LOCAL_DEVICES \ + POD_REMOTE_MACHINE POD_REMOTE_HOST POD_REMOTE_DEVICES \ + POD_REMOTE_USER POD_REMOTE_SSH_PORT \ + POD_L3_DAEMON_PORT POD_L3_SESSION_TIMEOUT_S POD_L3_SESSION_LISTEN_HOST \ + POD_REMOTE_STAGING_ROOT POD_REMOTE_STAGING_TTL_DAYS \ + POD_DAEMON_WAIT_S POD_SMOKE_TIMEOUT_S \ + POD_CANN_ENV POD_REMOTE_CANN_ENV \ + POD_HTTP_PROXY POD_HTTPS_PROXY POD_NO_PROXY \ + POD_REMOTE_HTTP_PROXY POD_REMOTE_HTTPS_PROXY POD_REMOTE_NO_PROXY; do + printf '%s=%s\n' "$key" "${!key-}" >> "$GITHUB_ENV" + done + + echo "pod machine=${POD_MACHINE} ip=${POD_LOCAL_IP:-unset} devices=${POD_LOCAL_DEVICES}" + echo "pod peer=${POD_REMOTE_MACHINE} host=${POD_REMOTE_HOST}:${POD_REMOTE_SSH_PORT} devices=${POD_REMOTE_DEVICES}" + + - name: Checkout target + uses: actions/checkout@v5 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || github.ref }} + persist-credentials: false + + - name: Set up environment + uses: ./.github/actions/setup-venv + with: + packages: ".[test]" + install-torch: "false" + system-site-packages: "true" + source-cann: "true" + cann-env: ${{ env.POD_CANN_ENV }} + http-proxy: ${{ env.POD_HTTP_PROXY }} + https-proxy: ${{ env.POD_HTTPS_PROXY }} + no-proxy: ${{ env.POD_NO_PROXY }} + + # Staging and the peer-side build are per-run: every example below runs + # against the same tree and the same venv, so re-staging per example + # would repeat the whole cost of the job. + - name: Stage the checkout on the peer + uses: ./.github/actions/pod-stage + + # continue-on-error so a second example still runs when the first fails — + # one round of this job occupies two machines, and finding out about only + # the first failure wastes the second half of it. The summary step below + # is what makes the job red. + - name: vector_add_mixed_l3 + id: vector-add-mixed-l3 + continue-on-error: true + uses: ./.github/actions/pod-run-example + with: + example: vector_add_mixed_l3 + env-prefix: SIMPLER_VECTOR_ADD_MIXED_L3 + parent-script: examples/workers/l4/vector_add_mixed_l3/run_parent.sh + + - name: Clear the pod staging tree + if: always() + uses: ./.github/actions/pod-teardown + + # continue-on-error rewrites a failed example's `conclusion` to success + # and leaves the truth in `outcome`, so this is the only step whose + # result means the examples passed. + - name: Report pod example results + if: always() + run: | + FAILED="" + [ "${{ steps.vector-add-mixed-l3.outcome }}" = "success" ] || FAILED="$FAILED vector_add_mixed_l3" + if [ -n "$FAILED" ]; then + echo "::error::pod examples failed:$FAILED" + exit 1 + fi + echo "all pod examples passed" + + # Both sides' device logs and each daemon's output land under the run + # directory and stay on the runner otherwise. A device-side failure — a + # scheduler timeout on the peer, say — names its sub-class only there, so + # without this the run page shows the host traceback and nothing that + # explains it. + - name: Upload pod logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: pod-ci-${{ github.run_id }}-${{ github.run_attempt }} + path: output/pod-ci-${{ github.run_id }}-${{ github.run_attempt }}/ + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55fded0864..633b62b1e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,18 @@ jobs: include_dfx_smokes: true a2a3_sdma_mode: marker + # The only job that spans two machines. It is arch-specific and the most + # expensive thing in this file, so it gates exactly as st-onboard-a2a3 does — + # never on a weaker signal than a cheaper job, per + # .claude/rules/ci-change-detection.md. + st-pod-onboard-a2a3: + needs: [detect-changes, pre-commit] + if: needs.detect-changes.outputs.a2a3_changed == 'true' && needs.detect-changes.outputs.st_affected == 'true' + uses: ./.github/workflows/_st-pod.yml + with: + platform: a2a3 + runs_on: '["self-hosted","Linux","ARM64","a2a3pod"]' + ut-a5: needs: [detect-changes, pre-commit] if: needs.detect-changes.outputs.non_code_only != 'true' && needs.detect-changes.outputs.ut_affected == 'true' diff --git a/docs/capability-survey.md b/docs/capability-survey.md index 152701aa00..2c28fd2d8c 100644 --- a/docs/capability-survey.md +++ b/docs/capability-survey.md @@ -34,7 +34,7 @@ be re-checked directly. The 7-level model (L6 Cluster … L0 Core) is declared in [hierarchical-level-runtime.md](hierarchical-level-runtime.md). Its own status -table is accurate: L3 implemented; L4 local implemented, remote simulation only; +table is accurate: L3 implemented; L4 local implemented, remote host_tcp shipped; L5/L6 untested. **The level is a label in C++ and a real branch in Python.** `Worker` stores @@ -98,7 +98,7 @@ AICPU directly: [l3-l2-orch-comm.md](l3-l2-orch-comm.md) and `examples/workers/l3/`. Parent-directed targeting of a specific child is [directed-next-level-scheduling.md](directed-next-level-scheduling.md). -### L4 — host → remote host (control plane shipped, data plane simulation only) +### L4 — host → remote host (host_tcp data plane shipped) Shipped: `add_remote_worker(RemoteWorkerSpec)` → JSON manifest over TCP → `simpler-remote-worker` daemon → session runner → C++ `RemoteL3Endpoint` @@ -107,13 +107,15 @@ registered as a NEXT_LEVEL endpoint and dispatched by the same Scheduler The remote session builds a real `Worker(level=3, device_ids=…)` and initialises its chip subtree before answering READY. -**Design only:** `transport` defaults to `"sim"` and the daemon raises on any -other value (`python/simpler/remote_l3_worker.py:66`). Remote buffers are -`multiprocessing.shared_memory`, so `REMOTE_WINDOW` / `UB_LDST` are protocol -placeholders. The A2 RoCE / A3 HCCS / A5 UB HCOMM profiles are documented -contracts marked hardware-gated -([remote-l3-worker-design.md](remote-l3-worker-design.md):71-77). There is no -L4 example and no CI job starts the daemon. +`transport` defaults to `"host_tcp"`, and the daemon rejects other +profiles. Remote buffers use session-managed host storage, so `REMOTE_WINDOW` / +`UB_LDST` remain protocol placeholders for future peer-memory transports. The +A2 RoCE / A3 HCCS / A5 UB HCOMM profiles are documented contracts marked +hardware-gated +([remote-l3-worker-design.md](remote-l3-worker-design.md):71-77). +`examples/workers/l4/vector_add_mixed_l3/` is the L4 example — one parent +driving a local L3 subtree and a remote one on a second machine — and the +`st-pod-onboard-a2a3` job starts the daemon on that peer. ## Launching AICore and AICPU work (the CANN surface) diff --git a/docs/ci.md b/docs/ci.md index 974af6747c..a43bd8165c 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -20,7 +20,7 @@ The complete test-type × hardware-tier matrix. Empty cells have no tests yet; o | Category | github-hosted (no hardware) | a2a3 runner | a5 runner | | -------- | --------------------------- | ----------- | --------- | | **ut** (py + cpp) | `ut` | `ut-a2a3` | `ut-a5` | -| **st** | `st-sim-a2a3`, `st-sim-a5` | `st-onboard-a2a3` | `st-onboard-a5` | +| **st** | `st-sim-a2a3`, `st-sim-a5` | `st-onboard-a2a3`, `st-pod-onboard-a2a3` | `st-onboard-a5` | ## GitHub Actions Jobs @@ -30,11 +30,11 @@ shape. The executable job bodies live in reusable workflows: `_detect-changes.yml`, `_pre-commit.yml`, `_ut-no-hardware.yml`, `_packaging.yml`, `_profiling-flags-smoke.yml`, `_st-sim-a2a3.yml`, `_st-sim-a5.yml`, `_ut-npu-a2a3.yml`, `_ut-npu-a5.yml`, -`_st-npu-a2a3.yml`, and `_st-npu-a5.yml`. The scene-test and NPU unit-test -bodies are split one workflow per architecture so each job renders only its own -steps. Shared step scaffolding that is safe to run +`_st-npu-a2a3.yml`, `_st-npu-a5.yml`, and `_st-pod.yml`. The scene-test and +NPU unit-test bodies are split one workflow per architecture so each job +renders only its own steps. Shared step scaffolding that is safe to run after checkout lives in composite actions under `.github/actions/` -(`cache-pip`, `setup-venv`). +(`cache-pip`, `setup-venv`, and the three `pod-*` actions). ```text PullRequest @@ -47,6 +47,7 @@ PullRequest ├── st-sim-a5 (ubuntu + macOS) — a5_changed && st_affected ├── ut-a2a3 (a2a3 self-hosted) — Python + C++ UT, a2a3 hardware [needs ut_affected] ├── st-onboard-a2a3 (a2a3 self-hosted) — a2a3_changed && st_affected + ├── st-pod-onboard-a2a3 (a2a3pod pair) — a2a3_changed && st_affected ├── ut-a5 (a5 self-hosted) — Python + C++ UT, a5 hardware [needs ut_affected] └── st-onboard-a5 (a5 self-hosted) — a5_changed && st_affected ``` @@ -60,6 +61,45 @@ PullRequest | `st-onboard-a2a3` | a2a3 self-hosted | `pytest examples tests/st -m "not sdma" --platform a2a3 --device ...`, then a separate `-m sdma` step, then the DFX per-feature smokes | | `ut-a5` | a5 self-hosted | `pytest tests/ut --platform a5` + `ctest -L "^requires_hardware(_a5)?$"` + build `tools/cann-examples/query` and run `query version` (no device) + build `tools/cann-examples/aicpu-device-query` and `tools/cann-examples/aicpu-kernel-launch` (link smoke only) | | `st-onboard-a5` | a5 self-hosted | `pytest examples tests/st --platform a5 --device ...` | +| `st-pod-onboard-a2a3` | a pair of `a2a3pod` machines | the L4 mixed local/remote examples, one L3 per machine | + +### Multi-machine pod jobs + +`st-pod-onboard-a2a3` is the only job spanning two machines. The runner it +lands on becomes the L4 parent and drives its peer entirely over ssh; the peer +runs no workflow code at all. Everything that identifies either machine — +addresses, the device split, ports, the staging root, proxies — comes from a +`.env` the runner carries, so `_st-pod.yml` holds no machine-specific value. +Adding or re-addressing a machine is an edit to that file. Only a machine +hosting a runner needs one. + +Its body splits by what is per-run and what is per-example: + +| Action | Called | What it does | +| ------ | ------ | ------------ | +| `pod-stage` | once | rsync this run's tree onto the peer and build it there | +| `pod-run-example` | once per example | start the peer's L3 daemon, run the example's parent, stop the daemon and pull its logs | +| `pod-teardown` | once, `if: always()` | remove the run's tree from the peer | + +Staging and the peer-side build are the job's whole cost, and every example +runs against that same tree and venv, so they happen once; only the daemon and +the parent repeat. A job-level matrix over examples would instead repeat the +staging and both venvs per branch. + +Examples run with `continue-on-error` and a summary step decides the result: +one round holds two machines, so learning about only the first failure wastes +the second half of it. **A step reporting green there has not necessarily +passed** — `continue-on-error` rewrites a failed step's `conclusion` to success +and leaves the truth in `outcome`, which only the summary step reads. + +Each example's logs go to `output/pod-ci--//` and the +whole directory is uploaded as one artifact. Reach for it first on a +device-side failure: the host traceback only says the peer's scheduler gave +up, and the sub-class saying why is printed on the device. + +Writing an example — the files, the entry module, the environment variables +`pod-run-example` sets — is covered in +[`examples/workers/README.md`](../examples/workers/README.md). ### Nightly sanitizer sweep diff --git a/docs/remote-l3-worker-design.md b/docs/remote-l3-worker-design.md index 3a9734889d..8acc27de3d 100644 --- a/docs/remote-l3-worker-design.md +++ b/docs/remote-l3-worker-design.md @@ -407,7 +407,7 @@ callable registry: inner L3 Worker registry: hashid -> ChipCallable register payload, when needed hashid -> Python import descriptor, when needed -comm policy: roce | hccs | ub | sim +comm policy: host_tcp | roce | hccs | ub feature flags ``` @@ -587,16 +587,16 @@ The recommended first cut is conservative: **Implemented for C++ submit, Python `TaskArgs.add_tensor(RemoteTensorRef(...))`, owner buffers, and imported simulation buffers.** 3. Add the versioned frame codec and the independent health-lane contract. - **Implemented for the socket-backed simulation transport.** + **Implemented for the socket-backed host_tcp transport.** 4. Add remote callable registration with all-or-nothing multi-endpoint visibility and final-unregister cleanup. **Implemented for dispatcher `PYTHON_IMPORT`, inner `PYTHON_IMPORT`, and inner inline `CHIP_CALLABLE`.** 5. Add the fork-safe simulation session runner with explicit prestart before `HELLO READY`. **Implemented.** -6. Prove local behavior is unchanged and remote sim behavior handles success, +6. Prove local behavior is unchanged and remote host_tcp behavior handles success, failure, hashid mapping, timeouts, health, and buffer cleanup. - **Focused Python remote sim and C++ no-hardware UT coverage is present.** + **Focused Python remote host_tcp and C++ no-hardware UT coverage is present.** 7. Add A2 RoCE, A3 HCCS, and A5 UB profiles behind the HCOMM adapter layer. **Pending.** diff --git a/docs/remote-l3-worker-design/implementation-plan.md b/docs/remote-l3-worker-design/implementation-plan.md index a8452004d1..f7685bbc77 100644 --- a/docs/remote-l3-worker-design/implementation-plan.md +++ b/docs/remote-l3-worker-design/implementation-plan.md @@ -5,13 +5,13 @@ fork/shm behavior working. Status for the local PR #866 cut: -- Steps 1-5 are implemented for the simulation transport, including an +- Steps 1-5 are implemented for the host_tcp transport, including an independent health lane. - Step 6 is implemented for dispatcher `PYTHON_IMPORT`, inner manifest/control `PYTHON_IMPORT`, and inner manifest/control inline `CHIP_CALLABLE`. `PYTHON_SERIALIZED` remains a negotiated future extension, and staged chip-callable blobs require a staged-blob adapter before use. -- Steps 7-9 are implemented for the two-process simulation runner and sim +- Steps 7-9 are implemented for the two-process host_tcp session runner and host_tcp remote buffers, including export/import/release-import. - Steps 10-12 remain pending hardware-profile work. Remote CommDomain controls remain reserved/unsupported in this cut. @@ -152,8 +152,8 @@ Status for the local PR #866 cut: do not add the hashid to the parent-facing dispatcher registry. - Release staged callable bytes after confirmed commit or abort cleanup. -7. Fork-safe simulation session runner. **Implemented for the socket-backed - simulation transport.** +7. Fork-safe host_tcp session runner. **Implemented for the socket-backed + host_tcp transport.** - Add `simpler-remote-worker` control entry point. - Add per-session `simpler-remote-l3-session` runner. - Pass the validated bootstrap manifest from daemon to runner through an @@ -163,14 +163,14 @@ Status for the local PR #866 cut: plus `_start_hierarchical()`: fork L3 chip/sub children, register local endpoints, and start the inner Scheduler before any remote transport or health threads are started. - - Start the sim transport only after the local L3 child tree is established, + - Start the host_tcp transport only after the local L3 child tree is established, then run the post-prestart `HELLO`/ready handshake. - Treat `HELLO ready_state=READY` as a scheduling barrier; the parent must not schedule an endpoint that is alive but not prestarted. - - Run TASK frames over the sim transport and return completions. + - Run TASK frames over the host_tcp transport and return completions. - Add localhost two-process integration tests. -8. Remote control-plane parity. **Implemented for the simulation transport; +8. Remote control-plane parity. **Implemented for the host_tcp transport; Remote CommDomain controls remain reserved/unsupported.** - Map existing NEXT_LEVEL controls onto typed remote frames: prepare, register, unregister, remote buffer allocation, remote buffer @@ -240,11 +240,11 @@ Status for the local PR #866 cut: | Local adapter regression | Existing L3/L4 fork/shm behavior unchanged. | | Endpoint eligibility | Exact target is rejected before enqueue when ineligible. | | Frame fuzz/bounds | Corrupt lengths and counts are rejected. | -| Remote sim hello | Parent bootstraps remote L3 and shuts down cleanly. | +| Remote host_tcp hello | Parent bootstraps remote L3 and shuts down cleanly. | | Manifest handoff | Runner reads manifest before transport starts. | | Prestart barrier | HELLO READY only after inner L3 scheduler is started. | -| Remote sim task | L4 parent dispatches one L3 orch task successfully. | -| Remote sim error | Remote orch raises; parent raises with host/seq/hashid. | +| Remote host_tcp task | L4 parent dispatches one L3 orch task successfully. | +| Remote host_tcp error | Remote orch raises; parent raises with host/seq/hashid. | | Failed dependency | Consumer of failed remote producer is not dispatched. | | Remote hashid mapping | Daemon resolves an outer remote-orch hashid. | | Remote dep key | Shared remote buffer serializes through TensorMap. | @@ -298,7 +298,7 @@ Status for the local PR #866 cut: The current cut lands endpoint abstraction, endpoint eligibility, remote callable identity, remote sidecars, frame codec, failure poisoning, the -fork-safe simulation runner, and sim buffer import/export. Hardware HCOMM +fork-safe host_tcp session runner, and host_tcp buffer import/export. Hardware HCOMM profiles and Remote CommDomain controls remain future work. ## Failure Poisoning Contract diff --git a/docs/remote-l3-worker-design/implementation-record.md b/docs/remote-l3-worker-design/implementation-record.md index 9958df3816..7c2c53b023 100644 --- a/docs/remote-l3-worker-design/implementation-record.md +++ b/docs/remote-l3-worker-design/implementation-record.md @@ -9,15 +9,15 @@ It is updated as each documented feature is completed and verified. | Step | Documented feature | Status | Notes | | ---- | ------------------ | ------ | ----- | -| 1 | Endpoint interface and local adapter | In progress | Local adapter and remote sim endpoint are implemented; HCOMM endpoint adapters remain. | +| 1 | Endpoint interface and local adapter | In progress | Local adapter and remote host_tcp endpoint are implemented; HCOMM endpoint adapters remain. | | 2 | Worker eligibility metadata | In progress | Callable worker-id sets are intersected with owner/imported remote sidecar eligibility. | | 3 | Remote task sidecars and dependency keys | In progress | Public `TaskArgs.add_tensor(RemoteTensorRef(...))` API, remote TensorMap keys, and remote payload-sidecar rejection are implemented. | | 4 | Failed task poisoning | In progress | Remote task-failure poisoning and session-exit endpoint failure are verified; explicit health-expiry-only coverage remains. | | 5 | Versioned remote frame codec | In progress | TASK/COMPLETION/CONTROL_REPLY/HELLO/CONTROL/HEALTH exist; core fuzz/bounds coverage is present, with more exhaustive corpus testing still possible. | | 6 | Remote callable registry | In progress | Dispatcher `PYTHON_IMPORT`, inner manifest/control `PYTHON_IMPORT`, and inner manifest/control inline `CHIP_CALLABLE` are implemented; serialized payloads and staged chip blobs remain negotiated extensions. | -| 7 | Fork-safe simulation session runner | In progress | Daemon/session bootstrap and HELLO READY barrier are implemented for sim transport. | -| 8 | Remote control-plane parity | In progress | Registry, alloc/free/copy, export/import/release-import controls are implemented for sim; Remote CommDomain controls are reserved/unsupported. | -| 9 | Remote buffer registry | In progress | Sim owner/imported buffers, TASK materialization, public memory API, opaque handles, slot/import-ref capture, and deferred free/release-import are implemented. | +| 7 | Fork-safe host_tcp session runner | In progress | Daemon/session bootstrap and HELLO READY barrier are implemented for host_tcp transport. | +| 8 | Remote control-plane parity | In progress | Registry, alloc/free/copy, export/import/release-import controls are implemented for host_tcp; Remote CommDomain controls are reserved/unsupported. | +| 9 | Remote buffer registry | In progress | host_tcp owner/imported buffers, TASK materialization, public memory API, opaque handles, slot/import-ref capture, and deferred free/release-import are implemented. | | 10 | A2 RoCE HCOMM profile | Pending | Hardware-gated profile. | | 11 | A3 HCCS HCOMM profile | Pending | Hardware-gated profile. | | 12 | A5 UB HCOMM profile | Pending | Hardware-gated profile. | @@ -59,7 +59,7 @@ It is updated as each documented feature is completed and verified. - Mapped `WorkerEndpoint::control_prepare(digest)` for remote endpoints to a typed `PREPARE_CALLABLE` control. The session runner accepts it only for a committed `REMOTE_TASK_DISPATCHER` / `PYTHON_IMPORT` digest. -- Added typed sim `ALLOC_REMOTE_BUFFER`, `FREE_REMOTE_BUFFER`, +- Added typed host_tcp `ALLOC_REMOTE_BUFFER`, `FREE_REMOTE_BUFFER`, `COPY_TO_REMOTE`, and `COPY_FROM_REMOTE` handling in the session runner. - Added `Worker.remote_malloc()`, `remote_free()`, `remote_copy_to()`, and `remote_copy_from()` public APIs. Remote handles are returned by the Worker @@ -72,7 +72,7 @@ It is updated as each documented feature is completed and verified. `drain()`. - Added session-side `RemoteTensorDesc` materialization before `inner_worker.run()`: `HOST_INLINE` descriptors become local ctypes-backed - tensors and sim `REMOTE_DEVICE` descriptors resolve through the live session + tensors and host_tcp `REMOTE_DEVICE` descriptors resolve through the live session buffer registry. - Removed the descriptor-dropping Python `task_args_from_wire()` helper so the session runner has a single registry-backed materialization path. @@ -84,7 +84,7 @@ It is updated as each documented feature is completed and verified. - Documented the v1 `EXPORT_BUFFER`, `IMPORT_BUFFER`, and `RELEASE_IMPORT` wire schema, imported-handle identity, release deferral, and partial-import rollback contract. -- Implemented sim `EXPORT_BUFFER`, `IMPORT_BUFFER`, and `RELEASE_IMPORT`. +- Implemented host_tcp `EXPORT_BUFFER`, `IMPORT_BUFFER`, and `RELEASE_IMPORT`. Imports use shared-memory backed mappings in the session runner, imported handles remain opaque on the parent, and owner frees wait for live imports and slot refs to drain. @@ -101,31 +101,31 @@ It is updated as each documented feature is completed and verified. - Focused endpoint/data eligibility tests: remote sidecar owner filtering and non-owner C++ direct-submit rejection passed. -- Remote sim daemon/session noop TASK integration: +- Remote host_tcp daemon/session noop TASK integration: noop TASK, prepare-callable control, error completion, post-init dynamic registration, unregister/reregister, health, buffer copy, failed dependency, session exit, input-free deferral, and `HOST_INLINE` integration passed with `11 passed` outside the network-restricted sandbox. The same tests skip inside the sandbox when local TCP sockets are denied. -- Remote sim long TASK health integration: +- Remote host_tcp long TASK health integration: a remote orch that keeps the command lane busy for 1 second completed while the independent health lane stayed live. -- Remote sim buffer copy integration: +- Remote host_tcp buffer copy integration: `remote_malloc` + `COPY_TO_REMOTE` + remote TASK materialization/write + - `COPY_FROM_REMOTE` passed on the sim backend. -- Remote sim input-only free deferral: + `COPY_FROM_REMOTE` passed on the host_tcp backend. +- Remote host_tcp input-only free deferral: freeing an input buffer immediately after submit kept the remote allocation alive until the captured slot ref dropped after drain. -- Remote sim `HOST_INLINE` descriptor integration: +- Remote host_tcp `HOST_INLINE` descriptor integration: inline TASK payload materialized into a session-local tensor and fed a remote output write. - C++ wire fuzz/bounds coverage: bad frame version/type/flags, truncated control payload, and invalid remote descriptor inline fields are rejected. -- Remote sim failed-dependency integration: +- Remote host_tcp failed-dependency integration: a failed remote producer poisoned a downstream same-buffer remote consumer, and the consumer did not dispatch or mutate the buffer. -- Remote sim endpoint-failure integration: +- Remote host_tcp endpoint-failure integration: a session runner exit during TASK returned a bounded endpoint failure to the parent instead of hanging `drain()`. - Remote callable unregister/reregister integration: diff --git a/docs/remote-l3-worker-design/pr-split-and-audit-artifacts.md b/docs/remote-l3-worker-design/pr-split-and-audit-artifacts.md index f98ca87ace..6a16a09265 100644 --- a/docs/remote-l3-worker-design/pr-split-and-audit-artifacts.md +++ b/docs/remote-l3-worker-design/pr-split-and-audit-artifacts.md @@ -66,33 +66,33 @@ been created by this audit. | Req | Source | Class | Implementation | Tests | Gap/Drift | PR | | --- | ------ | ----- | -------------- | ----- | --------- | -- | | R1 Endpoint abstraction keeps local mailbox local-only and routes remote L3 through `WorkerEndpoint` / `RemoteL3Endpoint` | `remote-l3-worker-design.md` §Target Architecture; `implementation-plan.md` step 1; `worker-manager.md` §4 | required in this PR cut | `WorkerEndpoint`, `LocalMailboxEndpoint`, `RemoteL3Endpoint`, `WorkerManager::add_next_level_endpoint`, `Worker::add_remote_l3_socket` | `test_remote_endpoint`, `test_scheduler`, `test_remote_sim_noop_task_roundtrip` | None found | PR 4 / PR 5 | -| R2 Endpoint outcomes distinguish success, task failure, endpoint failure, and skipped group members | `remote-l3-worker-design.md` §Failure Semantics; `implementation-plan.md` steps 1, 4 and Failure Poisoning Contract; `scheduler.md` §§2,6,9 | required in this PR cut | `EndpointOutcome`; endpoint completion mapping; scheduler failure poisoning and group skip state | `RemoteTaskErrorMapsToTaskFailure`, `FailedProducerPoisonsDependentTask`, `GroupFailureWaitsForRunningMembersThenConsumes`, remote sim error/exit tests | None found | PR 4 / PR 5 | +| R2 Endpoint outcomes distinguish success, task failure, endpoint failure, and skipped group members | `remote-l3-worker-design.md` §Failure Semantics; `implementation-plan.md` steps 1, 4 and Failure Poisoning Contract; `scheduler.md` §§2,6,9 | required in this PR cut | `EndpointOutcome`; endpoint completion mapping; scheduler failure poisoning and group skip state | `RemoteTaskErrorMapsToTaskFailure`, `FailedProducerPoisonsDependentTask`, `GroupFailureWaitsForRunningMembersThenConsumes`, remote host_tcp error/exit tests | None found | PR 4 / PR 5 | | R3 Scheduler dispatch preserves stable worker ids and `worker=` is validated against final eligibility | `remote-l3-worker-design.md` §Worker Identity and Callable Routing; `implementation-plan.md` step 2; `scheduler.md` §3 | required in this PR cut | `Orchestrator::validate_worker_eligibility`, directed NEXT_LEVEL queues, Python worker-id set intersection | `TargetMustBeInEligibleEndpointSet`, target-ID mapping tests, Python remote callable worker-id intersection tests | None found | PR 4 | | R4 Mixed local/remote pools are allowed only when callable and tensor representations are consumable by the selected worker | `remote-l3-worker-design.md` §Worker Identity and Callable Routing; `buffers-and-transports.md` §TaskArgs Sidecar Contract | required in this PR cut | Python `Orchestrator` only allows `RemoteTensorRef` for `RemoteCallable`; C++ rejects remote sidecars on local workers and non-owner remote-device dispatch without import | `RemoteSidecarRejectsLocalEndpointEligibility`, `RemoteSidecarRejectsNonOwnerEligibleEndpointWithoutImport`, Python RemoteCallable sidecar tests | No blocking drift; add a future end-to-end mixed local+remote smoke during split validation | PR 4 | | R5 Remote sidecars are hidden metadata aligned by tensor index; local endpoints reject sidecars; remote endpoints reject bare host pointers | `remote-l3-worker-design.md` §Remote TaskArgs Representation; `buffers-and-transports.md` §§Public Memory API, TaskArgs Sidecar Contract; `protocol.md` §TASK Payload | required in this PR cut | `RemoteTaskArgsSidecar`, Python `_remote_sidecar_for`, C++ `validate_remote_sidecars`, `LocalMailboxEndpoint::run`, `RemoteL3Endpoint::build_task_payload` | `TestRemoteTaskArgsSidecar`, `RemoteBarePayloadFailsBeforeSlotCommit`, `BareHostPointerWithoutSidecarIsEndpointFailure` | None found | PR 4 / PR 6 | -| R6 Remote null `OUTPUT` tensors fail fast unless the caller supplies an explicit `RemoteTensorRef` | `remote-l3-worker-design.md` §Remote TaskArgs Representation; `buffers-and-transports.md` §Remote OUTPUT Allocation Policy; `implementation-plan.md` step 3 | required in this PR cut | `Orchestrator::validate_remote_sidecars` requires sidecar for remote OUTPUT; `reserve_outputs_and_slot` skips local HeapRing only when sidecar present | `RemoteOutputSidecarSkipsLocalAutoAllocAndRegistersRemoteKey`, remote sim OUTPUT buffer tests | No code drift; add a narrow negative test for null remote OUTPUT without sidecar when carving PR 4 | PR 4 / PR 6 | +| R6 Remote null `OUTPUT` tensors fail fast unless the caller supplies an explicit `RemoteTensorRef` | `remote-l3-worker-design.md` §Remote TaskArgs Representation; `buffers-and-transports.md` §Remote OUTPUT Allocation Policy; `implementation-plan.md` step 3 | required in this PR cut | `Orchestrator::validate_remote_sidecars` requires sidecar for remote OUTPUT; `reserve_outputs_and_slot` skips local HeapRing only when sidecar present | `RemoteOutputSidecarSkipsLocalAutoAllocAndRegistersRemoteKey`, remote host_tcp OUTPUT buffer tests | No code drift; add a narrow negative test for null remote OUTPUT without sidecar when carving PR 4 | PR 4 / PR 6 | | R7 Remote dependency keys use `(address_kind, owner_worker_id, buffer_id, generation, offset)` and preserve exact-start semantics | `remote-l3-worker-design.md` §Remote TaskArgs Representation; `buffers-and-transports.md` §Dependency Keys; `orchestrator.md` §2 | required in this PR cut | `TensorKey::remote_buffer`, `Orchestrator::infer_deps` remote path, `TensorMap` remote keys | `RemoteInputSidecarUsesRemoteTensorMapKey`, `test_remote_sim_failed_dependency_skips_consumer` | Range-overlap support remains future/open decision | PR 4 | | R8 Remote callable identity uses canonical hashid descriptors, not target-private slots or cross-worker integer ids | `remote-l3-worker-design.md` §Worker Identity and Callable Routing; `task-flow.md` §Callable Identity; `implementation-plan.md` step 6 | required in this PR cut | `callable_identity.py`, `CallableHandle`, descriptor builders, `Worker._identity_registry` | hash stability, public export, forged/mutated handle, cleanup-uncertain tests | None found | PR 2 / PR 6 | | R9 `RemoteCallable("module:qualname")` is the required baseline with explicit non-empty `workers=[...]` | `remote-l3-worker-design.md` §Worker Identity and Callable Routing; `protocol.md` §CONTROL Payload; `implementation-plan.md` step 6 | required in this PR cut | `RemoteCallable`, `parse_python_import_target`, `_build_callable_registration`, explicit remote worker validation | target validation and explicit remote worker tests | None found | PR 2 / PR 6 | | R10 Multi-worker remote registration is all-or-nothing with prepare, commit, abort, cleanup-uncertain blocking, and final-unregister tombstones | `remote-l3-worker-design.md` §Worker Identity and Callable Routing; `protocol.md` §CONTROL Payload; `implementation-plan.md` step 6 | required in this PR cut | `Worker._post_start_register_remote`, `remote_prepare/commit/abort/unregister`, pending unregister tombstones, uncertain hashid guard | post-init register, unregister/reregister, inner register/unregister, uncertain cleanup guard | Partial-failure cleanup path is complex; keep as high-risk review item | PR 2 / PR 5 / PR 6 | | R11 `INNER_L3_WORKER` is remote-internal; parent TASK frames resolve only in `REMOTE_TASK_DISPATCHER` | `remote-l3-worker-design.md` §§Worker Identity and Callable Routing, Remote Worker Session; `protocol.md` §CONTROL Payload; `task-flow.md` §Callable Identity | required in this PR cut | session dispatcher registry vs inner registry; `_prepare_register_callable`; `get_inner_handle` | dispatcher rejects chip target, inner Python import/chip callable install, inner sub-task integration | None found | PR 5 / PR 6 | | R12 `PYTHON_SERIALIZED` remote callables reject unless serialized support is explicitly negotiated | `remote-l3-worker-design.md` §§Scope, Worker Identity and Callable Routing; `protocol.md` §CONTROL Payload; `implementation-plan.md` step 6 | required explicit unsupported behavior | `_prepare_register_callable` rejects `CallableKind.PYTHON_SERIALIZED` before install | `test_remote_register_rejects_python_serialized_without_negotiation` | Phase 3 added missing coverage | PR 6 | -| R13 `CHIP_CALLABLE` is valid only for `INNER_L3_WORKER`; inline blobs are supported by sim, staged blobs reject unless negotiated | `protocol.md` §CONTROL Payload; `implementation-plan.md` step 6; `implementation-record.md` §Completed Items | required in this PR cut plus required explicit unsupported behavior for staged blobs | `_prepare_inner_chip_callable`; dispatcher rejects chip target; staged blob reject | inner chip manifest install, dispatcher reject test, `test_remote_inner_chip_callable_rejects_staged_blob_without_negotiation` | Phase 3 added missing staged-blob coverage | PR 5 / PR 6 | -| R14 Bootstrap manifest installs dispatcher and inner registries before `HELLO READY`; unsupported negotiated manifest extensions reject rather than partially install | `remote-l3-worker-design.md` §Remote Worker Session; `protocol.md` §CONTROL Payload; `implementation-plan.md` steps 6-7 | required in this PR cut | `remote_l3_session.run_session`, `_install_manifest_inner_registry`, daemon manifest validation | manifest inner Python/chip install tests; remote sim roundtrip tests | None found | PR 6 | -| R15 Fork ordering preserves prestart before command/health transport threads; `HELLO READY` is a scheduling barrier | `remote-l3-worker-design.md` §Fork-Safe Remote Process Model; `protocol.md` §HELLO Payload; `hierarchical-level-runtime.md` §Process Model | required in this PR cut | daemon writes manifest, session prestarts `inner_worker`, then binds command/health and sends HELLO READY; parent `add_remote_l3_socket` waits for READY | remote unreachable daemon test; remote sim roundtrip/prep tests | None found | PR 6 | +| R13 `CHIP_CALLABLE` is valid only for `INNER_L3_WORKER`; inline blobs are supported by host_tcp, staged blobs reject unless negotiated | `protocol.md` §CONTROL Payload; `implementation-plan.md` step 6; `implementation-record.md` §Completed Items | required in this PR cut plus required explicit unsupported behavior for staged blobs | `_prepare_inner_chip_callable`; dispatcher rejects chip target; staged blob reject | inner chip manifest install, dispatcher reject test, `test_remote_inner_chip_callable_rejects_staged_blob_without_negotiation` | Phase 3 added missing staged-blob coverage | PR 5 / PR 6 | +| R14 Bootstrap manifest installs dispatcher and inner registries before `HELLO READY`; unsupported negotiated manifest extensions reject rather than partially install | `remote-l3-worker-design.md` §Remote Worker Session; `protocol.md` §CONTROL Payload; `implementation-plan.md` steps 6-7 | required in this PR cut | `remote_l3_session.run_session`, `_install_manifest_inner_registry`, daemon manifest validation | manifest inner Python/chip install tests; remote host_tcp roundtrip tests | None found | PR 6 | +| R15 Fork ordering preserves prestart before command/health transport threads; `HELLO READY` is a scheduling barrier | `remote-l3-worker-design.md` §Fork-Safe Remote Process Model; `protocol.md` §HELLO Payload; `hierarchical-level-runtime.md` §Process Model | required in this PR cut | daemon writes manifest, session prestarts `inner_worker`, then binds command/health and sends HELLO READY; parent `add_remote_l3_socket` waits for READY | remote unreachable daemon test; remote host_tcp roundtrip/prep tests | None found | PR 6 | | R16 Remote frame protocol uses versioned canonical little-endian encoding and never memcpy's C++ POD structs | `remote-l3-worker-design.md` §Protocol; `protocol.md` §§Frames, Wire Encoding; `implementation-plan.md` step 5 | required in this PR cut | `remote_wire.cpp` explicit put/get helpers; Python `_Reader`; local mailbox remains same-binary POD IPC only | `FrameRoundTripValidatesHeader`, Python decode tests | Found and fixed `CallConfigWire.enable_scope_stats` drift in Phase 3 | PR 3 | | R17 Frame codec rejects bad magic/version/type/flags, oversized/truncated payloads, unknown enums, non-zero reserved fields, and malformed counts | `protocol.md` §§Frames, Wire Encoding, Bounds and Fuzz Tests; `implementation-plan.md` step 5 | required in this PR cut | C++ and Python decode validation for headers, counts, enums, reserved fields, payload sizes | remote wire bad header/truncation/reserved tests; Python materialization/decode tests | No blocking drift found | PR 3 | | R18 TASK payload carries digest, `CallConfigWire`, and `RemoteTaskArgsWire`; tensor wire `data` must be zero; `HOST_INLINE` must be descriptor-backed and bounds-checked | `protocol.md` §§TASK Payload, RemoteTensorDesc, Bounds and Fuzz Tests; `buffers-and-transports.md` §TaskArgs Sidecar Contract | required in this PR cut | `encode_task_payload`, `decode_task_payload`, `RemoteL3Endpoint::build_task_payload`, Python `_materialize_task_args` | non-zero tensor data rejection, HOST_INLINE descriptor bounds, host-inline materialization/roundtrip | Phase 3 fixed and tested missing `enable_scope_stats` field | PR 3 / PR 5 | -| R19 COMPLETION and CONTROL_REPLY match sequence/name/version, bound error text, and fabricate failures on health expiry or process exit | `protocol.md` §§COMPLETION Payload, CONTROL_REPLY Payload, Ordering; `remote-l3-worker-design.md` §Failure Semantics | required in this PR cut | `decode_completion`, `decode_control_reply`, remote endpoint reply validation, session `_format_remote_error`, health monitor | sequence/name mismatch tests, remote sim error completion and process exit tests | None found | PR 3 / PR 5 / PR 6 | +| R19 COMPLETION and CONTROL_REPLY match sequence/name/version, bound error text, and fabricate failures on health expiry or process exit | `protocol.md` §§COMPLETION Payload, CONTROL_REPLY Payload, Ordering; `remote-l3-worker-design.md` §Failure Semantics | required in this PR cut | `decode_completion`, `decode_control_reply`, remote endpoint reply validation, session `_format_remote_error`, health monitor | sequence/name mismatch tests, remote host_tcp error completion and process exit tests | None found | PR 3 / PR 5 / PR 6 | | R20 Each endpoint has one ordered command lane; TASK, state-changing CONTROL, SHUTDOWN, replies, and visibility are sequence ordered | `remote-l3-worker-design.md` §§Remote Worker Session, Protocol; `protocol.md` §Ordering; `buffers-and-transports.md` §HCOMM Adapter Contract | required in this PR cut | `OrderedCommandLane`, endpoint command mutex, session command loop, SHUTDOWN frame path | `OrderedCommandLaneIsSingleFlight`, remote prepare/register tests | None found | PR 3 / PR 5 / PR 6 | | R21 Health/liveness is independent from command-lane progress; health expiry removes endpoint eligibility and fails pending/in-flight work | `remote-l3-worker-design.md` §Remote Worker Session; `protocol.md` §Ordering; `implementation-plan.md` steps 5,7 | required in this PR cut | separate health socket/thread, remote endpoint health monitor, process-exit endpoint failure | long-task health-lane test, process-exit endpoint failure test | Broader multi-endpoint health-removal coverage should remain in PR 5/6 verification | PR 5 / PR 6 | -| R22 Remote memory APIs expose opaque handles and simulation alloc/free/copy/export/import/release-import controls | `buffers-and-transports.md` §§Buffer Handles, Public Memory API, Required Controls; `implementation-plan.md` steps 8-9 | required in this PR cut | Python `RemoteBufferHandle`, Worker remote memory APIs, C++ controls, session sim buffer registry | opaque handle tests, buffer copy roundtrip, export/import controls roundtrip | None found | PR 5 / PR 6 | +| R22 Remote memory APIs expose opaque handles and host_tcp alloc/free/copy/export/import/release-import controls | `buffers-and-transports.md` §§Buffer Handles, Public Memory API, Required Controls; `implementation-plan.md` steps 8-9 | required in this PR cut | Python `RemoteBufferHandle`, Worker remote memory APIs, C++ controls, session host buffer registry | opaque handle tests, buffer copy roundtrip, export/import controls roundtrip | None found | PR 5 / PR 6 | | R23 Owner/imported handle semantics preserve owner identity, worker eligibility, access flags, and import rollback behavior | `remote-l3-worker-design.md` §Buffer Lifecycle; `buffers-and-transports.md` §§Export/Import Handle Semantics, Release Policy; `protocol.md` §Remote Buffer Export and Import Controls | required in this PR cut | export/import descriptors, access flag validation, owner/import worker routing, imported address-space materialization | remote buffer export/import wire tests, imported buffer runs on peer worker | Import rollback is implementation-reviewed but should get split-specific failure injection | PR 5 / PR 6 | | R24 Owner free and release-import defer physical cleanup until slot refs and imports drain; failed runs use the same post-drain cleanup path | `remote-l3-worker-design.md` §Buffer Lifecycle; `buffers-and-transports.md` §Release Policy; `implementation-plan.md` step 9 | required in this PR cut | Python slot refs, pending remote frees/import releases, `run`/`close` cleanup flush | owner-free-waits-for-import-release, input-free-deferred-until-slot-refs-drop | None found | PR 4 / PR 5 / PR 6 | | R25 Remote CommDomain controls `COMM_INIT`, `ALLOC_DOMAIN`, and `RELEASE_DOMAIN` are reserved and rejected as unsupported in this cut | `remote-l3-worker-design.md` §Scope; `protocol.md` §CONTROL Payload; `implementation-plan.md` steps 8,13 | required explicit unsupported behavior | session control loop returns error for reserved domain controls; local C++ mailbox CommDomain controls remain local-only | Implementation inspected; no direct UT found | Add direct reserved-control negative in PR 6 verification | PR 5 / PR 6 | -| R26 A2 RoCE, A3 HCCS, and A5 UB HCOMM profiles are documented contracts but pending hardware-profile work, not part of this split cut | `remote-l3-worker-design.md` §§Current Implementation Status, Rollout; `buffers-and-transports.md` §§HCOMM Adapter Contract, A2/A3/A5 Profiles; `implementation-plan.md` steps 10-12 | reserved or future work | daemon accepts only `transport="sim"`; export/import simulation rejects unsupported profiles | Implementation inspected; remote worker manifest validation covers sim-only | Future only; do not include HCOMM adapters in child PRs | Future | +| R26 A2 RoCE, A3 HCCS, and A5 UB HCOMM profiles are documented contracts but pending hardware-profile work, not part of this split cut | `remote-l3-worker-design.md` §§Current Implementation Status, Rollout; `buffers-and-transports.md` §§HCOMM Adapter Contract, A2/A3/A5 Profiles; `implementation-plan.md` steps 10-12 | reserved or future work | daemon accepts only `transport="host_tcp"`; export/import host_tcp rejects unsupported profiles | Implementation inspected; remote worker manifest validation covers host_tcp-only | Future only; do not include HCOMM adapters in child PRs | Future | | R27 Exact HCCS/UB HAL names, daemon auth/isolation, serialized compatibility metadata, and future CommContext split remain open decisions | `implementation-plan.md` §Open Decisions | open decision | Documented only | Not applicable | Future only | Future | -| R28 Top-of-stack verification excludes unavailable hardware except through `task-submit`, covers Python UT, C++ UT, pre-commit docs, sim ST, and remote sim integration | `pr-split-and-audit-plan.md` §Review and Verification; `.claude/rules/task-submit-isolation.md` | required in this PR cut | Audit follows local UT only here; hardware remains `task-submit` only | Current run log below | Full child PR verification remains to be run after split | All | +| R28 Top-of-stack verification excludes unavailable hardware except through `task-submit`, covers Python UT, C++ UT, pre-commit docs, sim ST, and remote host_tcp integration | `pr-split-and-audit-plan.md` §Review and Verification; `.claude/rules/task-submit-isolation.md` | required in this PR cut | Audit follows local UT only here; hardware remains `task-submit` only | Current run log below | Full child PR verification remains to be run after split | All | ## Phase 3 Drift Fixes @@ -111,7 +111,7 @@ been created by this audit. | `src/common/hierarchical/remote_wire.*`, `python/simpler/remote_l3_protocol.py`, remote-wire-focused tests | PR 3 | Stable cross-host protocol and codec validation | PR 1 docs; PR 2 enum/identity definitions | C++ `test_remote_wire`; Python protocol decode tests | | Scheduler and slot-state hunks in `src/common/hierarchical/types.*`, `orchestrator.*`, `scheduler.*`, `worker_manager.*` | PR 4 | Endpoint eligibility, outcomes, sidecars, dependency keys, failure poisoning | PR 2 identity; PR 3 types/protocol where referenced | C++ `test_orchestrator`, `test_scheduler` | | C++ remote endpoint and binding hunks in `remote_endpoint.*`, `worker_manager.*`, `worker.h`, `python/bindings/worker_bind.h` | PR 5 | Transport endpoint, command lane, controls, and Python C++ facade | PR 3 wire; PR 4 endpoint abstraction | C++ `test_remote_endpoint`, `test_remote_wire`; binding smoke | -| Python remote session/runtime hunks in `python/simpler/worker.py`, `orchestrator.py`, `task_interface.py`, `remote_l3_worker.py`, `remote_l3_session.py` | PR 6 | Remote daemon/session runner, registration choreography, sim memory controls, lifecycle cleanup | PR 2-5 | Python remote sim integration tests | +| Python remote session/runtime hunks in `python/simpler/worker.py`, `orchestrator.py`, `task_interface.py`, `remote_l3_worker.py`, `remote_l3_session.py` | PR 6 | Remote daemon/session runner, registration choreography, host_tcp memory controls, lifecycle cleanup | PR 2-5 | Python remote host_tcp integration tests | | Fixup commit `92709210` | Squash into PR 4 / PR 5 / PR 6 hunks | Stabilizes CI controls across Python orchestration, task interface, worker manager, and tests | Matching child PRs | Matching child PR tests | | Fixup commit `04de866b` | Squash into PR 3 / PR 5 / PR 6 hunks | Resolves protocol/session/binding/orchestrator CI failures | Matching child PRs | Matching child PR tests | | Fixup commit `d6500ea2` | Squash into PR 5 plus affected C++ tests | clang-format-only formatting for remote endpoint/worker manager/scheduler tests | Matching child PRs | formatting / C++ UT | @@ -123,7 +123,7 @@ been created by this audit. | ---- | -------- | ------------ | ---------- | ---- | | Protocol drift between C++ endpoint and Python session | `remote_wire.cpp`, `remote_l3_protocol.py`, `protocol.md` | Remote TASK decode shifts fields or drops config flags | Keep PR 3 protocol-first; add round-trip tests for every `CallConfigWire` field | Phase 3 `enable_scope_stats` tests; `test_remote_wire` | | Partial remote registration cleanup | `Worker._post_start_register_remote`, remote prepare/commit/abort controls | Some endpoints commit while others fail, leaving stale registry state | Preserve prepare/commit/abort/tombstone design; review cleanup-uncertain guard in PR 6 | post-init register/unregister tests; add failure injection in PR 6 | -| Endpoint failure poisoning | `scheduler.cpp`, `remote_endpoint.cpp`, `remote_l3_session.py` | Failed producer or dead endpoint lets consumers dispatch with stale data | First-error-wins, dependency poisoning, health/process-exit failures | scheduler failed producer; remote sim process-exit and failed-dependency tests | +| Endpoint failure poisoning | `scheduler.cpp`, `remote_endpoint.cpp`, `remote_l3_session.py` | Failed producer or dead endpoint lets consumers dispatch with stale data | First-error-wins, dependency poisoning, health/process-exit failures | scheduler failed producer; remote host_tcp process-exit and failed-dependency tests | | Remote buffer lifetime | `worker.py`, `task_interface.py`, `remote_l3_session.py` | Owner freed while imported/slot-ref still active | Slot refs, import refs, pending free/release queues | owner-free/import-release and slot-ref-deferred free tests | | Hidden sidecar integrity | `task_interface.py`, `orchestrator.cpp`, `remote_endpoint.cpp` | Bare host pointer or mismatched descriptor count reaches remote worker | Python sidecar storage, C++ validation, remote endpoint fail-fast | sidecar unit tests, bare host pointer endpoint failure | | Future controls accidentally treated as supported | `remote_l3_session.py`, HCOMM docs | Reviewer believes CommDomain/HCOMM or serialized/staged paths are ready | Explicit unsupported tests and Future Work list | Phase 3 serialized/staged tests; add CommDomain negative | @@ -138,7 +138,7 @@ been created by this audit. | PR 3 Protocol | C++ `test_remote_wire`; Python protocol decode/materialization tests; verify no POD memcpy in remote wire | | PR 4 Scheduler / Eligibility | C++ `test_orchestrator` and `test_scheduler`; negative tests for remote sidecar/local endpoint and null remote OUTPUT | | PR 5 C++ Remote Endpoint / Bindings | C++ `test_remote_endpoint`; binding import smoke; command lane/control reply checks | -| PR 6 Python Session / Sim Runtime | Python remote sim integration tests in `test_callable_identity.py`; memory alloc/copy/export/import/release tests; reserved-control negative | +| PR 6 Python Session / Host TCP Runtime | Python remote host_tcp integration tests in `test_callable_identity.py`; memory alloc/copy/export/import/release tests; reserved-control negative | | All | No hardware commands locally; hardware verification only through `task-submit` per repo rule | ## Future Work diff --git a/docs/remote-l3-worker-design/pr-split-and-audit-plan.md b/docs/remote-l3-worker-design/pr-split-and-audit-plan.md index 665b37ddc4..0c710646d6 100644 --- a/docs/remote-l3-worker-design/pr-split-and-audit-plan.md +++ b/docs/remote-l3-worker-design/pr-split-and-audit-plan.md @@ -366,7 +366,7 @@ Minimum verification: - PR 3: C++ remote wire tests and Python protocol codec tests. - PR 4: C++ scheduler, orchestrator, and worker-manager tests. - PR 5: C++ remote endpoint tests and binding smoke tests. -- PR 6: remote simulation integration tests, buffer export/import/release +- PR 6: remote host_tcp integration tests, buffer export/import/release tests, health lane tests, and session-exit failure tests. Top-of-stack verification: diff --git a/docs/user/how-to/run-on-multiple-chips.md b/docs/user/how-to/run-on-multiple-chips.md index 819c684ce7..d845ab4c21 100644 --- a/docs/user/how-to/run-on-multiple-chips.md +++ b/docs/user/how-to/run-on-multiple-chips.md @@ -90,11 +90,10 @@ implementations. explicitly; there is no cross-node collective path today. - **`--enable-chip-swimlane` does not work on L3.** It is rejected up front. To get a swimlane, scope the run to one chip with `--level 2`. -- **Multi-host (L4) is not usable for data movement yet.** The control plane is - real — you can register a remote worker, and the remote side builds a genuine - L3 subtree — but the transport is simulation-backed, and the daemon rejects - any other setting. The RoCE / HCCS / UB profiles are documented contracts, not - working paths. See [Remote L3 Worker Design](../../remote-l3-worker-design.md) +- **Multi-host (L4) remote L3 uses the shipped host_tcp data plane.** You + can register a remote worker, the remote side builds a genuine L3 subtree, + and remote buffers move through the host TCP session. The RoCE / HCCS / UB + peer-memory profiles are documented contracts, not working paths. See [Remote L3 Worker Design](../../remote-l3-worker-design.md) and the [Capability Survey](../../capability-survey.md) before designing against it. diff --git a/examples/workers/README.md b/examples/workers/README.md index 5203f53fc0..d321eb19e5 100644 --- a/examples/workers/README.md +++ b/examples/workers/README.md @@ -33,6 +33,8 @@ workers/ l3/ # Multi-chip examples (host-level DAG) multi_chip_dispatch/ # Worker(level=3) + orchestration + SubWorker child_memory/ # orch.malloc + child_memory=True, weight reuse across tasks + l4/ # Multi-machine examples (one L3 here, one over TCP) + vector_add_mixed_l3/ # Worker(level=4) + add_remote_worker, golden checked on both sides ``` Why no `tensormap_and_ringbuffer/` layer? Because every example here hard-codes @@ -40,6 +42,73 @@ Why no `tensormap_and_ringbuffer/` layer? Because every example here hard-codes default user-facing runtime. The other runtime (`host_build_graph`) is covered by scene tests under `tests/st/`, not here. +## L4: examples that span two machines + +An L4 example is the only kind here that needs **two hosts**. The parent runs +on one, holding a forked local L3, and attaches a second L3 running on the +other over TCP: + +```text +L4 parent on machine B ─┬─ local L3 on machine B → its own NPUs + └─ remote L3 on machine A → machine A's NPUs +``` + +Two processes, then, not one: a **daemon** on the peer and a **parent** here. +The daemon is `python -m simpler.remote_l3_worker --host H --port P` — generic, +identical for every example, nothing to write. All an example ships is the +parent side. + +### What a new L4 example needs + +```text +l4// + README.md + kernels/aiv/*.cpp + kernels/orchestration/*.cpp + main.py # entry point: argparse + main() + run_parent.sh # maps environment variables onto main.py's flags +``` + +Copy [`vector_add_mixed_l3/`](l4/vector_add_mixed_l3/) and work outwards from +it. Three things are load-bearing: + +- **`main.py` must not be named `test_*.py`.** `pyproject.toml` sets + `testpaths = ["tests", "examples"]`, so pytest imports anything matching that + name. A file with no test functions collects as zero tests and looks + harmless — until someone adds one, and the single-machine scene-test job + starts trying to run a two-machine example. +- **The remote imports your module by path.** `REMOTE_ORCH_TARGET` is a + `"package.module:function"` string the *peer* resolves, so renaming or moving + the file means updating that string in the same commit — nothing on the local + side will fail if you forget. +- **Exit non-zero when the golden check fails.** CI reads the exit code and + nothing else. + +### The environment-variable contract + +`run_parent.sh` exists to turn environment variables into `main.py`'s flags, +because that is the interface CI drives it through. Pick a prefix and read +these five; CI sets exactly them: + +| Variable | Meaning | +| -------- | ------- | +| `_REMOTE` | The peer's daemon, as `host:port` | +| `_LOCAL_DEVICES` | Device ids the local L3 owns | +| `_REMOTE_DEVICES` | Device ids the peer's L3 owns | +| `_SESSION_TIMEOUT` | Seconds to wait on the remote session | +| `_SESSION_LISTEN_HOST` | Interface the parent's session runner binds | + +Anything else — platform, runtime — defaults inside `run_parent.sh`. + +### Running it in CI + +The `st-pod-onboard-a2a3` job runs L4 examples across a pair of a2a3 machines. +Adding yours to it is one block plus one line, and the wiring, the log +artifact, and the failure semantics are described in +[`docs/ci.md`](../../docs/ci.md#multi-machine-pod-jobs). Read that before adding the +block — in particular why a step that reports green there may still have +failed. + ## Prerequisites Examples assume you have built and installed the package in a venv: @@ -76,6 +145,12 @@ Flags: Simulator (`a2a3sim`) works on any Linux host with gcc; hardware platforms require an Ascend NPU box with `ASCEND_HOME_PATH` set. +L2 and L3 examples follow that uniform CLI. **L4 examples do not** — they need +a peer's address and a device split on each side, so they take `--remote`, +`--local-devices` and `--remote-devices` instead of `-p`/`-d`, and are normally +launched through their `run_parent.sh`. See each L4 example's README for the +two-machine sequence. + ## Related documentation - [`docs/hierarchical-level-runtime.md`](../../docs/hierarchical-level-runtime.md) — the L0–L6 level model diff --git a/examples/workers/l4/vector_add_mixed_l3/README.md b/examples/workers/l4/vector_add_mixed_l3/README.md new file mode 100644 index 0000000000..31f0793e93 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/README.md @@ -0,0 +1,73 @@ +# Vector add mixed L3 example + +This example validates the mixed-L3 vector-add path for an L4 parent that combines one +forked local L3 worker and one TCP-connected remote L3 worker: + +```text +L4 parent on machine A -> local L3 on machine A -> local NPU 0 + NPU 1 vector group + -> remote L3 on machine B -> remote NPU 0 + NPU 1 vector group +``` + +The parent stages local inputs through fork-inherited shared host buffers and +remote inputs through `remote_malloc` / `remote_copy_to`. It dispatches both L3 +tasks in one L4 run, downloads the remote outputs with `remote_copy_from`, and +checks the golden result on both sides. No `mpirun` is used. + +This baseline proves remote startup, remote buffers, `RemoteTensorRef`, L4 task +dispatch to mixed local/remote L3 workers, local L3 group scheduling, NPU +execution, and result copy-back. It does not read peer-machine memory or +exercise Global CommDomain; those stay out of this mixed-L3 vector-add path. + +## Prepare both machines + +Use the same source revision on machine A and machine B: + +```bash +python3 -m venv --system-site-packages .venv +source .venv/bin/activate +pip install --no-build-isolation -e . + +export ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest +export PATH="$ASCEND_HOME_PATH/bin:$PATH" +``` + +## Start the remote L3 daemon + +Start this on machine B: + +The daemon is the generic session server — nothing about it is specific to this +example, so it is started directly rather than through a wrapper: + +`PEER_HOST` is machine B's address as machine A reaches it — `192.0.2.20` below +is a documentation placeholder, not a usable address. + +```bash +export ASCEND_PROCESS_LOG_PATH=/tmp/simpler-vector-add-mixed-l3-machine-b +source .venv/bin/activate + +PEER_HOST=192.0.2.20 # machine B's address +python -m simpler.remote_l3_worker --host "$PEER_HOST" --port 19073 +``` + +The daemon starts a session runner on random TCP command and health ports. +Firewalls between the parent and remote machine must allow those returned ports +in addition to the daemon port. + +## Run on the L4 parent + +Run this on machine A. The local device ids are owned by the forked local L3 +on machine A; the remote device ids are owned by the daemon-started L3 on +machine B. + +```bash +export ASCEND_PROCESS_LOG_PATH=/tmp/simpler-vector-add-mixed-l3-machine-a +PEER_HOST=192.0.2.20 # the same machine B address used above +export SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE="$PEER_HOST:19073" +export SIMPLER_VECTOR_ADD_MIXED_L3_LOCAL_DEVICES=0,1 +export SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE_DEVICES=0,1 + +bash examples/workers/l4/vector_add_mixed_l3/run_parent.sh +``` + +Each side must provide exactly two free A3 device ids. Success requires both +local outputs and both remote outputs to match the golden result. diff --git a/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add.cpp b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add.cpp new file mode 100644 index 0000000000..209e806ef1 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +/** + * Element-wise ChipTensor Addition Kernel + * + * Implements: out[i] = src0[i] + src1[i] + * + * This kernel performs element-wise addition of two tensors. It's compiled + * separately as a standalone kernel and linked with the dispatcher using + * function pointers, demonstrating the separation pattern used in production + * systems where kernel binaries are loaded dynamically. + */ + +#include +#include + +#include "tensor.h" + +using namespace pto; + +#include "pipe_sync.h" + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] +#endif + +/** + * Element-wise addition kernel implementation + * + * Unified signature: all arguments passed via int64_t array + * @param args Argument array: + * args[0] = src0 pointer (first input tensor) + * args[1] = src1 pointer (second input tensor) + * args[2] = out pointer (output tensor) + * + * The element count is not an argument: the kernel processes a fixed + * 128 x 128 float tile. + */ +extern "C" __aicore__ __attribute__((always_inline)) void kernel_entry(__gm__ int64_t *args) { + // Unpack arguments (ChipTensor* pointers from runtime) + __gm__ ChipTensor *src0_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[0]); + __gm__ ChipTensor *src1_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[1]); + __gm__ ChipTensor *out_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[2]); + __gm__ float *src0 = reinterpret_cast<__gm__ float *>(src0_tensor->buffer.addr) + src0_tensor->start_offset; + __gm__ float *src1 = reinterpret_cast<__gm__ float *>(src1_tensor->buffer.addr) + src1_tensor->start_offset; + __gm__ float *out = reinterpret_cast<__gm__ float *>(out_tensor->buffer.addr) + out_tensor->start_offset; + + // Configuration: float, 128, 128, 128, 128 + constexpr int kTRows_ = 128; + constexpr int kTCols_ = 128; + constexpr int vRows = 128; + constexpr int vCols = 128; + + using DynShapeDim5 = Shape<1, 1, 1, vRows, vCols>; + using DynStridDim5 = Stride<1, 1, 1, kTCols_, 1>; + using GlobalData = GlobalTensor; + using TileData = Tile; + + TileData src0Tile(vRows, vCols); + TileData src1Tile(vRows, vCols); + TileData dstTile(vRows, vCols); + TASSIGN(src0Tile, 0x0); + TASSIGN(src1Tile, 0x10000); + TASSIGN(dstTile, 0x20000); + + GlobalData src0Global(src0); + GlobalData src1Global(src1); + GlobalData dstGlobal(out); + + TLOAD(src0Tile, src0Global); + TLOAD(src1Tile, src1Global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + TADD(dstTile, src0Tile, src1Tile); + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(dstGlobal, dstTile); + + pipe_sync(); +} diff --git a/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add_scalar.cpp b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add_scalar.cpp new file mode 100644 index 0000000000..4982bd8ce5 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_add_scalar.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +/** + * Scalar Addition Kernel + * + * Implements: out[i] = src[i] + scalar + * + * This kernel adds a scalar value to each element of a tensor. It's compiled + * separately as a standalone kernel and linked with the dispatcher using + * function pointers, demonstrating the separation pattern used in production + * systems where kernel binaries are loaded dynamically. + */ + +#include +#include + +#include "tensor.h" // NOLINT(build/include_subdir) + +// NOLINTNEXTLINE(build/namespaces) +using namespace pto; + +#include "pipe_sync.h" + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] // NOLINT(whitespace/braces) +#endif + +/** + * Scalar addition kernel implementation + * + * Unified signature: all arguments passed via int64_t array + * @param args Argument array: + * args[0] = src pointer (input tensor) + * args[1] = out pointer (output tensor) + * args[2] = scalar value (as uint64_t, needs conversion to float) + * + * The element count is not an argument: the kernel processes a fixed + * 128 x 128 float tile. The orchestration DAG passes a fourth scalar that + * this kernel does not read. + */ +extern "C" __aicore__ __attribute__((always_inline)) void kernel_entry(__gm__ int64_t *args) { + // Unpack arguments (ChipTensor* pointers from runtime) + __gm__ ChipTensor *src_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[0]); + __gm__ ChipTensor *out_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[1]); + __gm__ float *src = reinterpret_cast<__gm__ float *>(src_tensor->buffer.addr) + src_tensor->start_offset; + __gm__ float *out = reinterpret_cast<__gm__ float *>(out_tensor->buffer.addr) + out_tensor->start_offset; + + // Convert scalar from uint64_t to float + float scalar = from_u64(static_cast(args[2])); + + // Configuration: float, 128, 128, 128, 128 + constexpr int kTRows_ = 128; + constexpr int kTCols_ = 128; + constexpr int vRows = 128; + constexpr int vCols = 128; + + using DynShapeDim5 = Shape<1, 1, 1, vRows, vCols>; + using DynStridDim5 = Stride<1, 1, 1, kTCols_, 1>; + using GlobalData = GlobalTensor; + using TileData = Tile; + + TileData srcTile(vRows, vCols); + TileData dstTile(vRows, vCols); + TASSIGN(srcTile, 0x0); + TASSIGN(dstTile, 0x10000); + + GlobalData srcGlobal(src); + GlobalData dstGlobal(out); + + TLOAD(srcTile, srcGlobal); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + TADDS(dstTile, srcTile, scalar); + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(dstGlobal, dstTile); + + pipe_sync(); +} diff --git a/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_mul.cpp b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_mul.cpp new file mode 100644 index 0000000000..b76af2c9d7 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/kernels/aiv/kernel_mul.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +/** + * Element-wise ChipTensor Multiplication Kernel + * + * Implements: out[i] = src0[i] * src1[i] + * + * This kernel performs element-wise multiplication of two tensors. It's + * compiled separately as a standalone kernel and linked with the dispatcher + * using function pointers, demonstrating the separation pattern used in + * production systems where kernel binaries are loaded dynamically. + */ + +#include +#include + +#include "tensor.h" + +using namespace pto; + +#include "pipe_sync.h" + +#ifndef __gm__ +#define __gm__ +#endif + +#ifndef __aicore__ +#define __aicore__ [aicore] +#endif + +/** + * Element-wise multiplication kernel implementation + * + * Unified signature: all arguments passed via int64_t array + * @param args Argument array: + * args[0] = src0 pointer (first input tensor) + * args[1] = src1 pointer (second input tensor) + * args[2] = out pointer (output tensor) + * + * The element count is not an argument: the kernel processes a fixed + * 128 x 128 float tile. + */ +extern "C" __aicore__ __attribute__((always_inline)) void kernel_entry(__gm__ int64_t *args) { + // Unpack arguments (ChipTensor* pointers from runtime) + __gm__ ChipTensor *src0_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[0]); + __gm__ ChipTensor *src1_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[1]); + __gm__ ChipTensor *out_tensor = reinterpret_cast<__gm__ ChipTensor *>(args[2]); + __gm__ float *src0 = reinterpret_cast<__gm__ float *>(src0_tensor->buffer.addr) + src0_tensor->start_offset; + __gm__ float *src1 = reinterpret_cast<__gm__ float *>(src1_tensor->buffer.addr) + src1_tensor->start_offset; + __gm__ float *out = reinterpret_cast<__gm__ float *>(out_tensor->buffer.addr) + out_tensor->start_offset; + + // Configuration: float, 128, 128, 128, 128 + constexpr int kTRows_ = 128; + constexpr int kTCols_ = 128; + constexpr int vRows = 128; + constexpr int vCols = 128; + + using DynShapeDim5 = Shape<1, 1, 1, vRows, vCols>; + using DynStridDim5 = Stride<1, 1, 1, kTCols_, 1>; + using GlobalData = GlobalTensor; + using TileData = Tile; + + TileData src0Tile(vRows, vCols); + TileData src1Tile(vRows, vCols); + TileData dstTile(vRows, vCols); + TASSIGN(src0Tile, 0x0); + TASSIGN(src1Tile, 0x10000); + TASSIGN(dstTile, 0x20000); + + GlobalData src0Global(src0); + GlobalData src1Global(src1); + GlobalData dstGlobal(out); + + TLOAD(src0Tile, src0Global); + TLOAD(src1Tile, src1Global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + TMUL(dstTile, src0Tile, src1Tile); + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(dstGlobal, dstTile); + + pipe_sync(); +} diff --git a/examples/workers/l4/vector_add_mixed_l3/kernels/orchestration/vector_add_mixed_l3_orchestration.cpp b/examples/workers/l4/vector_add_mixed_l3/kernels/orchestration/vector_add_mixed_l3_orchestration.cpp new file mode 100644 index 0000000000..e7699c9e01 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/kernels/orchestration/vector_add_mixed_l3_orchestration.cpp @@ -0,0 +1,114 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +/** + * Example: aicpu_orchestration_entry (device-side orchestration) + * + * DAG structure for formula: (a + b + 1)(a + b + 2) + (a + b) + * t0: c = a + b (func_id=0, kernel_add) [outer scope] + * t1: d = c + 1 (func_id=1, kernel_add_scalar) [inner scope] + * t2: e = c + 2 (func_id=1, kernel_add_scalar) [inner scope] + * t3: g = d * e (func_id=2, kernel_mul) [inner scope] + * t4: f = g + c (func_id=0, kernel_add) [inner scope] + * Dependencies: t0->t1, t0->t2, t1->t3, t2->t3, t0->t4, t3->t4 + * + * Nested scope demonstration: + * - Inner scope owns t1, t2, t3, t4; intermediates d, e, g release on inner scope end + * - Outer scope owns t0; c persists across inner scope for t1, t2, t4 + * - c flows from outer to inner scope (outer-scope tensors are visible to inner scopes) + * + * This file compiles as a standalone .so with zero runtime link dependencies. + * All runtime calls go through the PTO2RuntimeOps function-pointer table. + */ + +#include +#include + +#include "pto_orchestration_api.h" // NOLINT(build/include_subdir) + +extern "C" { + +/** + * Orchestration config — the executor reads these values to set up + * shared memory and runtime before calling aicpu_orchestration_entry. + */ +__attribute__((visibility("default"))) PTO2OrchestrationConfig +aicpu_orchestration_config(const ChipTaskArgs &orch_args) { + (void)orch_args; // NOLINT(readability/casting) + return PTO2OrchestrationConfig{ + .expected_arg_count = 3, + }; +} + +/** + * Orchestration entry — runtime is bound implicitly by the framework. + * The executor wraps this call in PTO2_SCOPE, so we are already inside + * the outer scope on entry. + */ +__attribute__((visibility("default"))) void aicpu_orchestration_entry(const ChipTaskArgs &orch_args) { + // golden shape = kernel shape, use orch_args.tensor(i).ref() directly + const ChipTensor &ext_a = orch_args.tensor(0).ref(); + const ChipTensor &ext_b = orch_args.tensor(1).ref(); + const ChipTensor &ext_f = orch_args.tensor(2).ref(); + + uint32_t SIZE = orch_args.tensor(0).ref().shapes[0]; + LOG_INFO("===============SIZE=%u", SIZE); + + uint32_t inter_shapes[1] = {SIZE}; + TensorCreateInfo inter_ci(inter_shapes, 1, DataType::FLOAT32); + + // t0: c = a + b (kernel_id=0, kernel_add) [outer scope] + CoreTaskArgs params_t0; + params_t0.add_input(ext_a); + params_t0.add_input(ext_b); + params_t0.add_output(inter_ci); + TaskOutputTensors outs_t0 = rt_submit_aiv_task(0, params_t0); // kernel_add + const ChipTensor &c = outs_t0.get_ref(0); + + // Inner scope: owns t1, t2, t3, t4; intermediates d, e, g release on scope end. + // c flows in from outer scope (outer-scope tensors are visible to inner scopes). + PTO2_SCOPE() { + // t1: d = c + 1 (kernel_id=1, kernel_add_scalar) + CoreTaskArgs params_t1; + params_t1.add_input(c); + params_t1.add_output(inter_ci); + params_t1.add_scalar(1.0f); + params_t1.add_scalar(3u); + TaskOutputTensors outs_t1 = rt_submit_aiv_task(1, params_t1); // kernel_add_scalar + const ChipTensor &d = outs_t1.get_ref(0); + + // t2: e = c + 2 (kernel_id=1, kernel_add_scalar) + CoreTaskArgs params_t2; + params_t2.add_input(c); + params_t2.add_output(inter_ci); + params_t2.add_scalar(2.0f); + params_t2.add_scalar(3u); + TaskOutputTensors outs_t2 = rt_submit_aiv_task(1, params_t2); // kernel_add_scalar + const ChipTensor &e = outs_t2.get_ref(0); + + // t3: g = d * e (kernel_id=2, kernel_mul) + CoreTaskArgs params_t3; + params_t3.add_input(d); + params_t3.add_input(e); + params_t3.add_output(inter_ci); + params_t3.add_scalar(3u); + TaskOutputTensors outs_t3 = rt_submit_aiv_task(2, params_t3); // kernel_mul + const ChipTensor &g = outs_t3.get_ref(0); + + // t4: f = g + c (kernel_id=0, kernel_add) + CoreTaskArgs params_t4; + params_t4.add_input(g); + params_t4.add_input(c); + params_t4.add_output(ext_f); + rt_submit_aiv_task(0, params_t4); // kernel_add + } // inner scope ends: releases d, e, g +} + +} // extern "C" diff --git a/examples/workers/l4/vector_add_mixed_l3/main.py b/examples/workers/l4/vector_add_mixed_l3/main.py new file mode 100644 index 0000000000..aeddb0bc52 --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/main.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Run L4 -> one local L3 and one remote L3, each executing a two-NPU vector group.""" + +from __future__ import annotations + +import argparse +import contextlib +import ctypes +from multiprocessing import shared_memory +from pathlib import Path +from typing import Any + +from simpler.callable_identity import CallableHandle +from simpler.remote_l3_protocol import HOST_TCP_TRANSPORT_PROFILE +from simpler.task_interface import ( + ArgDirection, + CallConfig, + ChipCallable, + ChipTensor, + CoreCallable, + DataType, + RemoteBufferHandle, + RemoteTensorRef, + TaskArgs, + TensorArgType, +) +from simpler.worker import RemoteCallable, RemoteWorkerSpec, Worker + +from simpler_setup.elf_parser import extract_text_section +from simpler_setup.kernel_compiler import KernelCompiler +from simpler_setup.pto_isa import ensure_pto_isa_root + +REMOTE_ORCH_TARGET = "examples.workers.l4.vector_add_mixed_l3.main:remote_l3_group_orch" +ELEMENTS = 128 * 128 +FLOAT_NBYTES = ctypes.sizeof(ctypes.c_float) +TENSOR_NBYTES = ELEMENTS * FLOAT_NBYTES +TENSOR_COUNT = 6 +FloatArray = ctypes.c_float * ELEMENTS +_LOCAL_GROUP_KEEPALIVE: list[TaskArgs] = [] +_LOCAL_CHIP_HANDLE: CallableHandle | None = None +_REMOTE_GROUP_KEEPALIVE: list[TaskArgs] = [] + + +def _digest_from_scalars(args: TaskArgs) -> bytes: + return b"".join(int(args.scalar(index)).to_bytes(8, "little") for index in range(4)) + + +def _submit_two_chip_group(orch, chip_handle: CallableHandle, args: TaskArgs, cfg: CallConfig) -> list[TaskArgs]: + if args.tensor_count() != TENSOR_COUNT or args.scalar_count() != 4: + raise ValueError("vector_add_mixed_l3 group task expects six tensors and four digest scalars") + + chip_args0 = TaskArgs() + chip_args0.add_tensor(args.tensor(0), TensorArgType.INPUT) + chip_args0.add_tensor(args.tensor(1), TensorArgType.INPUT) + chip_args0.add_tensor(args.tensor(2), TensorArgType.OUTPUT_EXISTING) + + chip_args1 = TaskArgs() + chip_args1.add_tensor(args.tensor(3), TensorArgType.INPUT) + chip_args1.add_tensor(args.tensor(4), TensorArgType.INPUT) + chip_args1.add_tensor(args.tensor(5), TensorArgType.OUTPUT_EXISTING) + + group_args = [chip_args0, chip_args1] + orch.submit_next_level_group(chip_handle, group_args, cfg, workers=[0, 1]) + return group_args + + +def local_l3_group_orch(orch, args: TaskArgs, cfg: CallConfig) -> None: + """Submit two local chip tasks from the forked local L3 worker.""" + if _LOCAL_CHIP_HANDLE is None: + raise RuntimeError("local L3 chip handle was not installed before fork") + _LOCAL_GROUP_KEEPALIVE[:] = _submit_two_chip_group(orch, _LOCAL_CHIP_HANDLE, args, cfg) + + +def remote_l3_group_orch(orch, args: TaskArgs, cfg: CallConfig) -> None: + """Submit two local chip tasks from the daemon-started remote L3 worker.""" + from simpler.remote_l3_session import get_inner_handle # noqa: PLC0415 + + chip_handle = get_inner_handle(_digest_from_scalars(args).hex()) + _REMOTE_GROUP_KEEPALIVE[:] = _submit_two_chip_group(orch, chip_handle, args, cfg) + + +def _build_vector_chip_callable(platform: str, runtime: str) -> ChipCallable: + kernels = Path(__file__).resolve().parent / "kernels" + orch_source = kernels / "orchestration" / "vector_add_mixed_l3_orchestration.cpp" + aiv_sources = ( + kernels / "aiv" / "kernel_add.cpp", + kernels / "aiv" / "kernel_add_scalar.cpp", + kernels / "aiv" / "kernel_mul.cpp", + ) + + compiler = KernelCompiler(platform=platform) + pto_isa_root = ensure_pto_isa_root() + include_dirs = compiler.get_orchestration_include_dirs(runtime) + include_dirs = list(include_dirs) + [str(compiler.project_root / "src" / "common")] + + def compile_aiv(source: Path) -> bytes: + binary = compiler.compile_incore( + source_path=str(source), + core_type="aiv", + pto_isa_root=pto_isa_root, + extra_include_dirs=include_dirs, + ) + return binary if platform.endswith("sim") else extract_text_section(binary) + + children = ( + ( + 0, + CoreCallable.build( + signature=[ArgDirection.IN, ArgDirection.IN, ArgDirection.OUT], + binary=compile_aiv(aiv_sources[0]), + ), + ), + ( + 1, + CoreCallable.build( + signature=[ArgDirection.IN, ArgDirection.OUT], + binary=compile_aiv(aiv_sources[1]), + ), + ), + ( + 2, + CoreCallable.build( + signature=[ArgDirection.IN, ArgDirection.IN, ArgDirection.OUT], + binary=compile_aiv(aiv_sources[2]), + ), + ), + ) + orch_binary = compiler.compile_orchestration(runtime_name=runtime, source_path=str(orch_source)) + return ChipCallable.build( + signature=[ArgDirection.IN, ArgDirection.IN, ArgDirection.OUT], + func_name="aicpu_orchestration_entry", + config_name="aicpu_orchestration_config", + binary=orch_binary, + children=list(children), + ) + + +def _add_digest_scalars(task_args: TaskArgs, digest: bytes) -> None: + if len(digest) != 32: + raise ValueError("inner chip callable digest must be 32 bytes") + for offset in range(0, 32, 8): + task_args.add_scalar(int.from_bytes(digest[offset : offset + 8], "little")) + + +def _parse_device_ids(value: str, *, label: str) -> tuple[int, int]: + device_ids = tuple(int(part.strip()) for part in value.split(",") if part.strip()) + if len(device_ids) != 2: + raise ValueError(f"{label} L3 group requires exactly two device ids") + if any(device_id < 0 for device_id in device_ids) or len(set(device_ids)) != 2: + raise ValueError(f"{label} device ids must be distinct and non-negative") + return device_ids + + +def _fill_array(array: Any, value: float) -> None: + for index in range(ELEMENTS): + array[index] = value + + +def _make_array(value: float) -> Any: + array = FloatArray() + _fill_array(array, value) + return array + + +def _expected(lhs: float, rhs: float) -> float: + summed = lhs + rhs + return (summed + 1.0) * (summed + 2.0) + summed + + +def _tensor_from_shm(shm: shared_memory.SharedMemory) -> ChipTensor: + buf = shm.buf + assert buf is not None + data_ptr = ctypes.addressof(ctypes.c_char.from_buffer(buf)) + return ChipTensor.make(data_ptr, (ELEMENTS,), DataType.FLOAT32) + + +def _make_local_group( + values: tuple[float, float, float, float], +) -> tuple[list[shared_memory.SharedMemory], list[Any], TaskArgs, dict[str, tuple[Any, float]]]: + a0_value, b0_value, a1_value, b1_value = values + initial_values = (a0_value, b0_value, 0.0, a1_value, b1_value, 0.0) + shms: list[shared_memory.SharedMemory] = [] + views: list[Any] = [] + args = TaskArgs() + for index, value in enumerate(initial_values): + shm = shared_memory.SharedMemory(create=True, size=TENSOR_NBYTES) + buf = shm.buf + assert buf is not None + view = FloatArray.from_buffer(buf) + _fill_array(view, value) + shms.append(shm) + views.append(view) + tag = TensorArgType.OUTPUT_EXISTING if index in (2, 5) else TensorArgType.INPUT + args.add_tensor(_tensor_from_shm(shm), tag) + return ( + shms, + views, + args, + { + "f0": (views[2], _expected(a0_value, b0_value)), + "f1": (views[5], _expected(a1_value, b1_value)), + }, + ) + + +def _make_remote_group_args(handles: list[RemoteBufferHandle], digest: bytes) -> TaskArgs: + if len(handles) != TENSOR_COUNT: + raise ValueError("remote L3 group requires six remote buffers") + args = TaskArgs() + for index, handle in enumerate(handles): + tag = TensorArgType.OUTPUT_EXISTING if index in (2, 5) else TensorArgType.INPUT + args.add_tensor(RemoteTensorRef(handle, shape=(ELEMENTS,), dtype=DataType.FLOAT32), tag) + _add_digest_scalars(args, digest) + return args + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--remote", required=True, help="remote L3 daemon endpoint, HOST:PORT") + parser.add_argument("--local-devices", default="0,1", help="two local device ids owned by the forked local L3") + parser.add_argument("--remote-devices", default="0,1", help="two remote device ids owned by the daemon L3") + parser.add_argument("--platform", default="a2a3") + parser.add_argument("--runtime", default="tensormap_and_ringbuffer") + parser.add_argument("--session-timeout", type=float, default=120.0) + parser.add_argument("--session-listen-host", default="0.0.0.0") + return parser.parse_args() + + +def _free_local_shms(shms: list[shared_memory.SharedMemory]) -> None: + for shm in reversed(shms): + with contextlib.suppress(BufferError): + shm.close() + with contextlib.suppress(FileNotFoundError): + shm.unlink() + + +def main() -> int: + # The local L3 is a fork of this process, so its orchestration function + # reaches the handle only through module state; a local would not survive + # into the child. + global _LOCAL_CHIP_HANDLE # noqa: PLW0603 + + args = _parse_args() + local_devices = _parse_device_ids(args.local_devices, label="local") + remote_devices = _parse_device_ids(args.remote_devices, label="remote") + + chip_callable = _build_vector_chip_callable(args.platform, args.runtime) + local_l3 = Worker(level=3, platform=args.platform, runtime=args.runtime, device_ids=local_devices) + _LOCAL_CHIP_HANDLE = local_l3.register(chip_callable) + + worker = Worker(level=4, num_sub_workers=0, remote_session_timeout_s=args.session_timeout) + remote_buffers: list[RemoteBufferHandle] = [] + local_shms: list[shared_memory.SharedMemory] = [] + local_views: list[Any] = [] + local_outputs: dict[str, tuple[Any, float]] = {} + parent_keepalive: list[TaskArgs] = [] + try: + local_worker = worker.add_worker(local_l3) + remote_worker = worker.add_remote_worker( + RemoteWorkerSpec( + endpoint=args.remote, + platform=args.platform, + runtime=args.runtime, + device_ids=remote_devices, + transport=HOST_TCP_TRANSPORT_PROFILE, + session_listen_host=args.session_listen_host, + allow_wildcard_session_bind=True, + ) + ) + chip_handle = worker.register(chip_callable) + local_handle = worker.register(local_l3_group_orch) + remote_handle = worker.register(RemoteCallable(REMOTE_ORCH_TARGET), workers=[remote_worker]) + + local_shms, local_views, local_args, local_outputs = _make_local_group((2.0, 3.0, 4.0, 5.0)) + _add_digest_scalars(local_args, chip_handle.digest) + worker.init() + + remote_handles = [worker.remote_malloc(worker=remote_worker, nbytes=TENSOR_NBYTES) for _ in range(TENSOR_COUNT)] + remote_buffers.extend(remote_handles) + remote_inputs = ( + _make_array(6.0), + _make_array(7.0), + _make_array(0.0), + _make_array(8.0), + _make_array(9.0), + _make_array(0.0), + ) + # zip() truncates to the shorter side, which would leave a remote input + # silently uninitialised if TENSOR_COUNT and this tuple ever disagree. + # zip(strict=True) says the same thing in one word but needs 3.10, and + # the pod runners are on 3.9. + if len(remote_inputs) != TENSOR_COUNT: + raise AssertionError(f"expected {TENSOR_COUNT} remote inputs, got {len(remote_inputs)}") + for handle, array in zip(remote_handles, remote_inputs): + worker.remote_copy_to(handle, array, TENSOR_NBYTES) + remote_outputs = { + "f0": (_make_array(0.0), _expected(6.0, 7.0)), + "f1": (_make_array(0.0), _expected(8.0, 9.0)), + } + + def parent_orch(orch, _args, cfg): + remote_args = _make_remote_group_args(remote_handles, chip_handle.digest) + parent_keepalive[:] = [local_args, remote_args] + orch.submit_next_level(local_handle, local_args, cfg, worker=local_worker) + orch.submit_next_level(remote_handle, remote_args, cfg, worker=remote_worker) + + config = CallConfig() + config.aicpu_thread_num = 4 + worker.run(parent_orch, args=None, config=config) + + worker.remote_copy_from(remote_handles[2], remote_outputs["f0"][0], TENSOR_NBYTES) + worker.remote_copy_from(remote_handles[5], remote_outputs["f1"][0], TENSOR_NBYTES) + + for worker_label, output_map in (("local", local_outputs), ("remote", remote_outputs)): + for name, (output_array, expected) in output_map.items(): + max_diff = max(abs(float(output_array[index]) - expected) for index in range(ELEMENTS)) + print(f"[vector-add-mixed-l3] {worker_label} output={name} max_diff={max_diff:.3e}") + if max_diff > 1e-4: + raise AssertionError(f"{worker_label} {name} golden mismatch: max_diff={max_diff}") + + print( + "vector_add_mixed_l3 passed: " + f"local[devices={args.local_devices}], remote={args.remote}[devices={args.remote_devices}], " + f"elements={ELEMENTS}" + ) + return 0 + finally: + parent_keepalive.clear() + _LOCAL_GROUP_KEEPALIVE.clear() + _REMOTE_GROUP_KEEPALIVE.clear() + for handle in reversed(remote_buffers): + try: + worker.remote_free(handle) + except Exception as exc: # noqa: BLE001 + # The pod job diagnoses this example from stdout alone, so a + # leaked peer buffer has to name itself here or leave no trace. + print(f"[vector-add-mixed-l3] remote_free failed: {exc}") + worker.close() + local_outputs.clear() + local_views.clear() + _free_local_shms(local_shms) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/workers/l4/vector_add_mixed_l3/run_parent.sh b/examples/workers/l4/vector_add_mixed_l3/run_parent.sh new file mode 100755 index 0000000000..70b95137db --- /dev/null +++ b/examples/workers/l4/vector_add_mixed_l3/run_parent.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" + +: "${SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE:?set remote L3 daemon as HOST:PORT}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_LOCAL_DEVICES:=0,1}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE_DEVICES:=0,1}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_PLATFORM:=a2a3}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_RUNTIME:=tensormap_and_ringbuffer}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_SESSION_LISTEN_HOST:=0.0.0.0}" +: "${SIMPLER_VECTOR_ADD_MIXED_L3_SESSION_TIMEOUT:=120}" + +cd "${ROOT_DIR}" +# The pod job builds the venv in an earlier step, so a staging failure reaches +# this script before it reaches python — name it here rather than let `set -e` +# abort on a bare "No such file or directory". +if [[ ! -f .venv/bin/activate ]]; then + echo "error: ${ROOT_DIR}/.venv/bin/activate not found; create the virtual environment first" >&2 + exit 1 +fi +# shellcheck source=/dev/null +source .venv/bin/activate + +exec python -m examples.workers.l4.vector_add_mixed_l3.main \ + --remote "${SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE}" \ + --local-devices "${SIMPLER_VECTOR_ADD_MIXED_L3_LOCAL_DEVICES}" \ + --remote-devices "${SIMPLER_VECTOR_ADD_MIXED_L3_REMOTE_DEVICES}" \ + --platform "${SIMPLER_VECTOR_ADD_MIXED_L3_PLATFORM}" \ + --runtime "${SIMPLER_VECTOR_ADD_MIXED_L3_RUNTIME}" \ + --session-listen-host "${SIMPLER_VECTOR_ADD_MIXED_L3_SESSION_LISTEN_HOST}" \ + --session-timeout "${SIMPLER_VECTOR_ADD_MIXED_L3_SESSION_TIMEOUT}" diff --git a/python/simpler/remote_l3_protocol.py b/python/simpler/remote_l3_protocol.py index 42500fd8e9..54e7b88109 100644 --- a/python/simpler/remote_l3_protocol.py +++ b/python/simpler/remote_l3_protocol.py @@ -28,6 +28,7 @@ MAX_INLINE_PAYLOAD_BYTES = 1024 * 1024 MAX_TRANSPORT_PROFILE_BYTES = 128 MAX_TRANSPORT_DESCRIPTOR_BYTES = 4096 +HOST_TCP_TRANSPORT_PROFILE = "host_tcp" MAX_CHIP_CALLABLE_DESCRIPTOR_BYTES = 4096 MAX_STAGED_BLOB_TOKEN_BYTES = 1024 REMOTE_BUFFER_ACCESS_READ = 1 << 0 diff --git a/python/simpler/remote_l3_session.py b/python/simpler/remote_l3_session.py index 20739ced86..e72cb63f95 100644 --- a/python/simpler/remote_l3_session.py +++ b/python/simpler/remote_l3_session.py @@ -43,6 +43,7 @@ validate_hashid, ) from .remote_l3_protocol import ( + HOST_TCP_TRANSPORT_PROFILE, PROTOCOL_VERSION, CallableKind, ChipCallableBlobLocation, @@ -75,7 +76,7 @@ send_frame, ) from .task_interface import ChipCallable, ChipTensor, TaskArgs -from .worker import Worker +from .worker import Worker, _NoHostBufferChildrenError sys.modules.setdefault("simpler.remote_l3_session", sys.modules[__name__]) @@ -111,6 +112,7 @@ class _RemoteBufferEntry: nbytes: int generation: int address_space: RemoteAddressSpace + owner: Worker | None = None offset: int = 0 released: bool = False @@ -120,6 +122,8 @@ def addr(self) -> int: buf = self.data.buf assert buf is not None return ctypes.addressof(ctypes.c_char.from_buffer(buf)) + if hasattr(self.data, "data_ptr"): + return int(self.data.data_ptr) return ctypes.addressof(self.data) @property @@ -130,6 +134,10 @@ def shm_name(self) -> str: def close(self, *, unlink: bool = False) -> None: if not isinstance(self.data, shared_memory.SharedMemory): + if self.owner is not None: + owner, self.owner = self.owner, None + self.data.buffer.release() + owner.free_host_buffer(self.data) return self.data.close() if unlink: @@ -656,9 +664,19 @@ def _run_command_loop( # noqa: PLR0912, PLR0915 buffer_id = next_buffer_id next_buffer_id += 1 generation = 1 - buf = shared_memory.SharedMemory(create=True, size=int(nbytes)) + try: + buf = inner_worker.create_host_buffer(int(nbytes)) + entry = _RemoteBufferEntry( + buf, + int(nbytes), + generation, + RemoteAddressSpace.REMOTE_DEVICE, + owner=inner_worker, + ) + except _NoHostBufferChildrenError: + buf = shared_memory.SharedMemory(create=True, size=int(nbytes)) + entry = _RemoteBufferEntry(buf, int(nbytes), generation, RemoteAddressSpace.REMOTE_DEVICE) key = _buffer_key(buffer_id, generation) - entry = _RemoteBufferEntry(buf, int(nbytes), generation, RemoteAddressSpace.REMOTE_DEVICE) buffers[key] = entry remote_addr = entry.addr result = struct.pack( @@ -752,8 +770,8 @@ def _run_command_loop( # noqa: PLR0912, PLR0915 raise ValueError("EXPORT_BUFFER names released buffer") if request.offset + request.nbytes > entry.nbytes: raise ValueError("EXPORT_BUFFER range exceeds buffer") - if request.transport_profile not in ("", "sim"): - raise ValueError("EXPORT_BUFFER transport_profile is not supported by sim") + if request.transport_profile not in ("", HOST_TCP_TRANSPORT_PROFILE): + raise ValueError("EXPORT_BUFFER transport_profile is not supported by host_tcp") export_id = next_export_id next_export_id += 1 result = ExportBufferResult( @@ -768,7 +786,7 @@ def _run_command_loop( # noqa: PLR0912, PLR0915 rkey_or_token=export_id, ub_ldst_va=0, access_flags=request.access_flags, - transport_profile="sim", + transport_profile=HOST_TCP_TRANSPORT_PROFILE, transport_descriptor=entry.shm_name.encode("utf-8"), ) payload = encode_control_reply( @@ -789,8 +807,8 @@ def _run_command_loop( # noqa: PLR0912, PLR0915 if request.importer_worker_id != worker_id: raise ValueError("IMPORT_BUFFER worker mismatch") export_desc = request.export_desc - if export_desc.transport_profile != "sim": - raise ValueError("IMPORT_BUFFER transport_profile is not supported by sim") + if export_desc.transport_profile != HOST_TCP_TRANSPORT_PROFILE: + raise ValueError("IMPORT_BUFFER transport_profile is not supported by host_tcp") shm_name = export_desc.transport_descriptor.decode("utf-8") shm = shared_memory.SharedMemory(name=shm_name) import_id = next_import_id @@ -816,7 +834,7 @@ def _run_command_loop( # noqa: PLR0912, PLR0915 rkey_or_token=import_id, ub_ldst_va=export_desc.ub_ldst_va, access_flags=request.requested_access_flags, - transport_profile="sim", + transport_profile=HOST_TCP_TRANSPORT_PROFILE, import_descriptor=b"", ) payload = encode_control_reply( diff --git a/python/simpler/remote_l3_worker.py b/python/simpler/remote_l3_worker.py index 3561ee95c4..ea25529b64 100644 --- a/python/simpler/remote_l3_worker.py +++ b/python/simpler/remote_l3_worker.py @@ -26,6 +26,8 @@ import time from typing import Any +from .remote_l3_protocol import HOST_TCP_TRANSPORT_PROFILE + def _read_exact(sock: socket.socket, n: int) -> bytes: data = bytearray() @@ -62,8 +64,8 @@ def _validate_manifest(manifest: dict[str, Any]) -> None: raise ValueError("manifest remote_worker_level must be 3") if not str(manifest["platform"]): raise ValueError("manifest platform must be non-empty") - if str(manifest["transport"]) != "sim": - raise ValueError("only sim transport is accepted by simpler-remote-worker") + if str(manifest["transport"]) != HOST_TCP_TRANSPORT_PROFILE: + raise ValueError(f"only {HOST_TCP_TRANSPORT_PROFILE} transport is accepted by simpler-remote-worker") def _session_timeout_s(manifest: dict[str, Any]) -> float: @@ -147,7 +149,8 @@ def _start_session(manifest: dict[str, Any]) -> tuple[dict[str, Any], subprocess # when no runner survives (a failed handshake has already been killed and # reaped here), so a failed send then leaves nothing to reclaim. A successful # send only means the bytes were queued locally, not that the parent read - # them; unobserved receipt would need an ACK / lease, which sim does not have. + # them; unobserved receipt would need an ACK / lease, which the + # host_tcp protocol does not have. _validate_manifest(manifest) # Both numeric timeouts are validated before any spawn resource (ready pipe, # manifest tempfile, runner Popen) exists: the runner is never launched only diff --git a/python/simpler/task_interface.py b/python/simpler/task_interface.py index 4926bb045c..c3c5dfb5af 100644 --- a/python/simpler/task_interface.py +++ b/python/simpler/task_interface.py @@ -175,7 +175,7 @@ class RemoteAddressSpace(IntEnum): ``HOST_INLINE`` carries the payload in the message itself rather than naming remote memory. ``REMOTE_WINDOW`` and ``UB_LDST`` are protocol - placeholders: the shipped transport is simulation-backed. + placeholders: the shipped host_tcp transport uses host-side session buffers. """ HOST_INLINE = 1 diff --git a/python/simpler/worker.py b/python/simpler/worker.py index 077d5243eb..bb7970331c 100644 --- a/python/simpler/worker.py +++ b/python/simpler/worker.py @@ -63,6 +63,7 @@ def my_l4_orch(orch, args, config): import contextlib import ctypes import enum +import hashlib import importlib import json import math @@ -113,6 +114,7 @@ def my_l4_orch(orch, args, config): parse_python_import_target, ) from .orchestrator import Orchestrator, _callback_run, direct_control +from .remote_l3_protocol import HOST_TCP_TRANSPORT_PROFILE from .task_interface import ( MAILBOX_ERROR_MSG_SIZE, MAILBOX_FRAME_SIZE, @@ -465,8 +467,8 @@ def qualname(self) -> str: class RemoteWorkerSpec: """Describes a remote L3 worker to attach via ``Worker.add_remote_worker``. - ``transport`` selects the data plane and is simulation-backed today; the - daemon rejects any other value. + ``transport`` selects the data plane. The shipped daemon accepts + only the host_tcp profile today. """ # endpoint is "host:port"; host must be a numeric IP (or "localhost"). @@ -477,7 +479,7 @@ class RemoteWorkerSpec: runtime: str = "tensormap_and_ringbuffer" device_ids: tuple[int, ...] = () num_sub_workers: int = 0 - transport: str = "sim" + transport: str = HOST_TCP_TRANSPORT_PROFILE session_listen_host: str | None = None allow_wildcard_session_bind: bool = False @@ -1224,6 +1226,10 @@ def _validate_domain_allocation( return resources +class _NoHostBufferChildrenError(RuntimeError): + """The Worker has no process child that can attach a host buffer.""" + + def _rewrite_blob_host_addrs(buf: memoryview, blob_off: int, ranges: list[tuple[int, int, int]]) -> None: """Redirect registered host pointers in a task-args blob to child mappings. @@ -1421,14 +1427,15 @@ def _pack_py_callable_payload(target) -> bytes: def _chip_descriptor_context(worker: Worker) -> tuple[str, str]: platform = str(worker._config.get("platform", "")) runtime = str(worker._config.get("runtime", "")) - if platform or runtime: - return platform, runtime - contexts: list[tuple[str, str]] = [] + if platform or runtime: + contexts.append((platform, runtime)) for child in getattr(worker, "_next_level_workers", []): child_context = _chip_descriptor_context(child) if child_context != ("", ""): contexts.append(child_context) + for spec in getattr(worker, "_remote_worker_specs", []): + contexts.append((str(spec.platform), str(spec.runtime))) if not contexts: return "", "" first = contexts[0] @@ -4246,6 +4253,50 @@ def _remote_dispatcher_entries_for_worker(self, worker_id: int) -> list[dict[str ) return entries + def _inner_registry_entries_for_spec(self, spec: RemoteWorkerSpec) -> list[dict[str, Any]]: + from .remote_l3_protocol import ( # noqa: PLC0415 + ChipCallableBlobLocation, + RemoteChipCallablePayload, + encode_remote_chip_callable_payload, + ) + + entries: list[dict[str, Any]] = [] + with self._registry_lock: + states = list(self._identity_registry.values()) + for state in states: + if state.target_namespace != "LOCAL_CHIP": + continue + if not isinstance(state.target, ChipCallable): + raise RuntimeError(f"inner chip hashid {state.hashid} does not carry a ChipCallable target") + descriptor = build_chip_callable_descriptor( + target=state.target, + platform=spec.platform, + runtime=spec.runtime, + ) + if descriptor != state.descriptor: + raise RuntimeError(f"inner chip hashid {state.hashid} was registered for a different platform/runtime") + blob = ctypes.string_at(int(state.target.buffer_ptr()), int(state.target.buffer_size())) + payload = encode_remote_chip_callable_payload( + RemoteChipCallablePayload( + descriptor_bytes=descriptor, + blob_location=ChipCallableBlobLocation.INLINE_BLOB, + blob_size=len(blob), + blob_sha256=hashlib.sha256(blob).digest(), + inline_blob=blob, + staged_blob_token=b"", + ) + ) + entries.append( + { + "hashid": state.digest.hex(), + "kind": "CHIP_CALLABLE", + "target_registry": "INNER_L3_WORKER", + "payload_version": 1, + "payload_hex": payload.hex(), + } + ) + return entries + def _build_remote_manifest( self, *, spec: RemoteWorkerSpec, worker_id: int, session_id: int, startup_remaining_s: float ) -> dict[str, Any]: @@ -4272,7 +4323,7 @@ def _build_remote_manifest( "listen_host": listen_host, "connect_host": daemon_host, "remote_task_dispatcher": self._remote_dispatcher_entries_for_worker(worker_id), - "inner_l3_worker": [], + "inner_l3_worker": self._inner_registry_entries_for_spec(spec), "feature_flags": [], } @@ -4578,7 +4629,7 @@ def remote_export( offset: int = 0, nbytes: int | None = None, access: str | int = "readwrite", - transport_profile: str = "sim", + transport_profile: str = HOST_TCP_TRANSPORT_PROFILE, ) -> RemoteBufferExport: """Export a range of an owner buffer so another worker can import it. @@ -4598,7 +4649,7 @@ def _remote_export_locked( offset: int = 0, nbytes: int | None = None, access: str | int = "readwrite", - transport_profile: str = "sim", + transport_profile: str = HOST_TCP_TRANSPORT_PROFILE, ) -> RemoteBufferExport: self._require_live_remote_buffer(handle) if handle.is_imported: @@ -6181,8 +6232,17 @@ def _eligible_target_need(self, namespace: str | None, eligible_worker_ids) -> s has_python_child = self._config.get("num_sub_workers", 0) > 0 or bool(self._next_level_workers) return None if has_python_child else "a SUB or next-level child" if namespace == "LOCAL_CHIP": - has_chip_child = bool(self._config.get("device_ids")) or bool(self._next_level_workers) - return None if has_chip_child else "a chip device (device_ids)" + # A chip target need not be this worker's own: an L4 parent carries + # no device_ids and reaches its chips through a next-level child or + # a remote spec, so the search walks the frozen topology. + def has_chip_target(worker: Worker) -> bool: + if worker._config.get("device_ids"): + return True + if any(spec.device_ids for spec in worker._remote_worker_specs): + return True + return any(has_chip_target(child) for child in worker._next_level_workers) + + return None if has_chip_target(self) else "a chip device (device_ids)" if namespace == "REMOTE_TASK_DISPATCHER": has_remote_workers = set(self._remote_worker_ids) ok = bool(has_remote_workers) and set(eligible_worker_ids) <= has_remote_workers @@ -8080,7 +8140,7 @@ def _create_host_buffer_locked(self, nbytes: int) -> HostBuffer: # and sub alike, via _broadcast_host_control). Only a truly childless L3 # has nowhere to attach it. if not self._chip_shms and not self._sub_shms: - raise RuntimeError( + raise _NoHostBufferChildrenError( "create_host_buffer requires at least one forked chip or sub child (this Worker has none)" ) assert self._worker is not None diff --git a/tests/ut/py/test_callable_identity.py b/tests/ut/py/test_callable_identity.py index ecba2d82b7..1dfd4f0260 100644 --- a/tests/ut/py/test_callable_identity.py +++ b/tests/ut/py/test_callable_identity.py @@ -1204,7 +1204,7 @@ def test_remote_sim_noop_task_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_noop_orch"), @@ -1228,7 +1228,7 @@ def test_remote_sim_prepare_callable_control_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_noop_orch"), @@ -1254,7 +1254,7 @@ def test_remote_sim_error_completion_raises_root_error(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_raises_orch"), @@ -1279,7 +1279,7 @@ def test_remote_sim_post_init_register_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) worker.init() handle = worker.register( @@ -1303,7 +1303,7 @@ def test_remote_sim_unregister_then_reregister_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) worker.init() @@ -1337,7 +1337,7 @@ def test_remote_sim_health_lane_stays_live_during_long_task(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_sleep_orch"), @@ -1366,7 +1366,7 @@ def test_remote_sim_inner_python_import_register_runs_sub_task(): RemoteWorkerSpec( endpoint=f"127.0.0.1:{port}", platform="a2a3sim", - transport="sim", + transport="host_tcp", num_sub_workers=1, ) ) @@ -1430,7 +1430,7 @@ def test_remote_sim_buffer_copy_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_increment_u8_orch"), @@ -1471,10 +1471,10 @@ def test_remote_sim_imported_buffer_runs_on_peer_worker(): owner_daemon.await_ready() peer_daemon.await_ready() owner_worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{owner_port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{owner_port}", platform="a2a3sim", transport="host_tcp") ) peer_worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{peer_port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{peer_port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_increment_u8_orch"), @@ -2453,7 +2453,7 @@ def test_remote_sim_failed_dependency_skips_consumer(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) fail_handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_fail_before_write_orch"), @@ -2502,7 +2502,7 @@ def test_remote_sim_session_exit_becomes_endpoint_failure(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_exit_orch"), @@ -2527,7 +2527,7 @@ def test_remote_sim_input_free_is_deferred_until_slot_refs_drop(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_sum_u8_orch"), @@ -2572,7 +2572,7 @@ def test_remote_sim_host_inline_descriptor_roundtrip(): try: daemon.await_ready() worker_id = worker.add_remote_worker( - RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="sim") + RemoteWorkerSpec(endpoint=f"127.0.0.1:{port}", platform="a2a3sim", transport="host_tcp") ) handle = worker.register( RemoteCallable("tests.ut.py.test_callable_identity:_remote_sum_u8_orch"), diff --git a/tests/ut/py/test_remote_l3_lifecycle.py b/tests/ut/py/test_remote_l3_lifecycle.py index acd644cb3d..e2e03e8381 100644 --- a/tests/ut/py/test_remote_l3_lifecycle.py +++ b/tests/ut/py/test_remote_l3_lifecycle.py @@ -27,7 +27,7 @@ def _manifest(**extra): "parent_worker_level": 4, "remote_worker_level": 3, "platform": "a2a3sim", - "transport": "sim", + "transport": "host_tcp", "listen_host": "127.0.0.1", "connect_host": "127.0.0.1", "session_timeout_s": 0.01, diff --git a/tests/ut/py/test_worker/test_remote_startup_budget.py b/tests/ut/py/test_worker/test_remote_startup_budget.py index 69535e668c..e0cffc6d45 100644 --- a/tests/ut/py/test_worker/test_remote_startup_budget.py +++ b/tests/ut/py/test_worker/test_remote_startup_budget.py @@ -212,7 +212,7 @@ def _manifest(**overrides) -> dict: "parent_worker_level": 4, "remote_worker_level": 3, "platform": "a2a3sim", - "transport": "sim", + "transport": "host_tcp", "session_timeout_s": 30.0, "startup_remaining_s": 10.0, }