diff --git a/benchpress/config/jobs.yml b/benchpress/config/jobs.yml index 365d84f5..73d39278 100644 --- a/benchpress/config/jobs.yml +++ b/benchpress/config/jobs.yml @@ -502,6 +502,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: # Hardcode num_instances=1: this job exists specifically for single-instance @@ -534,6 +535,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -592,6 +599,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -635,6 +643,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -695,6 +709,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -735,6 +750,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: copymove diff --git a/packages/feedsim/README.md b/packages/feedsim/README.md index 6dce669c..699831fc 100644 --- a/packages/feedsim/README.md +++ b/packages/feedsim/README.md @@ -55,7 +55,7 @@ taskset -c 0-15 ./benchpress_cli.py install feedsim_dlrm Unlike FeedSim v1 which spawns a new FeedSim instance per 100 CPU cores, `feedsim_dlrm` is pinned to **one FeedSim instance per host** because the redesigned threading model in FeedSim v2 has overcome the scalability issue -on ultra-high-core-count CPUs and ARM CPUs. +on ultra-high-core-count CPUs and ARM CPUs. The runner searches for the QPS that keeps 95th-percentile end-to-end latency at or below **700 ms**. When it converges it runs a final 5-minute @@ -64,7 +64,7 @@ counters) during that final window. We expect the total wall-clock runtime to be around 30 minutes. Please make sure to turn CPU turbo-boost on before starting, or FeedSim may -fail to converge and report a low QPS. +fail to converge and report a low QPS. ### Result report @@ -211,7 +211,7 @@ feedsim server, driver and mock_services instances. 2. Use the `feedsim_autoscale_dlrm` job. This autoscale job will spawn `ceil(nproc / 100)` FeedSim instances, each pinned to its own CPU range via `taskset`, plus one driver -and one `mock_services` process per instance (also `taskset`-isolated). For example: +and one `mock_services` process per instance (also `taskset`-isolated). For example: ``` ./benchpress_cli.py run feedsim_autoscale_dlrm ``` @@ -220,6 +220,41 @@ In multi-instance mode, the overall QPS is the sum across all instances. and the average latency will be the average of p95 latency values observed across all instances. +### Driver depth (fixing CPU/latency under-utilization) + +The `depth` parameter sets the driver's pipeline depth — the maximum number of +outstanding (in-flight) requests per driver connection. The driver's total +offered concurrency is `driver_threads × connections × depth`, so with the +default `depth=1` the driver can cap the achievable load below what the server +can actually handle. + +**Increase `depth` beyond 1 when the final benchmarking phase saturates neither +CPU nor latency** — i.e. the final achieved p95 latency is well below the SLA +limit (`sla_p95_ms`, default 700 ms) *and* the CPU utilization during the final +5-minute benchmarking phase is less than ~90%. In that situation the reported QPS +is limited by driver concurrency rather than by the server, so it understates the +hardware's true capacity. Raising `depth` (start with `2`) lets the driver offer +more concurrent load until the server becomes the bottleneck — either CPU-bound +(~100% utilization) or latency-bound (p95 ≈ SLA). **This is likely necessary on +high-performance ARM cores** (e.g. NVIDIA Grace), which can otherwise sit at +80–90% CPU with p95 far below the SLA at `depth=1`. + +``` +# Force driver depth 2 +./benchpress_cli.py run feedsim_dlrm -i '{"depth": 2}' +``` + +There is also an **adaptive depth** mechanism (on by default) that raises the +depth automatically during the peak-finding stage until the server saturates +(system CPU ≥ 95% or p95 ≥ SLA). It catches *severe* under-utilization early, but +because it evaluates saturation on the high-load peak/search probes rather than +on the final SLA-converged operating point, it **may not catch all +under-utilization cases**. If you still observe under-utilization in the final +result (low CPU + p95 well under SLA), increase `depth` manually as above. When +adaptive depth is on, a manually-set `depth` acts as the starting floor the +adaptive search raises from; to pin an exact fixed depth, also set the +`FEEDSIM_ADAPTIVE_DEPTH_MAX=0` environment variable to disable adaptive search. + ### Other parameters This section lists additional parameters in `feedsim_dlrm` benchmark. These parameters @@ -233,6 +268,7 @@ Job-level parameters (can be passed via `-i` flag in Benchpress CLI): |---|---|---| | `num_instances` | Number of FeedSim instances to run in parallel. Defaults to 1 in `feedsim_dlrm`; set to -1 to autoscale for `feedsim_autoscale_dlrm`. | `1` | | `sla_p95_ms` | SLA target in ms. The runner searches for the highest QPS keeping p95 ≤ this. | `700` | +| `depth` | Driver pipeline depth (max outstanding requests per connection; total in-flight = `driver_threads × connections × depth`). Raise (e.g. `2`) when the final phase saturates neither CPU nor latency — often needed on high-perf ARM. See [Driver depth](#driver-depth-fixing-cpulatency-under-utilization). | `1` | | `io_dist` | I/O latency distribution: `fixed`, `exponential`, or `lognormal`. | `fixed` | | `io_mean` | Mean I/O latency in ms. | `200` | | `workload` | Ranking workload: `pagerank` or `dlrm`. `dlrm` is v2. | `dlrm` | diff --git a/packages/feedsim/install_feedsim.sh b/packages/feedsim/install_feedsim.sh index e9bb855b..03c22fe4 100755 --- a/packages/feedsim/install_feedsim.sh +++ b/packages/feedsim/install_feedsim.sh @@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" +# Dependency versions are env-overridable so experiments can bump them without +# forking this script; defaults reproduce the v2 baseline exactly. +JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" +LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" +# Export so the aarch64 sub-installer (dispatched below) inherits the pins. +export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \ libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \ libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \ zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \ - xxhash xxhash-devel unzip rsync liburing-devel + xxhash xxhash-devel unzip rsync liburing-devel python3-pip # Creates feedsim directory under benchmarks/ mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim" @@ -178,30 +191,30 @@ else fi # Installing JEMalloc -if ! [ -d "jemalloc-5.3.0" ]; then - wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" - bunzip2 "jemalloc-5.3.0.tar.bz2" - tar -xvf "jemalloc-5.3.0.tar" - cd "jemalloc-5.3.0" +if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then + wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2" + tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar" + cd "jemalloc-${JEMALLOC_VERSION}" ./configure --enable-prof --enable-prof-libunwind make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] jemalloc-5.3.0" + msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}" fi # Installing libevent -if ! [ -d "libevent-2.1.12-stable" ]; then - wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz" - tar -xzf "libevent-2.1.12-stable.tar.gz" - cd "libevent-2.1.12-stable" +if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then + wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz" + tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz" + cd "libevent-${LIBEVENT_VERSION}" ./configure make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] libevent-2.1.12-stable" + msg "[SKIPPED] libevent-${LIBEVENT_VERSION}" fi msg "Installing third-party dependencies ... DONE" @@ -218,12 +231,33 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # pip on the box (3.9, or an internal stale mirror) can't see the cp310 + # 2.13 wheels, so resolve the wheel href straight from the PEP-503 index + # and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is + # Python-version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/install_feedsim_aarch64.sh b/packages/feedsim/install_feedsim_aarch64.sh index 59a238d1..acd0ce62 100755 --- a/packages/feedsim/install_feedsim_aarch64.sh +++ b/packages/feedsim/install_feedsim_aarch64.sh @@ -182,11 +182,15 @@ else msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}" fi -DEP_JEMALLOC_VERSION="5.3.0" +DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" # Installing JEMalloc if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" - verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then + verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + else + msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)" + fi bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar" cd "jemalloc-${DEP_JEMALLOC_VERSION}" @@ -198,11 +202,15 @@ else msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}" fi -DEP_LIBEVENT_VERSION="2.1.12-stable" +DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" # Installing libevent if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" - verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then + verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + else + msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)" + fi tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" cd "libevent-${DEP_LIBEVENT_VERSION}" ./configure @@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # unset reproduces the v2 baseline (latest). + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh index e6abb5ad..54565bfb 100755 --- a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh +++ b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh @@ -27,6 +27,7 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -34,7 +35,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync + sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing is # older than folly's minimum, so folly's io_uring integration links @@ -204,9 +205,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # empty falls back to pip's latest resolution. + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_ubuntu.sh b/packages/feedsim/install_feedsim_ubuntu.sh index f525f63e..4237ea6b 100755 --- a/packages/feedsim/install_feedsim_ubuntu.sh +++ b/packages/feedsim/install_feedsim_ubuntu.sh @@ -11,7 +11,14 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -30,7 +37,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq xxhash libxxhash-dev unzip rsync + sysstat jq xxhash libxxhash-dev unzip rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing (0.7 on # 20.04, 2.1 on 22.04) is older than folly's minimum, so folly's io_uring @@ -213,12 +220,31 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # The C++ libtorch inside (torch/lib, torch/share/cmake) is Python- + # version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/run.sh b/packages/feedsim/run.sh index 6ca879e0..2010f549 100644 --- a/packages/feedsim/run.sh +++ b/packages/feedsim/run.sh @@ -120,6 +120,7 @@ Usage: ${0##*/} [OPTION]... --rpc-fanout-scale Scale factor applied to per-session fanout counts. Default: 0.05 (t43 c7). --server-zstd Enable ZSTD compression on server-side response payloads (0=off, 1=on). Default: 0 (t43 c7). --sla-p95-ms search_qps SLA target (95th percentile latency in ms). Default: 700. + --depth Driver pipeline depth: max outstanding requests per driver connection (max in-flight = driver_threads * connections * depth). Default: 1 (or \$FEEDSIM_DRIVER_DEPTH). Raise (e.g. 2) when the final phase saturates neither CPU nor SLA latency; with adaptive depth on, this is the starting floor the peak search raises from. EOF } @@ -316,6 +317,12 @@ main() { local sla_p95_ms sla_p95_ms="700" + # Driver pipeline depth (max outstanding requests per driver connection). + # Env var FEEDSIM_DRIVER_DEPTH is the fallback default; the --depth CLI flag + # (forwarded from the benchpress `depth` job parameter) overrides it. + local driver_depth + driver_depth="${FEEDSIM_DRIVER_DEPTH:-1}" + if [ -z "$IS_AUTOSCALE_RUN" ]; then echo > $BREPS_LFILE fi @@ -644,6 +651,13 @@ main() { --sla-p95-ms=*) sla_p95_ms="${1#*=}" ;; + --depth) + driver_depth="$2" + shift + ;; + --depth=*) + driver_depth="${1#*=}" + ;; -h|--help) show_help >&2 exit 1 @@ -823,6 +837,47 @@ main() { if [ "$mock_tls" = "1" ]; then export MOCK_TLS=1 fi + # Driver↔Leaf TLS. When FEEDSIM_DRIVER_TLS=1 is passed in the environment, + # the DriverNodeRank client (reads FEEDSIM_DRIVER_TLS) wraps its + # bufferevents in OpenSSL, and the LeafNodeRank server needs a cert so + # FeedSimServer's accept path enables AsyncSSLSocket — wire the shared + # example cert/key into FEEDSIM_TLS_CERT/FEEDSIM_TLS_KEY. Both ends pin + # AES-GCM so the encrypted driver↔leaf volume registers as hardware crypto + # (matches prod's Rocket-over-TLS driver path). Default off. The LeafNodeRank + # server reads FEEDSIM_TLS_CERT/KEY from the exported env; the DriverNodeRank + # client is launched through search_qps.sh, which does not reliably forward + # the parent's exported env to the driver process, so FEEDSIM_DRIVER_TLS is + # also injected directly on the driver command line via `driver_bin`. + # Per-run driver knobs are injected on the DriverNodeRank command line via + # `env VAR=val` (accumulated in driver_env), because search_qps.sh launches + # the driver as a bare `$command &` that does not reliably inherit the + # parent shell's exported env. + driver_env="" + if [ "${FEEDSIM_DRIVER_TLS:-0}" = "1" ]; then + driver_cert_dir="${FEEDSIM_ROOT}/certs" + if [ ! -r "${driver_cert_dir}/example.crt" ] || [ ! -r "${driver_cert_dir}/example.key" ]; then + echo "ERROR: FEEDSIM_DRIVER_TLS=1 but ${driver_cert_dir}/example.{crt,key} not found" >&2 + exit 1 + fi + export FEEDSIM_TLS_CERT="${driver_cert_dir}/example.crt" + export FEEDSIM_TLS_KEY="${driver_cert_dir}/example.key" + export FEEDSIM_DRIVER_TLS=1 + driver_env="${driver_env} FEEDSIM_DRIVER_TLS=1" + echo "Driver↔Leaf TLS: ENABLED (cert=${driver_cert_dir}/example.crt, AES-GCM)" + fi + # FEEDSIM_STATS_WARMUP_SECS: drop the first N seconds of latency/throughput + # samples in each search_qps probe (DriverNodeRank resets its stats N secs + # in) so cold-start transients don't inflate the tail and make the search + # back off QPS prematurely. Default unset/0 (no warmup). + if [ -n "${FEEDSIM_STATS_WARMUP_SECS:-}" ] && [ "${FEEDSIM_STATS_WARMUP_SECS}" != "0" ]; then + driver_env="${driver_env} FEEDSIM_STATS_WARMUP_SECS=${FEEDSIM_STATS_WARMUP_SECS}" + echo "Driver stats warmup: ${FEEDSIM_STATS_WARMUP_SECS}s (dropping cold-start samples per probe)" + fi + if [ -n "$driver_env" ]; then + driver_bin="env${driver_env} build/workloads/ranking/DriverNodeRank" + else + driver_bin="build/workloads/ranking/DriverNodeRank" + fi # MOCK_ZSTD_FRAC env consumed by MockServicesClient::resolveZstdFraction. # Always export so the t43 c7 default 0.75 reaches the binary. export MOCK_ZSTD_FRAC="$mock_zstd_frac" @@ -955,16 +1010,32 @@ main() { # Override via --sla-p95-ms CLI flag (handled in arg parsing above). sla_arg="95p:${sla_p95_ms}" + # Adaptive driver depth (fleet default): search_qps raises the driver's + # pipeline --depth in the peak phase until the server saturates (system + # CPU>=95% or p95>=SLA), giving each platform just enough offered concurrency + # to reach a real bound instead of capping on driver concurrency (the t19 + # anti-pattern where big boxes sat at ~80% CPU with p95 far below SLA). + # Enabled by default up to depth 8; set FEEDSIM_ADAPTIVE_DEPTH_MAX=0 to + # disable and use the fixed driver_depth (--depth flag / FEEDSIM_DRIVER_DEPTH, + # default 1). When adaptive is on, driver_depth is the STARTING floor the peak + # search raises from (search_qps reads the --depth we pass below). + sqps_adaptive_arg="" + adaptive_depth_max="${FEEDSIM_ADAPTIVE_DEPTH_MAX:-8}" + if [ "$adaptive_depth_max" != "0" ]; then + sqps_adaptive_arg="-D ${adaptive_depth_max}" + fi + if [ -z "$fixed_qps" ] && [ "$auto_driver_threads" != "1" ]; then benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + scripts/search_qps.sh -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${driver_threads}" \ --connections=4 \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -973,10 +1044,11 @@ main() { benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -a -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + scripts/search_qps.sh -a -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + $driver_bin \ --monitor_port "$client_monitor_port" \ --server "0.0.0.0:$port" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -1000,11 +1072,12 @@ main() { -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" \ $qps_threshold_args $no_retry_args \ -o "${FEEDSIM_ROOT}/${result_filename}" \ - -- build/workloads/ranking/DriverNodeRank \ + -- $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${num_workers}" \ --connections="${num_connections}" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index b811794d..331446e6 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -31,6 +31,26 @@ echo "${SCRIPT_NAME}: DCPERF_PERF_RECORD=${DCPERF_PERF_RECORD}" function benchreps_tell_state () { date +"%Y-%m-%d_%T ${1}" >> $BREPS_LFILE } + +# ─── CPU utilization helpers (used by adaptive depth) ──────────────────────── +# Read /proc/stat's aggregate cpu line and echo "total idle_all" jiffies. +cpu_snapshot() { + local cpu u n s idle iow irq sirq st rest + read -r cpu u n s idle iow irq sirq st rest < /proc/stat + local idle_all=$((idle + iow)) + local total=$((u + n + s + idle + iow + irq + sirq + st)) + echo "$total $idle_all" +} +# System-wide CPU busy% (100 - idle%) measured over the next $1 seconds. +cpu_busy_over() { + local secs="$1" s1 s2 t1 i1 t2 i2 dt di + s1=$(cpu_snapshot); t1=${s1% *}; i1=${s1#* } + sleep "$secs" + s2=$(cpu_snapshot); t2=${s2% *}; i2=${s2#* } + dt=$((t2 - t1)); di=$((i2 - i1)) + if [ "$dt" -le 0 ]; then echo "0"; return; fi + echo "scale=1; (($dt - $di) * 100) / $dt" | bc +} # Source runtime breakdown utilities if they exist if [ -f "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" ]; then source "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" @@ -99,6 +119,12 @@ mutilate (EuroSys \'14) [https://github.com/leverich/mutilate] without retrying. Optional -P PID of the process to log runtime breakdowns. Optional -B Folder to log runtime breakdowns. Optional + -D Adaptive depth: max driver pipeline depth. When set, the peak + phase raises the driver's --depth until the server is saturated + (system CPU >= 95% OR achieved p95 >= SLA), then holds that depth + for the QPS search. The search STARTS from any --depth in the + driver command (default 1), so a manually-set --depth acts as a + floor. Optional. EOF } @@ -145,7 +171,7 @@ run_loadtest() { for r in $(seq 1 $load_test_retries); do # run the command, saving result to tmpfile local tmp_file=$(mktemp) - $command $threads_arg $qps_arg &>$tmp_file & + $command $threads_arg $qps_arg $adaptive_depth_arg &>$tmp_file & LOADTEST_PID=$! if [ "$no_retry_mode" = "1" ]; then @@ -301,14 +327,19 @@ max_warmup_iterations=10 no_retry_mode="" breakdown_pid="" breakdown_folder="" +adaptive_depth_max="" # -D: when set, search_qps raises driver --depth in the +adaptive_depth_arg="" # peak phase until the server saturates (CPU>=95% or p95>=SLA) OPTIND=1 # Reset is necessary if getopts was used previously in the script. It is a good idea to make this local in a function. -while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:" opt; do +while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:D:" opt; do case "$opt" in h) show_help exit 0 ;; + D) + adaptive_depth_max=$OPTARG + ;; t) experiment_time=$OPTARG ;; @@ -364,6 +395,20 @@ fi # remaining argument is loadtest command command=$@ +# In adaptive-depth mode, search_qps owns the driver's --depth: capture any fixed +# --depth as the STARTING depth (so a manually-set --depth acts as a floor the +# peak search raises from), then strip it so our per-attempt --depth is the only +# one on the command. Default start is depth=1 (unchanged behavior). +adaptive_start_depth=1 +if [ -n "$adaptive_depth_max" ]; then + fixed_depth=$(echo "$command" | grep -oE -- '--depth=[0-9]+' | head -1 | grep -oE '[0-9]+') + if [ -n "$fixed_depth" ] && [ "$fixed_depth" -gt 1 ]; then + adaptive_start_depth=$fixed_depth + fi + command=$(echo "$command" | sed -E 's/[[:space:]]*--depth=[0-9]+//g') + adaptive_depth_arg="--depth=$adaptive_start_depth" +fi + # make sure latency_type and latency_target are specified if [[ -z "$fixed_qps" ]] && ( [[ $latency_type = "" ]] || [[ $latency_target = "" ]] ); then echo 'error: -s metric:target must be specified' >&2; exit 1 @@ -513,7 +558,42 @@ fi # find peak QPS benchreps_tell_state "before peak_qps" -run_loadtest peak_qps measured_latency "" "" +if [ -n "$adaptive_depth_max" ]; then + # Adaptive depth: the peak load test offers at most threads*connections*depth + # concurrent requests. Starting at adaptive_start_depth, keep raising depth (and + # re-running peak) until the server saturates — system CPU >= 95% OR p95 >= SLA + # — so platforms that need more offered concurrency reach a real bound instead + # of capping on driver concurrency (the t19 anti-pattern). The selected depth + # is then held for the QPS search / tuning / final phases. The starting depth + # is the fixed --depth from the driver command (default 1), so a manually-set + # depth raises the floor. + cur_depth=$adaptive_start_depth + while : ; do + adaptive_depth_arg="--depth=$cur_depth" + # Sample system CPU busy% over a mid-run window while the peak load runs. + cpu_busy_file="/tmp/adaptive_cpu_busy_$$" + ( sleep 20; cpu_busy_over 40 > "$cpu_busy_file" ) & + cpu_sampler_pid=$! + run_loadtest peak_qps measured_latency "" "" + wait "$cpu_sampler_pid" 2>/dev/null + cpu_busy=$(cat "$cpu_busy_file" 2>/dev/null || echo 0) + rm -f "$cpu_busy_file" + cpu_sat=$(echo "${cpu_busy:-0} >= 95" | bc 2>/dev/null || echo 0) + lat_sat=$(echo "$measured_latency >= $latency_target" | bc 2>/dev/null || echo 0) + printf "adaptive-depth: depth=%d peak_qps=%.2f p95=%.2f cpu_busy=%s%% cpu_sat=%s lat_sat=%s\n" \ + "$cur_depth" "$peak_qps" "$measured_latency" "${cpu_busy:-0}" "$cpu_sat" "$lat_sat" + echo "adaptive-depth: depth=$cur_depth peak_qps=$peak_qps p95=$measured_latency cpu_busy=${cpu_busy}% cpu_sat=$cpu_sat lat_sat=$lat_sat" >> $BREPS_LFILE + if [ "$cpu_sat" -eq 1 ] || [ "$lat_sat" -eq 1 ] || [ "$cur_depth" -ge "$adaptive_depth_max" ]; then + break + fi + cur_depth=$((cur_depth + 1)) + sleep "$wait_time" + done + echo "adaptive-depth: SELECTED depth=$cur_depth (cpu_busy=${cpu_busy}%, p95=$measured_latency, sla=$latency_target)" >> $BREPS_LFILE + printf "adaptive-depth: selected depth=%d (cpu_busy=%s%%, p95=%.2f)\n" "$cur_depth" "${cpu_busy:-0}" "$measured_latency" +else + run_loadtest peak_qps measured_latency "" "" +fi printf "peak qps = %.2f, latency = %.2f\n" $peak_qps $measured_latency benchreps_tell_state "after peak_qps" diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc index 2d1119a4..65d700c4 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc @@ -246,6 +246,18 @@ SSL_CTX* getDriverSslCtxOrNull() { return nullptr; } SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, nullptr); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES (AES-NI on x86, ARMv8 crypto extensions on aarch64) via + // libcrypto, matching prod's cipher. Without this, OpenSSL may pick + // ChaCha20-Poly1305 (a NEON/integer cipher with no AES instructions), + // which reads as ~0% in the crypto instruction mix and runs the AEAD + // un-accelerated. Mirrors MockServicesClient's cipher pinning. + SSL_CTX_set_cipher_list( + ctx, + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx, "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 std::cout << "FeedSimDriver: TLS enabled via FEEDSIM_DRIVER_TLS=1" << std::endl; return ctx; @@ -852,6 +864,17 @@ void FeedSimDriver::enableMonitoring(uint16_t port) { impl_->monitor_port = port; } +namespace { +// One-shot timer callback used by FEEDSIM_STATS_WARMUP_SECS: drops all +// latency/throughput samples collected so far so the final stats reflect only +// the post-warmup (steady-state) window. Runs on the DriverThread's own event +// base, so it never races that thread's logRequest()/logResponse() writes to +// the same DriverStats. +void resetDriverStatsCb(evutil_socket_t, short, void* arg) { + reinterpret_cast(arg)->reset(); +} +} // namespace + void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, uint32_t num_connections_per_thread, uint32_t max_connection_depth) { @@ -864,6 +887,23 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, impl_->running = true; impl_->total_stats = std::make_unique(1000); + // FEEDSIM_STATS_WARMUP_SECS: when >0, each DriverThread drops the samples it + // collected in the first N seconds of the measurement window (a per-thread + // one-shot timer calls DriverStats::reset()). This excludes cold-start + // transients (connection ramp, cache/JIT warmup, first-touch faults) from + // the reported latency distribution, giving search_qps a more stable p95 so + // it doesn't back off QPS on a noisy tail (the t19 anti-pattern). + int stats_warmup_secs = 0; + { + const char* e = std::getenv("FEEDSIM_STATS_WARMUP_SECS"); + if (e != nullptr && e[0] != '\0') { + stats_warmup_secs = std::atoi(e); + if (stats_warmup_secs < 0) { + stats_warmup_secs = 0; + } + } + } + // Barrier for thread init synchronization pthread_barrier_t init_barrier; pthread_barrier_init(&init_barrier, nullptr, num_threads + 1); @@ -909,7 +949,7 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start thread dt->thread = std::thread([this, &dt_ref = *dt, &init_barrier, - thread_pinning, i]() { + thread_pinning, i, stats_warmup_secs]() { // CPU affinity if (thread_pinning) { cpu_set_t mask; @@ -941,6 +981,18 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start making requests TestDriver::Impl::makeRequests(*dt_ref.driver); + // Schedule the cold-start stats reset on this thread's own base (added + // here, before dispatch, so it fires from this thread — no data race with + // the stats writes). Fires once ~stats_warmup_secs into steady traffic. + if (stats_warmup_secs > 0) { + struct timeval warmup_tv { + stats_warmup_secs, 0 + }; + event_base_once( + dt_ref.base, -1, EV_TIMEOUT, resetDriverStatsCb, + &dt_ref.driver->impl_->current_stats, &warmup_tv); + } + // Run event loop event_base_dispatch(dt_ref.base); }); @@ -1058,7 +1110,14 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, } double end_time = getTimeSec(); - double elapsed = end_time - start_time; + // With a stats warmup, samples were reset ~stats_warmup_secs into the run, so + // the throughput denominator must be the post-warmup window (else QPS would + // be understated by counting post-warmup queries over the full duration). + double meas_start = start_time; + if (stats_warmup_secs > 0) { + meas_start = start_time + stats_warmup_secs; + } + double elapsed = end_time - meas_start; // Aggregate stats from all threads for (auto& dt : impl_->threads) { diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc index ff58f81a..fb92e243 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -35,17 +36,30 @@ #include #include +#include + namespace feedsim { +class ServerConnection; + // ─── RequestContext implementation ────────────────────────────────────────── struct RequestContext::Impl { - // The socket fd to write the response back on. - // We use raw fd + write() because the response is a single small write - // and we want to avoid the complexity of AsyncSocket write callbacks. + // Plaintext path: the socket fd to write the response back on. We use raw + // fd + writev() because the response is a single small write and we want to + // avoid the complexity of AsyncSocket write callbacks. int fd; uint64_t received_time; bool response_sent; + // TLS path: raw fd writes bypass the TLS layer (they hit the TCP socket + // underneath AsyncSSLSocket, so the peer receives plaintext on an encrypted + // connection). When tls is true, the response must be written THROUGH the + // AsyncSSLSocket on its owning EventBase instead. sendResponse marshals the + // write onto evb and targets the connection via a weak_ptr so a response + // that completes after the connection closed is dropped safely. + bool tls = false; + folly::EventBase* evb = nullptr; + std::weak_ptr conn; }; RequestContext::RequestContext( @@ -69,6 +83,17 @@ RequestContext::RequestContext(RequestContext&& other) noexcept RequestContext::~RequestContext() = default; +// Marshals a TLS response (header+payload already framed in buf) onto the +// connection's EventBase and writes it through the AsyncSSLSocket. Defined +// after ServerConnection (needs its full type); declared here so sendResponse +// can call it. Safe if the connection has already closed. +namespace { +void enqueueTlsResponse( + std::weak_ptr conn, + folly::EventBase* evb, + std::unique_ptr buf); +} // namespace + void RequestContext::sendResponse(const void* data, uint32_t data_length) { if (!impl_ || impl_->response_sent) return; impl_->response_sent = true; @@ -85,7 +110,21 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { ResponsePacketHeader net = responseToNetwork(hdr); - // Use writev to send header + payload atomically + if (impl_->tls) { + // Raw fd writes bypass TLS, so build one contiguous frame (copying the + // payload synchronously — the caller may free `data` after we return) and + // hand it to the connection's EventBase to write through AsyncSSLSocket. + auto buf = folly::IOBuf::create(sizeof(net) + data_length); + memcpy(buf->writableData(), &net, sizeof(net)); + if (data_length > 0) { + memcpy(buf->writableData() + sizeof(net), data, data_length); + } + buf->append(sizeof(net) + data_length); + enqueueTlsResponse(impl_->conn, impl_->evb, std::move(buf)); + return; + } + + // Plaintext path (unchanged): use writev to send header + payload atomically struct iovec iov[2]; iov[0].iov_base = &net; iov[0].iov_len = sizeof(net); @@ -122,15 +161,19 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { // ─── ServerConnection: handles framing for one client connection ──────────── -class ServerConnection : public folly::AsyncTransport::ReadCallback { +class ServerConnection + : public folly::AsyncTransport::ReadCallback, + public std::enable_shared_from_this { public: ServerConnection( folly::AsyncSocket::UniquePtr socket, int thread_id, - const folly::F14FastMap& callbacks) + const folly::F14FastMap& callbacks, + bool tls) : socket_(std::move(socket)), thread_id_(thread_id), callbacks_(callbacks), + tls_(tls), read_buf_(nullptr), read_buf_size_(0), data_offset_(0) { @@ -144,6 +187,17 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { delete[] read_buf_; } + // Called once right after construction so the connection keeps itself alive + // while registered as the socket's read callback (broken on EOF/error). + void attachSelf(std::shared_ptr self) { + self_ = std::move(self); + } + + // Writes a fully-framed TLS response through the AsyncSSLSocket. MUST be + // invoked on the socket's EventBase thread. Defined out-of-line below + // (needs TlsWriteCallback's full definition). + void writeResponse(std::unique_ptr buf); + // AsyncTransport::ReadCallback void getReadBuffer(void** bufReturn, size_t* lenReturn) override { // Grow buffer if needed @@ -165,15 +219,14 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } void readEOF() noexcept override { - // Client disconnected - socket_->close(); - // Self-delete via destroy callback (see below) - delete this; + // Client disconnected. Break the self-reference so the object is destroyed + // once any in-flight TLS write callbacks release their refs. Hold a local + // ref so `this` stays valid until we return from the callback. + close(); } void readErr(const folly::AsyncSocketException& ex) noexcept override { - socket_->close(); - delete this; + close(); } private: @@ -199,6 +252,13 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { impl->fd = socket_->getNetworkSocket().toFd(); impl->received_time = getTimeNano(); impl->response_sent = false; + impl->tls = tls_; + if (tls_) { + // The response may be produced asynchronously on a pool thread; it + // must be written back through the AsyncSSLSocket on this EventBase. + impl->evb = socket_->getEventBase(); + impl->conn = weak_from_this(); + } RequestContext ctx( hdr.type, hdr.request_id, hdr.start_time, @@ -216,14 +276,112 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } } + void close() { + socket_->close(); + // Hold a local ref so `this` survives until we return, then drop the + // self-reference. If TLS write callbacks are still outstanding they hold + // their own refs and the object lives until they complete. + auto keepalive = shared_from_this(); + self_.reset(); + } + folly::AsyncSocket::UniquePtr socket_; int thread_id_; const folly::F14FastMap& callbacks_; + bool tls_; + std::shared_ptr self_; uint8_t* read_buf_; size_t read_buf_size_; size_t data_offset_; }; +// ─── TLS response write path ──────────────────────────────────────────────── + +namespace { +// Keeps the connection alive (via shared_ptr) until the AsyncSSLSocket finishes +// encrypting and writing the response, then deletes itself. +class TlsWriteCallback : public folly::AsyncWriter::WriteCallback { + public: + explicit TlsWriteCallback(std::shared_ptr conn) + : conn_(std::move(conn)) {} + void writeSuccess() noexcept override { delete this; } + void writeErr( + size_t /*bytesWritten*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; + } + + private: + std::shared_ptr conn_; +}; + +void enqueueTlsResponse( + std::weak_ptr weak, + folly::EventBase* evb, + std::unique_ptr buf) { + if (evb == nullptr) { + return; + } + evb->runInEventBaseThread( + [weak = std::move(weak), buf = std::move(buf)]() mutable { + auto conn = weak.lock(); + if (!conn) { + return; // connection closed before the response was ready + } + conn->writeResponse(std::move(buf)); + }); +} +} // namespace + +void ServerConnection::writeResponse(std::unique_ptr buf) { + // Runs on the socket's EventBase thread. The callback holds a ref that keeps + // this connection alive until the encrypted write completes. + auto* cb = new TlsWriteCallback(shared_from_this()); + socket_->writeChain(cb, std::move(buf)); +} + +// ─── SslAcceptor: drives the server-side TLS handshake before wire reads ───── + +// A folly server-side AsyncSSLSocket does NOT auto-handshake when you merely +// setReadCB — sslAccept() must be called to run the handshake. Until it +// completes the socket can neither decrypt requests nor send responses. This +// helper owns the socket during the handshake and, on success, hands it to a +// ServerConnection (which then starts reading). It self-deletes either way. +class SslAcceptor : public folly::AsyncSSLSocket::HandshakeCB { + public: + SslAcceptor( + folly::AsyncSSLSocket::UniquePtr socket, + int thread_id, + const folly::F14FastMap& callbacks) + : socket_(std::move(socket)), + thread_id_(thread_id), + callbacks_(callbacks) {} + + void start() { + auto* raw = socket_.get(); + raw->sslAccept(this); + } + + void handshakeSuc(folly::AsyncSSLSocket* /*sock*/) noexcept override { + folly::AsyncSocket::UniquePtr base(socket_.release()); + auto conn = std::make_shared( + std::move(base), thread_id_, callbacks_, /*tls=*/true); + conn->attachSelf(conn); + delete this; + } + + void handshakeErr( + folly::AsyncSSLSocket* /*sock*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; // socket_ (and the fd) torn down with it + } + + private: + folly::AsyncSSLSocket::UniquePtr socket_; + int thread_id_; + const folly::F14FastMap& callbacks_; +}; + // ─── WorkerThread ─────────────────────────────────────────────────────────── struct WorkerThread { @@ -275,20 +433,25 @@ class AcceptCallback : public folly::AsyncServerSocket::AcceptCallback { worker->evb->runInEventBaseThread( [fd, thread_id = worker->thread_id, &callbacks = callbacks_, evb = worker->evb.get(), ssl_ctx]() { - folly::AsyncSocket::UniquePtr socket; if (ssl_ctx) { - // AsyncSSLSocket server-side: pass true for the server flag. - // The handshake is initiated lazily on first read/write, - // matching the existing client's connect-then-write pattern. + // AsyncSSLSocket server-side: pass true for the server flag, then + // drive the handshake via sslAccept (SslAcceptor). The + // ServerConnection is created only after the handshake succeeds — + // reading/writing before that would see undecrypted bytes. folly::AsyncSSLSocket::UniquePtr ssl_sock(new folly::AsyncSSLSocket( ssl_ctx, evb, folly::NetworkSocket::fromFd(fd), true)); - socket.reset(ssl_sock.release()); + auto* acceptor = + new SslAcceptor(std::move(ssl_sock), thread_id, callbacks); + acceptor->start(); } else { - socket = folly::AsyncSocket::newSocket( + folly::AsyncSocket::UniquePtr socket = folly::AsyncSocket::newSocket( evb, folly::NetworkSocket::fromFd(fd)); + // ServerConnection keeps itself alive via a self-reference (set by + // attachSelf) until EOF/error. + auto conn = std::make_shared( + std::move(socket), thread_id, callbacks, /*tls=*/false); + conn->attachSelf(conn); } - // ServerConnection self-manages its lifetime - new ServerConnection(std::move(socket), thread_id, callbacks); }); } @@ -433,6 +596,17 @@ void FeedSimServer::run() { auto ctx = std::make_shared(); ctx->loadCertificate(cert_env); ctx->loadPrivateKey(key_env); + // Restrict to AES-GCM so the negotiated cipher uses hardware AES + // (AES-NI / ARMv8 crypto extensions) via libcrypto, matching prod + // and the driver's pinned ciphers. Without this the server may accept + // ChaCha20-Poly1305, which has no AES instructions and reads as ~0% + // crypto in the instruction mix. + ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 // No ALPN — FeedSim uses its own custom binary protocol over the // TLS-wrapped socket, not Rocket. The client similarly does not // advertise ALPN. diff --git a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc index bfd10031..349e2fbe 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc @@ -28,6 +28,8 @@ #include #include +#include + #include #include @@ -191,6 +193,17 @@ MockServicesClient::MockServicesClient( // pinned fbthrift v2026.01.05.00). Without ALPN, the server may // reject the connection or fall back to the header-upgrade path. ssl_ctx->setAdvertisedNextProtocols({"rs"}); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES (ARMv8 crypto extensions via libcrypto), matching prod's + // cipher. Without this, OpenSSL may pick ChaCha20-Poly1305 (a NEON/integer + // cipher with no AES instructions), which reads as ~0% in the crypto + // instruction mix vs prod's ~0.88% and runs the AEAD un-accelerated. + SSL_CTX_set_ciphersuites( + ssl_ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 + ssl_ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 folly::AsyncSSLSocket::UniquePtr ssl_sock( new folly::AsyncSSLSocket(ssl_ctx, evb_)); // AsyncSSLSocket buffers writes until the TLS handshake completes,