From 62a4016847d94f632624f5104f68a2ae30d744ae Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 1/8] Make FeedSim dep versions env-overridable + add prebuilt libtorch-from-wheel path Reviewed By: YifanYuan3 Differential Revision: D112069862 --- packages/feedsim/install_feedsim.sh | 72 ++++++++++++++----- packages/feedsim/install_feedsim_aarch64.sh | 27 +++++-- .../feedsim/install_feedsim_aarch64_ubuntu.sh | 14 ++-- packages/feedsim/install_feedsim_ubuntu.sh | 42 ++++++++--- 4 files changed, 117 insertions(+), 38 deletions(-) diff --git a/packages/feedsim/install_feedsim.sh b/packages/feedsim/install_feedsim.sh index e9bb855b..03c22fe4 100755 --- a/packages/feedsim/install_feedsim.sh +++ b/packages/feedsim/install_feedsim.sh @@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" +# Dependency versions are env-overridable so experiments can bump them without +# forking this script; defaults reproduce the v2 baseline exactly. +JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" +LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" +# Export so the aarch64 sub-installer (dispatched below) inherits the pins. +export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \ libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \ libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \ zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \ - xxhash xxhash-devel unzip rsync liburing-devel + xxhash xxhash-devel unzip rsync liburing-devel python3-pip # Creates feedsim directory under benchmarks/ mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim" @@ -178,30 +191,30 @@ else fi # Installing JEMalloc -if ! [ -d "jemalloc-5.3.0" ]; then - wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" - bunzip2 "jemalloc-5.3.0.tar.bz2" - tar -xvf "jemalloc-5.3.0.tar" - cd "jemalloc-5.3.0" +if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then + wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2" + tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar" + cd "jemalloc-${JEMALLOC_VERSION}" ./configure --enable-prof --enable-prof-libunwind make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] jemalloc-5.3.0" + msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}" fi # Installing libevent -if ! [ -d "libevent-2.1.12-stable" ]; then - wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz" - tar -xzf "libevent-2.1.12-stable.tar.gz" - cd "libevent-2.1.12-stable" +if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then + wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz" + tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz" + cd "libevent-${LIBEVENT_VERSION}" ./configure make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] libevent-2.1.12-stable" + msg "[SKIPPED] libevent-${LIBEVENT_VERSION}" fi msg "Installing third-party dependencies ... DONE" @@ -218,12 +231,33 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # pip on the box (3.9, or an internal stale mirror) can't see the cp310 + # 2.13 wheels, so resolve the wheel href straight from the PEP-503 index + # and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is + # Python-version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/install_feedsim_aarch64.sh b/packages/feedsim/install_feedsim_aarch64.sh index 59a238d1..acd0ce62 100755 --- a/packages/feedsim/install_feedsim_aarch64.sh +++ b/packages/feedsim/install_feedsim_aarch64.sh @@ -182,11 +182,15 @@ else msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}" fi -DEP_JEMALLOC_VERSION="5.3.0" +DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" # Installing JEMalloc if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" - verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then + verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + else + msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)" + fi bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar" cd "jemalloc-${DEP_JEMALLOC_VERSION}" @@ -198,11 +202,15 @@ else msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}" fi -DEP_LIBEVENT_VERSION="2.1.12-stable" +DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" # Installing libevent if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" - verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then + verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + else + msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)" + fi tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" cd "libevent-${DEP_LIBEVENT_VERSION}" ./configure @@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # unset reproduces the v2 baseline (latest). + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh index e6abb5ad..54565bfb 100755 --- a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh +++ b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh @@ -27,6 +27,7 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -34,7 +35,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync + sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing is # older than folly's minimum, so folly's io_uring integration links @@ -204,9 +205,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # empty falls back to pip's latest resolution. + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_ubuntu.sh b/packages/feedsim/install_feedsim_ubuntu.sh index f525f63e..4237ea6b 100755 --- a/packages/feedsim/install_feedsim_ubuntu.sh +++ b/packages/feedsim/install_feedsim_ubuntu.sh @@ -11,7 +11,14 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -30,7 +37,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq xxhash libxxhash-dev unzip rsync + sysstat jq xxhash libxxhash-dev unzip rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing (0.7 on # 20.04, 2.1 on 22.04) is older than folly's minimum, so folly's io_uring @@ -213,12 +220,31 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # The C++ libtorch inside (torch/lib, torch/share/cmake) is Python- + # version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi From 0867a702c32583f29c0ea8c6bc792a4cc37c0496 Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 2/8] Force AES-GCM on mock_services TLS client for hardware crypto Reviewed By: YifanYuan3 Differential Revision: D112460168 --- .../src/workloads/ranking/MockServicesClient.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc index bfd10031..349e2fbe 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc @@ -28,6 +28,8 @@ #include #include +#include + #include #include @@ -191,6 +193,17 @@ MockServicesClient::MockServicesClient( // pinned fbthrift v2026.01.05.00). Without ALPN, the server may // reject the connection or fall back to the header-upgrade path. ssl_ctx->setAdvertisedNextProtocols({"rs"}); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES (ARMv8 crypto extensions via libcrypto), matching prod's + // cipher. Without this, OpenSSL may pick ChaCha20-Poly1305 (a NEON/integer + // cipher with no AES instructions), which reads as ~0% in the crypto + // instruction mix vs prod's ~0.88% and runs the AEAD un-accelerated. + SSL_CTX_set_ciphersuites( + ssl_ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 + ssl_ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 folly::AsyncSSLSocket::UniquePtr ssl_sock( new folly::AsyncSSLSocket(ssl_ctx, evb_)); // AsyncSSLSocket buffers writes until the TLS handshake completes, From 171de735e8bd17be83d18c70156deca19bd6e13b Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 3/8] =?UTF-8?q?Add=20working=20driver=E2=86=94leaf=20TLS?= =?UTF-8?q?=20(hardware=20AES-GCM)=20to=20match=20prod=20encryption?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed By: YifanYuan3 Differential Revision: D112488664 --- packages/feedsim/run.sh | 30 ++- .../src/workloads/ranking/FeedSimDriver.cc | 12 + .../src/workloads/ranking/FeedSimServer.cc | 214 ++++++++++++++++-- 3 files changed, 233 insertions(+), 23 deletions(-) diff --git a/packages/feedsim/run.sh b/packages/feedsim/run.sh index 6ca879e0..b006ff8d 100644 --- a/packages/feedsim/run.sh +++ b/packages/feedsim/run.sh @@ -823,6 +823,30 @@ main() { if [ "$mock_tls" = "1" ]; then export MOCK_TLS=1 fi + # Driver↔Leaf TLS. When FEEDSIM_DRIVER_TLS=1 is passed in the environment, + # the DriverNodeRank client (reads FEEDSIM_DRIVER_TLS) wraps its + # bufferevents in OpenSSL, and the LeafNodeRank server needs a cert so + # FeedSimServer's accept path enables AsyncSSLSocket — wire the shared + # example cert/key into FEEDSIM_TLS_CERT/FEEDSIM_TLS_KEY. Both ends pin + # AES-GCM so the encrypted driver↔leaf volume registers as hardware crypto + # (matches prod's Rocket-over-TLS driver path). Default off. The LeafNodeRank + # server reads FEEDSIM_TLS_CERT/KEY from the exported env; the DriverNodeRank + # client is launched through search_qps.sh, which does not reliably forward + # the parent's exported env to the driver process, so FEEDSIM_DRIVER_TLS is + # also injected directly on the driver command line via `driver_bin`. + driver_bin="build/workloads/ranking/DriverNodeRank" + if [ "${FEEDSIM_DRIVER_TLS:-0}" = "1" ]; then + driver_cert_dir="${FEEDSIM_ROOT}/certs" + if [ ! -r "${driver_cert_dir}/example.crt" ] || [ ! -r "${driver_cert_dir}/example.key" ]; then + echo "ERROR: FEEDSIM_DRIVER_TLS=1 but ${driver_cert_dir}/example.{crt,key} not found" >&2 + exit 1 + fi + export FEEDSIM_TLS_CERT="${driver_cert_dir}/example.crt" + export FEEDSIM_TLS_KEY="${driver_cert_dir}/example.key" + export FEEDSIM_DRIVER_TLS=1 + driver_bin="env FEEDSIM_DRIVER_TLS=1 build/workloads/ranking/DriverNodeRank" + echo "Driver↔Leaf TLS: ENABLED (cert=${driver_cert_dir}/example.crt, AES-GCM)" + fi # MOCK_ZSTD_FRAC env consumed by MockServicesClient::resolveZstdFraction. # Always export so the t43 c7 default 0.75 reaches the binary. export MOCK_ZSTD_FRAC="$mock_zstd_frac" @@ -960,7 +984,7 @@ main() { echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 scripts/search_qps.sh -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${driver_threads}" \ @@ -974,7 +998,7 @@ main() { echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 scripts/search_qps.sh -a -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + $driver_bin \ --monitor_port "$client_monitor_port" \ --server "0.0.0.0:$port" \ $client_feature_opts \ @@ -1000,7 +1024,7 @@ main() { -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" \ $qps_threshold_args $no_retry_args \ -o "${FEEDSIM_ROOT}/${result_filename}" \ - -- build/workloads/ranking/DriverNodeRank \ + -- $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${num_workers}" \ diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc index 2d1119a4..aa4467d5 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc @@ -246,6 +246,18 @@ SSL_CTX* getDriverSslCtxOrNull() { return nullptr; } SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, nullptr); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES (AES-NI on x86, ARMv8 crypto extensions on aarch64) via + // libcrypto, matching prod's cipher. Without this, OpenSSL may pick + // ChaCha20-Poly1305 (a NEON/integer cipher with no AES instructions), + // which reads as ~0% in the crypto instruction mix and runs the AEAD + // un-accelerated. Mirrors MockServicesClient's cipher pinning. + SSL_CTX_set_cipher_list( + ctx, + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx, "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 std::cout << "FeedSimDriver: TLS enabled via FEEDSIM_DRIVER_TLS=1" << std::endl; return ctx; diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc index ff58f81a..fb92e243 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -35,17 +36,30 @@ #include #include +#include + namespace feedsim { +class ServerConnection; + // ─── RequestContext implementation ────────────────────────────────────────── struct RequestContext::Impl { - // The socket fd to write the response back on. - // We use raw fd + write() because the response is a single small write - // and we want to avoid the complexity of AsyncSocket write callbacks. + // Plaintext path: the socket fd to write the response back on. We use raw + // fd + writev() because the response is a single small write and we want to + // avoid the complexity of AsyncSocket write callbacks. int fd; uint64_t received_time; bool response_sent; + // TLS path: raw fd writes bypass the TLS layer (they hit the TCP socket + // underneath AsyncSSLSocket, so the peer receives plaintext on an encrypted + // connection). When tls is true, the response must be written THROUGH the + // AsyncSSLSocket on its owning EventBase instead. sendResponse marshals the + // write onto evb and targets the connection via a weak_ptr so a response + // that completes after the connection closed is dropped safely. + bool tls = false; + folly::EventBase* evb = nullptr; + std::weak_ptr conn; }; RequestContext::RequestContext( @@ -69,6 +83,17 @@ RequestContext::RequestContext(RequestContext&& other) noexcept RequestContext::~RequestContext() = default; +// Marshals a TLS response (header+payload already framed in buf) onto the +// connection's EventBase and writes it through the AsyncSSLSocket. Defined +// after ServerConnection (needs its full type); declared here so sendResponse +// can call it. Safe if the connection has already closed. +namespace { +void enqueueTlsResponse( + std::weak_ptr conn, + folly::EventBase* evb, + std::unique_ptr buf); +} // namespace + void RequestContext::sendResponse(const void* data, uint32_t data_length) { if (!impl_ || impl_->response_sent) return; impl_->response_sent = true; @@ -85,7 +110,21 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { ResponsePacketHeader net = responseToNetwork(hdr); - // Use writev to send header + payload atomically + if (impl_->tls) { + // Raw fd writes bypass TLS, so build one contiguous frame (copying the + // payload synchronously — the caller may free `data` after we return) and + // hand it to the connection's EventBase to write through AsyncSSLSocket. + auto buf = folly::IOBuf::create(sizeof(net) + data_length); + memcpy(buf->writableData(), &net, sizeof(net)); + if (data_length > 0) { + memcpy(buf->writableData() + sizeof(net), data, data_length); + } + buf->append(sizeof(net) + data_length); + enqueueTlsResponse(impl_->conn, impl_->evb, std::move(buf)); + return; + } + + // Plaintext path (unchanged): use writev to send header + payload atomically struct iovec iov[2]; iov[0].iov_base = &net; iov[0].iov_len = sizeof(net); @@ -122,15 +161,19 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { // ─── ServerConnection: handles framing for one client connection ──────────── -class ServerConnection : public folly::AsyncTransport::ReadCallback { +class ServerConnection + : public folly::AsyncTransport::ReadCallback, + public std::enable_shared_from_this { public: ServerConnection( folly::AsyncSocket::UniquePtr socket, int thread_id, - const folly::F14FastMap& callbacks) + const folly::F14FastMap& callbacks, + bool tls) : socket_(std::move(socket)), thread_id_(thread_id), callbacks_(callbacks), + tls_(tls), read_buf_(nullptr), read_buf_size_(0), data_offset_(0) { @@ -144,6 +187,17 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { delete[] read_buf_; } + // Called once right after construction so the connection keeps itself alive + // while registered as the socket's read callback (broken on EOF/error). + void attachSelf(std::shared_ptr self) { + self_ = std::move(self); + } + + // Writes a fully-framed TLS response through the AsyncSSLSocket. MUST be + // invoked on the socket's EventBase thread. Defined out-of-line below + // (needs TlsWriteCallback's full definition). + void writeResponse(std::unique_ptr buf); + // AsyncTransport::ReadCallback void getReadBuffer(void** bufReturn, size_t* lenReturn) override { // Grow buffer if needed @@ -165,15 +219,14 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } void readEOF() noexcept override { - // Client disconnected - socket_->close(); - // Self-delete via destroy callback (see below) - delete this; + // Client disconnected. Break the self-reference so the object is destroyed + // once any in-flight TLS write callbacks release their refs. Hold a local + // ref so `this` stays valid until we return from the callback. + close(); } void readErr(const folly::AsyncSocketException& ex) noexcept override { - socket_->close(); - delete this; + close(); } private: @@ -199,6 +252,13 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { impl->fd = socket_->getNetworkSocket().toFd(); impl->received_time = getTimeNano(); impl->response_sent = false; + impl->tls = tls_; + if (tls_) { + // The response may be produced asynchronously on a pool thread; it + // must be written back through the AsyncSSLSocket on this EventBase. + impl->evb = socket_->getEventBase(); + impl->conn = weak_from_this(); + } RequestContext ctx( hdr.type, hdr.request_id, hdr.start_time, @@ -216,14 +276,112 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } } + void close() { + socket_->close(); + // Hold a local ref so `this` survives until we return, then drop the + // self-reference. If TLS write callbacks are still outstanding they hold + // their own refs and the object lives until they complete. + auto keepalive = shared_from_this(); + self_.reset(); + } + folly::AsyncSocket::UniquePtr socket_; int thread_id_; const folly::F14FastMap& callbacks_; + bool tls_; + std::shared_ptr self_; uint8_t* read_buf_; size_t read_buf_size_; size_t data_offset_; }; +// ─── TLS response write path ──────────────────────────────────────────────── + +namespace { +// Keeps the connection alive (via shared_ptr) until the AsyncSSLSocket finishes +// encrypting and writing the response, then deletes itself. +class TlsWriteCallback : public folly::AsyncWriter::WriteCallback { + public: + explicit TlsWriteCallback(std::shared_ptr conn) + : conn_(std::move(conn)) {} + void writeSuccess() noexcept override { delete this; } + void writeErr( + size_t /*bytesWritten*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; + } + + private: + std::shared_ptr conn_; +}; + +void enqueueTlsResponse( + std::weak_ptr weak, + folly::EventBase* evb, + std::unique_ptr buf) { + if (evb == nullptr) { + return; + } + evb->runInEventBaseThread( + [weak = std::move(weak), buf = std::move(buf)]() mutable { + auto conn = weak.lock(); + if (!conn) { + return; // connection closed before the response was ready + } + conn->writeResponse(std::move(buf)); + }); +} +} // namespace + +void ServerConnection::writeResponse(std::unique_ptr buf) { + // Runs on the socket's EventBase thread. The callback holds a ref that keeps + // this connection alive until the encrypted write completes. + auto* cb = new TlsWriteCallback(shared_from_this()); + socket_->writeChain(cb, std::move(buf)); +} + +// ─── SslAcceptor: drives the server-side TLS handshake before wire reads ───── + +// A folly server-side AsyncSSLSocket does NOT auto-handshake when you merely +// setReadCB — sslAccept() must be called to run the handshake. Until it +// completes the socket can neither decrypt requests nor send responses. This +// helper owns the socket during the handshake and, on success, hands it to a +// ServerConnection (which then starts reading). It self-deletes either way. +class SslAcceptor : public folly::AsyncSSLSocket::HandshakeCB { + public: + SslAcceptor( + folly::AsyncSSLSocket::UniquePtr socket, + int thread_id, + const folly::F14FastMap& callbacks) + : socket_(std::move(socket)), + thread_id_(thread_id), + callbacks_(callbacks) {} + + void start() { + auto* raw = socket_.get(); + raw->sslAccept(this); + } + + void handshakeSuc(folly::AsyncSSLSocket* /*sock*/) noexcept override { + folly::AsyncSocket::UniquePtr base(socket_.release()); + auto conn = std::make_shared( + std::move(base), thread_id_, callbacks_, /*tls=*/true); + conn->attachSelf(conn); + delete this; + } + + void handshakeErr( + folly::AsyncSSLSocket* /*sock*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; // socket_ (and the fd) torn down with it + } + + private: + folly::AsyncSSLSocket::UniquePtr socket_; + int thread_id_; + const folly::F14FastMap& callbacks_; +}; + // ─── WorkerThread ─────────────────────────────────────────────────────────── struct WorkerThread { @@ -275,20 +433,25 @@ class AcceptCallback : public folly::AsyncServerSocket::AcceptCallback { worker->evb->runInEventBaseThread( [fd, thread_id = worker->thread_id, &callbacks = callbacks_, evb = worker->evb.get(), ssl_ctx]() { - folly::AsyncSocket::UniquePtr socket; if (ssl_ctx) { - // AsyncSSLSocket server-side: pass true for the server flag. - // The handshake is initiated lazily on first read/write, - // matching the existing client's connect-then-write pattern. + // AsyncSSLSocket server-side: pass true for the server flag, then + // drive the handshake via sslAccept (SslAcceptor). The + // ServerConnection is created only after the handshake succeeds — + // reading/writing before that would see undecrypted bytes. folly::AsyncSSLSocket::UniquePtr ssl_sock(new folly::AsyncSSLSocket( ssl_ctx, evb, folly::NetworkSocket::fromFd(fd), true)); - socket.reset(ssl_sock.release()); + auto* acceptor = + new SslAcceptor(std::move(ssl_sock), thread_id, callbacks); + acceptor->start(); } else { - socket = folly::AsyncSocket::newSocket( + folly::AsyncSocket::UniquePtr socket = folly::AsyncSocket::newSocket( evb, folly::NetworkSocket::fromFd(fd)); + // ServerConnection keeps itself alive via a self-reference (set by + // attachSelf) until EOF/error. + auto conn = std::make_shared( + std::move(socket), thread_id, callbacks, /*tls=*/false); + conn->attachSelf(conn); } - // ServerConnection self-manages its lifetime - new ServerConnection(std::move(socket), thread_id, callbacks); }); } @@ -433,6 +596,17 @@ void FeedSimServer::run() { auto ctx = std::make_shared(); ctx->loadCertificate(cert_env); ctx->loadPrivateKey(key_env); + // Restrict to AES-GCM so the negotiated cipher uses hardware AES + // (AES-NI / ARMv8 crypto extensions) via libcrypto, matching prod + // and the driver's pinned ciphers. Without this the server may accept + // ChaCha20-Poly1305, which has no AES instructions and reads as ~0% + // crypto in the instruction mix. + ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 // No ALPN — FeedSim uses its own custom binary protocol over the // TLS-wrapped socket, not Rocket. The client similarly does not // advertise ALPN. From 3bc94268ccc39f1a35cc5c3084906bc511d97b81 Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 4/8] Adaptive driver depth in search_qps (fleet default) to fix the throughput anti-pattern Reviewed By: YifanYuan3 Differential Revision: D112694748 --- packages/feedsim/run.sh | 41 +++++++++- .../third_party/src/scripts/search_qps.sh | 77 ++++++++++++++++++- .../src/workloads/ranking/FeedSimDriver.cc | 51 +++++++++++- 3 files changed, 160 insertions(+), 9 deletions(-) diff --git a/packages/feedsim/run.sh b/packages/feedsim/run.sh index b006ff8d..06bcb161 100644 --- a/packages/feedsim/run.sh +++ b/packages/feedsim/run.sh @@ -834,7 +834,11 @@ main() { # client is launched through search_qps.sh, which does not reliably forward # the parent's exported env to the driver process, so FEEDSIM_DRIVER_TLS is # also injected directly on the driver command line via `driver_bin`. - driver_bin="build/workloads/ranking/DriverNodeRank" + # Per-run driver knobs are injected on the DriverNodeRank command line via + # `env VAR=val` (accumulated in driver_env), because search_qps.sh launches + # the driver as a bare `$command &` that does not reliably inherit the + # parent shell's exported env. + driver_env="" if [ "${FEEDSIM_DRIVER_TLS:-0}" = "1" ]; then driver_cert_dir="${FEEDSIM_ROOT}/certs" if [ ! -r "${driver_cert_dir}/example.crt" ] || [ ! -r "${driver_cert_dir}/example.key" ]; then @@ -844,9 +848,22 @@ main() { export FEEDSIM_TLS_CERT="${driver_cert_dir}/example.crt" export FEEDSIM_TLS_KEY="${driver_cert_dir}/example.key" export FEEDSIM_DRIVER_TLS=1 - driver_bin="env FEEDSIM_DRIVER_TLS=1 build/workloads/ranking/DriverNodeRank" + driver_env="${driver_env} FEEDSIM_DRIVER_TLS=1" echo "Driver↔Leaf TLS: ENABLED (cert=${driver_cert_dir}/example.crt, AES-GCM)" fi + # FEEDSIM_STATS_WARMUP_SECS: drop the first N seconds of latency/throughput + # samples in each search_qps probe (DriverNodeRank resets its stats N secs + # in) so cold-start transients don't inflate the tail and make the search + # back off QPS prematurely. Default unset/0 (no warmup). + if [ -n "${FEEDSIM_STATS_WARMUP_SECS:-}" ] && [ "${FEEDSIM_STATS_WARMUP_SECS}" != "0" ]; then + driver_env="${driver_env} FEEDSIM_STATS_WARMUP_SECS=${FEEDSIM_STATS_WARMUP_SECS}" + echo "Driver stats warmup: ${FEEDSIM_STATS_WARMUP_SECS}s (dropping cold-start samples per probe)" + fi + if [ -n "$driver_env" ]; then + driver_bin="env${driver_env} build/workloads/ranking/DriverNodeRank" + else + driver_bin="build/workloads/ranking/DriverNodeRank" + fi # MOCK_ZSTD_FRAC env consumed by MockServicesClient::resolveZstdFraction. # Always export so the t43 c7 default 0.75 reaches the binary. export MOCK_ZSTD_FRAC="$mock_zstd_frac" @@ -979,16 +996,30 @@ main() { # Override via --sla-p95-ms CLI flag (handled in arg parsing above). sla_arg="95p:${sla_p95_ms}" + # Adaptive driver depth (fleet default): search_qps raises the driver's + # pipeline --depth in the peak phase until the server saturates (system + # CPU>=95% or p95>=SLA), giving each platform just enough offered concurrency + # to reach a real bound instead of capping on driver concurrency (the t19 + # anti-pattern where big boxes sat at ~80% CPU with p95 far below SLA). + # Enabled by default up to depth 8; set FEEDSIM_ADAPTIVE_DEPTH_MAX=0 to + # disable and fall back to the fixed FEEDSIM_DRIVER_DEPTH (default 1). + sqps_adaptive_arg="" + adaptive_depth_max="${FEEDSIM_ADAPTIVE_DEPTH_MAX:-8}" + if [ "$adaptive_depth_max" != "0" ]; then + sqps_adaptive_arg="-D ${adaptive_depth_max}" + fi + if [ -z "$fixed_qps" ] && [ "$auto_driver_threads" != "1" ]; then benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + scripts/search_qps.sh -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${driver_threads}" \ --connections=4 \ + --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -997,10 +1028,11 @@ main() { benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -a -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + scripts/search_qps.sh -a -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ $driver_bin \ --monitor_port "$client_monitor_port" \ --server "0.0.0.0:$port" \ + --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -1029,6 +1061,7 @@ main() { --monitor_port "$client_monitor_port" \ --threads="${num_workers}" \ --connections="${num_connections}" \ + --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index b811794d..e51fbf6c 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -31,6 +31,26 @@ echo "${SCRIPT_NAME}: DCPERF_PERF_RECORD=${DCPERF_PERF_RECORD}" function benchreps_tell_state () { date +"%Y-%m-%d_%T ${1}" >> $BREPS_LFILE } + +# ─── CPU utilization helpers (used by adaptive depth) ──────────────────────── +# Read /proc/stat's aggregate cpu line and echo "total idle_all" jiffies. +cpu_snapshot() { + local cpu u n s idle iow irq sirq st rest + read -r cpu u n s idle iow irq sirq st rest < /proc/stat + local idle_all=$((idle + iow)) + local total=$((u + n + s + idle + iow + irq + sirq + st)) + echo "$total $idle_all" +} +# System-wide CPU busy% (100 - idle%) measured over the next $1 seconds. +cpu_busy_over() { + local secs="$1" s1 s2 t1 i1 t2 i2 dt di + s1=$(cpu_snapshot); t1=${s1% *}; i1=${s1#* } + sleep "$secs" + s2=$(cpu_snapshot); t2=${s2% *}; i2=${s2#* } + dt=$((t2 - t1)); di=$((i2 - i1)) + if [ "$dt" -le 0 ]; then echo "0"; return; fi + echo "scale=1; (($dt - $di) * 100) / $dt" | bc +} # Source runtime breakdown utilities if they exist if [ -f "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" ]; then source "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" @@ -99,6 +119,11 @@ mutilate (EuroSys \'14) [https://github.com/leverich/mutilate] without retrying. Optional -P PID of the process to log runtime breakdowns. Optional -B Folder to log runtime breakdowns. Optional + -D Adaptive depth: max driver pipeline depth. When set, the peak + phase raises the driver's --depth from 1 until the server is + saturated (system CPU >= 95% OR achieved p95 >= SLA), then holds + that depth for the QPS search. Overrides any --depth in the + driver command. Optional. EOF } @@ -145,7 +170,7 @@ run_loadtest() { for r in $(seq 1 $load_test_retries); do # run the command, saving result to tmpfile local tmp_file=$(mktemp) - $command $threads_arg $qps_arg &>$tmp_file & + $command $threads_arg $qps_arg $adaptive_depth_arg &>$tmp_file & LOADTEST_PID=$! if [ "$no_retry_mode" = "1" ]; then @@ -301,14 +326,19 @@ max_warmup_iterations=10 no_retry_mode="" breakdown_pid="" breakdown_folder="" +adaptive_depth_max="" # -D: when set, search_qps raises driver --depth in the +adaptive_depth_arg="" # peak phase until the server saturates (CPU>=95% or p95>=SLA) OPTIND=1 # Reset is necessary if getopts was used previously in the script. It is a good idea to make this local in a function. -while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:" opt; do +while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:D:" opt; do case "$opt" in h) show_help exit 0 ;; + D) + adaptive_depth_max=$OPTARG + ;; t) experiment_time=$OPTARG ;; @@ -364,6 +394,14 @@ fi # remaining argument is loadtest command command=$@ +# In adaptive-depth mode, search_qps owns the driver's --depth: strip any fixed +# --depth from the command so our per-attempt --depth is the only one, and start +# the peak search at depth=1. +if [ -n "$adaptive_depth_max" ]; then + command=$(echo "$command" | sed -E 's/[[:space:]]*--depth=[0-9]+//g') + adaptive_depth_arg="--depth=1" +fi + # make sure latency_type and latency_target are specified if [[ -z "$fixed_qps" ]] && ( [[ $latency_type = "" ]] || [[ $latency_target = "" ]] ); then echo 'error: -s metric:target must be specified' >&2; exit 1 @@ -513,7 +551,40 @@ fi # find peak QPS benchreps_tell_state "before peak_qps" -run_loadtest peak_qps measured_latency "" "" +if [ -n "$adaptive_depth_max" ]; then + # Adaptive depth: the peak load test offers at most threads*connections*depth + # concurrent requests. Starting at depth=1, keep raising depth (and re-running + # peak) until the server saturates — system CPU >= 95% OR achieved p95 >= SLA + # — so platforms that need more offered concurrency reach a real bound instead + # of capping on driver concurrency (the t19 anti-pattern). The selected depth + # is then held for the QPS search / tuning / final phases. + cur_depth=1 + while : ; do + adaptive_depth_arg="--depth=$cur_depth" + # Sample system CPU busy% over a mid-run window while the peak load runs. + cpu_busy_file="/tmp/adaptive_cpu_busy_$$" + ( sleep 20; cpu_busy_over 40 > "$cpu_busy_file" ) & + cpu_sampler_pid=$! + run_loadtest peak_qps measured_latency "" "" + wait "$cpu_sampler_pid" 2>/dev/null + cpu_busy=$(cat "$cpu_busy_file" 2>/dev/null || echo 0) + rm -f "$cpu_busy_file" + cpu_sat=$(echo "${cpu_busy:-0} >= 95" | bc 2>/dev/null || echo 0) + lat_sat=$(echo "$measured_latency >= $latency_target" | bc 2>/dev/null || echo 0) + printf "adaptive-depth: depth=%d peak_qps=%.2f p95=%.2f cpu_busy=%s%% cpu_sat=%s lat_sat=%s\n" \ + "$cur_depth" "$peak_qps" "$measured_latency" "${cpu_busy:-0}" "$cpu_sat" "$lat_sat" + echo "adaptive-depth: depth=$cur_depth peak_qps=$peak_qps p95=$measured_latency cpu_busy=${cpu_busy}% cpu_sat=$cpu_sat lat_sat=$lat_sat" >> $BREPS_LFILE + if [ "$cpu_sat" -eq 1 ] || [ "$lat_sat" -eq 1 ] || [ "$cur_depth" -ge "$adaptive_depth_max" ]; then + break + fi + cur_depth=$((cur_depth + 1)) + sleep "$wait_time" + done + echo "adaptive-depth: SELECTED depth=$cur_depth (cpu_busy=${cpu_busy}%, p95=$measured_latency, sla=$latency_target)" >> $BREPS_LFILE + printf "adaptive-depth: selected depth=%d (cpu_busy=%s%%, p95=%.2f)\n" "$cur_depth" "${cpu_busy:-0}" "$measured_latency" +else + run_loadtest peak_qps measured_latency "" "" +fi printf "peak qps = %.2f, latency = %.2f\n" $peak_qps $measured_latency benchreps_tell_state "after peak_qps" diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc index aa4467d5..65d700c4 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc @@ -864,6 +864,17 @@ void FeedSimDriver::enableMonitoring(uint16_t port) { impl_->monitor_port = port; } +namespace { +// One-shot timer callback used by FEEDSIM_STATS_WARMUP_SECS: drops all +// latency/throughput samples collected so far so the final stats reflect only +// the post-warmup (steady-state) window. Runs on the DriverThread's own event +// base, so it never races that thread's logRequest()/logResponse() writes to +// the same DriverStats. +void resetDriverStatsCb(evutil_socket_t, short, void* arg) { + reinterpret_cast(arg)->reset(); +} +} // namespace + void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, uint32_t num_connections_per_thread, uint32_t max_connection_depth) { @@ -876,6 +887,23 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, impl_->running = true; impl_->total_stats = std::make_unique(1000); + // FEEDSIM_STATS_WARMUP_SECS: when >0, each DriverThread drops the samples it + // collected in the first N seconds of the measurement window (a per-thread + // one-shot timer calls DriverStats::reset()). This excludes cold-start + // transients (connection ramp, cache/JIT warmup, first-touch faults) from + // the reported latency distribution, giving search_qps a more stable p95 so + // it doesn't back off QPS on a noisy tail (the t19 anti-pattern). + int stats_warmup_secs = 0; + { + const char* e = std::getenv("FEEDSIM_STATS_WARMUP_SECS"); + if (e != nullptr && e[0] != '\0') { + stats_warmup_secs = std::atoi(e); + if (stats_warmup_secs < 0) { + stats_warmup_secs = 0; + } + } + } + // Barrier for thread init synchronization pthread_barrier_t init_barrier; pthread_barrier_init(&init_barrier, nullptr, num_threads + 1); @@ -921,7 +949,7 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start thread dt->thread = std::thread([this, &dt_ref = *dt, &init_barrier, - thread_pinning, i]() { + thread_pinning, i, stats_warmup_secs]() { // CPU affinity if (thread_pinning) { cpu_set_t mask; @@ -953,6 +981,18 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start making requests TestDriver::Impl::makeRequests(*dt_ref.driver); + // Schedule the cold-start stats reset on this thread's own base (added + // here, before dispatch, so it fires from this thread — no data race with + // the stats writes). Fires once ~stats_warmup_secs into steady traffic. + if (stats_warmup_secs > 0) { + struct timeval warmup_tv { + stats_warmup_secs, 0 + }; + event_base_once( + dt_ref.base, -1, EV_TIMEOUT, resetDriverStatsCb, + &dt_ref.driver->impl_->current_stats, &warmup_tv); + } + // Run event loop event_base_dispatch(dt_ref.base); }); @@ -1070,7 +1110,14 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, } double end_time = getTimeSec(); - double elapsed = end_time - start_time; + // With a stats warmup, samples were reset ~stats_warmup_secs into the run, so + // the throughput denominator must be the post-warmup window (else QPS would + // be understated by counting post-warmup queries over the full duration). + double meas_start = start_time; + if (stats_warmup_secs > 0) { + meas_start = start_time + stats_warmup_secs; + } + double elapsed = end_time - meas_start; // Aggregate stats from all threads for (auto& dt : impl_->threads) { From fc32178354a76fc93c3c820ceefc07ecfe59937c Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 5/8] Expose driver pipeline depth as a benchpress job parameter Reviewed By: YifanYuan3 Differential Revision: D113146808 --- benchpress/config/jobs.yml | 21 ++++++++++ packages/feedsim/README.md | 42 +++++++++++++++++-- packages/feedsim/run.sh | 24 +++++++++-- .../third_party/src/scripts/search_qps.sh | 33 +++++++++------ 4 files changed, 101 insertions(+), 19 deletions(-) diff --git a/benchpress/config/jobs.yml b/benchpress/config/jobs.yml index 365d84f5..73d39278 100644 --- a/benchpress/config/jobs.yml +++ b/benchpress/config/jobs.yml @@ -502,6 +502,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: # Hardcode num_instances=1: this job exists specifically for single-instance @@ -534,6 +535,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -592,6 +599,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -635,6 +643,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -695,6 +709,7 @@ - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -735,6 +750,12 @@ - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: copymove diff --git a/packages/feedsim/README.md b/packages/feedsim/README.md index 6dce669c..699831fc 100644 --- a/packages/feedsim/README.md +++ b/packages/feedsim/README.md @@ -55,7 +55,7 @@ taskset -c 0-15 ./benchpress_cli.py install feedsim_dlrm Unlike FeedSim v1 which spawns a new FeedSim instance per 100 CPU cores, `feedsim_dlrm` is pinned to **one FeedSim instance per host** because the redesigned threading model in FeedSim v2 has overcome the scalability issue -on ultra-high-core-count CPUs and ARM CPUs. +on ultra-high-core-count CPUs and ARM CPUs. The runner searches for the QPS that keeps 95th-percentile end-to-end latency at or below **700 ms**. When it converges it runs a final 5-minute @@ -64,7 +64,7 @@ counters) during that final window. We expect the total wall-clock runtime to be around 30 minutes. Please make sure to turn CPU turbo-boost on before starting, or FeedSim may -fail to converge and report a low QPS. +fail to converge and report a low QPS. ### Result report @@ -211,7 +211,7 @@ feedsim server, driver and mock_services instances. 2. Use the `feedsim_autoscale_dlrm` job. This autoscale job will spawn `ceil(nproc / 100)` FeedSim instances, each pinned to its own CPU range via `taskset`, plus one driver -and one `mock_services` process per instance (also `taskset`-isolated). For example: +and one `mock_services` process per instance (also `taskset`-isolated). For example: ``` ./benchpress_cli.py run feedsim_autoscale_dlrm ``` @@ -220,6 +220,41 @@ In multi-instance mode, the overall QPS is the sum across all instances. and the average latency will be the average of p95 latency values observed across all instances. +### Driver depth (fixing CPU/latency under-utilization) + +The `depth` parameter sets the driver's pipeline depth — the maximum number of +outstanding (in-flight) requests per driver connection. The driver's total +offered concurrency is `driver_threads × connections × depth`, so with the +default `depth=1` the driver can cap the achievable load below what the server +can actually handle. + +**Increase `depth` beyond 1 when the final benchmarking phase saturates neither +CPU nor latency** — i.e. the final achieved p95 latency is well below the SLA +limit (`sla_p95_ms`, default 700 ms) *and* the CPU utilization during the final +5-minute benchmarking phase is less than ~90%. In that situation the reported QPS +is limited by driver concurrency rather than by the server, so it understates the +hardware's true capacity. Raising `depth` (start with `2`) lets the driver offer +more concurrent load until the server becomes the bottleneck — either CPU-bound +(~100% utilization) or latency-bound (p95 ≈ SLA). **This is likely necessary on +high-performance ARM cores** (e.g. NVIDIA Grace), which can otherwise sit at +80–90% CPU with p95 far below the SLA at `depth=1`. + +``` +# Force driver depth 2 +./benchpress_cli.py run feedsim_dlrm -i '{"depth": 2}' +``` + +There is also an **adaptive depth** mechanism (on by default) that raises the +depth automatically during the peak-finding stage until the server saturates +(system CPU ≥ 95% or p95 ≥ SLA). It catches *severe* under-utilization early, but +because it evaluates saturation on the high-load peak/search probes rather than +on the final SLA-converged operating point, it **may not catch all +under-utilization cases**. If you still observe under-utilization in the final +result (low CPU + p95 well under SLA), increase `depth` manually as above. When +adaptive depth is on, a manually-set `depth` acts as the starting floor the +adaptive search raises from; to pin an exact fixed depth, also set the +`FEEDSIM_ADAPTIVE_DEPTH_MAX=0` environment variable to disable adaptive search. + ### Other parameters This section lists additional parameters in `feedsim_dlrm` benchmark. These parameters @@ -233,6 +268,7 @@ Job-level parameters (can be passed via `-i` flag in Benchpress CLI): |---|---|---| | `num_instances` | Number of FeedSim instances to run in parallel. Defaults to 1 in `feedsim_dlrm`; set to -1 to autoscale for `feedsim_autoscale_dlrm`. | `1` | | `sla_p95_ms` | SLA target in ms. The runner searches for the highest QPS keeping p95 ≤ this. | `700` | +| `depth` | Driver pipeline depth (max outstanding requests per connection; total in-flight = `driver_threads × connections × depth`). Raise (e.g. `2`) when the final phase saturates neither CPU nor latency — often needed on high-perf ARM. See [Driver depth](#driver-depth-fixing-cpulatency-under-utilization). | `1` | | `io_dist` | I/O latency distribution: `fixed`, `exponential`, or `lognormal`. | `fixed` | | `io_mean` | Mean I/O latency in ms. | `200` | | `workload` | Ranking workload: `pagerank` or `dlrm`. `dlrm` is v2. | `dlrm` | diff --git a/packages/feedsim/run.sh b/packages/feedsim/run.sh index 06bcb161..2010f549 100644 --- a/packages/feedsim/run.sh +++ b/packages/feedsim/run.sh @@ -120,6 +120,7 @@ Usage: ${0##*/} [OPTION]... --rpc-fanout-scale Scale factor applied to per-session fanout counts. Default: 0.05 (t43 c7). --server-zstd Enable ZSTD compression on server-side response payloads (0=off, 1=on). Default: 0 (t43 c7). --sla-p95-ms search_qps SLA target (95th percentile latency in ms). Default: 700. + --depth Driver pipeline depth: max outstanding requests per driver connection (max in-flight = driver_threads * connections * depth). Default: 1 (or \$FEEDSIM_DRIVER_DEPTH). Raise (e.g. 2) when the final phase saturates neither CPU nor SLA latency; with adaptive depth on, this is the starting floor the peak search raises from. EOF } @@ -316,6 +317,12 @@ main() { local sla_p95_ms sla_p95_ms="700" + # Driver pipeline depth (max outstanding requests per driver connection). + # Env var FEEDSIM_DRIVER_DEPTH is the fallback default; the --depth CLI flag + # (forwarded from the benchpress `depth` job parameter) overrides it. + local driver_depth + driver_depth="${FEEDSIM_DRIVER_DEPTH:-1}" + if [ -z "$IS_AUTOSCALE_RUN" ]; then echo > $BREPS_LFILE fi @@ -644,6 +651,13 @@ main() { --sla-p95-ms=*) sla_p95_ms="${1#*=}" ;; + --depth) + driver_depth="$2" + shift + ;; + --depth=*) + driver_depth="${1#*=}" + ;; -h|--help) show_help >&2 exit 1 @@ -1002,7 +1016,9 @@ main() { # to reach a real bound instead of capping on driver concurrency (the t19 # anti-pattern where big boxes sat at ~80% CPU with p95 far below SLA). # Enabled by default up to depth 8; set FEEDSIM_ADAPTIVE_DEPTH_MAX=0 to - # disable and fall back to the fixed FEEDSIM_DRIVER_DEPTH (default 1). + # disable and use the fixed driver_depth (--depth flag / FEEDSIM_DRIVER_DEPTH, + # default 1). When adaptive is on, driver_depth is the STARTING floor the peak + # search raises from (search_qps reads the --depth we pass below). sqps_adaptive_arg="" adaptive_depth_max="${FEEDSIM_ADAPTIVE_DEPTH_MAX:-8}" if [ "$adaptive_depth_max" != "0" ]; then @@ -1019,7 +1035,7 @@ main() { --monitor_port "$client_monitor_port" \ --threads="${driver_threads}" \ --connections=4 \ - --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -1032,7 +1048,7 @@ main() { $driver_bin \ --monitor_port "$client_monitor_port" \ --server "0.0.0.0:$port" \ - --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -1061,7 +1077,7 @@ main() { --monitor_port "$client_monitor_port" \ --threads="${num_workers}" \ --connections="${num_connections}" \ - --depth="${FEEDSIM_DRIVER_DEPTH:-1}" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index e51fbf6c..331446e6 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -120,10 +120,11 @@ mutilate (EuroSys \'14) [https://github.com/leverich/mutilate] -P PID of the process to log runtime breakdowns. Optional -B Folder to log runtime breakdowns. Optional -D Adaptive depth: max driver pipeline depth. When set, the peak - phase raises the driver's --depth from 1 until the server is - saturated (system CPU >= 95% OR achieved p95 >= SLA), then holds - that depth for the QPS search. Overrides any --depth in the - driver command. Optional. + phase raises the driver's --depth until the server is saturated + (system CPU >= 95% OR achieved p95 >= SLA), then holds that depth + for the QPS search. The search STARTS from any --depth in the + driver command (default 1), so a manually-set --depth acts as a + floor. Optional. EOF } @@ -394,12 +395,18 @@ fi # remaining argument is loadtest command command=$@ -# In adaptive-depth mode, search_qps owns the driver's --depth: strip any fixed -# --depth from the command so our per-attempt --depth is the only one, and start -# the peak search at depth=1. +# In adaptive-depth mode, search_qps owns the driver's --depth: capture any fixed +# --depth as the STARTING depth (so a manually-set --depth acts as a floor the +# peak search raises from), then strip it so our per-attempt --depth is the only +# one on the command. Default start is depth=1 (unchanged behavior). +adaptive_start_depth=1 if [ -n "$adaptive_depth_max" ]; then + fixed_depth=$(echo "$command" | grep -oE -- '--depth=[0-9]+' | head -1 | grep -oE '[0-9]+') + if [ -n "$fixed_depth" ] && [ "$fixed_depth" -gt 1 ]; then + adaptive_start_depth=$fixed_depth + fi command=$(echo "$command" | sed -E 's/[[:space:]]*--depth=[0-9]+//g') - adaptive_depth_arg="--depth=1" + adaptive_depth_arg="--depth=$adaptive_start_depth" fi # make sure latency_type and latency_target are specified @@ -553,12 +560,14 @@ fi benchreps_tell_state "before peak_qps" if [ -n "$adaptive_depth_max" ]; then # Adaptive depth: the peak load test offers at most threads*connections*depth - # concurrent requests. Starting at depth=1, keep raising depth (and re-running - # peak) until the server saturates — system CPU >= 95% OR achieved p95 >= SLA + # concurrent requests. Starting at adaptive_start_depth, keep raising depth (and + # re-running peak) until the server saturates — system CPU >= 95% OR p95 >= SLA # — so platforms that need more offered concurrency reach a real bound instead # of capping on driver concurrency (the t19 anti-pattern). The selected depth - # is then held for the QPS search / tuning / final phases. - cur_depth=1 + # is then held for the QPS search / tuning / final phases. The starting depth + # is the fixed --depth from the driver command (default 1), so a manually-set + # depth raises the floor. + cur_depth=$adaptive_start_depth while : ; do adaptive_depth_arg="--depth=$cur_depth" # Sample system CPU busy% over a mid-run window while the peak load runs. From 7cc9abaad79aad280a0d69703a5170c2f6020d65 Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 6/8] Use measured_qps as search bounds in search_qps.sh Reviewed By: YifanYuan3 Differential Revision: D113433316 --- .../third_party/src/scripts/search_qps.sh | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index 331446e6..2ebdf71e 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -619,19 +619,49 @@ while [[ $loop_cond -eq 1 ]]; do run_loadtest measured_qps measured_latency $cur_qps "" printf "requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $cur_qps $measured_qps $measured_latency - # set new QPS ranges + # Update search bounds using measured_qps (what the server actually sustained) + # rather than cur_qps (what the driver requested). On a server that saturates + # below cur_qps, using cur_qps as the "achievable" lower bound falsely inflates + # low_qps to a value the server never delivered. + # + # latency BAD: + # - measured_qps >= cur_qps: high_qps = measured_qps (server delivered at + # least what was asked; latency is the ceiling at that measured rate). + # - measured_qps < cur_qps: high_qps = measured_qps + (cur_qps - measured_qps) * SLA / latency + # Rationale: when cur_qps is much larger than what the server can fulfill, + # the surplus offered load queues up and exaggerates latency. The server + # may deliver ~measured_qps at acceptable latency if the offered rate is + # just slightly higher than measured (not far above it). SLA/latency scales + # the search headroom by how badly latency violated SLA — barely-over-SLA + # probes leave more room; badly-over-SLA probes shrink toward measured. + # + # latency GOOD: + # - low_qps = measured_qps (measured is a sustained floor). + # - If cur_qps > 1.01 * measured_qps (any noticeable gap between requested + # and delivered — server at or near its ceiling), shrink high_qps by 0.96, + # but never let high_qps drop below low_qps. Capping at low_qps ensures the + # search terminates cleanly when it has converged (loop condition + # high > low * 1.02 becomes false) instead of oscillating in a narrow + # window near max_iters. latency_good=$(echo "$measured_latency <= $latency_target" | bc) if [[ $latency_good -eq 0 ]]; then - high_qps=$cur_qps - else - low_qps=$cur_qps - measured_qps_is_higher=$(echo "$measured_qps > $low_qps" | bc) - if [[ $measured_qps_is_higher -eq 1 ]] ; then - low_qps=$measured_qps + measured_ge_requested=$(echo "$measured_qps >= $cur_qps" | bc) + if [[ $measured_ge_requested -eq 1 ]]; then + high_qps=$measured_qps + else + high_qps=$(echo "scale=5; $measured_qps + ($cur_qps - $measured_qps) * $latency_target / $measured_latency" | bc) fi - measured_qps_gap=$(echo "$cur_qps > $measured_qps * 1.02" | bc) + else + low_qps=$measured_qps + measured_qps_gap=$(echo "$cur_qps > $measured_qps * 1.01" | bc) if [[ $measured_qps_gap -eq 1 ]] ; then - high_qps=$(echo "scale=5; $high_qps*0.96" | bc) + new_high_qps=$(echo "scale=5; $high_qps * 0.96" | bc) + high_below_low=$(echo "$new_high_qps < $low_qps" | bc) + if [[ $high_below_low -eq 1 ]]; then + high_qps=$low_qps + else + high_qps=$new_high_qps + fi fi fi From 16155813bd1106226329576f995ae71184b415be Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 7/8] Log main_benchmark start/end for the final measurement in search_qps.sh Reviewed By: YifanYuan3 Differential Revision: D113485984 --- packages/feedsim/third_party/src/scripts/search_qps.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index 2ebdf71e..fcc617b7 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -546,7 +546,7 @@ if [[ -n "$fixed_qps" ]]; then else for fixed_qps_el in $fixed_qps_array; do benchreps_tell_state "before fixed_qps_iter $fixed_qps_el" - run_loadtest measured_qps measured_latency $fixed_qps_el "" + run_loadtest measured_qps measured_latency $fixed_qps_el "$main_operation_name" printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $fixed_qps_el $measured_qps $measured_latency echo "final requested_qps = $fixed_qps_el, measured_qps = $measured_qps, latency = $measured_latency" >> $BREPS_LFILE benchreps_tell_state "after fixed_qps_iter $fixed_qps_el" @@ -721,13 +721,17 @@ if [[ -n "$IS_AUTOSCALE_RUN" ]] && [[ "$IS_AUTOSCALE_RUN" -gt 1 ]]; then fi fi -# do final measurement +# do final measurement — this is the actual reported experiment window; log it +# as main_benchmark in breakdown.csv so perfpub sees a valid time window (the +# search/warmup/tuning probes above are informational and should not be logged +# as the primary benchmark window). $main_operation_name comes from +# packages/common/runtime_breakdown_utils.sh (sourced at the top of this script). benchreps_tell_state "before final_qps" experiment_time=$final_experiment_time if [ "${DCPERF_PERF_RECORD}" = 1 ] && ! [ -f "perf.data" ]; then collect_perf_record & fi -run_loadtest measured_qps measured_latency $cur_qps "" +run_loadtest measured_qps measured_latency $cur_qps "$main_operation_name" printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $cur_qps $measured_qps $measured_latency # report non-converging error if iteration reaches max tries From ccd7b3ddd7ee659eaf959de821dcdea39726463a Mon Sep 17 00:00:00 2001 From: Wei Su Date: Wed, 12 Aug 2026 16:55:33 -0700 Subject: [PATCH 8/8] Add memory-streaming stride-sweep to feature extractors (prod DRAM/LLC match) Differential Revision: D114964801 --- .../FeatureExtractorSuite.cpp | 10 +++ .../generated/extractor_helpers.cpp | 82 +++++++++++++++++++ .../generated/extractor_helpers.h | 18 ++++ 3 files changed, 110 insertions(+) diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp index f5d4203e..fbabeb45 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp @@ -9,6 +9,8 @@ #include #include +#include "generated/extractor_helpers.h" + void FeatureExtractorSuite::addExtractor( std::unique_ptr extractor) { extractors_.push_back(std::move(extractor)); @@ -191,7 +193,15 @@ void FeatureExtractorSuite::runFlatExtractors( size_t start = flat_pos_.fetch_add(static_cast(count), std::memory_order_relaxed) % total; + // Memory-streaming lever: FEEDSIM_SWEEP_N reads per call over a large + // read-only buffer, folded into live state so it can't be elided. No-op + // when FEEDSIM_SWEEP_N=0. Hoist the enabled check out of the loop. + const int sweep_n = dcperf::feature_extractors::helpers::sweepReadsPerCall(); for (int i = 0; i < count; ++i) { flat_copies_[(start + static_cast(i)) % total](&ctx); + if (sweep_n > 0) { + local_struct[0] += dcperf::feature_extractors::helpers::runStrideSweep( + start + static_cast(i)); + } } } diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp index 8866e510..9b2b67fa 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp @@ -5,7 +5,11 @@ #include "extractor_helpers.h" #include +#include +#include #include +#include +#include namespace dcperf { namespace feature_extractors { @@ -280,6 +284,84 @@ float cappedConvertFloat(double value, float min_val, float max_val) { return result; } +// ====================================================================== +// Memory-streaming stride sweep (backend/DRAM-pressure lever) +// ====================================================================== + +namespace { + +struct SweepConfig { + std::vector buf; // process-wide, read-only after init + size_t size = 0; // element count + int n = 0; // reads per extractor call (FEEDSIM_SWEEP_N) + size_t stride = 16; // element stride (FEEDSIM_SWEEP_STRIDE); 16 = 64B line +}; + +SweepConfig g_sweep; +std::once_flag g_sweep_once; + +int envInt(const char* name, int fallback) { + const char* v = std::getenv(name); + if (v == nullptr || v[0] == '\0') { + return fallback; + } + int parsed = std::atoi(v); + return parsed; +} + +void initSweep() { + // Defaults are the validated winning config (CPL/BGM/Grace, 2026-08-04): + // 16 strided reads/call over a 64 MB DRAM-resident buffer at a 64 B (1 cache + // line) stride. This closes most of the DRAM-bandwidth / LLC / L1-D / IPC gap + // to prod (BGM mean uArch err 27%->18%, Grace 34%->23%). Override any knob via + // the FEEDSIM_SWEEP_* env vars; set FEEDSIM_SWEEP_N=0 to disable entirely. + int mb = envInt("FEEDSIM_SWEEP_MB", 64); + if (mb < 1) { + mb = 1; + } + g_sweep.n = envInt("FEEDSIM_SWEEP_N", 16); + if (g_sweep.n < 0) { + g_sweep.n = 0; + } + int stride = envInt("FEEDSIM_SWEEP_STRIDE", 16); + g_sweep.stride = stride < 1 ? 1 : static_cast(stride); + g_sweep.size = static_cast(mb) * 1024 * 1024 / sizeof(float); + g_sweep.buf.resize(g_sweep.size); + // Fill with pseudo-random data so the compiler can't fold the buffer away. + uint64_t s = 0x9E3779B97F4A7C15ULL; + for (size_t i = 0; i < g_sweep.size; ++i) { + s = s * 6364136223846793005ULL + 1442695040888963407ULL; + g_sweep.buf[i] = static_cast((s >> 40) & 0xFFFF) * 1e-3f; + } +} + +} // namespace + +int sweepReadsPerCall() { + std::call_once(g_sweep_once, initSweep); + return g_sweep.n; +} + +float runStrideSweep(uint64_t seed) { + const SweepConfig& c = g_sweep; + if (c.n == 0 || c.size == 0) { + return 0.0f; + } + // Rotate the start offset per call so successive calls cover the whole + // buffer rather than re-touching one region (that was the flaw in the + // earlier size-only bumps and the B3 random gather). + size_t off = (seed * 2654435761ULL) % c.size; + float acc = 0.0f; + for (int i = 0; i < c.n; ++i) { + acc += c.buf[off]; + off += c.stride; + if (off >= c.size) { + off -= c.size; + } + } + return acc; +} + } // namespace helpers } // namespace feature_extractors } // namespace dcperf diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h index 3c3b32a8..cf857df1 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h @@ -115,6 +115,24 @@ bool validateFeatureValue(float value, int validation_type); __attribute__((noinline)) float cappedConvertFloat(double value, float min_val, float max_val); +// ====================================================================== +// Memory-streaming stride sweep (backend/DRAM-pressure lever) +// ====================================================================== +// Reads FEEDSIM_SWEEP_N elements, FEEDSIM_SWEEP_STRIDE floats apart, from a +// process-wide read-only buffer of FEEDSIM_SWEEP_MB megabytes. Adds genuine +// memory-level-parallelism / working-set pressure per extractor call to close +// the backend-bound / DRAM-bandwidth gap vs prod. All three knobs are read +// from the environment once, so one build sweeps the full parameter space; +// FEEDSIM_SWEEP_N=0 (default) makes it a no-op. Returns FEEDSIM_SWEEP_N (cached +// after first call, which initializes the buffer). +int sweepReadsPerCall(); + +// Accumulate a strided walk seeded by `seed` (rotates the start offset so +// successive calls cover the whole buffer). Caller folds the result into live +// state to defeat dead-code elimination. Assumes sweepReadsPerCall() ran first. +__attribute__((noinline)) +float runStrideSweep(uint64_t seed); + } // namespace helpers } // namespace feature_extractors } // namespace dcperf