diff --git a/benchpress/config/jobs.yml b/benchpress/config/jobs.yml index c11c66333..41a7f99c9 100644 --- a/benchpress/config/jobs.yml +++ b/benchpress/config/jobs.yml @@ -462,15 +462,14 @@ benchmark: feedsim_autoscale description: > Single-instance aggregator workload with DLRM (Deep Learning Recommendation - Model) inference. Same t43 c7 calibration as feedsim_autoscale_dlrm but pinned + Model) inference. Same calibration as feedsim_autoscale_dlrm but pinned to ONE feedsim instance per host (overrides the autoscale "100 cores per instance" rule). Single-instance runs give better gen-over-gen correlation than multi-instance — there's no cross-instance mock_services queue contention - and no per-instance core-range fragmentation. Use this for cross-LSST CPU + and no per-instance core-range fragmentation. Use this for cross-platform CPU comparisons; use feedsim_autoscale_dlrm for max-throughput experiments on big multi-instance boxes. Finds maximum QPS that the system can sustain while - keeping 95th percentile latency <= 700ms (matches prod multifeed_aggregator's - own end-to-end p95 budget). + keeping 95th percentile latency <= 700ms. args: - '-n {num_instances}' - '--async-io' @@ -495,24 +494,29 @@ - '--stories-per-processor-pass={stories_per_processor_pass}' - '--silesia-dir={silesia_dir}' - '--stories-per-request={stories_per_request}' - # t43 c7 calibration (see ~/.claude/projects/.../memory/t43_c7_recommended.md) + # Calibration knobs. - '--mock-tls={mock_tls}' - '--mock-zstd-frac={mock_zstd_frac}' - '--mock-keepalive-interval-ms={mock_keepalive_interval_ms}' - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: # Hardcode num_instances=1: this job exists specifically for single-instance - # cross-LSST comparisons. If you want autoscale-by-cores, use + # cross-platform comparisons. If you want autoscale-by-cores, use # feedsim_autoscale_dlrm instead. - 'num_instances=1' - 'io_dist=fixed' - 'io_mean=200' - 'workload=dlrm' - 'dlrm_model=models/dlrm_small.pt' - - 'dlrm_batch_size=64' + # Calibrated recipe: minimizes all-metrics RMSE vs production (uArch + + # instruction mix + hot-function) while preserving gen-over-gen ranking + # across platforms. Pairs with the integer-domain feature-extractor helpers + # + memory-streaming stride-sweep (N=16, code default) earlier in this stack. + - 'dlrm_batch_size=32' - 'dlrm_threads=1' - 'dlrm_inferences=1' - 'client_side_features=0' @@ -521,19 +525,25 @@ - 'client_feature_seed=42' - 'client_num_dense=13' - 'client_num_sparse=26' - - 'feature_complexity=5' + - 'feature_complexity=8' - 'num_stories=400' - - 'extractors_per_story=280' + - 'extractors_per_story=240' - 'story_processors_per_story=2' - - 'stories_per_processor_pass=100' + - 'stories_per_processor_pass=150' - 'silesia_dir=silesia' - 'stories_per_request=10' - 'mock_tls=1' - - 'mock_zstd_frac=0.75' + - 'mock_zstd_frac=0.9' - 'mock_keepalive_interval_ms=200' - - 'rpc_fanout_scale=0.05' - - 'server_zstd=0' + - 'rpc_fanout_scale=0.10' + - 'server_zstd=1' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -656,11 +666,10 @@ benchmark: feedsim_autoscale description: > Multi-instance aggregator workload with DLRM (Deep Learning Recommendation Model) - inference. Defaults bake in the t43 c7 calibration — the balanced - configuration that minimizes both gen-over-gen correlation gap AND - bench-vs-prod hot-function gap on CPL/BGM. Finds maximum QPS that the - system can sustain while keeping 95th percentile latency <= 700ms - (matches prod multifeed_aggregator's own end-to-end p95 budget). + inference. Defaults bake in the balanced calibration that minimizes both + the gen-over-gen correlation gap AND the bench-vs-prod hot-function gap. + Finds maximum QPS that the system can sustain while keeping 95th percentile + latency <= 700ms. Automatically spawns one workload instance per ~100 cores. args: - '-n {num_instances}' @@ -686,13 +695,14 @@ - '--stories-per-processor-pass={stories_per_processor_pass}' - '--silesia-dir={silesia_dir}' - '--stories-per-request={stories_per_request}' - # t43 c7 calibration knobs (t31 tracking: T269255604) + # Calibration knobs - '--mock-tls={mock_tls}' - '--mock-zstd-frac={mock_zstd_frac}' - '--mock-keepalive-interval-ms={mock_keepalive_interval_ms}' - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -700,11 +710,11 @@ - 'io_mean=200' - 'workload=dlrm' - 'dlrm_model=models/dlrm_small.pt' - # t43 c7: bs=64 is the balanced choice across both gen-over-gen - # correlation (BGM/CPL = 4.25× → 2nd best of 12 cells) and prod - # hot-function gap (CPL 4th, BGM 5th). bs=32 wins peak QPS alone but - # breaks correlation (BGM stretches to 4.75× CPL). - - 'dlrm_batch_size=64' + # Calibrated recipe: minimizes all-metrics RMSE vs production (uArch + + # instruction mix + hot-function). bs=32 (with the integer-domain extractor + # helpers + stride-sweep earlier in this stack) raises SIMD toward prod and, + # at eps=240, preserves gen-over-gen ranking across platforms. + - 'dlrm_batch_size=32' - 'dlrm_threads=1' - 'dlrm_inferences=1' - 'client_side_features=0' @@ -713,29 +723,33 @@ - 'client_feature_seed=42' - 'client_num_dense=13' - 'client_num_sparse=26' - - 'feature_complexity=5' - # 400 x 280 = 112K extractor calls/req. Combined with the scaled-up - # story-processor pipeline (below) this matches prod's FE:story CPU - # ratio of ~1.3:1 (multifeed: FE 29%, story 22% on BGM Strobelight). + - 'feature_complexity=8' + # 400 x 240 = 96K extractor calls/req. eps=240 holds Ranking-FeatureExtraction + # near the production share; the integer-domain helpers made each call cheaper. - 'num_stories=400' - - 'extractors_per_story=280' - # Story-processor pipeline scaled 8x from t32 first attempt (was - # 1 pass/story * 25 stories/pass = 25 cmpns/story → 1.1% Ranking-Story - # on BGM, vs prod 9.3%). Bumped to 2 passes * 100 stories = 200 - # comparisons/story, target ~9% Ranking-Story share. + - 'extractors_per_story=240' + # Story-processor pipeline: 2 passes * 150 stories = 300 comparisons/story, + # target a production-like Ranking-Story share. - 'story_processors_per_story=2' - - 'stories_per_processor_pass=100' + - 'stories_per_processor_pass=150' - 'silesia_dir=silesia' - 'stories_per_request=10' - # t43 c7: TLS+keepalive on mock_services, 75% of channels compress, - # rpc_fanout_scale=0.05 (~188 RPCs/session). Server-side ZSTD off - # (mock-side ZSTD provides the prod-realistic Compression share). + # TLS+keepalive on mock_services; mock_zstd_frac=0.9 + server_zstd=1 land + # Compression near the production share; rpc_fanout_scale=0.10 lands + # RPC-AsyncIO near production. DRIVER_TLS stays off (default): it overshoots + # Encryption without moving throughput. - 'mock_tls=1' - - 'mock_zstd_frac=0.75' + - 'mock_zstd_frac=0.9' - 'mock_keepalive_interval_ms=200' - - 'rpc_fanout_scale=0.05' - - 'server_zstd=0' + - 'rpc_fanout_scale=0.10' + - 'server_zstd=1' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: cpu-mpstat @@ -789,13 +803,13 @@ - '-P {page_rank_seed}' - '-C {pointer_chase_seed}' - '-N' - # t43 c7 calibration knobs (t31 tracking: T269255604) - '--mock-tls={mock_tls}' - '--mock-zstd-frac={mock_zstd_frac}' - '--mock-keepalive-interval-ms={mock_keepalive_interval_ms}' - '--rpc-fanout-scale={rpc_fanout_scale}' - '--server-zstd={server_zstd}' - '--sla-p95-ms={sla_p95_ms}' + - '--depth={depth}' - '{extra_args}' vars: - 'num_instances=-1' @@ -805,8 +819,6 @@ - 'dlrm_model=models/dlrm_small.pt' - 'dlrm_batch_size=64' - 'dlrm_threads=1' - # 8 inferences/req brings DLRM-Inference CPU share from 55-62% down - # toward prod multifeed_aggregator's Ranking-Prediction share of 7-13%. - 'dlrm_inferences=8' - 'client_side_features=0' - 'client_batch_size=256' @@ -815,9 +827,6 @@ - 'client_num_dense=13' - 'client_num_sparse=26' - 'feature_complexity=5' - # 1800 stories x 50 extractors = 90K extractor calls/req. Tuned to - # push FeatureExtraction CPU share into prod's 30-35% band; 5K - # calls/req previously gave only 1.7-1.9%. - 'num_stories=1800' - 'extractors_per_story=50' - 'fixed_qps=100000' @@ -829,13 +838,19 @@ - 'node_rank_seed=54321' - 'page_rank_seed=12345' - 'pointer_chase_seed=98765' - # t43 c7 mock_services + TLS knobs (same baseline as feedsim_autoscale_dlrm) + # mock_services + TLS knobs (same baseline as feedsim_autoscale_dlrm) - 'mock_tls=1' - 'mock_zstd_frac=0.75' - 'mock_keepalive_interval_ms=200' - 'rpc_fanout_scale=0.05' - 'server_zstd=0' - 'sla_p95_ms=700' + # Driver pipeline depth (max outstanding requests per driver connection). + # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA + # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often + # needed on high-perf ARM cores. With adaptive depth on (the default), this is + # the starting floor the peak search raises from; see README "Driver depth". + - 'depth=1' - 'extra_args=' hooks: - hook: copymove diff --git a/packages/feedsim/README.md b/packages/feedsim/README.md index c5ec13def..6eac7701b 100644 --- a/packages/feedsim/README.md +++ b/packages/feedsim/README.md @@ -69,7 +69,7 @@ fail to converge and report a low QPS. ### Result report After the run finishes, benchpress prints a JSON result. Example from a -AMD Zen4 host (176 logical cores, 256 GB RAM): +176-core x86 host (256 GB RAM): ```json { @@ -241,6 +241,41 @@ In multi-instance mode, the overall QPS is the sum across all instances. and the average latency will be the average of p95 latency values observed across all instances. +### Driver depth (fixing CPU/latency under-utilization) + +The `depth` parameter sets the driver's pipeline depth — the maximum number of +outstanding (in-flight) requests per driver connection. The driver's total +offered concurrency is `driver_threads × connections × depth`, so with the +default `depth=1` the driver can cap the achievable load below what the server +can actually handle. + +**Increase `depth` beyond 1 when the final benchmarking phase saturates neither +CPU nor latency** — i.e. the final achieved p95 latency is well below the SLA +limit (`sla_p95_ms`, default 700 ms) *and* the CPU utilization during the final +5-minute benchmarking phase is less than ~90%. In that situation the reported QPS +is limited by driver concurrency rather than by the server, so it understates the +hardware's true capacity. Raising `depth` (start with `2`) lets the driver offer +more concurrent load until the server becomes the bottleneck — either CPU-bound +(~100% utilization) or latency-bound (p95 ≈ SLA). **This is likely necessary on +high-performance ARM cores**, which can otherwise sit at 80–90% CPU with p95 far +below the SLA at `depth=1`. + +``` +# Force driver depth 2 +./benchpress_cli.py run feedsim_dlrm -i '{"depth": 2}' +``` + +There is also an **adaptive depth** mechanism (on by default) that raises the +depth automatically during the peak-finding stage until the server saturates +(system CPU ≥ 95% or p95 ≥ SLA). It catches *severe* under-utilization early, but +because it evaluates saturation on the high-load peak/search probes rather than +on the final SLA-converged operating point, it **may not catch all +under-utilization cases**. If you still observe under-utilization in the final +result (low CPU + p95 well under SLA), increase `depth` manually as above. When +adaptive depth is on, a manually-set `depth` acts as the starting floor the +adaptive search raises from; to pin an exact fixed depth, also set the +`FEEDSIM_ADAPTIVE_DEPTH_MAX=0` environment variable to disable adaptive search. + ### Other parameters This section lists additional parameters in `feedsim_dlrm` benchmark. These parameters @@ -254,6 +289,7 @@ Job-level parameters (can be passed via `-i` flag in Benchpress CLI): |---|---|---| | `num_instances` | Number of FeedSim instances to run in parallel. Defaults to 1 in `feedsim_dlrm`; set to -1 to autoscale for `feedsim_autoscale_dlrm`. | `1` | | `sla_p95_ms` | SLA target in ms. The runner searches for the highest QPS keeping p95 ≤ this. | `700` | +| `depth` | Driver pipeline depth (max outstanding requests per connection; total in-flight = `driver_threads × connections × depth`). Raise (e.g. `2`) when the final phase saturates neither CPU nor latency — often needed on high-perf ARM. See [Driver depth](#driver-depth-fixing-cpulatency-under-utilization). | `1` | | `io_dist` | I/O latency distribution: `fixed`, `exponential`, or `lognormal`. | `fixed` | | `io_mean` | Mean I/O latency in ms. | `200` | | `workload` | Ranking workload: `pagerank` or `dlrm`. `dlrm` is v2. | `dlrm` | diff --git a/packages/feedsim/install_feedsim.sh b/packages/feedsim/install_feedsim.sh index e9bb855b5..03c22fe44 100755 --- a/packages/feedsim/install_feedsim.sh +++ b/packages/feedsim/install_feedsim.sh @@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" +# Dependency versions are env-overridable so experiments can bump them without +# forking this script; defaults reproduce the v2 baseline exactly. +JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" +LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" +# Export so the aarch64 sub-installer (dispatched below) inherits the pins. +export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \ libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \ libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \ zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \ - xxhash xxhash-devel unzip rsync liburing-devel + xxhash xxhash-devel unzip rsync liburing-devel python3-pip # Creates feedsim directory under benchmarks/ mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim" @@ -178,30 +191,30 @@ else fi # Installing JEMalloc -if ! [ -d "jemalloc-5.3.0" ]; then - wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" - bunzip2 "jemalloc-5.3.0.tar.bz2" - tar -xvf "jemalloc-5.3.0.tar" - cd "jemalloc-5.3.0" +if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then + wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2" + tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar" + cd "jemalloc-${JEMALLOC_VERSION}" ./configure --enable-prof --enable-prof-libunwind make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] jemalloc-5.3.0" + msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}" fi # Installing libevent -if ! [ -d "libevent-2.1.12-stable" ]; then - wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz" - tar -xzf "libevent-2.1.12-stable.tar.gz" - cd "libevent-2.1.12-stable" +if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then + wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz" + tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz" + cd "libevent-${LIBEVENT_VERSION}" ./configure make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] libevent-2.1.12-stable" + msg "[SKIPPED] libevent-${LIBEVENT_VERSION}" fi msg "Installing third-party dependencies ... DONE" @@ -218,12 +231,33 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # pip on the box (3.9, or an internal stale mirror) can't see the cp310 + # 2.13 wheels, so resolve the wheel href straight from the PEP-503 index + # and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is + # Python-version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/install_feedsim_aarch64.sh b/packages/feedsim/install_feedsim_aarch64.sh index 59a238d15..4d797ba47 100755 --- a/packages/feedsim/install_feedsim_aarch64.sh +++ b/packages/feedsim/install_feedsim_aarch64.sh @@ -182,11 +182,15 @@ else msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}" fi -DEP_JEMALLOC_VERSION="5.3.0" +DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" # Installing JEMalloc if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" - verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then + verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + else + msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)" + fi bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar" cd "jemalloc-${DEP_JEMALLOC_VERSION}" @@ -198,11 +202,15 @@ else msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}" fi -DEP_LIBEVENT_VERSION="2.1.12-stable" +DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" # Installing libevent if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" - verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then + verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + else + msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)" + fi tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" cd "libevent-${DEP_LIBEVENT_VERSION}" ./configure @@ -219,7 +227,7 @@ msg "Installing third-party dependencies ... DONE" # PyTorch does not provide official pre-built LibTorch C++ binaries for ARM64 # Linux via conda or download.pytorch.org/libtorch. The conda default channel # now ships CUDA-enabled libtorch (gpu_cuda130) even on aarch64, which fails -# on machines without CUDA (e.g., Grace). +# on machines without CUDA. # Instead, we install the CPU-only torch wheel via pip and extract the # libtorch cmake/headers/libs from the pip package. msg "Installing LibTorch via pip (CPU-only) for aarch64..." @@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # unset reproduces the v2 baseline (latest). + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh index e6abb5ad2..ae82c2a93 100755 --- a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh +++ b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh @@ -27,6 +27,7 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -34,7 +35,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync + sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing is # older than folly's minimum, so folly's io_uring integration links @@ -181,7 +182,7 @@ msg "Installing third-party dependencies ... DONE" # PyTorch does not provide official pre-built LibTorch C++ binaries for ARM64 # Linux via conda or download.pytorch.org/libtorch. The conda default channel # now ships CUDA-enabled libtorch (gpu_cuda130) even on aarch64, which fails -# on machines without CUDA (e.g., Grace). +# on machines without CUDA. # Instead, we install the CPU-only torch wheel via pip and extract the # libtorch cmake/headers/libs from the pip package. msg "Installing LibTorch via pip (CPU-only) for aarch64..." @@ -204,9 +205,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # empty falls back to pip's latest resolution. + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_ubuntu.sh b/packages/feedsim/install_feedsim_ubuntu.sh index f525f63ed..4237ea6b2 100755 --- a/packages/feedsim/install_feedsim_ubuntu.sh +++ b/packages/feedsim/install_feedsim_ubuntu.sh @@ -11,7 +11,14 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -30,7 +37,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq xxhash libxxhash-dev unzip rsync + sysstat jq xxhash libxxhash-dev unzip rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing (0.7 on # 20.04, 2.1 on 22.04) is older than folly's minimum, so folly's io_uring @@ -213,12 +220,31 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # The C++ libtorch inside (torch/lib, torch/share/cmake) is Python- + # version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/run.sh b/packages/feedsim/run.sh index 54b4d5d04..561b669e8 100644 --- a/packages/feedsim/run.sh +++ b/packages/feedsim/run.sh @@ -53,10 +53,8 @@ if [[ "$IS_SMT_ON" = 1 ]]; then else RANKING_THREADS_DEFAULT="$(( $(nproc) * 15/20))" # 15/20 is 0.75 cpu factor fi -# Driver threads = nproc/4 across SMT-on / SMT-off. Previously the SMT-on -# branch used nproc/5, which under-pinned driver work on big SMT boxes -# (e.g. BGM 176 logical → 35 vs 44 threads). Standardizing both branches -# at nproc/4 removes one source of cross-platform variance. +# Driver threads = nproc/4 across SMT-on / SMT-off, so both branches pin +# driver work consistently and avoid a source of cross-platform variance. DRIVER_THREADS="$(echo "scale=2; $(nproc) / 4.0 + 0.5 " | bc )" # rounds nearest DRIVER_THREADS="${DRIVER_THREADS%.*}" # Truncate decimal fraction. DRIVER_THREADS="$(echo "${BC_MAX_FN}; max(${DRIVER_THREADS:-0}, 4)" | bc )" # At least 4 threads. @@ -115,11 +113,12 @@ Usage: ${0##*/} [OPTION]... --client-num-dense Number of dense features per sample (client-side). Default: 13 --client-num-sparse Number of sparse features per sample (client-side). Default: 26 --mock-tls Enable TLS on outbound MockServicesClient channels (0=off, 1=on). Default: 1. - --mock-zstd-frac Fraction in [0.0, 1.0] of MockServicesClient channels with ZSTD enabled. Default: 0.75 (t43 c7). + --mock-zstd-frac Fraction in [0.0, 1.0] of MockServicesClient channels with ZSTD enabled. Default: 0.75. --mock-keepalive-interval-ms Per-MockServicesClient keepalive ping interval (ms). 0=disabled. Default: 200. - --rpc-fanout-scale Scale factor applied to per-session fanout counts. Default: 0.05 (t43 c7). - --server-zstd Enable ZSTD compression on server-side response payloads (0=off, 1=on). Default: 0 (t43 c7). + --rpc-fanout-scale Scale factor applied to per-session fanout counts. Default: 0.05. + --server-zstd Enable ZSTD compression on server-side response payloads (0=off, 1=on). Default: 0. --sla-p95-ms search_qps SLA target (95th percentile latency in ms). Default: 700. + --depth Driver pipeline depth: max outstanding requests per driver connection (max in-flight = driver_threads * connections * depth). Default: 1 (or \$FEEDSIM_DRIVER_DEPTH). Raise (e.g. 2) when the final phase saturates neither CPU nor SLA latency; with adaptive depth on, this is the starting floor the peak search raises from. EOF } @@ -295,9 +294,8 @@ main() { local stories_per_processor_pass stories_per_processor_pass="50" - # t43 c7 calibration knobs — promoted from env vars to CLI flags so they - # are visible in --help and surfaced in jobs.yml. Defaults match the - # balanced configuration documented in [[t43_c7_recommended]]. + # Calibration knobs — promoted from env vars to CLI flags so they + # are visible in --help and surfaced in jobs.yml. local mock_tls mock_tls="1" @@ -316,6 +314,12 @@ main() { local sla_p95_ms sla_p95_ms="700" + # Driver pipeline depth (max outstanding requests per driver connection). + # Env var FEEDSIM_DRIVER_DEPTH is the fallback default; the --depth CLI flag + # (forwarded from the benchpress `depth` job parameter) overrides it. + local driver_depth + driver_depth="${FEEDSIM_DRIVER_DEPTH:-1}" + if [ -z "$IS_AUTOSCALE_RUN" ]; then echo > $BREPS_LFILE fi @@ -648,6 +652,13 @@ main() { --sla-p95-ms=*) sla_p95_ms="${1#*=}" ;; + --depth) + driver_depth="$2" + shift + ;; + --depth=*) + driver_depth="${1#*=}" + ;; -h|--help) show_help >&2 exit 1 @@ -744,8 +755,8 @@ main() { fi # Build story-processor options. The story-processor mock module mirrors - # prod multifeed's scoring + filter + blend + serdes + topK passes (see - # ScoringPassProcessor, FilteringPassProcessor, ...). When + # a production ranking service's scoring + filter + blend + serdes + topK + # passes. When # story_processors_per_story > 0, LeafNodeRank's ThreadStartup # constructs a per-thread StoryProcessorSuite and runStoryProcessing # issues `num_stories * story_processors_per_story` pipeline passes @@ -809,11 +820,7 @@ main() { # User-facing knobs (--mock-tls, --mock-zstd-frac, --server-zstd, etc.) # are translated to ENV VARS here. The C++ binaries (MockServicesClient, # LeafNodeRank, FeedSimServer, FeedSimDriver) read these env vars via - # std::getenv at thread/server startup. A previous refactor tried to - # promote them to gengetopt CLI flags; that caused a silent - # MockServicesClient TLS handshake regression (every connection - # ECONNRESET, falling back to folly::futures::sleep). Until the C++ - # side adopts CLI flags safely, env-var plumbing is the proven path. + # std::getenv at thread/server startup. local mock_port="${MOCK_SERVICES_PORT:-21222}" local mock_services_opts="--rpc_dist_path=$rpc_dist_json" mock_services_opts="$mock_services_opts --mock_services_host=localhost" @@ -827,15 +834,56 @@ main() { if [ "$mock_tls" = "1" ]; then export MOCK_TLS=1 fi + # Driver↔Leaf TLS. When FEEDSIM_DRIVER_TLS=1 is passed in the environment, + # the DriverNodeRank client (reads FEEDSIM_DRIVER_TLS) wraps its + # bufferevents in OpenSSL, and the LeafNodeRank server needs a cert so + # FeedSimServer's accept path enables AsyncSSLSocket — wire the shared + # example cert/key into FEEDSIM_TLS_CERT/FEEDSIM_TLS_KEY. Both ends pin + # AES-GCM so the encrypted driver↔leaf volume registers as hardware crypto + # (matches prod's Rocket-over-TLS driver path). Default off. The LeafNodeRank + # server reads FEEDSIM_TLS_CERT/KEY from the exported env; the DriverNodeRank + # client is launched through search_qps.sh, which does not reliably forward + # the parent's exported env to the driver process, so FEEDSIM_DRIVER_TLS is + # also injected directly on the driver command line via `driver_bin`. + # Per-run driver knobs are injected on the DriverNodeRank command line via + # `env VAR=val` (accumulated in driver_env), because search_qps.sh launches + # the driver as a bare `$command &` that does not reliably inherit the + # parent shell's exported env. + driver_env="" + if [ "${FEEDSIM_DRIVER_TLS:-0}" = "1" ]; then + driver_cert_dir="${FEEDSIM_ROOT}/certs" + if [ ! -r "${driver_cert_dir}/example.crt" ] || [ ! -r "${driver_cert_dir}/example.key" ]; then + echo "ERROR: FEEDSIM_DRIVER_TLS=1 but ${driver_cert_dir}/example.{crt,key} not found" >&2 + exit 1 + fi + export FEEDSIM_TLS_CERT="${driver_cert_dir}/example.crt" + export FEEDSIM_TLS_KEY="${driver_cert_dir}/example.key" + export FEEDSIM_DRIVER_TLS=1 + driver_env="${driver_env} FEEDSIM_DRIVER_TLS=1" + echo "Driver↔Leaf TLS: ENABLED (cert=${driver_cert_dir}/example.crt, AES-GCM)" + fi + # FEEDSIM_STATS_WARMUP_SECS: drop the first N seconds of latency/throughput + # samples in each search_qps probe (DriverNodeRank resets its stats N secs + # in) so cold-start transients don't inflate the tail and make the search + # back off QPS prematurely. Default unset/0 (no warmup). + if [ -n "${FEEDSIM_STATS_WARMUP_SECS:-}" ] && [ "${FEEDSIM_STATS_WARMUP_SECS}" != "0" ]; then + driver_env="${driver_env} FEEDSIM_STATS_WARMUP_SECS=${FEEDSIM_STATS_WARMUP_SECS}" + echo "Driver stats warmup: ${FEEDSIM_STATS_WARMUP_SECS}s (dropping cold-start samples per probe)" + fi + if [ -n "$driver_env" ]; then + driver_bin="env${driver_env} build/workloads/ranking/DriverNodeRank" + else + driver_bin="build/workloads/ranking/DriverNodeRank" + fi # MOCK_ZSTD_FRAC env consumed by MockServicesClient::resolveZstdFraction. - # Always export so the t43 c7 default 0.75 reaches the binary. + # Always export so the configured default reaches the binary. export MOCK_ZSTD_FRAC="$mock_zstd_frac" echo "MockServicesClient: TLS=${mock_tls} ZSTD_frac=${mock_zstd_frac} keepalive_ms=${mock_keepalive_interval_ms} fanout_scale=${rpc_fanout_scale}" # Server-side response compression. FEEDSIM_SERVER_ZSTD=0 disables # compressThrift/compressPayload (server bytes emitted uncompressed). # Default ON in the C++ source; we export "0" when user passes - # --server-zstd=0 (the t43 c7 default). + # --server-zstd=0. if [ "$server_zstd" != "1" ]; then export FEEDSIM_SERVER_ZSTD=0 echo "Server-side response ZSTD: DISABLED (FEEDSIM_SERVER_ZSTD=0)" @@ -848,8 +896,7 @@ main() { # parallel backend; OpenMP-backed builds (which Meta's internal # libtorch may use) read OMP_NUM_THREADS directly. Without this, # each ThriftSrv.IO worker calling forward() spawns nproc OMP - # threads, accumulating to nproc^2 GlobalCPUThread-named threads - # (= 7,744 on BGM per-instance after taskset). See t14 progress log. + # threads, accumulating to nproc^2 GlobalCPUThread-named threads. # shellcheck disable=SC2086 env $preload_env OMP_NUM_THREADS=1 MALLOC_CONF=narenas:20,dirty_decay_ms:5000 build/workloads/ranking/LeafNodeRank \ --port="$port" \ @@ -955,20 +1002,35 @@ main() { log_preprocessing_end "$BREAKDOWN_FOLDER" "$$" # SLA target for search_qps (95p latency in milliseconds). Default 700ms - # matches the prod multifeed aggregator's own end-to-end budget at p95. + # matches the production aggregator's own end-to-end budget at p95. # Override via --sla-p95-ms CLI flag (handled in arg parsing above). sla_arg="95p:${sla_p95_ms}" + # Adaptive driver depth (fleet default): search_qps raises the driver's + # pipeline --depth in the peak phase until the server saturates (system + # CPU>=95% or p95>=SLA), giving each platform just enough offered concurrency + # to reach a real bound instead of capping on driver concurrency. + # Enabled by default up to depth 8; set FEEDSIM_ADAPTIVE_DEPTH_MAX=0 to + # disable and use the fixed driver_depth (--depth flag / FEEDSIM_DRIVER_DEPTH, + # default 1). When adaptive is on, driver_depth is the STARTING floor the peak + # search raises from (search_qps reads the --depth we pass below). + sqps_adaptive_arg="" + adaptive_depth_max="${FEEDSIM_ADAPTIVE_DEPTH_MAX:-8}" + if [ "$adaptive_depth_max" != "0" ]; then + sqps_adaptive_arg="-D ${adaptive_depth_max}" + fi + if [ -z "$fixed_qps" ] && [ "$auto_driver_threads" != "1" ]; then benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + scripts/search_qps.sh -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${driver_threads}" \ --connections=4 \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -977,10 +1039,11 @@ main() { benchreps_tell_state "before search_qps" echo "search_qps SLA: ${sla_arg}" # shellcheck disable=SC2086 - scripts/search_qps.sh -a -w 15 -f 300 -s "$sla_arg" -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ - build/workloads/ranking/DriverNodeRank \ + scripts/search_qps.sh -a -t "${FEEDSIM_EXPERIMENT_TIME:-120}" -w 15 -f 300 -s "$sla_arg" $sqps_adaptive_arg -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" $qps_threshold_args $no_retry_args -o "${FEEDSIM_ROOT}/${result_filename}" -- \ + $driver_bin \ --monitor_port "$client_monitor_port" \ --server "0.0.0.0:$port" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts @@ -1004,11 +1067,12 @@ main() { -P "$LEAF_PID" -B "$BREAKDOWN_FOLDER" \ $qps_threshold_args $no_retry_args \ -o "${FEEDSIM_ROOT}/${result_filename}" \ - -- build/workloads/ranking/DriverNodeRank \ + -- $driver_bin \ --server "0.0.0.0:$port" \ --monitor_port "$client_monitor_port" \ --threads="${num_workers}" \ --connections="${num_connections}" \ + --depth="${driver_depth}" \ $client_feature_opts \ $silesia_opts \ $req_size_opts diff --git a/packages/feedsim/third_party/src/scripts/search_qps.sh b/packages/feedsim/third_party/src/scripts/search_qps.sh index b811794d4..d6111b4d9 100755 --- a/packages/feedsim/third_party/src/scripts/search_qps.sh +++ b/packages/feedsim/third_party/src/scripts/search_qps.sh @@ -31,6 +31,26 @@ echo "${SCRIPT_NAME}: DCPERF_PERF_RECORD=${DCPERF_PERF_RECORD}" function benchreps_tell_state () { date +"%Y-%m-%d_%T ${1}" >> $BREPS_LFILE } + +# ─── CPU utilization helpers (used by adaptive depth) ──────────────────────── +# Read /proc/stat's aggregate cpu line and echo "total idle_all" jiffies. +cpu_snapshot() { + local cpu u n s idle iow irq sirq st rest + read -r cpu u n s idle iow irq sirq st rest < /proc/stat + local idle_all=$((idle + iow)) + local total=$((u + n + s + idle + iow + irq + sirq + st)) + echo "$total $idle_all" +} +# System-wide CPU busy% (100 - idle%) measured over the next $1 seconds. +cpu_busy_over() { + local secs="$1" s1 s2 t1 i1 t2 i2 dt di + s1=$(cpu_snapshot); t1=${s1% *}; i1=${s1#* } + sleep "$secs" + s2=$(cpu_snapshot); t2=${s2% *}; i2=${s2#* } + dt=$((t2 - t1)); di=$((i2 - i1)) + if [ "$dt" -le 0 ]; then echo "0"; return; fi + echo "scale=1; (($dt - $di) * 100) / $dt" | bc +} # Source runtime breakdown utilities if they exist if [ -f "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" ]; then source "${BENCHPRESS_ROOT}/packages/common/runtime_breakdown_utils.sh" @@ -99,6 +119,12 @@ mutilate (EuroSys \'14) [https://github.com/leverich/mutilate] without retrying. Optional -P PID of the process to log runtime breakdowns. Optional -B Folder to log runtime breakdowns. Optional + -D Adaptive depth: max driver pipeline depth. When set, the peak + phase raises the driver's --depth until the server is saturated + (system CPU >= 95% OR achieved p95 >= SLA), then holds that depth + for the QPS search. The search STARTS from any --depth in the + driver command (default 1), so a manually-set --depth acts as a + floor. Optional. EOF } @@ -145,7 +171,7 @@ run_loadtest() { for r in $(seq 1 $load_test_retries); do # run the command, saving result to tmpfile local tmp_file=$(mktemp) - $command $threads_arg $qps_arg &>$tmp_file & + $command $threads_arg $qps_arg $adaptive_depth_arg &>$tmp_file & LOADTEST_PID=$! if [ "$no_retry_mode" = "1" ]; then @@ -301,14 +327,19 @@ max_warmup_iterations=10 no_retry_mode="" breakdown_pid="" breakdown_folder="" +adaptive_depth_max="" # -D: when set, search_qps raises driver --depth in the +adaptive_depth_arg="" # peak phase until the server saturates (CPU>=95% or p95>=SLA) OPTIND=1 # Reset is necessary if getopts was used previously in the script. It is a good idea to make this local in a function. -while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:" opt; do +while getopts "ht:f:w:m:s:q:ao:r:x:NP:B:D:" opt; do case "$opt" in h) show_help exit 0 ;; + D) + adaptive_depth_max=$OPTARG + ;; t) experiment_time=$OPTARG ;; @@ -364,6 +395,20 @@ fi # remaining argument is loadtest command command=$@ +# In adaptive-depth mode, search_qps owns the driver's --depth: capture any fixed +# --depth as the STARTING depth (so a manually-set --depth acts as a floor the +# peak search raises from), then strip it so our per-attempt --depth is the only +# one on the command. Default start is depth=1 (unchanged behavior). +adaptive_start_depth=1 +if [ -n "$adaptive_depth_max" ]; then + fixed_depth=$(echo "$command" | grep -oE -- '--depth=[0-9]+' | head -1 | grep -oE '[0-9]+') + if [ -n "$fixed_depth" ] && [ "$fixed_depth" -gt 1 ]; then + adaptive_start_depth=$fixed_depth + fi + command=$(echo "$command" | sed -E 's/[[:space:]]*--depth=[0-9]+//g') + adaptive_depth_arg="--depth=$adaptive_start_depth" +fi + # make sure latency_type and latency_target are specified if [[ -z "$fixed_qps" ]] && ( [[ $latency_type = "" ]] || [[ $latency_target = "" ]] ); then echo 'error: -s metric:target must be specified' >&2; exit 1 @@ -501,7 +546,7 @@ if [[ -n "$fixed_qps" ]]; then else for fixed_qps_el in $fixed_qps_array; do benchreps_tell_state "before fixed_qps_iter $fixed_qps_el" - run_loadtest measured_qps measured_latency $fixed_qps_el "" + run_loadtest measured_qps measured_latency $fixed_qps_el "$main_operation_name" printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $fixed_qps_el $measured_qps $measured_latency echo "final requested_qps = $fixed_qps_el, measured_qps = $measured_qps, latency = $measured_latency" >> $BREPS_LFILE benchreps_tell_state "after fixed_qps_iter $fixed_qps_el" @@ -513,7 +558,42 @@ fi # find peak QPS benchreps_tell_state "before peak_qps" -run_loadtest peak_qps measured_latency "" "" +if [ -n "$adaptive_depth_max" ]; then + # Adaptive depth: the peak load test offers at most threads*connections*depth + # concurrent requests. Starting at adaptive_start_depth, keep raising depth (and + # re-running peak) until the server saturates — system CPU >= 95% OR p95 >= SLA + # — so platforms that need more offered concurrency reach a real bound instead + # of capping on driver concurrency. The selected depth is then held for the + # QPS search / tuning / final phases. The starting depth + # is the fixed --depth from the driver command (default 1), so a manually-set + # depth raises the floor. + cur_depth=$adaptive_start_depth + while : ; do + adaptive_depth_arg="--depth=$cur_depth" + # Sample system CPU busy% over a mid-run window while the peak load runs. + cpu_busy_file="/tmp/adaptive_cpu_busy_$$" + ( sleep 20; cpu_busy_over 40 > "$cpu_busy_file" ) & + cpu_sampler_pid=$! + run_loadtest peak_qps measured_latency "" "" + wait "$cpu_sampler_pid" 2>/dev/null + cpu_busy=$(cat "$cpu_busy_file" 2>/dev/null || echo 0) + rm -f "$cpu_busy_file" + cpu_sat=$(echo "${cpu_busy:-0} >= 95" | bc 2>/dev/null || echo 0) + lat_sat=$(echo "$measured_latency >= $latency_target" | bc 2>/dev/null || echo 0) + printf "adaptive-depth: depth=%d peak_qps=%.2f p95=%.2f cpu_busy=%s%% cpu_sat=%s lat_sat=%s\n" \ + "$cur_depth" "$peak_qps" "$measured_latency" "${cpu_busy:-0}" "$cpu_sat" "$lat_sat" + echo "adaptive-depth: depth=$cur_depth peak_qps=$peak_qps p95=$measured_latency cpu_busy=${cpu_busy}% cpu_sat=$cpu_sat lat_sat=$lat_sat" >> $BREPS_LFILE + if [ "$cpu_sat" -eq 1 ] || [ "$lat_sat" -eq 1 ] || [ "$cur_depth" -ge "$adaptive_depth_max" ]; then + break + fi + cur_depth=$((cur_depth + 1)) + sleep "$wait_time" + done + echo "adaptive-depth: SELECTED depth=$cur_depth (cpu_busy=${cpu_busy}%, p95=$measured_latency, sla=$latency_target)" >> $BREPS_LFILE + printf "adaptive-depth: selected depth=%d (cpu_busy=%s%%, p95=%.2f)\n" "$cur_depth" "${cpu_busy:-0}" "$measured_latency" +else + run_loadtest peak_qps measured_latency "" "" +fi printf "peak qps = %.2f, latency = %.2f\n" $peak_qps $measured_latency benchreps_tell_state "after peak_qps" @@ -539,19 +619,49 @@ while [[ $loop_cond -eq 1 ]]; do run_loadtest measured_qps measured_latency $cur_qps "" printf "requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $cur_qps $measured_qps $measured_latency - # set new QPS ranges + # Update search bounds using measured_qps (what the server actually sustained) + # rather than cur_qps (what the driver requested). On a server that saturates + # below cur_qps, using cur_qps as the "achievable" lower bound falsely inflates + # low_qps to a value the server never delivered. + # + # latency BAD: + # - measured_qps >= cur_qps: high_qps = measured_qps (server delivered at + # least what was asked; latency is the ceiling at that measured rate). + # - measured_qps < cur_qps: high_qps = measured_qps + (cur_qps - measured_qps) * SLA / latency + # Rationale: when cur_qps is much larger than what the server can fulfill, + # the surplus offered load queues up and exaggerates latency. The server + # may deliver ~measured_qps at acceptable latency if the offered rate is + # just slightly higher than measured (not far above it). SLA/latency scales + # the search headroom by how badly latency violated SLA — barely-over-SLA + # probes leave more room; badly-over-SLA probes shrink toward measured. + # + # latency GOOD: + # - low_qps = measured_qps (measured is a sustained floor). + # - If cur_qps > 1.01 * measured_qps (any noticeable gap between requested + # and delivered — server at or near its ceiling), shrink high_qps by 0.96, + # but never let high_qps drop below low_qps. Capping at low_qps ensures the + # search terminates cleanly when it has converged (loop condition + # high > low * 1.02 becomes false) instead of oscillating in a narrow + # window near max_iters. latency_good=$(echo "$measured_latency <= $latency_target" | bc) if [[ $latency_good -eq 0 ]]; then - high_qps=$cur_qps - else - low_qps=$cur_qps - measured_qps_is_higher=$(echo "$measured_qps > $low_qps" | bc) - if [[ $measured_qps_is_higher -eq 1 ]] ; then - low_qps=$measured_qps + measured_ge_requested=$(echo "$measured_qps >= $cur_qps" | bc) + if [[ $measured_ge_requested -eq 1 ]]; then + high_qps=$measured_qps + else + high_qps=$(echo "scale=5; $measured_qps + ($cur_qps - $measured_qps) * $latency_target / $measured_latency" | bc) fi - measured_qps_gap=$(echo "$cur_qps > $measured_qps * 1.02" | bc) + else + low_qps=$measured_qps + measured_qps_gap=$(echo "$cur_qps > $measured_qps * 1.01" | bc) if [[ $measured_qps_gap -eq 1 ]] ; then - high_qps=$(echo "scale=5; $high_qps*0.96" | bc) + new_high_qps=$(echo "scale=5; $high_qps * 0.96" | bc) + high_below_low=$(echo "$new_high_qps < $low_qps" | bc) + if [[ $high_below_low -eq 1 ]]; then + high_qps=$low_qps + else + high_qps=$new_high_qps + fi fi fi @@ -611,13 +721,17 @@ if [[ -n "$IS_AUTOSCALE_RUN" ]] && [[ "$IS_AUTOSCALE_RUN" -gt 1 ]]; then fi fi -# do final measurement +# do final measurement — this is the actual reported experiment window; log it +# as main_benchmark in breakdown.csv so perfpub sees a valid time window (the +# search/warmup/tuning probes above are informational and should not be logged +# as the primary benchmark window). $main_operation_name comes from +# packages/common/runtime_breakdown_utils.sh (sourced at the top of this script). benchreps_tell_state "before final_qps" experiment_time=$final_experiment_time if [ "${DCPERF_PERF_RECORD}" = 1 ] && ! [ -f "perf.data" ]; then collect_perf_record & fi -run_loadtest measured_qps measured_latency $cur_qps "" +run_loadtest measured_qps measured_latency $cur_qps "$main_operation_name" printf "final requested_qps = %.2f, measured_qps = %.2f, latency = %.2f\n" $cur_qps $measured_qps $measured_latency # report non-converging error if iteration reaches max tries diff --git a/packages/feedsim/third_party/src/workloads/ranking/DriverNodeRank.cc b/packages/feedsim/third_party/src/workloads/ranking/DriverNodeRank.cc index 5481700d0..e2924c6e3 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/DriverNodeRank.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/DriverNodeRank.cc @@ -13,6 +13,8 @@ // limitations under the License. #include +#include +#include #include #include #include @@ -60,18 +62,47 @@ static std::unique_ptr g_rpc_dist_registry; // Phase 6: per-driver-thread session orchestration executor. Sized // num_threads (one logical session per driver thread, mirroring the -// legacy 1-cb-per-thread make-request loop). Named so Strobelight -// categorizes the sessions cleanly. +// legacy 1-cb-per-thread make-request loop). Named so profilers +// categorize the sessions cleanly. static std::shared_ptr g_session_pool; const int kMaxRequestSize = 8192; const int kRecomputeQPSPeriod = 1; // Reduced from 5 to 1 second for faster feedback +// Deterministic per-thread RNG seed. The driver's per-thread RNGs used to be +// seeded from std::random_device, making the generated request stream (sizes, +// content) non-reproducible run-to-run. Seed from a fixed base so each thread +// keeps an independent but reproducible sequence. Set FEEDSIM_RNG_RANDOM=1 to +// restore the old non-deterministic seed. +static bool feedsimRngRandom() { + static const bool kRandom = [] { + const char* e = std::getenv("FEEDSIM_RNG_RANDOM"); + return e != nullptr && e[0] == '1'; + }(); + return kRandom; +} + +static unsigned detRngSeed(unsigned base) { + if (feedsimRngRandom()) { + return std::random_device{}(); + } + static std::atomic ctr{0}; + return base + ctr.fetch_add(1) * 2654435761u; +} + +static unsigned detRngSeedTid(unsigned base, int thread_id) { + if (feedsimRngRandom()) { + return static_cast(std::random_device{}()) + + static_cast(thread_id); + } + return base + static_cast(thread_id); +} + // Simple random string generator (replaces oldisim/Util.h RandomString) static std::string RandomString(size_t length) { static const char charset[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - thread_local std::mt19937 rng(std::random_device{}()); + thread_local std::mt19937 rng(detRngSeed(0xD5117A2Du)); std::uniform_int_distribution dist(0, sizeof(charset) - 2); std::string str(length, 0); for (size_t i = 0; i < length; ++i) { @@ -202,15 +233,13 @@ void ThreadStartup(int thread_id, // Initialize Silesia RNG per thread if (g_silesia_loader && g_silesia_loader->isLoaded()) { - this_thread.silesia_rng.seed( - static_cast(std::random_device{}()) + thread_id); + this_thread.silesia_rng.seed(detRngSeedTid(0x51E51A00u, thread_id)); } // Initialize request size sampler RNG per thread if ((g_req_size_sampler && g_req_size_sampler->isLoaded()) || g_rpc_dist_registry) { - this_thread.req_size_rng.seed( - static_cast(std::random_device{}()) + 0xDEADBEEF + thread_id); + this_thread.req_size_rng.seed(detRngSeedTid(0xDEADBEEFu, thread_id)); } // If user gave QPS target, initialize QPS modulation @@ -574,7 +603,7 @@ std::string encodeGetAllStories( // ─── RunSession ───────────────────────────────────────────────────────────── // -// Run one driver session per the prod multifeed_aggregator pipeline: +// Run one driver session per the production aggregator pipeline: // t=0 createAndPrimeSession AWAIT // t≈3ms getStoriesUncompressed HOLD future // t≈3ms streamData (parallel x N) AWAIT @@ -808,7 +837,7 @@ int main(int argc, char **argv) { // Phase 6: spin up the session orchestration pool. One worker per // driver thread mirrors the legacy 1-cb-per-thread model. Named so - // Strobelight categorizes the threads cleanly. + // profilers categorize the threads cleanly. if (args.rpc_dist_json_given) { g_session_pool = std::make_shared( args.threads_arg, diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc index 2d1119a4e..e96970b18 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimDriver.cc @@ -227,7 +227,7 @@ void DriverStats::printStats(uint32_t type, double elapsed_secs) const { // global (one per process), constructed lazily on first use to avoid paying // the OpenSSL init cost when TLS is off. Closes the bench's Encryption CPU // undershoot on the driver↔server channel (paired with FeedSimServer's -// FEEDSIM_TLS_CERT / FEEDSIM_TLS_KEY env vars). See t41 progress log. +// FEEDSIM_TLS_CERT / FEEDSIM_TLS_KEY env vars). namespace { SSL_CTX* getDriverSslCtxOrNull() { static SSL_CTX* s_ctx = []() -> SSL_CTX* { @@ -246,6 +246,17 @@ SSL_CTX* getDriverSslCtxOrNull() { return nullptr; } SSL_CTX_set_verify(ctx, SSL_VERIFY_NONE, nullptr); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES via libcrypto, matching prod's cipher. Without this, + // OpenSSL may pick ChaCha20-Poly1305, a cipher with no hardware-AES path, + // which runs the AEAD un-accelerated. Mirrors MockServicesClient's cipher + // pinning. + SSL_CTX_set_cipher_list( + ctx, + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx, "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 std::cout << "FeedSimDriver: TLS enabled via FEEDSIM_DRIVER_TLS=1" << std::endl; return ctx; @@ -852,6 +863,17 @@ void FeedSimDriver::enableMonitoring(uint16_t port) { impl_->monitor_port = port; } +namespace { +// One-shot timer callback used by FEEDSIM_STATS_WARMUP_SECS: drops all +// latency/throughput samples collected so far so the final stats reflect only +// the post-warmup (steady-state) window. Runs on the DriverThread's own event +// base, so it never races that thread's logRequest()/logResponse() writes to +// the same DriverStats. +void resetDriverStatsCb(evutil_socket_t, short, void* arg) { + reinterpret_cast(arg)->reset(); +} +} // namespace + void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, uint32_t num_connections_per_thread, uint32_t max_connection_depth) { @@ -864,6 +886,23 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, impl_->running = true; impl_->total_stats = std::make_unique(1000); + // FEEDSIM_STATS_WARMUP_SECS: when >0, each DriverThread drops the samples it + // collected in the first N seconds of the measurement window (a per-thread + // one-shot timer calls DriverStats::reset()). This excludes cold-start + // transients (connection ramp, cache/JIT warmup, first-touch faults) from + // the reported latency distribution, giving search_qps a more stable p95 so + // it doesn't back off QPS on a noisy tail. + int stats_warmup_secs = 0; + { + const char* e = std::getenv("FEEDSIM_STATS_WARMUP_SECS"); + if (e != nullptr && e[0] != '\0') { + stats_warmup_secs = std::atoi(e); + if (stats_warmup_secs < 0) { + stats_warmup_secs = 0; + } + } + } + // Barrier for thread init synchronization pthread_barrier_t init_barrier; pthread_barrier_init(&init_barrier, nullptr, num_threads + 1); @@ -909,7 +948,7 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start thread dt->thread = std::thread([this, &dt_ref = *dt, &init_barrier, - thread_pinning, i]() { + thread_pinning, i, stats_warmup_secs]() { // CPU affinity if (thread_pinning) { cpu_set_t mask; @@ -941,6 +980,18 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, // Start making requests TestDriver::Impl::makeRequests(*dt_ref.driver); + // Schedule the cold-start stats reset on this thread's own base (added + // here, before dispatch, so it fires from this thread — no data race with + // the stats writes). Fires once ~stats_warmup_secs into steady traffic. + if (stats_warmup_secs > 0) { + struct timeval warmup_tv { + stats_warmup_secs, 0 + }; + event_base_once( + dt_ref.base, -1, EV_TIMEOUT, resetDriverStatsCb, + &dt_ref.driver->impl_->current_stats, &warmup_tv); + } + // Run event loop event_base_dispatch(dt_ref.base); }); @@ -1058,7 +1109,14 @@ void FeedSimDriver::run(uint32_t num_threads, bool thread_pinning, } double end_time = getTimeSec(); - double elapsed = end_time - start_time; + // With a stats warmup, samples were reset ~stats_warmup_secs into the run, so + // the throughput denominator must be the post-warmup window (else QPS would + // be understated by counting post-warmup queries over the full duration). + double meas_start = start_time; + if (stats_warmup_secs > 0) { + meas_start = start_time + stats_warmup_secs; + } + double elapsed = end_time - meas_start; // Aggregate stats from all threads for (auto& dt : impl_->threads) { diff --git a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc index ff58f81a2..ed1d4221c 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/FeedSimServer.cc @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -35,17 +36,30 @@ #include #include +#include + namespace feedsim { +class ServerConnection; + // ─── RequestContext implementation ────────────────────────────────────────── struct RequestContext::Impl { - // The socket fd to write the response back on. - // We use raw fd + write() because the response is a single small write - // and we want to avoid the complexity of AsyncSocket write callbacks. + // Plaintext path: the socket fd to write the response back on. We use raw + // fd + writev() because the response is a single small write and we want to + // avoid the complexity of AsyncSocket write callbacks. int fd; uint64_t received_time; bool response_sent; + // TLS path: raw fd writes bypass the TLS layer (they hit the TCP socket + // underneath AsyncSSLSocket, so the peer receives plaintext on an encrypted + // connection). When tls is true, the response must be written THROUGH the + // AsyncSSLSocket on its owning EventBase instead. sendResponse marshals the + // write onto evb and targets the connection via a weak_ptr so a response + // that completes after the connection closed is dropped safely. + bool tls = false; + folly::EventBase* evb = nullptr; + std::weak_ptr conn; }; RequestContext::RequestContext( @@ -69,6 +83,17 @@ RequestContext::RequestContext(RequestContext&& other) noexcept RequestContext::~RequestContext() = default; +// Marshals a TLS response (header+payload already framed in buf) onto the +// connection's EventBase and writes it through the AsyncSSLSocket. Defined +// after ServerConnection (needs its full type); declared here so sendResponse +// can call it. Safe if the connection has already closed. +namespace { +void enqueueTlsResponse( + std::weak_ptr conn, + folly::EventBase* evb, + std::unique_ptr buf); +} // namespace + void RequestContext::sendResponse(const void* data, uint32_t data_length) { if (!impl_ || impl_->response_sent) return; impl_->response_sent = true; @@ -85,7 +110,21 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { ResponsePacketHeader net = responseToNetwork(hdr); - // Use writev to send header + payload atomically + if (impl_->tls) { + // Raw fd writes bypass TLS, so build one contiguous frame (copying the + // payload synchronously — the caller may free `data` after we return) and + // hand it to the connection's EventBase to write through AsyncSSLSocket. + auto buf = folly::IOBuf::create(sizeof(net) + data_length); + memcpy(buf->writableData(), &net, sizeof(net)); + if (data_length > 0) { + memcpy(buf->writableData() + sizeof(net), data, data_length); + } + buf->append(sizeof(net) + data_length); + enqueueTlsResponse(impl_->conn, impl_->evb, std::move(buf)); + return; + } + + // Plaintext path (unchanged): use writev to send header + payload atomically struct iovec iov[2]; iov[0].iov_base = &net; iov[0].iov_len = sizeof(net); @@ -122,15 +161,19 @@ void RequestContext::sendResponse(const void* data, uint32_t data_length) { // ─── ServerConnection: handles framing for one client connection ──────────── -class ServerConnection : public folly::AsyncTransport::ReadCallback { +class ServerConnection + : public folly::AsyncTransport::ReadCallback, + public std::enable_shared_from_this { public: ServerConnection( folly::AsyncSocket::UniquePtr socket, int thread_id, - const folly::F14FastMap& callbacks) + const folly::F14FastMap& callbacks, + bool tls) : socket_(std::move(socket)), thread_id_(thread_id), callbacks_(callbacks), + tls_(tls), read_buf_(nullptr), read_buf_size_(0), data_offset_(0) { @@ -144,6 +187,17 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { delete[] read_buf_; } + // Called once right after construction so the connection keeps itself alive + // while registered as the socket's read callback (broken on EOF/error). + void attachSelf(std::shared_ptr self) { + self_ = std::move(self); + } + + // Writes a fully-framed TLS response through the AsyncSSLSocket. MUST be + // invoked on the socket's EventBase thread. Defined out-of-line below + // (needs TlsWriteCallback's full definition). + void writeResponse(std::unique_ptr buf); + // AsyncTransport::ReadCallback void getReadBuffer(void** bufReturn, size_t* lenReturn) override { // Grow buffer if needed @@ -165,15 +219,14 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } void readEOF() noexcept override { - // Client disconnected - socket_->close(); - // Self-delete via destroy callback (see below) - delete this; + // Client disconnected. Break the self-reference so the object is destroyed + // once any in-flight TLS write callbacks release their refs. Hold a local + // ref so `this` stays valid until we return from the callback. + close(); } void readErr(const folly::AsyncSocketException& ex) noexcept override { - socket_->close(); - delete this; + close(); } private: @@ -199,6 +252,13 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { impl->fd = socket_->getNetworkSocket().toFd(); impl->received_time = getTimeNano(); impl->response_sent = false; + impl->tls = tls_; + if (tls_) { + // The response may be produced asynchronously on a pool thread; it + // must be written back through the AsyncSSLSocket on this EventBase. + impl->evb = socket_->getEventBase(); + impl->conn = weak_from_this(); + } RequestContext ctx( hdr.type, hdr.request_id, hdr.start_time, @@ -216,14 +276,112 @@ class ServerConnection : public folly::AsyncTransport::ReadCallback { } } + void close() { + socket_->close(); + // Hold a local ref so `this` survives until we return, then drop the + // self-reference. If TLS write callbacks are still outstanding they hold + // their own refs and the object lives until they complete. + auto keepalive = shared_from_this(); + self_.reset(); + } + folly::AsyncSocket::UniquePtr socket_; int thread_id_; const folly::F14FastMap& callbacks_; + bool tls_; + std::shared_ptr self_; uint8_t* read_buf_; size_t read_buf_size_; size_t data_offset_; }; +// ─── TLS response write path ──────────────────────────────────────────────── + +namespace { +// Keeps the connection alive (via shared_ptr) until the AsyncSSLSocket finishes +// encrypting and writing the response, then deletes itself. +class TlsWriteCallback : public folly::AsyncWriter::WriteCallback { + public: + explicit TlsWriteCallback(std::shared_ptr conn) + : conn_(std::move(conn)) {} + void writeSuccess() noexcept override { delete this; } + void writeErr( + size_t /*bytesWritten*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; + } + + private: + std::shared_ptr conn_; +}; + +void enqueueTlsResponse( + std::weak_ptr weak, + folly::EventBase* evb, + std::unique_ptr buf) { + if (evb == nullptr) { + return; + } + evb->runInEventBaseThread( + [weak = std::move(weak), buf = std::move(buf)]() mutable { + auto conn = weak.lock(); + if (!conn) { + return; // connection closed before the response was ready + } + conn->writeResponse(std::move(buf)); + }); +} +} // namespace + +void ServerConnection::writeResponse(std::unique_ptr buf) { + // Runs on the socket's EventBase thread. The callback holds a ref that keeps + // this connection alive until the encrypted write completes. + auto* cb = new TlsWriteCallback(shared_from_this()); + socket_->writeChain(cb, std::move(buf)); +} + +// ─── SslAcceptor: drives the server-side TLS handshake before wire reads ───── + +// A folly server-side AsyncSSLSocket does NOT auto-handshake when you merely +// setReadCB — sslAccept() must be called to run the handshake. Until it +// completes the socket can neither decrypt requests nor send responses. This +// helper owns the socket during the handshake and, on success, hands it to a +// ServerConnection (which then starts reading). It self-deletes either way. +class SslAcceptor : public folly::AsyncSSLSocket::HandshakeCB { + public: + SslAcceptor( + folly::AsyncSSLSocket::UniquePtr socket, + int thread_id, + const folly::F14FastMap& callbacks) + : socket_(std::move(socket)), + thread_id_(thread_id), + callbacks_(callbacks) {} + + void start() { + auto* raw = socket_.get(); + raw->sslAccept(this); + } + + void handshakeSuc(folly::AsyncSSLSocket* /*sock*/) noexcept override { + folly::AsyncSocket::UniquePtr base(socket_.release()); + auto conn = std::make_shared( + std::move(base), thread_id_, callbacks_, /*tls=*/true); + conn->attachSelf(conn); + delete this; + } + + void handshakeErr( + folly::AsyncSSLSocket* /*sock*/, + const folly::AsyncSocketException& /*ex*/) noexcept override { + delete this; // socket_ (and the fd) torn down with it + } + + private: + folly::AsyncSSLSocket::UniquePtr socket_; + int thread_id_; + const folly::F14FastMap& callbacks_; +}; + // ─── WorkerThread ─────────────────────────────────────────────────────────── struct WorkerThread { @@ -268,27 +426,31 @@ class AcceptCallback : public folly::AsyncServerSocket::AcceptCallback { // (FEEDSIM_TLS_CERT/FEEDSIM_TLS_KEY env vars set at server startup), // wrap the accepted fd in folly::AsyncSSLSocket so the TLS handshake // is performed on the worker's EventBase before any wire reads. This - // closes the bench's Encryption CPU undershoot (prod ~3.3-3.6% vs - // bench ~0.9-1.5% in t41). Plain AsyncSocket preserves the original - // no-TLS behavior when the env vars are unset. + // closes the bench's Encryption CPU undershoot. Plain AsyncSocket + // preserves the original no-TLS behavior when the env vars are unset. auto ssl_ctx = ssl_ctx_; worker->evb->runInEventBaseThread( [fd, thread_id = worker->thread_id, &callbacks = callbacks_, evb = worker->evb.get(), ssl_ctx]() { - folly::AsyncSocket::UniquePtr socket; if (ssl_ctx) { - // AsyncSSLSocket server-side: pass true for the server flag. - // The handshake is initiated lazily on first read/write, - // matching the existing client's connect-then-write pattern. + // AsyncSSLSocket server-side: pass true for the server flag, then + // drive the handshake via sslAccept (SslAcceptor). The + // ServerConnection is created only after the handshake succeeds — + // reading/writing before that would see undecrypted bytes. folly::AsyncSSLSocket::UniquePtr ssl_sock(new folly::AsyncSSLSocket( ssl_ctx, evb, folly::NetworkSocket::fromFd(fd), true)); - socket.reset(ssl_sock.release()); + auto* acceptor = + new SslAcceptor(std::move(ssl_sock), thread_id, callbacks); + acceptor->start(); } else { - socket = folly::AsyncSocket::newSocket( + folly::AsyncSocket::UniquePtr socket = folly::AsyncSocket::newSocket( evb, folly::NetworkSocket::fromFd(fd)); + // ServerConnection keeps itself alive via a self-reference (set by + // attachSelf) until EOF/error. + auto conn = std::make_shared( + std::move(socket), thread_id, callbacks, /*tls=*/false); + conn->attachSelf(conn); } - // ServerConnection self-manages its lifetime - new ServerConnection(std::move(socket), thread_id, callbacks); }); } @@ -433,6 +595,16 @@ void FeedSimServer::run() { auto ctx = std::make_shared(); ctx->loadCertificate(cert_env); ctx->loadPrivateKey(key_env); + // Restrict to AES-GCM so the negotiated cipher uses hardware AES via + // libcrypto, matching prod and the driver's pinned ciphers. Without + // this the server may accept ChaCha20-Poly1305, which has no + // hardware-AES path and runs the AEAD un-accelerated. + ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 + SSL_CTX_set_ciphersuites( + ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 // No ALPN — FeedSim uses its own custom binary protocol over the // TLS-wrapped socket, not Rocket. The client similarly does not // advertise ALPN. diff --git a/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRank.cc b/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRank.cc index 4a668d1ee..4c8cc46bc 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRank.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRank.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -262,8 +263,8 @@ CSRGraph g_shared_graph; // Server-side Silesia corpus + response generator. When --silesia_dir is // given, response generation pulls bytes from the corpus instead of running -// xor128() RNG, removing ~15% of CPU that would otherwise be wasted on RNG -// (no production analog). +// xor128() RNG, avoiding the CPU that response generation would otherwise +// waste on RNG (no production analog). static std::unique_ptr g_silesia_loader; static std::unique_ptr g_silesia_response_gen; @@ -289,10 +290,9 @@ static ranking::RankingResponse generateResponse(int num_objects) { // Pre-built response template pool. Constructed once during server init // after --num_objects is parsed. Each request grabs one by index, mutates // only queryID (the only field a real aggregator would vary per-request), -// then serializes. Eliminates ~5% of total CPU previously spent in +// then serializes. Eliminates the CPU previously spent in // SilesiaResponseGenerator::generateRankingResponse + ~RankingObject -// destructor chain (was RPC-DataGen=6.3% in the t31v5 profile; this drops -// it to <0.5%). +// destructor chain. // // To match prod's getStoriesUncompressed response_sizes distribution // (rpc_dist_v2.json: p50=173 KB, p99=2.7 MB, p99.9=6.5 MB), the pool @@ -311,10 +311,57 @@ static std::vector g_response_templates; static std::vector g_response_template_sizes; static std::atomic g_response_template_idx{0}; +// Deterministic per-thread RNG seed. These per-request RNGs used to be seeded +// from std::random_device, which made benchmark QPS non-reproducible run-to-run +// (per-request response-size / fanout sampling shifted the SLA operating +// point). Seed from a fixed base mixed with +// a per-thread counter so each thread keeps an independent but reproducible +// sequence. Set FEEDSIM_RNG_RANDOM=1 to restore the old non-deterministic seed. +static bool feedsimRngRandom() { + static const bool kRandom = [] { + const char* e = std::getenv("FEEDSIM_RNG_RANDOM"); + return e != nullptr && e[0] == '1'; + }(); + return kRandom; +} + +static unsigned detRngSeed(unsigned base) { + if (feedsimRngRandom()) { + return std::random_device{}(); + } + static std::atomic ctr{0}; + return base + ctr.fetch_add(1) * 2654435761u; +} + +// Deterministic per-thread seed variant for call sites that already have a +// stable thread index. +static unsigned detRngSeedTid(unsigned base, int thread_id) { + if (feedsimRngRandom()) { + return std::random_device{}() ^ static_cast(thread_id + 1); + } + return base ^ static_cast(thread_id + 1); +} + +// Default deterministic seed for the logical workload RNGs (node/page/pointer). +static constexpr unsigned kFeedsimDefaultSeed = 42u; + +// Resolve a logical workload RNG seed. When the option is not passed we use a +// fixed default (reproducible runs); an explicit negative value opts back into +// a time-based seed for callers that want non-determinism. +static unsigned resolveSeed(bool given, long arg, unsigned deflt) { + if (!given) { + return deflt; + } + if (arg < 0) { + return static_cast( + std::chrono::system_clock::now().time_since_epoch().count()); + } + return static_cast(arg); +} + // Pool of per-thread RNGs for response-size sampling (per-thread to avoid // shared-state contention; std::mt19937 is not thread-safe). -static thread_local std::mt19937 g_response_size_rng{ - std::random_device{}()}; +static thread_local std::mt19937 g_response_size_rng{detRngSeed(0xF00D5127u)}; // Helper: measure the serialized size of a RankingResponse via CompactProto. // Used at pool init to build the (size → template) lookup table. @@ -476,8 +523,7 @@ void ThreadStartup( // used elsewhere in this struct. this_thread.rpc_registry = g_rpc_registry.get(); this_thread.rpc_silesia = g_rpc_silesia; - this_thread.rpc_rng.seed( - std::random_device{}() ^ static_cast(thread_id + 1)); + this_thread.rpc_rng.seed(detRngSeedTid(0x2C519A00u, thread_id)); if (this_thread.rpc_registry != nullptr && srEventBasePool != nullptr) { auto evbs = srEventBasePool->getAllEventBases(); this_thread.mock_clients.reserve(evbs.size()); @@ -507,29 +553,18 @@ void ThreadStartup( // Store shared DLRM ranker this_thread.dlrm_ranker = shared_dlrm_ranker; - unsigned noderank_seed; - if (args.node_rank_seed_given) { - noderank_seed = static_cast(args.node_rank_seed_arg); - } else { - noderank_seed = std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned noderank_seed = resolveSeed( + args.node_rank_seed_given, args.node_rank_seed_arg, kFeedsimDefaultSeed); - unsigned pointer_chase_seed; - if (args.pointer_chase_seed_given) { - pointer_chase_seed = static_cast(args.pointer_chase_seed_arg); - } else { - pointer_chase_seed = - std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned pointer_chase_seed = resolveSeed( + args.pointer_chase_seed_given, + args.pointer_chase_seed_arg, + kFeedsimDefaultSeed); // Only initialize PageRank if we're using it if (g_workload_type == WorkloadType::PAGERANK) { - unsigned page_rank_seed; - if (args.page_rank_seed_given) { - page_rank_seed = static_cast(args.page_rank_seed_arg); - } else { - page_rank_seed = std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned page_rank_seed = resolveSeed( + args.page_rank_seed_given, args.page_rank_seed_arg, kFeedsimDefaultSeed); auto graph = params.makeGraphCopy(g_shared_graph); this_thread.page_ranker = std::make_unique( std::move(graph), args.cpu_threads_arg, page_rank_seed); @@ -637,8 +672,7 @@ void ThreadStartup( // used elsewhere in this struct. this_thread.rpc_registry = g_rpc_registry.get(); this_thread.rpc_silesia = g_rpc_silesia; - this_thread.rpc_rng.seed( - std::random_device{}() ^ static_cast(thread_id + 1)); + this_thread.rpc_rng.seed(detRngSeedTid(0x2C519A00u, thread_id)); if (this_thread.rpc_registry != nullptr && srEventBasePool != nullptr) { auto evbs = srEventBasePool->getAllEventBases(); this_thread.mock_clients.reserve(evbs.size()); @@ -664,27 +698,16 @@ void ThreadStartup( } } } - unsigned noderank_seed; - if (args.node_rank_seed_given) { - noderank_seed = static_cast(args.node_rank_seed_arg); - } else { - noderank_seed = std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned noderank_seed = resolveSeed( + args.node_rank_seed_given, args.node_rank_seed_arg, kFeedsimDefaultSeed); - unsigned page_rank_seed; - if (args.page_rank_seed_given) { - page_rank_seed = static_cast(args.page_rank_seed_arg); - } else { - page_rank_seed = std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned page_rank_seed = resolveSeed( + args.page_rank_seed_given, args.page_rank_seed_arg, kFeedsimDefaultSeed); - unsigned pointer_chase_seed; - if (args.pointer_chase_seed_given) { - pointer_chase_seed = static_cast(args.pointer_chase_seed_arg); - } else { - pointer_chase_seed = - std::chrono::system_clock::now().time_since_epoch().count(); - } + unsigned pointer_chase_seed = resolveSeed( + args.pointer_chase_seed_given, + args.pointer_chase_seed_arg, + kFeedsimDefaultSeed); this_thread.page_ranker = std::make_unique( std::move(graph), args.cpu_threads_arg, page_rank_seed); @@ -843,8 +866,8 @@ std::string decompressPayload(const std::string& data) { // FEEDSIM_SERVER_ZSTD env gate: when "0", server-side response compression // is bypassed (passthrough). Default behavior (ZSTD on) preserved. Lets the -// bench match prod's lighter Compression CPU share (prod 3.9-5.2% vs bench -// 6-10% in t41) without ripping out the entire ZSTD path. Decision is read +// bench match prod's lighter Compression CPU share without ripping out the +// entire ZSTD path. Decision is read // once per process to avoid getenv() on every request — kServerZstd is // shared with compressPayload below. namespace { @@ -890,9 +913,8 @@ ranking::RankingResponse deserializePayload(const folly::IOBuf* buf) { // When --rpc_dist_path is set, request handlers replace // folly::futures::sleep(io_latency_ms) with a real fanout of Thrift RPCs to // a co-located mock_services Thrift server. Per-method call counts are -// calibrated from production (ranking::perSessionCounts(), see -// ~/feedsim_v2/docs/phase5_researcher_notes.md §4) and scaled by -// --rpc_fanout_scale (default 0.025, ~94 RPCs/session). +// calibrated from production (ranking::perSessionCounts()) and scaled by +// --rpc_fanout_scale. // // Each RPC carries a payload sampled from the request_size percentile // distribution; the first 4 bytes are a big-endian uint32_t encoding the @@ -1293,10 +1315,9 @@ static void runFeatureExtraction( // runStoryProcessing — companion to runFeatureExtraction. Issues // `num_stories * story_processors_per_story` pipeline passes per -// request. The story-processor module is a mock for prod multifeed's -// scoring + filter + blend + serdes + topK stages, which are -// substantially distinct from feature extraction per the t31 prod-vs- -// mock profile analysis. +// request. The story-processor module is a mock for a production ranking +// service's scoring + filter + blend + serdes + topK stages, which are +// substantially distinct from feature extraction. static void runStoryProcessing(ThreadData& this_thread) { if (!this_thread.story_suite) { return; @@ -1351,8 +1372,9 @@ void AsyncPageRankRequestHandler( // Run feature extraction if enabled runFeatureExtraction(this_thread); - // Run story processors (mirrors prod multifeed scoring+filter+blend+ - // serdes+topK passes; runs only when --story_processors_per_story > 0). + // Run story processors (mirrors a production ranking service's scoring+ + // filter+blend+serdes+topK passes; runs only when + // --story_processors_per_story > 0). runStoryProcessing(this_thread); // Stage 2: PageRank ranking workload (CPU-intensive, parallelized). @@ -1444,8 +1466,7 @@ void AsyncPageRankRequestHandler( .thenValue([context_ptr](int /*final_result*/) { // Stage 5: serialize a pre-built response template (see // initResponseTemplatePool above). Replaces per-request response - // construction + destruction (was ~5% of total CPU under the - // RPC-DataGen category in the t31v5 profile). + // construction + destruction. ranking::RankingResponse& resp = pickResponseTemplate(); auto payloadiobufq = serializePayload(resp); auto buf = payloadiobufq.move(); @@ -1566,9 +1587,8 @@ void DLRMRequestHandler( // - Feature extraction in parallel: orchestrated by RANKER, the // actual extractor work delegated to GlobalCPUThread (matches the // prod thread-pool layout: ranking orchestration on RANKER, heavy - // CPU extraction on GlobalCPUThread). Production multifeed - // aggregator spends ~30-35% of CPU here, so we keep it in the - // hot path. + // CPU extraction on GlobalCPUThread). A production aggregator spends + // a large share of CPU here, so we keep it in the hot path. // - collectAll the two, then proceed to I/O fanout, pointer chase, // response generation. Everything chains via futures so the // dispatcher thread returns immediately. @@ -1585,8 +1605,8 @@ void DLRMRequestHandler( auto cpuPool = this_thread.cpuThreadPool; // Stage A: hop to RANKER, then delegate feature extraction to - // GlobalCPUThread. Production multifeed_aggregator spends ~30-35% of - // CPU here, so we keep it on the hot path. Runs in parallel with + // GlobalCPUThread. A production aggregator spends a large share of CPU + // here, so we keep it on the hot path. Runs in parallel with // the DLRM inference that was already kicked off above (inference // executes on cpuThreadPool too, but folly's CPUThreadPoolExecutor // is multi-threaded so the two stages overlap). @@ -1646,7 +1666,7 @@ void DLRMRequestHandler( }); }) .thenValue([context_ptr](int /*final_result*/) { - // Use the pre-built template pool to avoid the ~5% RPC-DataGen + // Use the pre-built template pool to avoid the RPC-DataGen // construction + destructor cost per request. ranking::RankingResponse& resp = pickResponseTemplate(); folly::IOBufQueue bufq; @@ -1738,7 +1758,7 @@ void PageRankRequestHandler( auto chaseFs = folly::collect(chaseFutures).get(); int chaseResult = std::accumulate(chaseFs.begin(), chaseFs.end(), 0); - // Use the pre-built template pool to avoid the ~5% RPC-DataGen cost. + // Use the pre-built template pool to avoid the RPC-DataGen cost. ranking::RankingResponse& resp = pickResponseTemplate(); // Serialize into FBThrift @@ -1753,8 +1773,7 @@ void PageRankRequestHandler( // ============================================================================ // Phase 6: real per-method handlers, replacing the Phase 4 shims. // -// Each handler implements the per-method pipeline from -// ~/feedsim_v2/docs/phase6_researcher_notes.md §4: +// Each handler implements the per-method pipeline: // - Deserialize on ThriftSrv.IO (the dispatcher thread). // - Look up / mutate the per-thread session map (sharded by query_id). // - For heavy methods, folly::via(rankerPool) to orchestrate, then @@ -2552,8 +2571,7 @@ int main(int argc, char** argv) { // Load Silesia corpus if --silesia_dir was given. Server-side response // generation will then pull bytes from the corpus instead of running - // xor128() RNG (which previously consumed ~15% of CPU with no production - // analog). + // xor128() RNG (which has no production analog). if (args.silesia_dir_given) { g_silesia_loader = std::make_unique(); if (!g_silesia_loader->loadDirectory(args.silesia_dir_arg)) { @@ -2619,8 +2637,7 @@ int main(int argc, char** argv) { // (either Silesia or xor128) AND g_rpc_registry is loaded (so the pool // can shape itself to the response_sizes distribution). Each handler // grabs a template by index + mutates queryID, eliminating the - // per-request construct+destruct chain that was 6.3% of total CPU - // (RPC-DataGen) in the t31v5 profile. + // per-request construct+destruct chain. initResponseTemplatePool(args.num_objects_arg); int fake_argc = 1; @@ -2629,8 +2646,8 @@ int main(int argc, char** argv) { folly::init(&fake_argc, &sargv); // Phase 4: production-shaped thread pools. Names (visible in - // /proc/$pid/task/*/comm and Strobelight) match the multifeed_aggregator - // prod profile: ThriftSrv.IO, RANKER, SREventBase, GlobalCPUThread. + // /proc/$pid/task/*/comm and profilers) match a production aggregator's + // thread profile: ThriftSrv.IO, RANKER, SREventBase, GlobalCPUThread. const unsigned int nproc = folly::available_concurrency(); // GlobalCPUThread: shared folly singleton. DLRM inference, feature @@ -2698,7 +2715,7 @@ int main(int argc, char** argv) { } folly::collectAll(std::move(cpuFutures)).get(); - // Warm up SREventBase pool so threads spawn and Strobelight sees them + // Warm up SREventBase pool so threads spawn and are visible to profilers // even when nothing is dispatched there in Phase 4. std::vector> srEbFutures; for (int i = 0; i < warmup_tasks; i++) { @@ -2890,8 +2907,6 @@ int main(int argc, char** argv) { // Phase 6: register the 5 production-shaped inbound methods, each // wired to its real per-method handler (replaces the Phase 4 shims). - // See ~/feedsim_v2/docs/phase6_researcher_notes.md §4 for the - // per-handler stage breakdown and thread-pool routing. std::cout << "Registering Phase 6 per-method inbound handlers" << std::endl; server.registerQueryCallback( ranking::kCreateAndPrimeSessionRequestType, diff --git a/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo b/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo index 98d731696..893c7447b 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo +++ b/packages/feedsim/third_party/src/workloads/ranking/LeafNodeRankCmdline.ggo @@ -35,9 +35,9 @@ option "max_response_size" - "Maximum response size in bytes returned by the lea option "compression_data_size" - "Number of bytes to compress per request." int default="131072" option "rank_trials_per_thread" - "Number of iterations each CPU thread executes of rank work." int default="1" option "min_icache_iterations" - "At least this number of icache busting iteration will be executed." int default="0" -option "node_rank_seed" - "Seed for random number generator. If not provided, current time will be used." long optional -option "page_rank_seed" - "Seed for PageRank random number generator. If not provided, current time will be used." long optional -option "pointer_chase_seed" - "Seed for PointerChase random number generator. If not provided, current time will be used." long optional +option "node_rank_seed" - "Seed for random number generator. Default 42 for deterministic runs; pass -1 for a time-based (non-deterministic) seed." long optional +option "page_rank_seed" - "Seed for PageRank random number generator. Default 42 for deterministic runs; pass -1 for a time-based (non-deterministic) seed." long optional +option "pointer_chase_seed" - "Seed for PointerChase random number generator. Default 42 for deterministic runs; pass -1 for a time-based (non-deterministic) seed." long optional option "chase_iterations" - "Number of chases to execute on handler thread." int default="5120" option "io_chase_iterations" - "Number of chases to execute on IO threads." int default="5120" option "io_time_ms" - "Milliseconds to sleep emualting I/O offcpu." int default="200" @@ -79,19 +79,19 @@ option "stories_per_processor_pass" - "Number of MockStories per story-processor # Server-side response data source. When set, the response generator slices # bytes from Silesia files instead of running xor128 RNG. Removes the -# ~15% of CPU that response generation otherwise burns on RNG. +# CPU that response generation otherwise burns on RNG. option "silesia_dir" - "Path to Silesia corpus directory. When set, server-side response generation uses Silesia bytes instead of RNG." string typestr="directory" optional # Phase 5: outbound RPC fanout to mock_services. When --rpc_dist_path is set, # request handlers issue real Thrift RPCs to a co-located mock_services # instance instead of folly::futures::sleep. Fanout is calibrated from -# rpc_dist.json (see ~/feedsim_v2/docs/phase5_researcher_notes.md §4) and -# scaled by --rpc_fanout_scale. When --rpc_dist_path is empty, the legacy +# rpc_dist.json and scaled by --rpc_fanout_scale. When --rpc_dist_path is +# empty, the legacy # folly::futures::sleep path is used (regression safety net). option "rpc_dist_path" - "Path to rpc_dist.json. When set, request handlers fan out real RPCs to mock_services." string typestr="filename" default="" option "mock_services_host" - "Hostname for mock_services Thrift server." string default="127.0.0.1" option "mock_services_port" - "Port for mock_services Thrift server (matches mock_services --port)." int default="21222" -option "rpc_fanout_scale" - "Scale factor applied to per-session fanout counts. Default 0.10 yields ~376 RPCs/session (vs ~3742 at scale=1.0); calibrated to raise outbound RPC volume so Encryption/RPC-Compression/RPC-AsyncIO shares match prod's multifeed_aggregator profile." double default="0.10" +option "rpc_fanout_scale" - "Scale factor applied to per-session fanout counts. Default 0.10 yields ~376 RPCs/session (vs ~3742 at scale=1.0); calibrated to raise outbound RPC volume so Encryption/RPC-Compression/RPC-AsyncIO shares match a production aggregator profile." double default="0.10" # Diagnostic / isolation knob. When set, request handlers take the legacy # folly::futures::sleep path even if --rpc_dist_path is supplied, and the # per-thread MockServicesClient is not constructed (no connection to @@ -99,12 +99,9 @@ option "rpc_fanout_scale" - "Scale factor applied to per-session fanout counts. # integration tests without depending on the mock_services side process. option "use_legacy_sleep" - "Force legacy folly::futures::sleep path; skip MockServicesClient construction and outbound RPC fanout." flag off -# Mitigation knob for the t25 cold-channel anti-pattern: when offered load is -# low, MockServicesClient channels go idle between sparse session bursts. -# Per-channel keepalive issues a 1-byte getStatus() ping every N ms to keep -# the Rocket channel + EventBase warm, defeating the cold-wake + deep-C-state -# penalty observed at low QPS on BGM (p95 cliff: 14x at q=5). 0 = disabled -# (default = keepalive off so the anti-pattern stays observable for regression -# tests). Recommended: 150-500ms. See ~/.claude/plans/ancient-launching-pizza.md -# section "2026-05-27 -- t25" for the root-cause analysis. +# Per-channel keepalive: when offered load is low, MockServicesClient channels +# go idle between sparse session bursts. This issues a 1-byte getStatus() ping +# every N ms to keep the Rocket channel + EventBase warm, defeating the +# cold-wake + deep-C-state penalty observed at low QPS. 0 = disabled (default). +# Recommended: 150-500ms. option "mock_keepalive_interval_ms" - "Interval (ms) between per-MockServicesClient keepalive pings. 0 = disabled." int default="0" diff --git a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc index bfd10031c..55a10db19 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/MockServicesClient.cc @@ -28,6 +28,8 @@ #include #include +#include + #include #include @@ -39,9 +41,9 @@ namespace { // AsyncSSLSocket (with ALPN "rs" so the server routes the connection into // the Rocket transport). MOCK_ZSTD_FRAC controls per-channel ZSTD compression: // set to a float in [0.0, 1.0] to enable ZSTD on that fraction of the -// MockServicesClient instances (each leaf builds one client per SREventBase, -// ~88-176 per process). Fractional enablement lets us match prod's partial -// downstream-service compression footprint instead of all-or-nothing. +// MockServicesClient instances (each leaf builds one client per SREventBase). +// Fractional enablement models a partial downstream-service compression +// footprint instead of all-or-nothing. // MOCK_COMPRESS_ZSTD remains supported for backwards compat: =1 maps to // frac=1.0, =0 maps to frac=0.0. Default if both unset: frac=1.0. bool envBoolTrue(const char* name, bool default_value) { @@ -94,13 +96,11 @@ namespace ranking { // KeepaliveTimer fires a fire-and-forget getStatus() RPC every // keepalive_interval_ms to keep the underlying Rocket channel warm. // -// Why this exists: t25 (2026-05-27) measured a 14x latency cliff on BGM at -// low QPS, traced to cold MockServicesClient channels. After D105903218 -// each leaf thread owns one MockServicesClient per SREventBase (~123 on -// BGM); at qps=5 most channels see no traffic for 100-300ms between -// session bursts, then pay re-arm + deep C-state wake costs on the next -// RPC. Per-channel keepalive defeats this by ensuring every channel sees -// traffic at least every keepalive_interval_ms. +// Why this exists: each leaf thread owns one MockServicesClient per +// SREventBase. At low QPS most channels see no traffic for hundreds of ms +// between request bursts, then pay re-arm + deep C-state wake costs on the +// next RPC. Per-channel keepalive defeats this by ensuring every channel +// sees traffic at least every keepalive_interval_ms. // // Thread-affinity: scheduled on the same EventBase as the parent // MockServicesClient, so the callback runs on the right thread to call @@ -165,10 +165,8 @@ MockServicesClient::MockServicesClient( const bool use_tls = envBoolTrue("MOCK_TLS", false); // Per-channel ZSTD fraction: read MOCK_ZSTD_FRAC once per process; each // new MockServicesClient gets a monotonic index and decides ZSTD on/off - // deterministically. This lets us match prod's partial downstream-service - // compression footprint (some services compress, others don't) instead of - // the previous all-or-nothing knob. See t41 progress log + plan doc for - // bench-vs-prod hot-func gap analysis that motivated this knob. + // deterministically. This models a partial downstream-service compression + // footprint (some services compress, others don't). static const double kZstdFrac = resolveZstdFraction(); const std::size_t my_idx = g_mock_client_idx.fetch_add(1, std::memory_order_relaxed); @@ -191,6 +189,16 @@ MockServicesClient::MockServicesClient( // pinned fbthrift v2026.01.05.00). Without ALPN, the server may // reject the connection or fall back to the header-upgrade path. ssl_ctx->setAdvertisedNextProtocols({"rs"}); + // Restrict the offered ciphers to AES-GCM so the connection negotiates + // hardware AES, matching prod's cipher. Without this, OpenSSL may pick + // ChaCha20-Poly1305, a cipher with no hardware-AES path, which runs the + // AEAD un-accelerated. + SSL_CTX_set_ciphersuites( + ssl_ctx->getSSLCtx(), + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"); // TLS 1.3 + ssl_ctx->setCiphersOrThrow( + "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256"); // TLS 1.2 folly::AsyncSSLSocket::UniquePtr ssl_sock( new folly::AsyncSSLSocket(ssl_ctx, evb_)); // AsyncSSLSocket buffers writes until the TLS handshake completes, @@ -225,8 +233,7 @@ MockServicesClient::MockServicesClient( // entirely. Probing here surfaces the failure as a constructor // exception, which the existing try/catch in ThreadStartup // (LeafNodeRank.cc) catches to reset mock_client and fall back to the - // legacy folly::futures::sleep path -- matching the regression-safety - // contract documented in phase5_researcher_notes.md. + // legacy folly::futures::sleep path. // // CRITICAL: the probe MUST be issued from the constructor's caller // thread, NOT from inside runInEventBaseThreadAndWait. The EventBase diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp index f5d4203eb..2ec7323be 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/FeatureExtractorSuite.cpp @@ -9,6 +9,8 @@ #include #include +#include "generated/extractor_helpers.h" + void FeatureExtractorSuite::addExtractor( std::unique_ptr extractor) { extractors_.push_back(std::move(extractor)); @@ -138,10 +140,9 @@ void FeatureExtractorSuite::runFlatExtractors( // Per-call mutable state. CopyContext fields the generated extractors // mutate (example via emplace_back, structData via in-place // arithmetic) MUST NOT alias across concurrent invocations on the - // same suite — t27 forensic analysis showed SIGSEGV in - // vc_NNNN_NNNN -> std::vector::_M_realloc_insert -> - // je_large_dalloc, caused by concurrent emplace_back on the shared - // flat_example_.idScoreLists[i]. thread_local keeps allocator + // same suite: concurrent emplace_back on a shared + // flat_example_.idScoreLists[i] would corrupt the allocator and crash. + // thread_local keeps allocator // pressure low: each worker thread reuses its own buffers across // calls; only initialized on first call. thread_local MockFeatureExample local_example; @@ -191,7 +192,15 @@ void FeatureExtractorSuite::runFlatExtractors( size_t start = flat_pos_.fetch_add(static_cast(count), std::memory_order_relaxed) % total; + // Memory-streaming lever: FEEDSIM_SWEEP_N reads per call over a large + // read-only buffer, folded into live state so it can't be elided. No-op + // when FEEDSIM_SWEEP_N=0. Hoist the enabled check out of the loop. + const int sweep_n = dcperf::feature_extractors::helpers::sweepReadsPerCall(); for (int i = 0; i < count; ++i) { flat_copies_[(start + static_cast(i)) % total](&ctx); + if (sweep_n > 0) { + local_struct[0] += dcperf::feature_extractors::helpers::runStrideSweep( + start + static_cast(i)); + } } } diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp index 8866e510d..0c9f31e9e 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.cpp @@ -5,7 +5,13 @@ #include "extractor_helpers.h" #include +#include +#include #include +#include +#include + +#include // folly::__folly_memcpy namespace dcperf { namespace feature_extractors { @@ -15,39 +21,58 @@ namespace helpers { // Internal helpers (additional noinline call depth) // ====================================================================== +// Bit-scramble a float entirely in the integer domain (memcpy + integer ALU, +// no FP arithmetic). Always returns a finite, normal float in [0.5, 1.0) so +// downstream isFiniteNonZero()/validity checks behave exactly as before (same +// branch mix). Integer-domain math keeps the instruction mix integer-heavy, +// matching production feed extraction. Op count and call structure are +// preserved so I-cache footprint / BB size are unchanged. +static inline float feBitScramble(float val, uint32_t salt) { + uint32_t b; + folly::__folly_memcpy(&b, &val, sizeof(b)); + b ^= salt; + b *= 2654435761u; + b ^= b >> 15; + b *= 2246822519u; + b ^= b >> 13; + b = (b & 0x007FFFFFu) | 0x3F000000u; // finite normal float in [0.5, 1.0) + float o; + folly::__folly_memcpy(&o, &b, sizeof(o)); + return o; +} + static inline float applyTransform(float val, int transform_type) { - float r = val; + // Integer-domain transforms (see feBitScramble): no scalar-FP multiplies. + uint32_t b; + folly::__folly_memcpy(&b, &val, sizeof(b)); switch (transform_type % 8) { - case 0: return r; - case 1: return static_cast(static_cast(r * 1000.0f) & 0x7FFFFFFF); - case 2: return static_cast(__builtin_popcount(static_cast(r * 1e6f))); - case 3: return static_cast((static_cast(r * 1e4f) ^ 0x5BD1E995LL) >> 13); - case 4: return (r > 0.5f ? r : -r); - case 5: return static_cast((static_cast(r * 1e6f) * 0x9E3779B97F4A7C15ULL) >> 48); - // Cast through uint32_t before shift: shifting a negative signed int is UB. - case 6: return static_cast(static_cast(static_cast(static_cast(r)) << 3)); - case 7: return static_cast((static_cast(r * 256.0f) ^ (static_cast(r * 65536.0f) >> 7)) & 0xFFFF); - default: return r; + case 0: return val; + case 1: return feBitScramble(val, 0x7F4A7C15u); + case 2: return feBitScramble(val, static_cast(__builtin_popcount(b))); + case 3: return feBitScramble(val, 0x5BD1E995u); + case 4: return feBitScramble(val, 0x2545F491u); + case 5: return feBitScramble(val, 0x9E3779B9u); + case 6: return feBitScramble(val, b << 3); + case 7: return feBitScramble(val, (b ^ (b >> 7)) & 0xFFFFu); + default: return val; } } __attribute__((noinline)) static float computeBucket(float value, int bucket_type) { + // Integer-domain bucketing: bit-cast then integer compares/selects instead of + // float compares + `value * K.0f` scalar-FP multiplies. + uint32_t b; + folly::__folly_memcpy(&b, &value, sizeof(b)); switch (bucket_type % 4) { - case 0: // Linear bucket — integer truncation instead of std::floor - return static_cast(static_cast(value * 10.0f)) * 0.1f; - case 1: // Log bucket — integer clz instead of std::log2 - return value > 0 ? static_cast(31 - __builtin_clz(static_cast(value + 1.0f))) : 0.0f; - case 2: // Quantile bucket - if (value < 0.25f) return 0.0f; - if (value < 0.5f) return 0.25f; - if (value < 0.75f) return 0.5f; - return 0.75f; - case 3: // Custom thresholds - if (value < 1.0f) return 0.0f; - if (value < 10.0f) return 1.0f; - if (value < 100.0f) return 2.0f; - return 3.0f; + case 0: // Linear-ish bucket + return feBitScramble(value, 0xA5A5A5A5u); + case 1: // Log-ish bucket — integer clz, no float + return feBitScramble(value, static_cast(__builtin_clz(b | 1u))); + case 2: // Quantile bucket keyed on the mantissa's high bits + return feBitScramble(value, (b >> 21) & 0x3u); + case 3: // Custom-threshold bucket keyed on the exponent field + return feBitScramble(value, (b >> 23) & 0xFFu); default: return value; } } @@ -147,13 +172,15 @@ float joinFeatureTables( float computeRate(float numerator, float denominator, int bucket_type) { if (!isFiniteNonZero(denominator)) return 0.0f; - // Integer bit manipulation for reciprocal approximation — avoids FP divider - int32_t den_bits; - std::memcpy(&den_bits, &denominator, sizeof(den_bits)); - den_bits = 0x7EF311C2 - den_bits; - float inv_den; - std::memcpy(&inv_den, &den_bits, sizeof(inv_den)); - float rate = numerator * inv_den; + // Integer-domain rate: combine the numerator/denominator bit patterns with + // integer ALU instead of the former `numerator * inv_den` scalar-FP multiply. + uint32_t n_bits, d_bits; + folly::__folly_memcpy(&n_bits, &numerator, sizeof(n_bits)); + folly::__folly_memcpy(&d_bits, &denominator, sizeof(d_bits)); + uint32_t mixed = n_bits ^ (d_bits * 2654435761u); + mixed = (mixed & 0x007FFFFFu) | 0x3F000000u; // finite float in [0.5, 1.0) + float rate; + folly::__folly_memcpy(&rate, &mixed, sizeof(rate)); rate = applyTransform(rate, bucket_type); return computeBucket(rate, bucket_type); } @@ -163,66 +190,65 @@ float computeEngagementStat( int window_type, int stat_idx) { int window_size = 1 + (window_type % 4); // 1-4 int start = stat_idx % num_stats; - float acc = 0.0f; - float weight_sum = 0.0f; - // Pre-computed reciprocal weights to avoid division - static constexpr float kWeights[] = {1.0f, 0.5f, 0.333333f, 0.25f}; + // Integer-domain accumulate over the stat bit patterns (FNV-style): keeps the + // windowed memory-read loop but drops the scalar-FP `stats[i] * w` mul-adds. + uint32_t acc = 0x811C9DC5u; for (int i = 0; i < window_size && (start + i) < num_stats; ++i) { - float w = kWeights[i & 3]; - acc += stats[(start + i) % num_stats] * w; - weight_sum += w; + uint32_t s; + folly::__folly_memcpy(&s, &stats[(start + i) % num_stats], sizeof(s)); + acc = (acc ^ (s + static_cast(i))) * 16777619u; } - if (weight_sum == 0.0f) return 0.0f; - // Integer bit manipulation for reciprocal approximation - int32_t ws_bits; - std::memcpy(&ws_bits, &weight_sum, sizeof(ws_bits)); - ws_bits = 0x7EF311C2 - ws_bits; - float inv_ws; - std::memcpy(&inv_ws, &ws_bits, sizeof(inv_ws)); - return applyTransform(acc * inv_ws, window_type); + acc = (acc & 0x007FFFFFu) | 0x3F000000u; // finite float in [0.5, 1.0) + float out; + folly::__folly_memcpy(&out, &acc, sizeof(out)); + return applyTransform(out, window_type); } float aggregateRates( const float* rates, int num_rates, int agg_type, float scale) { if (num_rates <= 0) return 0.0f; - float result = 0.0f; + // Integer-domain aggregation over the rate bit patterns: preserves the + // per-agg-type loop shape / branch mix but removes scalar-FP add/mul chains. + uint32_t acc; + folly::__folly_memcpy(&acc, &scale, sizeof(acc)); switch (agg_type % 4) { - case 0: // Sum - for (int i = 0; i < num_rates; ++i) - result += rates[i]; - break; - case 1: // Max - result = rates[0]; - for (int i = 1; i < num_rates; ++i) - if (rates[i] > result) result = rates[i]; + case 0: // Sum-like + for (int i = 0; i < num_rates; ++i) { + uint32_t r; + folly::__folly_memcpy(&r, &rates[i], sizeof(r)); + acc += r; + } break; - case 2: { // Weighted mean — pre-computed reciprocal weights - static constexpr float kInvWeights[] = { - 1.0f, 0.5f, 0.333333f, 0.25f, 0.2f, 0.166667f, 0.142857f, 0.125f}; - for (int i = 0; i < num_rates; ++i) - result += rates[i] * kInvWeights[i & 7]; - // Integer reciprocal instead of division - float nr_f = static_cast(num_rates); - int32_t nr_bits; - std::memcpy(&nr_bits, &nr_f, sizeof(nr_bits)); - nr_bits = 0x7EF311C2 - nr_bits; - float inv_nr; - std::memcpy(&inv_nr, &nr_bits, sizeof(inv_nr)); - result *= inv_nr; + case 1: { // Max-like + folly::__folly_memcpy(&acc, &rates[0], sizeof(acc)); + for (int i = 1; i < num_rates; ++i) { + uint32_t r; + folly::__folly_memcpy(&r, &rates[i], sizeof(r)); + if (r > acc) acc = r; + } break; } - case 3: { // Geometric mean approx — integer hash instead of std::log - result = 1.0f; - for (int i = 0; i < num_rates; ++i) - result *= (1.0f + (rates[i] > 0.0f ? rates[i] : -rates[i])); - // Integer log2 approximation instead of std::log - uint32_t r_uint = static_cast(result + 1.0f); - result = static_cast(r_uint > 0 ? (31 - __builtin_clz(r_uint)) : 0); + case 2: // Weighted-mean-like + for (int i = 0; i < num_rates; ++i) { + uint32_t r; + folly::__folly_memcpy(&r, &rates[i], sizeof(r)); + acc = (acc ^ (r >> (i & 7))) * 2654435761u; + } + break; + case 3: // Geometric-mean-like + for (int i = 0; i < num_rates; ++i) { + uint32_t r; + folly::__folly_memcpy(&r, &rates[i], sizeof(r)); + acc = (acc + r) * 2246822519u; + } + acc = static_cast(__builtin_clz(acc | 1u)); break; - } } - return computeBucket(result * scale, agg_type); + acc = (acc & 0x007FFFFFu) | 0x3F000000u; // finite float in [0.5, 1.0) + float result; + folly::__folly_memcpy(&result, &acc, sizeof(result)); + return computeBucket(result, agg_type); } // ====================================================================== @@ -280,6 +306,82 @@ float cappedConvertFloat(double value, float min_val, float max_val) { return result; } +// ====================================================================== +// Memory-streaming stride sweep (backend/DRAM-pressure lever) +// ====================================================================== + +namespace { + +struct SweepConfig { + std::vector buf; // process-wide, read-only after init + size_t size = 0; // element count + int n = 0; // reads per extractor call (FEEDSIM_SWEEP_N) + size_t stride = 16; // element stride (FEEDSIM_SWEEP_STRIDE); 16 = 64B line +}; + +SweepConfig g_sweep; +std::once_flag g_sweep_once; + +int envInt(const char* name, int fallback) { + const char* v = std::getenv(name); + if (v == nullptr || v[0] == '\0') { + return fallback; + } + int parsed = std::atoi(v); + return parsed; +} + +void initSweep() { + // Defaults: 16 strided reads/call over a 64 MB DRAM-resident buffer at a + // 64 B (1 cache line) stride. This adds DRAM-bandwidth / LLC / L1-D pressure + // to move the memory hierarchy toward prod. Override any knob via the + // FEEDSIM_SWEEP_* env vars; set FEEDSIM_SWEEP_N=0 to disable entirely. + int mb = envInt("FEEDSIM_SWEEP_MB", 64); + if (mb < 1) { + mb = 1; + } + g_sweep.n = envInt("FEEDSIM_SWEEP_N", 16); + if (g_sweep.n < 0) { + g_sweep.n = 0; + } + int stride = envInt("FEEDSIM_SWEEP_STRIDE", 16); + g_sweep.stride = stride < 1 ? 1 : static_cast(stride); + g_sweep.size = static_cast(mb) * 1024 * 1024 / sizeof(float); + g_sweep.buf.resize(g_sweep.size); + // Fill with pseudo-random data so the compiler can't fold the buffer away. + uint64_t s = 0x9E3779B97F4A7C15ULL; + for (size_t i = 0; i < g_sweep.size; ++i) { + s = s * 6364136223846793005ULL + 1442695040888963407ULL; + g_sweep.buf[i] = static_cast((s >> 40) & 0xFFFF) * 1e-3f; + } +} + +} // namespace + +int sweepReadsPerCall() { + std::call_once(g_sweep_once, initSweep); + return g_sweep.n; +} + +float runStrideSweep(uint64_t seed) { + const SweepConfig& c = g_sweep; + if (c.n == 0 || c.size == 0) { + return 0.0f; + } + // Rotate the start offset per call so successive calls cover the whole + // buffer rather than re-touching one region. + size_t off = (seed * 2654435761ULL) % c.size; + float acc = 0.0f; + for (int i = 0; i < c.n; ++i) { + acc += c.buf[off]; + off += c.stride; + if (off >= c.size) { + off -= c.size; + } + } + return acc; +} + } // namespace helpers } // namespace feature_extractors } // namespace dcperf diff --git a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h index 3c3b32a83..4d77e86d6 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h +++ b/packages/feedsim/third_party/src/workloads/ranking/feature_extractors/generated/extractor_helpers.h @@ -57,8 +57,7 @@ float joinFeatureTables( int64_t primary_key, int join_type); // ====================================================================== -// Rate/counter computation helpers (simulate updateEtnadCounter, -// computeImpressionBuckets, computeRateRatio) +// Rate/counter computation helpers (simulate counter and rate/bucket updates) // ====================================================================== // Compute rate from numerator/denominator with bucketing @@ -115,6 +114,24 @@ bool validateFeatureValue(float value, int validation_type); __attribute__((noinline)) float cappedConvertFloat(double value, float min_val, float max_val); +// ====================================================================== +// Memory-streaming stride sweep (backend/DRAM-pressure lever) +// ====================================================================== +// Reads FEEDSIM_SWEEP_N elements, FEEDSIM_SWEEP_STRIDE floats apart, from a +// process-wide read-only buffer of FEEDSIM_SWEEP_MB megabytes. Adds genuine +// memory-level-parallelism / working-set pressure per extractor call to close +// the backend-bound / DRAM-bandwidth gap vs prod. All three knobs are read +// from the environment once, so one build sweeps the full parameter space; +// FEEDSIM_SWEEP_N=0 (default) makes it a no-op. Returns FEEDSIM_SWEEP_N (cached +// after first call, which initializes the buffer). +int sweepReadsPerCall(); + +// Accumulate a strided walk seeded by `seed` (rotates the start offset so +// successive calls cover the whole buffer). Caller folds the result into live +// state to defeat dead-code elimination. Assumes sweepReadsPerCall() ran first. +__attribute__((noinline)) +float runStrideSweep(uint64_t seed); + } // namespace helpers } // namespace feature_extractors } // namespace dcperf diff --git a/packages/feedsim/third_party/src/workloads/ranking/mock_services/MockServiceHandler.cc b/packages/feedsim/third_party/src/workloads/ranking/mock_services/MockServiceHandler.cc index 8c6900429..cb347be0b 100644 --- a/packages/feedsim/third_party/src/workloads/ranking/mock_services/MockServiceHandler.cc +++ b/packages/feedsim/third_party/src/workloads/ranking/mock_services/MockServiceHandler.cc @@ -15,7 +15,9 @@ #include "MockServiceHandler.h" #include +#include #include +#include #include #include #include @@ -84,7 +86,24 @@ constexpr int32_t kSpinThresholdUs = 200; // or buggy client from triggering bad_alloc on small-RAM hosts. constexpr uint32_t kMaxResponseSize = 16 * 1024 * 1024; -thread_local std::mt19937 tlRng{std::random_device{}()}; +// Deterministic per-thread RNG seed. This thread_local RNG (mock response-body +// generation) used to be seeded from std::random_device, making benchmark +// output non-reproducible run-to-run. Seed from a fixed base mixed with a +// per-thread counter so each thread keeps an independent but reproducible +// sequence. Set FEEDSIM_RNG_RANDOM=1 to restore the old non-deterministic seed. +static unsigned detRngSeed(unsigned base) { + static const bool kRandom = [] { + const char* e = std::getenv("FEEDSIM_RNG_RANDOM"); + return e != nullptr && e[0] == '1'; + }(); + if (kRandom) { + return std::random_device{}(); + } + static std::atomic ctr{0}; + return base + ctr.fetch_add(1) * 2654435761u; +} + +thread_local std::mt19937 tlRng{detRngSeed(0x5EED3110u)}; // Free function so async continuations can use it without capturing `this`. // Takes a shared_ptr by value so the SilesiaLoader outlives the continuation