diff --git a/packages/feedsim/install_feedsim.sh b/packages/feedsim/install_feedsim.sh index e9bb855b..03c22fe4 100755 --- a/packages/feedsim/install_feedsim.sh +++ b/packages/feedsim/install_feedsim.sh @@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" +# Dependency versions are env-overridable so experiments can bump them without +# forking this script; defaults reproduce the v2 baseline exactly. +JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" +LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" +# Export so the aarch64 sub-installer (dispatched below) inherits the pins. +export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \ libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \ libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \ zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \ - xxhash xxhash-devel unzip rsync liburing-devel + xxhash xxhash-devel unzip rsync liburing-devel python3-pip # Creates feedsim directory under benchmarks/ mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim" @@ -178,30 +191,30 @@ else fi # Installing JEMalloc -if ! [ -d "jemalloc-5.3.0" ]; then - wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2" - bunzip2 "jemalloc-5.3.0.tar.bz2" - tar -xvf "jemalloc-5.3.0.tar" - cd "jemalloc-5.3.0" +if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then + wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" + bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2" + tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar" + cd "jemalloc-${JEMALLOC_VERSION}" ./configure --enable-prof --enable-prof-libunwind make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] jemalloc-5.3.0" + msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}" fi # Installing libevent -if ! [ -d "libevent-2.1.12-stable" ]; then - wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz" - tar -xzf "libevent-2.1.12-stable.tar.gz" - cd "libevent-2.1.12-stable" +if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then + wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz" + tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz" + cd "libevent-${LIBEVENT_VERSION}" ./configure make -j"$(nproc)" make install cd ../ else - msg "[SKIPPED] libevent-2.1.12-stable" + msg "[SKIPPED] libevent-${LIBEVENT_VERSION}" fi msg "Installing third-party dependencies ... DONE" @@ -218,12 +231,33 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # pip on the box (3.9, or an internal stale mirror) can't see the cp310 + # 2.13 wheels, so resolve the wheel href straight from the PEP-503 index + # and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is + # Python-version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi diff --git a/packages/feedsim/install_feedsim_aarch64.sh b/packages/feedsim/install_feedsim_aarch64.sh index 59a238d1..acd0ce62 100755 --- a/packages/feedsim/install_feedsim_aarch64.sh +++ b/packages/feedsim/install_feedsim_aarch64.sh @@ -182,11 +182,15 @@ else msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}" fi -DEP_JEMALLOC_VERSION="5.3.0" +DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}" # Installing JEMalloc if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" - verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then + verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa" + else + msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)" + fi bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar" cd "jemalloc-${DEP_JEMALLOC_VERSION}" @@ -198,11 +202,15 @@ else msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}" fi -DEP_LIBEVENT_VERSION="2.1.12-stable" +DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}" # Installing libevent if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" - verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then + verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb" + else + msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)" + fi tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" cd "libevent-${DEP_LIBEVENT_VERSION}" ./configure @@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # unset reproduces the v2 baseline (latest). + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh index e6abb5ad..54565bfb 100755 --- a/packages/feedsim/install_feedsim_aarch64_ubuntu.sh +++ b/packages/feedsim/install_feedsim_aarch64_ubuntu.sh @@ -27,6 +27,7 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -34,7 +35,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync + sysstat jq unzip xxhash libxxhash-dev libboost-all-dev rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing is # older than folly's minimum, so folly's io_uring integration links @@ -204,9 +205,14 @@ if ! [ -d "libtorch" ]; then export PATH="${CONDA_DIR}/bin:${PATH}" # Install CPU-only PyTorch via pip — this is the only reliable way to get - # CPU-only libtorch on aarch64 - msg "Installing PyTorch CPU-only via pip..." - pip install torch --index-url https://download.pytorch.org/whl/cpu + # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version; + # empty falls back to pip's latest resolution. + msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..." + if [ -n "${LIBTORCH_VERSION:-}" ]; then + pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu + else + pip install torch --index-url https://download.pytorch.org/whl/cpu + fi # Also install libstdcxx-ng to ensure compatible C++ runtime eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)" diff --git a/packages/feedsim/install_feedsim_ubuntu.sh b/packages/feedsim/install_feedsim_ubuntu.sh index f525f63e..4237ea6b 100755 --- a/packages/feedsim/install_feedsim_ubuntu.sh +++ b/packages/feedsim/install_feedsim_ubuntu.sh @@ -11,7 +11,14 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")" FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim" FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party" -LIBTORCH_VERSION="2.8.0" +LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}" +# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel +# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip. +# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no +# standalone zip); harmless for older versions. Default 1 pairs with the +# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without +# additional env overrides. +LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}" DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz" echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}" @@ -30,7 +37,7 @@ apt install -y bc cmake ninja-build flex bison texinfo binutils-dev \ libunwind-dev bzip2 libbz2-dev libsodium-dev libghc-double-conversion-dev \ libzstd-dev lz4 liblz4-dev xzip libsnappy-dev libtool libssl-dev \ zlib1g-dev libdwarf-dev libaio-dev libatomic1 patch perl libiberty-dev \ - sysstat jq xxhash libxxhash-dev unzip rsync + sysstat jq xxhash libxxhash-dev unzip rsync curl # Install liburing >= 2.6 from source. Ubuntu's apt-shipped liburing (0.7 on # 20.04, 2.1 on 22.04) is older than folly's minimum, so folly's io_uring @@ -213,12 +220,31 @@ else fi if ! [ -d "libtorch" ]; then - msg "Downloading LibTorch ${LIBTORCH_VERSION}..." - wget "${LIBTORCH_URL}" -O libtorch.zip - msg "Extracting LibTorch..." - unzip -q libtorch.zip - rm libtorch.zip - msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then + # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's + # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as + # the standalone libtorch zip, so we just rename it to libtorch/. + msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..." + # The C++ libtorch inside (torch/lib, torch/share/cmake) is Python- + # version independent, so the cp310 wheel is fine for our C++ link. + WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \ + | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \ + | head -1)" + [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index" + msg "Wheel: ${WHEEL_HREF}" + wget "${WHEEL_HREF}" -O torch.whl + unzip -q torch.whl -d ./_torch_whl_x + mv ./_torch_whl_x/torch libtorch + rm -rf ./_torch_whl_x torch.whl + msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + else + msg "Downloading LibTorch ${LIBTORCH_VERSION}..." + wget "${LIBTORCH_URL}" -O libtorch.zip + msg "Extracting LibTorch..." + unzip -q libtorch.zip + rm libtorch.zip + msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch" + fi else msg "[SKIPPED] LibTorch already installed" fi