From 75c6b72053f77c5e42f7812937b273c05be9ea2d Mon Sep 17 00:00:00 2001 From: Yorick Downe Date: Thu, 2 Jul 2026 12:29:53 +0100 Subject: [PATCH] Nimbus archive node improvements --- default.env | 5 +- nimbus-allin1.yml | 1 + nimbus-cl-only.yml | 1 + nimbus-el/docker-entrypoint-unified.sh | 112 +++++++++++++++++++++++-- nimbus-unified.yml | 1 + nimbus.yml | 1 + nimbus/Dockerfile.binary | 1 + nimbus/Dockerfile.source | 1 + nimbus/Dockerfile.sourcegnosis | 1 + nimbus/docker-entrypoint.sh | 110 ++++++++++++++++++++++-- 10 files changed, 219 insertions(+), 15 deletions(-) diff --git a/default.env b/default.env index 1563bcee1..7b3b04060 100644 --- a/default.env +++ b/default.env @@ -283,6 +283,9 @@ NETHERMIND_FLATDB= # If this URL is provided, an EL that supports EraE import will use it when fresh syncing # "urls.txt", "checksums.txt" and "checksums_sha256.txt" must exist at the target url ERE_URL= +# EraC URL, see https://ethresear.ch/t/era-archival-files-for-block-and-consensus-data/13526 +# If this URL is provided, a CL that supports EraC import will use it when fresh syncing +ERC_URL= # If you want debug logs, set this to debug instead of info LOG_LEVEL=info @@ -576,4 +579,4 @@ DOCKER_SOCK=/var/run/docker.sock ALLOY_PROJECT_NAME= # Used by ethd update - please do not adjust -ENV_VERSION=63 +ENV_VERSION=64 diff --git a/nimbus-allin1.yml b/nimbus-allin1.yml index c40559923..3b01c6f19 100644 --- a/nimbus-allin1.yml +++ b/nimbus-allin1.yml @@ -43,6 +43,7 @@ services: - jwtsecret:/var/lib/nimbus/ee-secret environment: - CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL} + - ERC_URL=${ERC_URL:-} - NETWORK=${NETWORK} - JWT_SECRET=${JWT_SECRET} - MEV_BOOST=${MEV_BOOST} diff --git a/nimbus-cl-only.yml b/nimbus-cl-only.yml index 8338a2c2d..574d53af3 100644 --- a/nimbus-cl-only.yml +++ b/nimbus-cl-only.yml @@ -33,6 +33,7 @@ services: - jwtsecret:/var/lib/nimbus/ee-secret environment: - CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL} + - ERC_URL=${ERC_URL:-} - NETWORK=${NETWORK} - JWT_SECRET=${JWT_SECRET} - MEV_BOOST=${MEV_BOOST} diff --git a/nimbus-el/docker-entrypoint-unified.sh b/nimbus-el/docker-entrypoint-unified.sh index 0d4a0bbf0..bccd3f865 100755 --- a/nimbus-el/docker-entrypoint-unified.sh +++ b/nimbus-el/docker-entrypoint-unified.sh @@ -46,7 +46,7 @@ __download_ere_files() { download_dir="$2" mkdir -p "${download_dir}" - cd "${download_dir}" || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; } + pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; } # 🔧 Normalize base URL (handle trailing slash) case "${download_url}" in @@ -54,7 +54,7 @@ __download_ere_files() { *) base_url="${download_url}" ;; esac - curl -sS -O "${base_url}/urls.txt" + curl -fsS -O "${base_url}/urls.txt" total_files=$(wc -l < urls.txt) aria2c -x 8 -j 5 -c -i urls.txt \ --dir="." \ @@ -87,9 +87,93 @@ __download_ere_files() { echo "✅ All files downloaded to: ${download_dir}" echo "Verifying checksums" - curl -sS -O "${base_url}/checksums_sha256.txt" + curl -fsS -O "${base_url}/checksums_sha256.txt" sha256sum -c checksums_sha256.txt --ignore-missing echo "✅ All checksums verified" + + popd > /dev/null +} + + +__download_erc_files() { +# Copyright (c) 2025 Status Research & Development GmbH and 2026 Eth Docker maintainers. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# Usage: __download_erc_files + + local download_url + local download_dir + local base_url + local completed + local percent + local total_files + local aria_pid + + if [[ $# -ne 2 ]]; then + echo "__download_erc_files called without . This is a bug." + exit 70 + fi + + download_url="$1" + download_dir="$2" + + mkdir -p "${download_dir}" + pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; } + + # 🔧 Normalize base URL (handle trailing slash) + case "${download_url}" in + */) base_url="${download_url%/}" ;; + *) base_url="${download_url}" ;; + esac + + curl -fsS "${base_url}/" | sed -n 's/.*href="\([^"]*\.\(era\|erc\)\)".*/\1/p' > erc_files.txt + total_files=$(wc -l < erc_files.txt) + sed -i "s|^|${base_url}/|" erc_files.txt + aria2c -x 8 -j 5 -c -i erc_files.txt \ + --dir="." \ + --console-log-level=warn \ + --quiet=true \ + --summary-interval=0 \ + --continue=true \ + > /dev/null 2>&1 & + + aria_pid=$! + + echo "Downloading EraC history files" + echo "📥 Starting download of ${total_files} files..." + while kill -0 "${aria_pid}" 2> /dev/null; do + completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l) + percent=$(awk "BEGIN { printf \"%.1f\", (${completed}/${total_files})*100 }") + echo "📦 Download Progress: ${percent}% complete (${completed} / ${total_files} files)" + sleep 10 + done + + wait "${aria_pid}" && exitstatus=0 || exitstatus=$? + if [[ "${exitstatus}" -ne 0 ]]; then + echo "EraC download failed with exit code ${exitstatus}" + exit "${exitstatus}" + fi + + completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l) + echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)" + + echo "✅ All files downloaded to: ${download_dir}" + + rm -f erc_files.txt + + echo "Verifying checksums" + if curl -fsS -O "${base_url}/checksums_sha256.txt" 2>/dev/null; then + sha256sum -c checksums_sha256.txt --ignore-missing + echo "✅ All checksums verified" + else + echo "No checksums file available, skipping verification" + fi + + popd > /dev/null } @@ -133,7 +217,7 @@ case "${EL_NODE_TYPE}" in esac case "${CL_NODE_TYPE}" in - archive) + archive|blob-archive) echo "Nimbus Unified archive consensus node without history pruning" __prune+=" --history=archive --reindex" ;; @@ -201,9 +285,11 @@ if [[ -n "${ERE_URL}" && ! -f /var/lib/nimbus/ere-import-complete && ! "${NETWOR fi fi -if [[ -n "${CHECKPOINT_SYNC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then - if [[ "${CL_NODE_TYPE}" = "archive" ]]; then +if [[ -n "${CHECKPOINT_SYNC_URL}" && -z "${ERC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then + if [[ "${CL_NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then echo "Starting checkpoint sync with backfill and archive reindex. Nimbus will restart when done." + echo "NB: For this to work, the checkpoint server has to serve blocks" + echo "Alternatively, use \"ERC_URL\" to download EraC files" # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 nimbus trustedNodeSync --backfill=true --reindex ${__network} --data-dir=/var/lib/nimbus --trusted-node-url="${CHECKPOINT_SYNC_URL}" @@ -217,6 +303,18 @@ if [[ -n "${CHECKPOINT_SYNC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then fi fi +__erc_dir="" +if [[ -n "${ERC_URL}" && ! -f /var/lib/nimbus/erc-download-complete && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network + if [[ "${CL_NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then + echo "Starting EraC history file download from ${ERC_URL}" + __download_erc_files "${ERC_URL}" /var/lib/nimbus/erc + touch /var/lib/nimbus/erc-download-complete + __erc_dir="--era-dir=/var/lib/nimbus/erc" + else + echo "Nimbus is not an archive node, it uses ${CL_NODE_TYPE}. Skipping EraC import." + fi +fi + # Check whether we should use MEV Boost if [[ "${MEV_BOOST}" = "true" ]]; then __mev_boost="--payload-builder=true --payload-builder-url=${MEV_NODE:-http://mev-boost:18550}" @@ -230,4 +328,4 @@ set -- "${__args[@]}" # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 -exec "$@" ${__prune} ${__mev_boost} ${__network} ${EL_EXTRAS} ${CL_EXTRAS} +exec "$@" ${__prune} ${__mev_boost} ${__network} ${__erc_dir} ${EL_EXTRAS} ${CL_EXTRAS} diff --git a/nimbus-unified.yml b/nimbus-unified.yml index 424f8fadf..9e328308d 100644 --- a/nimbus-unified.yml +++ b/nimbus-unified.yml @@ -32,6 +32,7 @@ services: - NETWORK=${NETWORK} - CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL} - ERE_URL=${ERE_URL:-} + - ERC_URL=${ERC_URL:-} volumes: - nimbus-unified-data:/var/lib/nimbus - /etc/localtime:/etc/localtime:ro diff --git a/nimbus.yml b/nimbus.yml index a197fe445..7c38dbe1d 100644 --- a/nimbus.yml +++ b/nimbus.yml @@ -33,6 +33,7 @@ services: - jwtsecret:/var/lib/nimbus/ee-secret environment: - CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL} + - ERC_URL=${ERC_URL:-} - NETWORK=${NETWORK} - JWT_SECRET=${JWT_SECRET} - MEV_BOOST=${MEV_BOOST} diff --git a/nimbus/Dockerfile.binary b/nimbus/Dockerfile.binary index 535c4504d..df7d8f7fa 100644 --- a/nimbus/Dockerfile.binary +++ b/nimbus/Dockerfile.binary @@ -24,6 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install git \ git-lfs \ curl \ + aria2 \ && gosu nobody true \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/nimbus/Dockerfile.source b/nimbus/Dockerfile.source index 3f2f8dfb2..153957a0c 100644 --- a/nimbus/Dockerfile.source +++ b/nimbus/Dockerfile.source @@ -54,6 +54,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install git \ git-lfs \ curl \ + aria2 \ && gosu nobody true \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/nimbus/Dockerfile.sourcegnosis b/nimbus/Dockerfile.sourcegnosis index d10e3521c..2c039dffd 100644 --- a/nimbus/Dockerfile.sourcegnosis +++ b/nimbus/Dockerfile.sourcegnosis @@ -54,6 +54,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install git \ git-lfs \ curl \ + aria2 \ && gosu nobody true \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/nimbus/docker-entrypoint.sh b/nimbus/docker-entrypoint.sh index d563300cc..0ff5d02e8 100755 --- a/nimbus/docker-entrypoint.sh +++ b/nimbus/docker-entrypoint.sh @@ -27,6 +27,88 @@ __normalize_int() { } +__download_erc_files() { +# Copyright (c) 2025 Status Research & Development GmbH and 2026 Eth Docker maintainers. +# Licensed under either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# Usage: __download_erc_files + + local download_url + local download_dir + local base_url + local completed + local percent + local total_files + local aria_pid + + if [[ $# -ne 2 ]]; then + echo "__download_erc_files called without . This is a bug." + exit 70 + fi + + download_url="$1" + download_dir="$2" + + mkdir -p "${download_dir}" + pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; } + + # 🔧 Normalize base URL (handle trailing slash) + case "${download_url}" in + */) base_url="${download_url%/}" ;; + *) base_url="${download_url}" ;; + esac + + curl -fsS "${base_url}/" | sed -n 's/.*href="\([^"]*\.\(era\|erc\)\)".*/\1/p' > erc_files.txt + total_files=$(wc -l < erc_files.txt) + sed -i "s|^|${base_url}/|" erc_files.txt + aria2c -x 8 -j 5 -c -i erc_files.txt \ + --dir="." \ + --console-log-level=warn \ + --quiet=true \ + --summary-interval=0 \ + --continue=true \ + > /dev/null 2>&1 & + + aria_pid=$! + + echo "Downloading EraC history files" + echo "📥 Starting download of ${total_files} files..." + while kill -0 "${aria_pid}" 2> /dev/null; do + completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l) + percent=$(awk "BEGIN { printf \"%.1f\", (${completed}/${total_files})*100 }") + echo "📦 Download Progress: ${percent}% complete (${completed} / ${total_files} files)" + sleep 10 + done + + wait "${aria_pid}" && exitstatus=0 || exitstatus=$? + if [[ "${exitstatus}" -ne 0 ]]; then + echo "EraC download failed with exit code ${exitstatus}" + exit "${exitstatus}" + fi + + completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l) + echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)" + + echo "✅ All files downloaded to: ${download_dir}" + + rm -f erc_files.txt + + echo "Verifying checksums" + if curl -fsS -O "${base_url}/checksums_sha256.txt" 2>/dev/null; then + sha256sum -c checksums_sha256.txt --ignore-missing + echo "✅ All checksums verified" + else + echo "No checksums file available, skipping verification" + fi + + popd > /dev/null +} + + if [[ ! -f /var/lib/nimbus/api-token.txt ]]; then token=api-token-0x$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32)$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32) echo "${token}" > /var/lib/nimbus/api-token.txt @@ -69,9 +151,11 @@ else __network="--network=${NETWORK}" fi -if [[ -n "${CHECKPOINT_SYNC_URL:+x}" && ! -f /var/lib/nimbus/setupdone ]]; then - if [[ "${NODE_TYPE}" = "archive" ]]; then +if [[ -n "${CHECKPOINT_SYNC_URL}" && -z "${ERC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then + if [[ "${NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then echo "Starting checkpoint sync with backfill and archive reindex. Nimbus will restart when done." + echo "NB: For this to work, the checkpoint server has to serve blocks" + echo "Alternatively, use \"ERC_URL\" to download EraC files" # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 /usr/local/bin/nimbus_beacon_node trustedNodeSync --backfill=true --reindex ${__network} --data-dir=/var/lib/nimbus --trusted-node-url="${CHECKPOINT_SYNC_URL}" @@ -85,6 +169,18 @@ if [[ -n "${CHECKPOINT_SYNC_URL:+x}" && ! -f /var/lib/nimbus/setupdone ]]; then fi fi +__erc_dir="" +if [[ -n "${ERC_URL}" && ! -f /var/lib/nimbus/erc-download-complete && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network + if [[ "${NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then + echo "Starting EraC history file download from ${ERC_URL}" + __download_erc_files "${ERC_URL}" /var/lib/nimbus/erc + touch /var/lib/nimbus/erc-download-complete + __erc_dir="--era-dir=/var/lib/nimbus/erc" + else + echo "Nimbus is not an archive node, it uses ${NODE_TYPE}. Skipping EraC import." + fi +fi + # Check whether we should use MEV Boost if [[ "${MEV_BOOST}" = "true" ]]; then __mev_boost="--payload-builder=true --payload-builder-url=${MEV_NODE:-http://mev-boost:18550}" @@ -132,9 +228,9 @@ else fi case "${NODE_TYPE}" in - archive) - echo "Nimbus archive node without history pruning" - __prune="--history=archive" + archive|blob-archive) + echo "Nimbus archive node without history or blob pruning." + __prune="--history=archive --reindex" ;; full) __prune="" @@ -189,9 +285,9 @@ done if [[ "${DEFAULT_GRAFFITI}" = "true" ]]; then # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 - exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${CL_EXTRAS} ${VC_EXTRAS} + exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${__erc_dir} ${CL_EXTRAS} ${VC_EXTRAS} else # Word splitting is desired for the command line parameters # shellcheck disable=SC2086 - exec "$@" ${__network} ${__w3s_url} "--graffiti=${GRAFFITI}" ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${CL_EXTRAS} ${VC_EXTRAS} + exec "$@" ${__network} ${__w3s_url} "--graffiti=${GRAFFITI}" ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${__erc_dir} ${CL_EXTRAS} ${VC_EXTRAS} fi