Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion default.env
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,9 @@ NETHERMIND_FLATDB=
# If this URL is provided, an EL that supports EraE import will use it when fresh syncing
# "urls.txt", "checksums.txt" and "checksums_sha256.txt" must exist at the target url
ERE_URL=
# EraC URL, see https://ethresear.ch/t/era-archival-files-for-block-and-consensus-data/13526
# If this URL is provided, a CL that supports EraC import will use it when fresh syncing
ERC_URL=

# If you want debug logs, set this to debug instead of info
LOG_LEVEL=info
Expand Down Expand Up @@ -576,4 +579,4 @@ DOCKER_SOCK=/var/run/docker.sock
ALLOY_PROJECT_NAME=

# Used by ethd update - please do not adjust
ENV_VERSION=63
ENV_VERSION=64
1 change: 1 addition & 0 deletions nimbus-allin1.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ services:
- jwtsecret:/var/lib/nimbus/ee-secret
environment:
- CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL}
- ERC_URL=${ERC_URL:-}
- NETWORK=${NETWORK}
- JWT_SECRET=${JWT_SECRET}
- MEV_BOOST=${MEV_BOOST}
Expand Down
1 change: 1 addition & 0 deletions nimbus-cl-only.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ services:
- jwtsecret:/var/lib/nimbus/ee-secret
environment:
- CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL}
- ERC_URL=${ERC_URL:-}
- NETWORK=${NETWORK}
- JWT_SECRET=${JWT_SECRET}
- MEV_BOOST=${MEV_BOOST}
Expand Down
112 changes: 105 additions & 7 deletions nimbus-el/docker-entrypoint-unified.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ __download_ere_files() {
download_dir="$2"

mkdir -p "${download_dir}"
cd "${download_dir}" || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; }
pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; }

# 🔧 Normalize base URL (handle trailing slash)
case "${download_url}" in
*/) base_url="${download_url%/}" ;;
*) base_url="${download_url}" ;;
esac

curl -sS -O "${base_url}/urls.txt"
curl -fsS -O "${base_url}/urls.txt"
total_files=$(wc -l < urls.txt)
aria2c -x 8 -j 5 -c -i urls.txt \
--dir="." \
Expand Down Expand Up @@ -87,9 +87,93 @@ __download_ere_files() {
echo "✅ All files downloaded to: ${download_dir}"

echo "Verifying checksums"
curl -sS -O "${base_url}/checksums_sha256.txt"
curl -fsS -O "${base_url}/checksums_sha256.txt"
sha256sum -c checksums_sha256.txt --ignore-missing
echo "✅ All checksums verified"

popd > /dev/null
}


__download_erc_files() {
# Copyright (c) 2025 Status Research & Development GmbH and 2026 Eth Docker maintainers.
# Licensed under either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.

# Usage: __download_erc_files <download_url> <download_path>

local download_url
local download_dir
local base_url
local completed
local percent
local total_files
local aria_pid

if [[ $# -ne 2 ]]; then
echo "__download_erc_files called without <download_url> <download_path>. This is a bug."
exit 70
fi

download_url="$1"
download_dir="$2"

mkdir -p "${download_dir}"
pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; }

# 🔧 Normalize base URL (handle trailing slash)
case "${download_url}" in
*/) base_url="${download_url%/}" ;;
*) base_url="${download_url}" ;;
esac

curl -fsS "${base_url}/" | sed -n 's/.*href="\([^"]*\.\(era\|erc\)\)".*/\1/p' > erc_files.txt
total_files=$(wc -l < erc_files.txt)
sed -i "s|^|${base_url}/|" erc_files.txt
aria2c -x 8 -j 5 -c -i erc_files.txt \
--dir="." \
--console-log-level=warn \
--quiet=true \
--summary-interval=0 \
--continue=true \
> /dev/null 2>&1 &

aria_pid=$!

echo "Downloading EraC history files"
echo "📥 Starting download of ${total_files} files..."
while kill -0 "${aria_pid}" 2> /dev/null; do
completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l)
percent=$(awk "BEGIN { printf \"%.1f\", (${completed}/${total_files})*100 }")
echo "📦 Download Progress: ${percent}% complete (${completed} / ${total_files} files)"
sleep 10
done

wait "${aria_pid}" && exitstatus=0 || exitstatus=$?
if [[ "${exitstatus}" -ne 0 ]]; then
echo "EraC download failed with exit code ${exitstatus}"
exit "${exitstatus}"
fi

completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l)
echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)"

echo "✅ All files downloaded to: ${download_dir}"

rm -f erc_files.txt

echo "Verifying checksums"
if curl -fsS -O "${base_url}/checksums_sha256.txt" 2>/dev/null; then
sha256sum -c checksums_sha256.txt --ignore-missing
echo "✅ All checksums verified"
else
echo "No checksums file available, skipping verification"
fi

popd > /dev/null
}


Expand Down Expand Up @@ -133,7 +217,7 @@ case "${EL_NODE_TYPE}" in
esac

case "${CL_NODE_TYPE}" in
archive)
archive|blob-archive)
echo "Nimbus Unified archive consensus node without history pruning"
__prune+=" --history=archive --reindex"
;;
Expand Down Expand Up @@ -201,9 +285,11 @@ if [[ -n "${ERE_URL}" && ! -f /var/lib/nimbus/ere-import-complete && ! "${NETWOR
fi
fi

if [[ -n "${CHECKPOINT_SYNC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then
if [[ "${CL_NODE_TYPE}" = "archive" ]]; then
if [[ -n "${CHECKPOINT_SYNC_URL}" && -z "${ERC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then
if [[ "${CL_NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then
echo "Starting checkpoint sync with backfill and archive reindex. Nimbus will restart when done."
echo "NB: For this to work, the checkpoint server has to serve blocks"
echo "Alternatively, use \"ERC_URL\" to download EraC files"
# Word splitting is desired for the command line parameters
# shellcheck disable=SC2086
nimbus trustedNodeSync --backfill=true --reindex ${__network} --data-dir=/var/lib/nimbus --trusted-node-url="${CHECKPOINT_SYNC_URL}"
Expand All @@ -217,6 +303,18 @@ if [[ -n "${CHECKPOINT_SYNC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then
fi
fi

__erc_dir=""
if [[ -n "${ERC_URL}" && ! -f /var/lib/nimbus/erc-download-complete && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network
if [[ "${CL_NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then
echo "Starting EraC history file download from ${ERC_URL}"
__download_erc_files "${ERC_URL}" /var/lib/nimbus/erc
touch /var/lib/nimbus/erc-download-complete
__erc_dir="--era-dir=/var/lib/nimbus/erc"
else
echo "Nimbus is not an archive node, it uses ${CL_NODE_TYPE}. Skipping EraC import."
fi
fi

# Check whether we should use MEV Boost
if [[ "${MEV_BOOST}" = "true" ]]; then
__mev_boost="--payload-builder=true --payload-builder-url=${MEV_NODE:-http://mev-boost:18550}"
Expand All @@ -230,4 +328,4 @@ set -- "${__args[@]}"

# Word splitting is desired for the command line parameters
# shellcheck disable=SC2086
exec "$@" ${__prune} ${__mev_boost} ${__network} ${EL_EXTRAS} ${CL_EXTRAS}
exec "$@" ${__prune} ${__mev_boost} ${__network} ${__erc_dir} ${EL_EXTRAS} ${CL_EXTRAS}
1 change: 1 addition & 0 deletions nimbus-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ services:
- NETWORK=${NETWORK}
- CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL}
- ERE_URL=${ERE_URL:-}
- ERC_URL=${ERC_URL:-}
volumes:
- nimbus-unified-data:/var/lib/nimbus
- /etc/localtime:/etc/localtime:ro
Expand Down
1 change: 1 addition & 0 deletions nimbus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ services:
- jwtsecret:/var/lib/nimbus/ee-secret
environment:
- CHECKPOINT_SYNC_URL=${CHECKPOINT_SYNC_URL}
- ERC_URL=${ERC_URL:-}
- NETWORK=${NETWORK}
- JWT_SECRET=${JWT_SECRET}
- MEV_BOOST=${MEV_BOOST}
Expand Down
1 change: 1 addition & 0 deletions nimbus/Dockerfile.binary
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
git \
git-lfs \
curl \
aria2 \
&& gosu nobody true \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
1 change: 1 addition & 0 deletions nimbus/Dockerfile.source
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
git \
git-lfs \
curl \
aria2 \
&& gosu nobody true \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
1 change: 1 addition & 0 deletions nimbus/Dockerfile.sourcegnosis
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
git \
git-lfs \
curl \
aria2 \
&& gosu nobody true \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
110 changes: 103 additions & 7 deletions nimbus/docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,88 @@ __normalize_int() {
}


__download_erc_files() {
# Copyright (c) 2025 Status Research & Development GmbH and 2026 Eth Docker maintainers.
# Licensed under either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.

# Usage: __download_erc_files <download_url> <download_path>

local download_url
local download_dir
local base_url
local completed
local percent
local total_files
local aria_pid

if [[ $# -ne 2 ]]; then
echo "__download_erc_files called without <download_url> <download_path>. This is a bug."
exit 70
fi

download_url="$1"
download_dir="$2"

mkdir -p "${download_dir}"
pushd "${download_dir}" > /dev/null || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; }

# 🔧 Normalize base URL (handle trailing slash)
case "${download_url}" in
*/) base_url="${download_url%/}" ;;
*) base_url="${download_url}" ;;
esac

curl -fsS "${base_url}/" | sed -n 's/.*href="\([^"]*\.\(era\|erc\)\)".*/\1/p' > erc_files.txt
total_files=$(wc -l < erc_files.txt)
sed -i "s|^|${base_url}/|" erc_files.txt
aria2c -x 8 -j 5 -c -i erc_files.txt \
--dir="." \
--console-log-level=warn \
--quiet=true \
--summary-interval=0 \
--continue=true \
> /dev/null 2>&1 &

aria_pid=$!

echo "Downloading EraC history files"
echo "📥 Starting download of ${total_files} files..."
while kill -0 "${aria_pid}" 2> /dev/null; do
completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l)
percent=$(awk "BEGIN { printf \"%.1f\", (${completed}/${total_files})*100 }")
echo "📦 Download Progress: ${percent}% complete (${completed} / ${total_files} files)"
sleep 10
done

wait "${aria_pid}" && exitstatus=0 || exitstatus=$?
if [[ "${exitstatus}" -ne 0 ]]; then
echo "EraC download failed with exit code ${exitstatus}"
exit "${exitstatus}"
fi

completed=$(find . -type f \( -name '*.era' -o -name '*.erc' \) | wc -l)
echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)"

echo "✅ All files downloaded to: ${download_dir}"

rm -f erc_files.txt

echo "Verifying checksums"
if curl -fsS -O "${base_url}/checksums_sha256.txt" 2>/dev/null; then
sha256sum -c checksums_sha256.txt --ignore-missing
echo "✅ All checksums verified"
else
echo "No checksums file available, skipping verification"
fi

popd > /dev/null
}


if [[ ! -f /var/lib/nimbus/api-token.txt ]]; then
token=api-token-0x$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32)$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32)
echo "${token}" > /var/lib/nimbus/api-token.txt
Expand Down Expand Up @@ -69,9 +151,11 @@ else
__network="--network=${NETWORK}"
fi

if [[ -n "${CHECKPOINT_SYNC_URL:+x}" && ! -f /var/lib/nimbus/setupdone ]]; then
if [[ "${NODE_TYPE}" = "archive" ]]; then
if [[ -n "${CHECKPOINT_SYNC_URL}" && -z "${ERC_URL}" && ! -f /var/lib/nimbus/setupdone ]]; then
if [[ "${NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then
echo "Starting checkpoint sync with backfill and archive reindex. Nimbus will restart when done."
echo "NB: For this to work, the checkpoint server has to serve blocks"
echo "Alternatively, use \"ERC_URL\" to download EraC files"
# Word splitting is desired for the command line parameters
# shellcheck disable=SC2086
/usr/local/bin/nimbus_beacon_node trustedNodeSync --backfill=true --reindex ${__network} --data-dir=/var/lib/nimbus --trusted-node-url="${CHECKPOINT_SYNC_URL}"
Expand All @@ -85,6 +169,18 @@ if [[ -n "${CHECKPOINT_SYNC_URL:+x}" && ! -f /var/lib/nimbus/setupdone ]]; then
fi
fi

__erc_dir=""
if [[ -n "${ERC_URL}" && ! -f /var/lib/nimbus/erc-download-complete && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network
if [[ "${NODE_TYPE}" =~ ^(archive|blob-archive)$ ]]; then
echo "Starting EraC history file download from ${ERC_URL}"
__download_erc_files "${ERC_URL}" /var/lib/nimbus/erc
touch /var/lib/nimbus/erc-download-complete
__erc_dir="--era-dir=/var/lib/nimbus/erc"
else
echo "Nimbus is not an archive node, it uses ${NODE_TYPE}. Skipping EraC import."
fi
fi

# Check whether we should use MEV Boost
if [[ "${MEV_BOOST}" = "true" ]]; then
__mev_boost="--payload-builder=true --payload-builder-url=${MEV_NODE:-http://mev-boost:18550}"
Expand Down Expand Up @@ -132,9 +228,9 @@ else
fi

case "${NODE_TYPE}" in
archive)
echo "Nimbus archive node without history pruning"
__prune="--history=archive"
archive|blob-archive)
echo "Nimbus archive node without history or blob pruning."
__prune="--history=archive --reindex"
;;
full)
__prune=""
Expand Down Expand Up @@ -189,9 +285,9 @@ done
if [[ "${DEFAULT_GRAFFITI}" = "true" ]]; then
# Word splitting is desired for the command line parameters
# shellcheck disable=SC2086
exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${CL_EXTRAS} ${VC_EXTRAS}
exec "$@" ${__network} ${__w3s_url} ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${__erc_dir} ${CL_EXTRAS} ${VC_EXTRAS}
else
# Word splitting is desired for the command line parameters
# shellcheck disable=SC2086
exec "$@" ${__network} ${__w3s_url} "--graffiti=${GRAFFITI}" ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${CL_EXTRAS} ${VC_EXTRAS}
exec "$@" ${__network} ${__w3s_url} "--graffiti=${GRAFFITI}" ${__mev_boost} ${__mev_factor} ${__doppel} ${__prune} ${__erc_dir} ${CL_EXTRAS} ${VC_EXTRAS}
fi
Loading