Skip to content

Commit 68671e0

Browse files
ananas-blockananas-block
committed
refactor: caching
1 parent 18c9244 commit 68671e0

36 files changed

Lines changed: 704 additions & 605 deletions

‎.github/actions/setup-and-build/action.yml‎

Lines changed: 139 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -21,119 +21,133 @@ runs:
2121
- name: Checkout sources
2222
uses: actions/checkout@v4
2323

24-
- name: Restore .local directory cache
25-
id: restore-local-cache
26-
uses: actions/cache@v4
27-
with:
28-
path: |
29-
.local
30-
prover/server/proving-keys
31-
key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }}
32-
restore-keys: |
33-
${{ runner.os }}-local-v3-
34-
${{ runner.os }}-local-v2-
35-
3624
- name: Install system dependencies
3725
shell: bash
3826
run: |
3927
sudo apt-get update
4028
sudo apt-get install -y libudev-dev pkg-config build-essential
4129
42-
- name: Install dependencies
43-
if: steps.restore-local-cache.outputs.cache-hit != 'true'
30+
- name: Load versions
31+
id: versions
4432
shell: bash
4533
run: |
46-
if [ -n "${{ inputs.skip-components }}" ]; then
47-
./scripts/install.sh --skip-components "${{ inputs.skip-components }}"
48-
else
49-
./scripts/install.sh
50-
fi
34+
source scripts/devenv/versions.sh
35+
echo "rust=${RUST_VERSION}" >> $GITHUB_OUTPUT
36+
echo "go=${GO_VERSION}" >> $GITHUB_OUTPUT
37+
echo "node=${NODE_VERSION}" >> $GITHUB_OUTPUT
38+
echo "pnpm=${PNPM_VERSION}" >> $GITHUB_OUTPUT
39+
echo "solana=${SOLANA_VERSION}" >> $GITHUB_OUTPUT
40+
echo "anchor=${ANCHOR_VERSION}" >> $GITHUB_OUTPUT
41+
echo "photon=${PHOTON_VERSION}" >> $GITHUB_OUTPUT
42+
echo "photon_commit=${PHOTON_COMMIT}" >> $GITHUB_OUTPUT
43+
echo "redis=${REDIS_VERSION}" >> $GITHUB_OUTPUT
5144
52-
- name: Verify and repair installation
53-
shell: bash
54-
run: |
55-
# Even if cache was restored, run install script to check and install missing components
56-
# The install script now checks for actual file existence, so it will only install what's missing
57-
echo "=== Verifying installation integrity ==="
58-
if [ -n "${{ inputs.skip-components }}" ]; then
59-
./scripts/install.sh --no-reset --skip-components "${{ inputs.skip-components }}"
60-
else
61-
./scripts/install.sh --no-reset
62-
fi
45+
- name: Setup Rust toolchain
46+
uses: actions-rust-lang/setup-rust-toolchain@v1
47+
with:
48+
toolchain: ${{ steps.versions.outputs.rust }}
49+
components: rustfmt, clippy
50+
cache-workspaces: |
51+
. -> target
52+
cli -> cli/target
6353
64-
- name: Validate environment setup
65-
shell: bash
66-
run: |
67-
echo "=== Validating environment setup ==="
68-
source ./scripts/devenv.sh
69-
70-
# Check critical binaries exist
71-
echo "Checking Go installation..."
72-
if [ -f "$GOROOT/bin/go" ]; then
73-
echo "✓ Go found at: $GOROOT/bin/go"
74-
$GOROOT/bin/go version || echo "⚠ Go binary exists but failed to run"
75-
else
76-
echo "✗ Go not found at expected location: $GOROOT/bin/go"
77-
exit 1
78-
fi
54+
- name: Setup Go
55+
uses: actions/setup-go@v5
56+
with:
57+
go-version: ${{ steps.versions.outputs.go }}
58+
cache-dependency-path: prover/server/go.sum
7959

80-
echo "Checking Rust installation..."
81-
if [ -f "$CARGO_HOME/bin/cargo" ]; then
82-
echo "✓ Cargo found at: $CARGO_HOME/bin/cargo"
83-
$CARGO_HOME/bin/cargo --version || echo "⚠ Cargo binary exists but failed to run"
84-
else
85-
echo "✗ Cargo not found at expected location: $CARGO_HOME/bin/cargo"
86-
exit 1
87-
fi
60+
- name: Setup Node.js
61+
uses: actions/setup-node@v4
62+
with:
63+
node-version: ${{ steps.versions.outputs.node }}
8864

89-
echo "Checking Node installation..."
90-
which node && node --version || echo "⚠ Node not in PATH"
65+
- name: Setup pnpm
66+
uses: pnpm/action-setup@v4
67+
with:
68+
version: ${{ steps.versions.outputs.pnpm }}
69+
run_install: false
9170

92-
echo "Checking pnpm installation..."
93-
which pnpm && pnpm --version || echo "⚠ pnpm not in PATH"
71+
- name: Cache Solana CLI
72+
id: cache-solana
73+
uses: actions/cache@v4
74+
with:
75+
path: |
76+
.local/bin/solana*
77+
.local/bin/agave*
78+
.local/bin/cargo-*
79+
key: ${{ runner.os }}-solana-${{ steps.versions.outputs.solana }}
9480

95-
echo "Checking Solana installation..."
96-
which solana && solana --version || echo "⚠ Solana not in PATH"
81+
- name: Install Solana CLI
82+
if: steps.cache-solana.outputs.cache-hit != 'true'
83+
shell: bash
84+
run: bash scripts/devenv/install-solana.sh
9785

98-
echo "Checking Anchor installation..."
99-
which anchor && anchor --version || echo "⚠ Anchor not in PATH"
86+
- name: Cache Anchor CLI
87+
id: cache-anchor
88+
uses: actions/cache@v4
89+
with:
90+
path: |
91+
.local/cargo/bin/avm
92+
.local/cargo/bin/anchor
93+
key: ${{ runner.os }}-anchor-${{ steps.versions.outputs.anchor }}
10094

101-
# Check proving keys
102-
if [ -d "prover/server/proving-keys" ] && [ -n "$(ls -A prover/server/proving-keys 2>/dev/null)" ]; then
103-
echo "✓ Proving keys found: $(ls prover/server/proving-keys | wc -l) files"
104-
else
105-
echo "✗ Proving keys directory missing or empty"
106-
exit 1
107-
fi
95+
- name: Install Anchor CLI
96+
if: steps.cache-anchor.outputs.cache-hit != 'true'
97+
shell: bash
98+
run: bash scripts/devenv/install-anchor.sh
10899

109-
echo "=== Environment validation complete ==="
100+
- name: Cache Photon indexer
101+
id: cache-photon
102+
uses: actions/cache@v4
103+
with:
104+
path: .local/cargo/bin/photon
105+
key: ${{ runner.os }}-photon-${{ steps.versions.outputs.photon }}-${{ steps.versions.outputs.photon_commit }}
110106

111-
- name: Set local environment
107+
- name: Install Photon indexer
108+
if: steps.cache-photon.outputs.cache-hit != 'true'
112109
shell: bash
113-
run: |
114-
source ./scripts/devenv.sh
115-
# Export critical environment variables for subsequent steps
116-
echo "GOROOT=$GOROOT" >> $GITHUB_ENV
117-
echo "CARGO_HOME=$CARGO_HOME" >> $GITHUB_ENV
118-
echo "RUSTUP_HOME=$RUSTUP_HOME" >> $GITHUB_ENV
119-
echo "PATH=$PATH" >> $GITHUB_ENV
120-
echo "CARGO_FEATURES=$CARGO_FEATURES" >> $GITHUB_ENV
121-
echo "REDIS_URL=$REDIS_URL" >> $GITHUB_ENV
122-
123-
- name: Rust cache
124-
uses: swatinem/rust-cache@v2
110+
run: bash scripts/devenv/install-photon.sh
111+
112+
- name: Cache Redis
113+
if: "!contains(inputs.skip-components, 'redis')"
114+
id: cache-redis
115+
uses: actions/cache@v4
125116
with:
126-
cache-all-crates: true
127-
cache-on-failure: true
128-
# Add workspace target directory
129-
workspaces: |
130-
. -> target
131-
cli -> cli/target
117+
path: |
118+
.local/bin/redis-*
119+
.local/etc/redis.conf
120+
key: ${{ runner.os }}-redis-${{ steps.versions.outputs.redis }}
132121

133-
- name: Setup pnpm
134-
uses: pnpm/action-setup@v4
122+
- name: Install Redis
123+
if: "!contains(inputs.skip-components, 'redis') && steps.cache-redis.outputs.cache-hit != 'true'"
124+
shell: bash
125+
run: bash scripts/devenv/install-redis.sh
126+
127+
- name: Cache proving keys
128+
id: cache-keys
129+
uses: actions/cache@v4
135130
with:
136-
run_install: false
131+
path: prover/server/proving-keys
132+
key: ${{ runner.os }}-proving-keys-${{ hashFiles('prover/server/scripts/download_keys.sh') }}
133+
134+
- name: Download proving keys
135+
if: steps.cache-keys.outputs.cache-hit != 'true'
136+
shell: bash
137+
run: bash scripts/devenv/download-gnark-keys.sh
138+
139+
- name: Set Light Protocol environment variables
140+
shell: bash
141+
run: |
142+
# Add .local/bin to PATH for Solana, Anchor, Photon, Redis
143+
echo "${PWD}/.local/bin" >> $GITHUB_PATH
144+
echo "${PWD}/.local/cargo/bin" >> $GITHUB_PATH
145+
146+
# Light Protocol-specific environment variables
147+
# (Official actions handle Rust, Go, Node.js environment setup)
148+
echo "CARGO_FEATURES=v2_ix" >> $GITHUB_ENV
149+
echo "REDIS_URL=redis://localhost:6379" >> $GITHUB_ENV
150+
echo "SBF_OUT_DIR=target/deploy" >> $GITHUB_ENV
137151
138152
- name: Get pnpm store directory
139153
id: pnpm-store
@@ -176,7 +190,6 @@ runs:
176190
- name: Generate Solana keypair
177191
shell: bash
178192
run: |
179-
source ./scripts/devenv.sh
180193
mkdir -p /home/runner/.config/solana/
181194
if [ ! -f /home/runner/.config/solana/id.json ]; then
182195
solana-keygen new --no-bip39-passphrase -o /home/runner/.config/solana/id.json
@@ -188,25 +201,50 @@ runs:
188201
mkdir -p ./target/deploy
189202
cp ./third-party/solana-program-library/spl_noop.so ./target/deploy/spl_noop.so
190203
204+
- name: Cache Solana programs
205+
id: cache-programs
206+
uses: actions/cache@v4
207+
with:
208+
path: |
209+
target/deploy/*.so
210+
target/idl
211+
target/types
212+
key: ${{ runner.os }}-programs-${{ hashFiles('programs/**/Cargo.toml', 'programs/**/Cargo.lock', 'programs/**/*.rs', 'program-libs/**/*.rs') }}
213+
restore-keys: |
214+
${{ runner.os }}-programs-
215+
216+
- name: Cache Solana program-tests
217+
id: cache-program-tests
218+
uses: actions/cache@v4
219+
with:
220+
path: |
221+
target/deploy/create_address_test_program.so
222+
target/deploy/sdk_anchor_test.so
223+
key: ${{ runner.os }}-program-tests-${{ hashFiles('program-tests/**/Cargo.toml', 'program-tests/**/Cargo.lock', 'program-tests/**/*.rs', 'test-programs/**/Cargo.toml', 'test-programs/**/*.rs') }}
224+
restore-keys: |
225+
${{ runner.os }}-program-tests-
226+
191227
- name: Build Rust programs
228+
if: steps.cache-programs.outputs.cache-hit != 'true'
192229
shell: bash
193230
run: |
194-
source ./scripts/devenv.sh
195231
echo "Building Rust programs..."
196232
npx nx build @lightprotocol/programs || {
197233
echo "Failed to build programs, retrying with verbose output..."
198234
npx nx build @lightprotocol/programs --verbose
199235
exit 1
200236
}
201237
202-
- name: Save .local directory cache
203-
if: steps.restore-local-cache.outputs.cache-hit != 'true'
204-
uses: actions/cache/save@v4
205-
with:
206-
path: |
207-
.local
208-
prover/server/proving-keys
209-
key: ${{ runner.os }}-local-v3-${{ hashFiles('scripts/install.sh', 'prover/server/scripts/download_keys.sh') }}
238+
- name: Build Rust program-tests
239+
if: steps.cache-program-tests.outputs.cache-hit != 'true'
240+
shell: bash
241+
run: |
242+
echo "Building Rust program-tests..."
243+
npx nx build @lightprotocol/program-tests || {
244+
echo "Failed to build program-tests, retrying with verbose output..."
245+
npx nx build @lightprotocol/program-tests --verbose
246+
exit 1
247+
}
210248
211249
- name: Check for git changes
212250
shell: bash

‎.github/workflows/cli-v1.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,24 +60,20 @@ jobs:
6060

6161
- name: Build stateless.js with V1
6262
run: |
63-
source ./scripts/devenv.sh
6463
cd js/stateless.js
6564
pnpm build:v1
6665
6766
- name: Build compressed-token with V1
6867
run: |
69-
source ./scripts/devenv.sh
7068
cd js/compressed-token
7169
pnpm build:v1
7270
7371
- name: Build CLI with V1 (CI mode - Linux x64 only)
7472
run: |
75-
source ./scripts/devenv.sh
7673
npx nx build-ci @lightprotocol/zk-compression-cli
7774
7875
- name: Run CLI tests with V1
7976
run: |
80-
source ./scripts/devenv.sh
8177
npx nx test @lightprotocol/zk-compression-cli
8278
8379
- name: Display prover logs on failure

‎.github/workflows/cli-v2.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,24 +60,20 @@ jobs:
6060

6161
- name: Build stateless.js with V2
6262
run: |
63-
source ./scripts/devenv.sh
6463
cd js/stateless.js
6564
pnpm build:v2
6665
6766
- name: Build compressed-token with V2
6867
run: |
69-
source ./scripts/devenv.sh
7068
cd js/compressed-token
7169
pnpm build:v2
7270
7371
- name: Build CLI with V2 (CI mode - Linux x64 only)
7472
run: |
75-
source ./scripts/devenv.sh
7673
npx nx build-ci @lightprotocol/zk-compression-cli
7774
7875
- name: Run CLI tests with V2
7976
run: |
80-
source ./scripts/devenv.sh
8177
npx nx test @lightprotocol/zk-compression-cli
8278
8379
- name: Display prover logs on failure

‎.github/workflows/forester-tests.yml‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,29 +67,17 @@ jobs:
6767
with:
6868
skip-components: "redis"
6969

70-
- name: Clean build artifacts before tests
71-
shell: bash
72-
run: |
73-
cargo clean
74-
rm -rf target/debug/deps/*
75-
7670
- name: Check available disk space
7771
shell: bash
7872
run: |
7973
df -h /
8074
du -sh /home/runner/work/* | sort -hr | head -n 10
8175
8276
- name: Build CLI (CI mode - Linux x64 only)
83-
run: |
84-
source ./scripts/devenv.sh
85-
npx nx build-ci @lightprotocol/zk-compression-cli
77+
run: npx nx build-ci @lightprotocol/zk-compression-cli
8678

8779
- name: Build test program
88-
run: |
89-
source ./scripts/devenv.sh
90-
cargo test-sbf -p create-address-test-program
80+
run: cargo test-sbf -p create-address-test-program
9181

9282
- name: Test
93-
run: |
94-
source ./scripts/devenv.sh
95-
cargo test --package forester e2e_test -- --nocapture
83+
run: cargo test --package forester e2e_test -- --nocapture

0 commit comments

Comments
 (0)