Skip to content

Commit f90b24c

Browse files
trxcllnttrxcllnt
authored
Use sccache-dist build cluster for conda and wheel builds (#5332)
RAPIDS has deployed an autoscaling cloud build cluster that can be used to accelerate building large RAPIDS projects. This PR updates the conda and wheel builds to use the build cluster. This contributes to rapidsai/build-planning#228. Authors: - Paul Taylor (https://github.com/trxcllnt) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Bradley Dice (https://github.com/bdice) URL: #5332
1 parent 0f1afef commit f90b24c

File tree

11 files changed

+102
-25
lines changed

11 files changed

+102
-25
lines changed

‎.devcontainer/Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ ENV HISTFILE="/home/coder/.cache/._bash_history"
4040
ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
4141
ENV SCCACHE_REGION="us-east-2"
4242
ENV SCCACHE_BUCKET="rapids-sccache-devs"
43+
ENV SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
4344
ENV SCCACHE_IDLE_TIMEOUT=0
4445

4546
###

‎.github/workflows/build.yaml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ jobs:
3939
build_type: ${{ inputs.build_type || 'branch' }}
4040
branch: ${{ inputs.branch }}
4141
date: ${{ inputs.date }}
42+
node_type: cpu16
4243
script: ci/build_cpp.sh
4344
sha: ${{ inputs.sha }}
44-
node_type: cpu32
45+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4546
python-build:
4647
needs: [cpp-build]
4748
secrets: inherit
@@ -52,6 +53,7 @@ jobs:
5253
date: ${{ inputs.date }}
5354
script: ci/build_python.sh
5455
sha: ${{ inputs.sha }}
56+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
5557
upload-conda:
5658
needs: [cpp-build, python-build]
5759
secrets: inherit
@@ -86,9 +88,10 @@ jobs:
8688
sha: ${{ inputs.sha }}
8789
date: ${{ inputs.date }}
8890
script: ci/build_wheel_libcugraph.sh
89-
node_type: cpu32
91+
node_type: cpu16
9092
package-name: libcugraph
9193
package-type: cpp
94+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
9295
wheel-publish-libcugraph:
9396
needs: wheel-build-libcugraph
9497
secrets: inherit
@@ -109,9 +112,11 @@ jobs:
109112
branch: ${{ inputs.branch }}
110113
sha: ${{ inputs.sha }}
111114
date: ${{ inputs.date }}
115+
node_type: cpu8
112116
script: ci/build_wheel_pylibcugraph.sh
113117
package-name: pylibcugraph
114118
package-type: python
119+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
115120
wheel-publish-pylibcugraph:
116121
needs: wheel-build-pylibcugraph
117122
secrets: inherit
@@ -132,9 +137,11 @@ jobs:
132137
branch: ${{ inputs.branch }}
133138
sha: ${{ inputs.sha }}
134139
date: ${{ inputs.date }}
140+
node_type: cpu8
135141
script: ci/build_wheel_cugraph.sh
136142
package-name: cugraph
137143
package-type: python
144+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
138145
wheel-publish-cugraph:
139146
needs: wheel-build-cugraph
140147
secrets: inherit

‎.github/workflows/pr.yaml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,9 @@ jobs:
9797
uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@release/25.12
9898
with:
9999
build_type: pull-request
100-
node_type: cpu32
100+
node_type: cpu16
101101
script: ci/build_cpp.sh
102+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
102103
conda-cpp-tests:
103104
needs: [conda-cpp-build, changed-files]
104105
secrets: inherit
@@ -107,6 +108,7 @@ jobs:
107108
with:
108109
build_type: pull-request
109110
script: ci/test_cpp.sh
111+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
110112
conda-cpp-checks:
111113
needs: conda-cpp-build
112114
secrets: inherit
@@ -121,6 +123,7 @@ jobs:
121123
with:
122124
build_type: pull-request
123125
script: ci/build_python.sh
126+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
124127
conda-python-tests:
125128
needs: [conda-python-build, changed-files]
126129
secrets: inherit
@@ -129,6 +132,7 @@ jobs:
129132
with:
130133
build_type: pull-request
131134
script: ci/test_python.sh
135+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
132136
conda-notebook-tests:
133137
needs: [conda-python-build, changed-files]
134138
secrets: inherit
@@ -158,18 +162,22 @@ jobs:
158162
# build for every combination of arch and CUDA version, but only for the latest Python
159163
matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
160164
build_type: pull-request
165+
node_type: cpu16
161166
script: ci/build_wheel_libcugraph.sh
162167
package-name: libcugraph
163168
package-type: cpp
169+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
164170
wheel-build-pylibcugraph:
165171
needs: wheel-build-libcugraph
166172
secrets: inherit
167173
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/25.12
168174
with:
169175
build_type: pull-request
176+
node_type: cpu8
170177
script: ci/build_wheel_pylibcugraph.sh
171178
package-name: pylibcugraph
172179
package-type: python
180+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
173181
wheel-tests-pylibcugraph:
174182
needs: [wheel-build-pylibcugraph, changed-files]
175183
secrets: inherit
@@ -178,15 +186,18 @@ jobs:
178186
with:
179187
build_type: pull-request
180188
script: ci/test_wheel_pylibcugraph.sh
189+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
181190
wheel-build-cugraph:
182191
needs: wheel-build-pylibcugraph
183192
secrets: inherit
184193
uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@release/25.12
185194
with:
186195
build_type: pull-request
196+
node_type: cpu8
187197
script: ci/build_wheel_cugraph.sh
188198
package-name: cugraph
189199
package-type: python
200+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
190201
wheel-tests-cugraph:
191202
needs: [wheel-build-cugraph, changed-files]
192203
secrets: inherit
@@ -195,6 +206,7 @@ jobs:
195206
with:
196207
build_type: pull-request
197208
script: ci/test_wheel_cugraph.sh
209+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
198210
devcontainer:
199211
secrets: inherit
200212
needs: telemetry-setup

‎.github/workflows/test.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ jobs:
4141
date: ${{ inputs.date }}
4242
script: ci/test_cpp.sh
4343
sha: ${{ inputs.sha }}
44+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
4445
conda-python-tests:
4546
secrets: inherit
4647
uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@release/25.12
@@ -50,6 +51,7 @@ jobs:
5051
date: ${{ inputs.date }}
5152
script: ci/test_python.sh
5253
sha: ${{ inputs.sha }}
54+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
5355
wheel-tests-pylibcugraph:
5456
secrets: inherit
5557
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@release/25.12
@@ -59,6 +61,7 @@ jobs:
5961
date: ${{ inputs.date }}
6062
sha: ${{ inputs.sha }}
6163
script: ci/test_wheel_pylibcugraph.sh
64+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN
6265
wheel-tests-cugraph:
6366
secrets: inherit
6467
uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@release/25.12
@@ -68,3 +71,4 @@ jobs:
6871
date: ${{ inputs.date }}
6972
sha: ${{ inputs.sha }}
7073
script: ci/test_wheel_cugraph.sh
74+
sccache-dist-token-secret-name: GIST_REPO_READ_ORG_GITHUB_TOKEN

‎ci/build_cpp.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -14,7 +13,7 @@ rapids-print-env
1413

1514
rapids-logger "Begin cpp build"
1615

17-
sccache --zero-stats
16+
sccache --stop-server 2>/dev/null || true
1817

1918
RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
2019
export RAPIDS_PACKAGE_VERSION
@@ -30,6 +29,7 @@ rattler-build build --recipe conda/recipes/libcugraph \
3029
"${RATTLER_CHANNELS[@]}"
3130

3231
sccache --show-adv-stats
32+
sccache --stop-server >/dev/null 2>&1 || true
3333

3434
# remove build_cache directory to avoid uploading the entire source tree
3535
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_python.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
set -euo pipefail
66

77
source rapids-configure-sccache
8-
98
source rapids-date-string
109

1110
export CMAKE_GENERATOR=Ninja
@@ -26,7 +25,7 @@ rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS"
2625

2726
RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}")
2827

29-
sccache --zero-stats
28+
sccache --stop-server 2>/dev/null || true
3029

3130
rapids-logger "Building pylibcugraph"
3231

@@ -38,7 +37,7 @@ rattler-build build --recipe conda/recipes/pylibcugraph \
3837
"${RATTLER_CHANNELS[@]}"
3938

4039
sccache --show-adv-stats
41-
sccache --zero-stats
40+
sccache --stop-server >/dev/null 2>&1 || true
4241

4342
rapids-logger "Building cugraph"
4443

@@ -47,6 +46,7 @@ rattler-build build --recipe conda/recipes/cugraph \
4746
"${RATTLER_CHANNELS[@]}"
4847

4948
sccache --show-adv-stats
49+
sccache --stop-server >/dev/null 2>&1 || true
5050

5151
# remove build_cache directory to avoid uploading the entire source tree
5252
# tracked in https://github.com/prefix-dev/rattler-build/issues/1424

‎ci/build_wheel.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ source rapids-configure-sccache
1111
source rapids-date-string
1212
source rapids-init-pip
1313

14+
export SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX="${package_name}/${RAPIDS_CONDA_ARCH}/cuda${RAPIDS_CUDA_VERSION%%.*}/wheel/preprocessor-cache"
15+
export SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE=true
16+
1417
rapids-generate-version > ./VERSION
1518

1619
cd "${package_dir}"
1720

18-
sccache --zero-stats
21+
sccache --stop-server 2>/dev/null || true
1922

2023
rapids-logger "Building '${package_name}' wheel"
2124

@@ -28,6 +31,7 @@ rapids-pip-retry wheel \
2831
.
2932

3033
sccache --show-adv-stats
34+
sccache --stop-server >/dev/null 2>&1 || true
3135

3236
EXCLUDE_ARGS=(
3337
--exclude "libraft.so"

‎cmake/rapids_config.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# =============================================================================
2+
# cmake-format: off
23
# SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION.
34
# SPDX-License-Identifier: Apache-2.0
5+
# cmake-format: on
46
# =============================================================================
57
file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version)
68
if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]])
@@ -32,3 +34,6 @@ if(NOT rapids-cmake-branch)
3234
set(rapids-cmake-branch "${_rapids_branch}")
3335
endif()
3436
include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake")
37+
38+
# Don't use sccache-dist for CMake's compiler tests
39+
set(ENV{SCCACHE_NO_DIST_COMPILE} "1")

‎conda/recipes/cugraph/recipe.yaml‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,32 @@ build:
3535
- AWS_ACCESS_KEY_ID
3636
- AWS_SECRET_ACCESS_KEY
3737
- AWS_SESSION_TOKEN
38+
- SCCACHE_DIST_AUTH_TOKEN
3839
env:
3940
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
4041
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
4142
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4243
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
43-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
44-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
45-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
46-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
47-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
44+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
45+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
46+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
47+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
48+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
49+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
50+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
51+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
52+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
53+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
54+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
55+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
56+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
57+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
4858
SCCACHE_S3_KEY_PREFIX: cugraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
59+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
60+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: cugraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
61+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
62+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
63+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
4964
requirements:
5065
build:
5166
- cmake ${{ cmake_version }}

‎conda/recipes/libcugraph/recipe.yaml‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,33 @@ cache:
3333
- AWS_ACCESS_KEY_ID
3434
- AWS_SECRET_ACCESS_KEY
3535
- AWS_SESSION_TOKEN
36+
- SCCACHE_DIST_AUTH_TOKEN
3637
env:
3738
CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
3839
CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }}
3940
CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
4041
CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
4142
EXTRA_CMAKE_ARGS: "-DCUGRAPH_COMPILE_CUVS=ON"
42-
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }}
43-
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
44-
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
45-
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
46-
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
47-
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
43+
NVCC_APPEND_FLAGS: ${{ env.get("NVCC_APPEND_FLAGS", default="") }}
44+
PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL", default="8") }}
45+
RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR", default="") }}
46+
SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET", default="") }}
47+
SCCACHE_DIST_AUTH_TYPE: ${{ env.get("SCCACHE_DIST_AUTH_TYPE", default="token") }}
48+
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE: ${{ env.get("SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE", default="false") }}
49+
SCCACHE_DIST_MAX_RETRIES: ${{ env.get("SCCACHE_DIST_MAX_RETRIES", default="inf") }}
50+
SCCACHE_DIST_REQUEST_TIMEOUT: ${{ env.get("SCCACHE_DIST_REQUEST_TIMEOUT", default="7140") }}
51+
SCCACHE_DIST_SCHEDULER_URL: ${{ env.get("SCCACHE_DIST_SCHEDULER_URL", default="") }}
52+
SCCACHE_ERROR_LOG: ${{ env.get("SCCACHE_ERROR_LOG", default="/tmp/sccache.log") }}
53+
SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT", default="0") }}
54+
SCCACHE_NO_CACHE: ${{ env.get("SCCACHE_NO_CACHE", default="") }}
55+
SCCACHE_RECACHE: ${{ env.get("SCCACHE_RECACHE", default="") }}
56+
SCCACHE_REGION: ${{ env.get("SCCACHE_REGION", default="") }}
4857
SCCACHE_S3_KEY_PREFIX: libcugraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}
58+
SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS", default="false") }}
59+
SCCACHE_S3_PREPROCESSOR_CACHE_KEY_PREFIX: libcugraph/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }}/conda/preprocessor-cache
60+
SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE: ${{ env.get("SCCACHE_S3_USE_PREPROCESSOR_CACHE_MODE", default="true") }}
61+
SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL", default="true") }}
62+
SCCACHE_SERVER_LOG: ${{ env.get("SCCACHE_SERVER_LOG", default="sccache=debug") }}
4963

5064
requirements:
5165
build:

0 commit comments

Comments
 (0)