minimize optimization in regular CI and GHA

juliasloan25 · juliasloan25 · commit f956b7c22b0f · 2025-11-14T14:25:08.000-08:00
diff --git a/.buildkite/benchmarks/pipeline.yml b/.buildkite/benchmarks/pipeline.yml
@@ -4,7 +4,6 @@ agents:
   modules: climacommon/2025_05_15
 
 env:
-  JULIA_NVTX_CALLBACKS: gc
   OPENBLAS_NUM_THREADS: 1
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   SLURM_KILL_BAD_EXIT: 1
diff --git a/.buildkite/hierarchies/pipeline.yml b/.buildkite/hierarchies/pipeline.yml
@@ -6,7 +6,6 @@ agents:
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   OPENBLAS_NUM_THREADS: 1
-  JULIA_NVTX_CALLBACKS: gc
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
@@ -6,7 +6,6 @@ agents:
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   OPENBLAS_NUM_THREADS: 1
-  JULIA_NVTX_CALLBACKS: gc
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -7,7 +7,6 @@ env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
   OPENBLAS_NUM_THREADS: 1
-  JULIA_NVTX_CALLBACKS: gc
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
@@ -21,30 +20,30 @@ steps:
     key: "init_cpu_env"
     command:
       - echo "--- Instantiate package env"
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project -e 'using Pkg; Pkg.status()'"
+      - "julia -O0 --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia -O0 --project -e 'using Pkg; Pkg.precompile()'"
+      - "julia -O0 --project -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate ClimaEarth experiments env"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.develop(path=\".\")'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(\"MPI\")'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
+      - "julia -O0 --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.develop(path=\".\")'"
+      - "julia -O0 --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia -O0 --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(\"MPI\")'"
+      - "julia -O0 --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia -O0 --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate ClimaCore experiments env"
-      - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.develop(path=\".\")'"
-      - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.add(\"MPI\")'"
-      - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.status()'"
+      - "julia -O0 --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.develop(path=\".\")'"
+      - "julia -O0 --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia -O0 --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.add(\"MPI\")'"
+      - "julia -O0 --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia -O0 --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.status()'"
 
       - echo "--- Instantiate test env"
-      - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
-      - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=test/ -e 'using Pkg; Pkg.add(\"MPI\")'"
-      - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
+      - "julia -O0 --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
+      - "julia -O0 --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia -O0 --project=test/ -e 'using Pkg; Pkg.add(\"MPI\")'"
+      - "julia -O0 --project=test/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia -O0 --project=test/ -e 'using Pkg; Pkg.status()'"
 
     concurrency: 1
     concurrency_group: 'depot/climacoupler-ci'
@@ -62,7 +61,7 @@ steps:
 
       - label: "MPI Utilities unit tests"
         key: "utilities_mpi_tests"
-        command: "srun julia --color=yes --project=test/ test/utilities_tests.jl"
+        command: "srun julia -O0 --color=yes --project=test/ test/utilities_tests.jl"
         timeout_in_minutes: 5
         env:
           CLIMACOMMS_CONTEXT: "MPI"
@@ -72,7 +71,7 @@ steps:
 
       - label: "MPI Interfacer unit tests"
         key: "interfacer_mpi_tests"
-        command: "srun julia --color=yes --project=test/ test/interfacer_tests.jl"
+        command: "srun julia -O0 --color=yes --project=test/ test/interfacer_tests.jl"
         timeout_in_minutes: 5
         env:
           CLIMACOMMS_CONTEXT: "MPI"
@@ -83,7 +82,7 @@ steps:
   - group: "GPU: unit tests"
     steps:
       - label: "GPU runtests"
-        command: "julia --color=yes --project=test/ test/runtests.jl"
+        command: "julia -O0 --color=yes --project=test/ test/runtests.jl"
         timeout_in_minutes: 10
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -95,7 +94,7 @@ steps:
   - group: "ClimaEarth tests"
     steps:
       - label: "ClimaEarth runtests"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/runtests.jl"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/runtests.jl"
         agents:
           slurm_mem: 16GB
 
@@ -113,7 +112,7 @@ steps:
           slurm_mem: 32GB
 
       - label: "GPU restarts"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/restart.jl"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/restart.jl"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         agents:
@@ -122,7 +121,7 @@ steps:
           slurm_mem: 32GB
 
       - label: "GPU restarts"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/restart_state_only.jl"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/test/restart_state_only.jl"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         agents:
@@ -164,14 +163,14 @@ steps:
 
       - label: "Slabplanet terra: atmos and bucket"
         key: "slabplanet_terra"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml --job_id slabplanet_terra"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml --job_id slabplanet_terra"
         artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/artifacts/*"
         agents:
           slurm_mem: 20GB
 
       - label: "Slabplanet aqua: atmos and slab ocean"
         key: "slabplanet_aqua"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua.yml --job_id slabplanet_aqua"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua.yml --job_id slabplanet_aqua"
         artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/artifacts/*"
         agents:
           slurm_mem: 20GB
@@ -181,28 +180,28 @@ steps:
       # Test default behavior with no config file or job ID provided
       - label: "AMIP: default"
         key: "amip_default"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl"
         artifact_paths: "experiments/ClimaEarth/output/amip_default/artifacts/*"
         agents:
           slurm_mem: 20GB
 
       - label: "AMIP: bucket initial condition test"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_bucket_ic.yml --job_id amip_bucket_ic"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_bucket_ic.yml --job_id amip_bucket_ic"
         artifact_paths: "experiments/ClimaEarth/output/amip_bucket_ic/artifacts/*"
         agents:
           slurm_ntasks: 1
           slurm_mem: 20GB
 
       - label: "AMIP: integrated land non-spun up initial condition test"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_land_ic.yml --job_id amip_land_ic"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_land_ic.yml --job_id amip_land_ic"
         artifact_paths: "experiments/ClimaEarth/output/amip_land_ic/artifacts/*"
         agents:
           slurm_ntasks: 1
           slurm_mem: 20GB
 
       - label: "AMIP - Float64 + hourly checkpoint"
         key: "amip"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_ft64_hourly_checkpoints.yml --job_id amip_coarse_ft64_hourly_checkpoints"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_ft64_hourly_checkpoints.yml --job_id amip_coarse_ft64_hourly_checkpoints"
         artifact_paths: "experiments/ClimaEarth/output/amip_coarse_ft64_hourly_checkpoints/artifacts/*"
         env:
           FLAME_PLOT: ""
@@ -212,14 +211,14 @@ steps:
           slurm_mem: 20GB
 
       - label: "AMIP - Component dts test"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_component_dts.yml --job_id target_amip_component_dts"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_component_dts.yml --job_id target_amip_component_dts"
         artifact_paths: "experiments/ClimaEarth/output/target_amip_component_dts/artifacts/*"
         agents:
           slurm_ntasks: 1
           slurm_mem: 20GB
 
       - label: "MPI AMIP"
-        command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_mpi.yml --job_id amip_coarse_mpi"
+        command: "srun julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_mpi.yml --job_id amip_coarse_mpi"
         artifact_paths: "experiments/ClimaEarth/output/amip_coarse_mpi/artifacts/*"
         timeout_in_minutes: 30
         env:
@@ -231,7 +230,7 @@ steps:
       # short high-res performance test
       - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph
         key: "unthreaded_amip_fine"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_n1_shortrun.yml --job_id target_amip_n1_shortrun"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_n1_shortrun.yml --job_id target_amip_n1_shortrun"
         artifact_paths: "experiments/ClimaEarth/output/target_amip_n1_shortrun/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
@@ -257,7 +256,7 @@ steps:
       # GPU RUNS: slabplanet
       - label: "GPU Slabplanet: albedo from function"
         key: "gpu_slabplanet_albedo_function"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml --job_id gpu_slabplanet_albedo_function"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml --job_id gpu_slabplanet_albedo_function"
         artifact_paths: "experiments/ClimaEarth/output/gpu_slabplanet_albedo_function/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -268,7 +267,7 @@ steps:
       # GPU RUNS: AMIP
       - label: "GPU AMIP: ED only + integrated land"
         key: "gpu_amip_edonly_integrated_land"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_integrated_land.yml --job_id gpu_amip_edonly_integrated_land"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_integrated_land.yml --job_id gpu_amip_edonly_integrated_land"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_edonly_integrated_land/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -278,7 +277,7 @@ steps:
 
       - label: "GPU AMIP: ED only + bucket"
         key: "gpu_amip_edonly_bucket"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_bucket.yml --job_id gpu_amip_edonly_bucket"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_edonly_bucket.yml --job_id gpu_amip_edonly_bucket"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_edonly_bucket/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -288,7 +287,7 @@ steps:
 
       - label: "GPU AMIP: diag. EDMF + integrated land"
         key: "gpu_amip_diagedmf_integrated_land"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_integrated_land.yml --job_id gpu_amip_diagedmf_integrated_land"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_integrated_land.yml --job_id gpu_amip_diagedmf_integrated_land"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_diagedmf_integrated_land/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -298,7 +297,7 @@ steps:
 
       - label: "GPU AMIP: diag. EDMF + bucket"
         key: "gpu_amip_diagedmf_bucket"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_bucket.yml --job_id gpu_amip_diagedmf_bucket"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_diagedmf_bucket.yml --job_id gpu_amip_diagedmf_bucket"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_diagedmf_bucket/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -315,11 +314,11 @@ steps:
         agents:
           slurm_mem: 20GB
           slurm_gpus: 1
-      
+
 
       - label: "GPU AMIP test: albedo from function"
         key: "gpu_amip_albedo_function"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_function.yml --job_id gpu_amip_albedo_function"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_function.yml --job_id gpu_amip_albedo_function"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_albedo_function/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -329,7 +328,7 @@ steps:
 
       - label: "GPU AMIP: albedo from temporal map + 0M"
         key: "gpu_amip_albedo_temporal_map"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_temporal_map.yml --job_id gpu_amip_albedo_temporal_map"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_temporal_map.yml --job_id gpu_amip_albedo_temporal_map"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_albedo_temporal_map/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -339,7 +338,7 @@ steps:
 
       - label: "GPU AMIP: albedo from temporal map + 1M"
         key: "gpu_amip_albedo_temporal_map_1M"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_temporal_map_1M.yml --job_id gpu_amip_albedo_temporal_map_1M"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_albedo_temporal_map_1M.yml --job_id gpu_amip_albedo_temporal_map_1M"
         artifact_paths: "experiments/ClimaEarth/output/gpu_amip_albedo_temporal_map_1M/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -352,7 +351,7 @@ steps:
 
       - label: "GPU CMIP"
         key: "gpu_my_first_cmip"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/my_first_cmip.yml --job_id my_first_cmip"
+        command: "julia -O0 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/my_first_cmip.yml --job_id my_first_cmip"
         artifact_paths: "experiments/ClimaEarth/output/my_first_cmip/artifacts/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -365,7 +364,7 @@ steps:
       - label: "Perfect model calibration test"
         key: "amip_pm_calibration"
         command:
-          - "julia --color=yes --project=experiments/ClimaEarth experiments/calibration/run_calibration.jl"
+          - "julia -O0 --color=yes --project=experiments/ClimaEarth experiments/calibration/run_calibration.jl"
         artifact_paths: "experiments/calibration/output/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml
@@ -20,9 +20,9 @@ jobs:
         with:
           version: '1.11'
       - name: Install dependencies
-        run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+        run: julia -O0 --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # For authentication with GitHub Actions token
           DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
-        run: julia --project=docs/ docs/make.jl
+        run: julia -O0 --project=docs/ docs/make.jl
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
