diff --git a/copydata.sh b/copydata.sh index 20ec184434..fa34840bfb 100755 --- a/copydata.sh +++ b/copydata.sh @@ -47,8 +47,7 @@ for folder in ${!foldermap[@]}; do # In GCS, there are /internal folders that should not be overwritten. # TODO(shifucun): Handle copy GCS reference. In this case, user would # store a link but not actual data in this repo. - gsutil cp -r "$folder/$import_group/data/*" "$gcs_path/$import_group/data/" + gcloud storage cp --recursive "$folder/$import_group/data/*" "$gcs_path/$import_group/data/" done fi done - diff --git a/scripts/us_census/decennial/download.sh b/scripts/us_census/decennial/download.sh index f8b2739717..6104b635ef 100755 --- a/scripts/us_census/decennial/download.sh +++ b/scripts/us_census/decennial/download.sh @@ -1,5 +1,5 @@ #!/bin/bash mkdir -p scratch -gsutil -m cp -R gs://datcom-csv/census/decennial/* scratch/ +gcloud storage cp --recursive gs://datcom-csv/census/decennial/* scratch/ mkdir -p output diff --git a/scripts/us_nces/demographics/private_school/run.sh b/scripts/us_nces/demographics/private_school/run.sh index 915675c314..f4dfb017f2 100644 --- a/scripts/us_nces/demographics/private_school/run.sh +++ b/scripts/us_nces/demographics/private_school/run.sh @@ -1,2 +1,2 @@ mkdir -p gcs_folder/input_files -gsutil -m cp -r gs://unresolved_mcf/us_nces/demographics/private_school/semi_automation_input_files/* gcs_folder/input_files/ +gcloud storage cp --recursive gs://unresolved_mcf/us_nces/demographics/private_school/semi_automation_input_files/* gcs_folder/input_files/ diff --git a/scripts/us_nces/demographics/public_school/run.sh b/scripts/us_nces/demographics/public_school/run.sh index fdb4960d15..2aff12ab42 100644 --- a/scripts/us_nces/demographics/public_school/run.sh +++ b/scripts/us_nces/demographics/public_school/run.sh @@ -1,2 +1,2 @@ mkdir -p gcs_folder/input_files -gsutil -m cp -r gs://unresolved_mcf/us_nces/demographics/public_school/semi_automation_input_files/* gcs_folder/input_files/ +gcloud storage cp --recursive gs://unresolved_mcf/us_nces/demographics/public_school/semi_automation_input_files/* gcs_folder/input_files/ diff --git a/scripts/us_nces/demographics/school_district/run.sh b/scripts/us_nces/demographics/school_district/run.sh index 1e3f8dd0ef..59f2112a70 100644 --- a/scripts/us_nces/demographics/school_district/run.sh +++ b/scripts/us_nces/demographics/school_district/run.sh @@ -1,2 +1,2 @@ mkdir -p gcs_folder/input_files -gsutil -m cp -r gs://unresolved_mcf/us_nces/demographics/school_district/semi_automation_input_files/* gcs_folder/input_files/ +gcloud storage cp --recursive gs://unresolved_mcf/us_nces/demographics/school_district/semi_automation_input_files/* gcs_folder/input_files/ diff --git a/statvar_imports/cdc/social_vulnerability_index/README.md b/statvar_imports/cdc/social_vulnerability_index/README.md index d71f4570f0..2a9077dcfe 100644 --- a/statvar_imports/cdc/social_vulnerability_index/README.md +++ b/statvar_imports/cdc/social_vulnerability_index/README.md @@ -35,7 +35,7 @@ Copies all latest input .csv files from GCS using: bash -gsutil cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/* source_files/ +gcloud storage cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/* source_files/ Adds a year column using add_years.py: @@ -59,7 +59,7 @@ Download the latest .csv/xlsx file manually from the CDC SVI source page. bash -gsutil cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/ +gcloud storage cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/ ## Ensure the file is named using the format: SVI__US_county.csv (e.g., SVI_2024_US_county.csv) diff --git a/statvar_imports/cdc/social_vulnerability_index/run.sh b/statvar_imports/cdc/social_vulnerability_index/run.sh index e9faeeb826..2d6070d3d4 100755 --- a/statvar_imports/cdc/social_vulnerability_index/run.sh +++ b/statvar_imports/cdc/social_vulnerability_index/run.sh @@ -4,10 +4,9 @@ set -e # Exit on error echo "Step 1: Copying SVI files from GCS..." mkdir -p source_files -gsutil cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/*.csv source_files/ +gcloud storage cp gs://unresolved_mcf/cdc/social_vulnerability_index/latest/input_files/*.csv source_files/ echo "Step 2: Adding 'year' column to all files except 2010..." python3 add_years.py echo "Preprocessing done. StatVar processing will now continue as per manifest." - diff --git a/statvar_imports/southkorea_statistics/run.sh b/statvar_imports/southkorea_statistics/run.sh index ac059af2e4..2d549747e8 100755 --- a/statvar_imports/southkorea_statistics/run.sh +++ b/statvar_imports/southkorea_statistics/run.sh @@ -9,4 +9,4 @@ GCS_PATH=$1 DESTINATION_DIR="source_files" mkdir -p "${DESTINATION_DIR}" -gsutil -m cp "${GCS_PATH}*.csv" "${DESTINATION_DIR}/" \ No newline at end of file +gcloud storage cp "${GCS_PATH}*.csv" "${DESTINATION_DIR}/" \ No newline at end of file diff --git a/statvar_imports/us_cdc/single_race/download.sh b/statvar_imports/us_cdc/single_race/download.sh index 969700ca9c..2a712aa2b1 100755 --- a/statvar_imports/us_cdc/single_race/download.sh +++ b/statvar_imports/us_cdc/single_race/download.sh @@ -3,4 +3,4 @@ set -e -o pipefail mkdir -p input_files -gsutil -m cp gs://unresolved_mcf/cdc/UnderlyingCause/Single_Race/latest/input_files/*.csv input_files/ +gcloud storage cp gs://unresolved_mcf/cdc/UnderlyingCause/Single_Race/latest/input_files/*.csv input_files/ diff --git a/statvar_imports/us_newyork/ny_diabetes/run.sh b/statvar_imports/us_newyork/ny_diabetes/run.sh index 1d93b705f0..514c337f9d 100644 --- a/statvar_imports/us_newyork/ny_diabetes/run.sh +++ b/statvar_imports/us_newyork/ny_diabetes/run.sh @@ -4,7 +4,6 @@ set -e # Exit on error echo "Step 1: Copying SVI files from GCS..." mkdir -p input_files -gsutil cp gs://unresolved_mcf/nyu_diabetes/adult_diabetes/input_files/*.csv input_files/ +gcloud storage cp gs://unresolved_mcf/nyu_diabetes/adult_diabetes/input_files/*.csv input_files/ echo "Preprocessing done. StatVar processing will now continue as per manifest." -