From 4a3040988b113147d2b0a08f756040df854bb322 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 13:40:10 -0600 Subject: [PATCH 01/19] Add initial unit tests for subset_data point. Failing. --- python/ctsm/test/test_sys_subset_data.py | 89 +++++++++++++++++++ ...P_amazon_hist_16pfts_CMIP6_2000_c250617.nc | 3 + .../test_subset_data_pt_amazon_type360 | 1 + 3 files changed, 93 insertions(+) create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc create mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index bc73c8c41d..7010e9d405 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -185,6 +185,95 @@ def test_subset_data_reg_infile_detect180_error(self): ): subset_data.main() + def test_subset_data_pt_amazon_type360(self): + """ + Test subset_data for Amazon point with longitude type 360 + """ + cfg_file = os.path.join( + self.inputdata_dir, + "ctsm", + "test", + "testinputs", + "subset_data_amazon.cfg", + ) + print(cfg_file) + sys.argv = [ + "subset_data", + "point", + "--lat", + "-12", + "--lon", + "291", + "--site", + "TMP", + "--create-domain", + "--create-surface", + "--surf-year", + "2000", + "--create-user-mods", + "--outdir", + self.temp_dir_out.name, + "--user-mods-dir", + self.temp_dir_umd.name, + "--inputdata-dir", + self.inputdata_dir, + "--cfg-file", + cfg_file, + "--overwrite", + ] + subset_data.main() + + # Loop through all the output files, making sure they match what we expect. + daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + expected_output_files = [ + f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", + ] + self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + + def test_subset_data_pt_amazon_type180(self): + """ + Test subset_data for Amazon point with longitude type 180 + """ + cfg_file = os.path.join( + self.inputdata_dir, + "ctsm", + "test", + "testinputs", + "subset_data_amazon.cfg", + ) + print(cfg_file) + sys.argv = [ + "subset_data", + "point", + "--lat", + "-12", + "--lon", + "-69", + "--site", + "TMP", + "--create-surface", + "--surf-year", + "2000", + "--create-user-mods", + "--outdir", + self.temp_dir_out.name, + "--user-mods-dir", + self.temp_dir_umd.name, + "--inputdata-dir", + self.inputdata_dir, + "--cfg-file", + cfg_file, + "--overwrite", + ] + subset_data.main() + + # Loop through all the output files, making sure they match what we expect. + daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + expected_output_files = [ + f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", + ] + self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc new file mode 100644 index 0000000000..6e742560d0 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e694ca46925fbe07270b5468fe3899ead98dcc7d41353a6551dcc1ec92a9f9e0 +size 27740 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 new file mode 120000 index 0000000000..997afa40bb --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 @@ -0,0 +1 @@ +test_subset_data_pt_amazon_type180 \ No newline at end of file From 367317ecbb1e3e92dc948797503fe233046d8045 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 14:56:02 -0600 Subject: [PATCH 02/19] subset_data point: Fix --create-surface Longitude TypeError. --- .../site_and_regional/single_point_case.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index bd16bae226..1bd5581310 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -15,6 +15,7 @@ # -- import local classes for this script from ctsm.site_and_regional.base_case import BaseCase, USRDAT_DIR, DatmFiles from ctsm.utils import add_tag_to_filename, ensure_iterable +from ctsm.longitude import _detect_lon_type logger = logging.getLogger(__name__) @@ -151,6 +152,26 @@ def __init__( # self.check_nonveg() self.check_pct_pft() + def convert_plon_to_filetype_if_needed(self, input_ds): + """ + Check that point and input file longitude types are equal. If not, convert point to match + file. + """ + plon_in = self.plon + f_lon_type = _detect_lon_type(input_ds["lsmlon"]) + plon_type = plon_in.lon_type() + if f_lon_type == plon_type: + plon_out = plon_in.get(plon_type) + else: + plon_orig = plon_in.get(plon_type) + plon_out = plon_in.get(f_lon_type) + if plon_orig != plon_out: + print( + f"Converted plon from type {plon_type} (value {plon_orig}) " + f"to type {f_lon_type} (value {plon_out})" + ) + return plon_out + def create_tag(self): """ Create a tag for single point which is the site name @@ -498,8 +519,11 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out # create 1d coordinate variables to enable sel() method f_in = self.create_1d_coord(fsurf_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") + # get point longitude, converting to match file type if needed + plon_converted = self.convert_plon_to_filetype_if_needed(f_in) + # extract gridcell closest to plon/plat - f_tmp = f_in.sel(lsmlon=self.plon, lsmlat=self.plat, method="nearest") + f_tmp = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") # expand dimensions f_tmp = f_tmp.expand_dims(["lsmlat", "lsmlon"]).copy(deep=True) @@ -525,10 +549,10 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out # update lsmlat and lsmlon to match site specific instead of the nearest point # we do this so that if we create user_mods the PTS_LON and PTS_LAT in CIME match # the surface data coordinates - which is required - f_out["lsmlon"] = np.atleast_1d(self.plon) + f_out["lsmlon"] = np.atleast_1d(plon_converted) f_out["lsmlat"] = np.atleast_1d(self.plat) f_out["LATIXY"][:, :] = self.plat - f_out["LONGXY"][:, :] = self.plon + f_out["LONGXY"][:, :] = plon_converted # update attributes self.update_metadata(f_out) From daa218c092919970c4ae05ff0bac005d2ed14f33 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 15:21:41 -0600 Subject: [PATCH 03/19] subset_data point: Fix --create-landuse Longitude TypeError. --- python/ctsm/site_and_regional/single_point_case.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index 1bd5581310..5eefc1bd89 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -384,8 +384,11 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): # create 1d coordinate variables to enable sel() method f_in = self.create_1d_coord(fluse_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") + # get point longitude, converting to match file type if needed + plon_converted = self.convert_plon_to_filetype_if_needed(f_in) + # extract gridcell closest to plon/plat - f_out = f_in.sel(lsmlon=self.plon, lsmlat=self.plat, method="nearest") + f_out = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["lsmlat", "lsmlon"]) From 3f5d157acf70b0901cb4c01d2951152be8e0d1d7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 15:26:23 -0600 Subject: [PATCH 04/19] subset_data point: Fix --create-datm Longitude TypeError. --- python/ctsm/site_and_regional/single_point_case.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index 5eefc1bd89..e97b1f3baa 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -152,13 +152,13 @@ def __init__( # self.check_nonveg() self.check_pct_pft() - def convert_plon_to_filetype_if_needed(self, input_ds): + def convert_plon_to_filetype_if_needed(self, lon_da): """ Check that point and input file longitude types are equal. If not, convert point to match file. """ plon_in = self.plon - f_lon_type = _detect_lon_type(input_ds["lsmlon"]) + f_lon_type = _detect_lon_type(lon_da) plon_type = plon_in.lon_type() if f_lon_type == plon_type: plon_out = plon_in.get(plon_type) @@ -385,7 +385,7 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): f_in = self.create_1d_coord(fluse_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in) + plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) # extract gridcell closest to plon/plat f_out = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") @@ -523,7 +523,7 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out f_in = self.create_1d_coord(fsurf_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in) + plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) # extract gridcell closest to plon/plat f_tmp = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") @@ -595,8 +595,11 @@ def create_datmdomain_at_point(self, datm_tuple: DatmFiles): # create 1d coordinate variables to enable sel() method f_in = self.create_1d_coord(fdatmdomain_in, "xc", "yc", "ni", "nj") + # get point longitude, converting to match file type if needed + plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lon"]) + # extract gridcell closest to plon/plat - f_out = f_in.sel(ni=self.plon, nj=self.plat, method="nearest") + f_out = f_in.sel(ni=plon_converted, nj=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["nj", "ni"]) From 8e2df09282cb6ba6c9a9b8eaf652eeffef167aeb Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 16:02:09 -0600 Subject: [PATCH 05/19] subset_data point: Fix filenames for --create-datm. Resolves ESCOMP/CTSM#3260. --- .../site_and_regional/single_point_case.py | 83 +++++++++---------- 1 file changed, 38 insertions(+), 45 deletions(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index e97b1f3baa..ac0d5c1456 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -23,10 +23,6 @@ NUM_PFT = 17 # for runs with generic crops MAX_PFT = 78 # for runs with explicit crops -# -- constants to represent months of year -FIRST_MONTH = 1 -LAST_MONTH = 12 - class SinglePointCase(BaseCase): """ @@ -621,14 +617,17 @@ def extract_datm_at(self, file_in, file_out): # create 1d coordinate variables to enable sel() method f_in = self.create_1d_coord(file_in, "LONGXY", "LATIXY", "lon", "lat") + # get point longitude, converting to match file type if needed + plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lon"]) + # extract gridcell closest to plon/plat - f_out = f_in.sel(lon=self.plon, lat=self.plat, method="nearest") + f_out = f_in.sel(lon=plon_converted, lat=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["lat", "lon"]) # specify dimension order - f_out = f_out.transpose("scalar", "time", "lat", "lon") + f_out = f_out.transpose("time", "lat", "lon") # update attributes self.update_metadata(f_out) @@ -683,46 +682,40 @@ def create_datm_at_point(self, datm_tuple: DatmFiles, datm_syr, datm_eyr, datm_s tpqwfiles = [] for year in range(datm_syr, datm_eyr + 1): ystr = str(year) - for month in range(FIRST_MONTH, LAST_MONTH + 1): - mstr = str(month) - if month < 10: - mstr = "0" + mstr - dtag = ystr + "-" + mstr - - fsolar = os.path.join( - datm_tuple.indir, - datm_tuple.dir_solar, - "{}{}.nc".format(datm_tuple.tag_solar, dtag), - ) - fsolar2 = "{}{}.{}.nc".format(datm_tuple.tag_solar, self.tag, dtag) - fprecip = os.path.join( - datm_tuple.indir, - datm_tuple.dir_prec, - "{}{}.nc".format(datm_tuple.tag_prec, dtag), - ) - fprecip2 = "{}{}.{}.nc".format(datm_tuple.tag_prec, self.tag, dtag) - ftpqw = os.path.join( - datm_tuple.indir, - datm_tuple.dir_tpqw, - "{}{}.nc".format(datm_tuple.tag_tpqw, dtag), - ) - ftpqw2 = "{}{}.{}.nc".format(datm_tuple.tag_tpqw, self.tag, dtag) - - outdir = os.path.join(self.out_dir, datm_tuple.outdir) - infile += [fsolar, fprecip, ftpqw] - outfile += [ - os.path.join(outdir, fsolar2), - os.path.join(outdir, fprecip2), - os.path.join(outdir, ftpqw2), - ] - solarfiles.append( - os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fsolar2) - ) - precfiles.append( - os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fprecip2) - ) - tpqwfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, ftpqw2)) + fsolar = os.path.join( + datm_tuple.indir, + datm_tuple.dir_solar, + "{}{}.nc".format(datm_tuple.tag_solar, ystr), + ) + fsolar2 = "{}{}.{}.nc".format(datm_tuple.tag_solar, self.tag, ystr) + fprecip = os.path.join( + datm_tuple.indir, + datm_tuple.dir_prec, + "{}{}.nc".format(datm_tuple.tag_prec, ystr), + ) + fprecip2 = "{}{}.{}.nc".format(datm_tuple.tag_prec, self.tag, ystr) + ftpqw = os.path.join( + datm_tuple.indir, + datm_tuple.dir_tpqw, + "{}{}.nc".format(datm_tuple.tag_tpqw, ystr), + ) + ftpqw2 = "{}{}.{}.nc".format(datm_tuple.tag_tpqw, self.tag, ystr) + + outdir = os.path.join(self.out_dir, datm_tuple.outdir) + infile += [fsolar, fprecip, ftpqw] + outfile += [ + os.path.join(outdir, fsolar2), + os.path.join(outdir, fprecip2), + os.path.join(outdir, ftpqw2), + ] + solarfiles.append( + os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fsolar2) + ) + precfiles.append( + os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fprecip2) + ) + tpqwfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, ftpqw2)) for idx, out_f in enumerate(outfile): logger.debug(out_f) From 7d651f0929a2db6da4c3bc3e12ff72e3ebd5c181 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 16:54:45 -0600 Subject: [PATCH 06/19] Add Python system test for subset_data point --create-datm. --- python/ctsm/test/test_sys_subset_data.py | 61 ++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 7010e9d405..3812cf5168 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -185,9 +185,9 @@ def test_subset_data_reg_infile_detect180_error(self): ): subset_data.main() - def test_subset_data_pt_amazon_type360(self): + def test_subset_data_pt_amazon_type360_surface(self): """ - Test subset_data for Amazon point with longitude type 360 + Test subset_data --create-surface for Amazon point with longitude type 360 """ cfg_file = os.path.join( self.inputdata_dir, @@ -230,9 +230,9 @@ def test_subset_data_pt_amazon_type360(self): ] self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) - def test_subset_data_pt_amazon_type180(self): + def test_subset_data_pt_amazon_type180_surface(self): """ - Test subset_data for Amazon point with longitude type 180 + Test subset_data --create-surface for Amazon point with longitude type 180 """ cfg_file = os.path.join( self.inputdata_dir, @@ -274,6 +274,59 @@ def test_subset_data_pt_amazon_type180(self): ] self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + def test_subset_data_pt_amazon_type360_datm(self): + """ + Test subset_data --create-datm for Amazon point with longitude type 360 + FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER + """ + start_year = 1986 + end_year = 1988 + sitename = "TMP" + # outdir = self.temp_dir_out.name + outdir = "/glade/work/samrabin/ctsm/python/abc456" + sys.argv = [ + "subset_data", + "point", + "--lat", + "-12", + "--lon", + "291", + "--site", + sitename, + "--create-datm", + "--datm-syr", + str(start_year), + "--datm-eyr", + str(end_year), + "--create-user-mods", + "--outdir", + outdir, + "--user-mods-dir", + self.temp_dir_umd.name, + "--overwrite", + ] + subset_data.main() + + # Loop through all the output files, making sure they exist. + daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + expected_output_files = [ + f"domain.crujra_v2.3_0.5x0.5_{sitename}_c{daystr}.nc", + ] + for year in list(range(start_year, end_year + 1)): + for forcing in ["Solr", "Prec", "TPQWL"]: + expected_output_files.append( + f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename}.{year}.nc" + ) + for file_basename in expected_output_files: + file_path = os.path.join(outdir, "datmdata", file_basename) + # The below will error if exactly one matching file isn't found + try: + find_one_file_matching_pattern(file_path) + except FileNotFoundError as e: + raise AssertionError(str(e)) from e + except RuntimeError as e: + raise AssertionError(str(e)) from e + if __name__ == "__main__": unit_testing.setup_for_tests() From 4ad46f46de7dde753b4653c15f05326f55116b73 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 16:55:27 -0600 Subject: [PATCH 07/19] Reformat with black. --- python/ctsm/site_and_regional/single_point_case.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index ac0d5c1456..ed8b4b5562 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -709,12 +709,8 @@ def create_datm_at_point(self, datm_tuple: DatmFiles, datm_syr, datm_eyr, datm_s os.path.join(outdir, fprecip2), os.path.join(outdir, ftpqw2), ] - solarfiles.append( - os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fsolar2) - ) - precfiles.append( - os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fprecip2) - ) + solarfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fsolar2)) + precfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, fprecip2)) tpqwfiles.append(os.path.join("${}".format(USRDAT_DIR), datm_tuple.outdir, ftpqw2)) for idx, out_f in enumerate(outfile): From 4400458dbce9682be3e6323d7470084f0bec4438 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 16:56:00 -0600 Subject: [PATCH 08/19] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 7ea285a6bc..6cffe9dd35 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -67,3 +67,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 3b7a2876933263f8986e4069f5d23bd45635756f 3dd489af7ebe06566e2c6a1c7ade18550f1eb4ba 742cfa606039ab89602fde5fef46458516f56fd4 +4ad46f46de7dde753b4653c15f05326f55116b73 From ad40e112671489b4317ecbc614d7b5c54877ede7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 17:07:15 -0600 Subject: [PATCH 09/19] Fix test_sys_subset_data.py. --- python/ctsm/test/test_sys_subset_data.py | 3 +-- .../surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc | 0 .../expected_result_files/test_subset_data_pt_amazon_type360 | 1 - .../test_subset_data_pt_amazon_type360_surface | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) rename python/ctsm/test/testinputs/expected_result_files/{test_subset_data_pt_amazon_type180 => test_subset_data_pt_amazon_type180_surface}/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc (100%) delete mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 create mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 3812cf5168..f12e7ad45e 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -282,8 +282,7 @@ def test_subset_data_pt_amazon_type360_datm(self): start_year = 1986 end_year = 1988 sitename = "TMP" - # outdir = self.temp_dir_out.name - outdir = "/glade/work/samrabin/ctsm/python/abc456" + outdir = self.temp_dir_out.name sys.argv = [ "subset_data", "point", diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180_surface/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc similarity index 100% rename from python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc rename to python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180_surface/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 deleted file mode 120000 index 997afa40bb..0000000000 --- a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360 +++ /dev/null @@ -1 +0,0 @@ -test_subset_data_pt_amazon_type180 \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface new file mode 120000 index 0000000000..7bcdf69458 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface @@ -0,0 +1 @@ +test_subset_data_pt_amazon_type180_surface \ No newline at end of file From 944bf273914e9bf473a0cd885fb7a08f0c6e4c61 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 17:09:00 -0600 Subject: [PATCH 10/19] Add test_subset_data_pt_amazon_type180_datm --- python/ctsm/test/test_sys_subset_data.py | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index f12e7ad45e..d33a00e80f 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -326,6 +326,58 @@ def test_subset_data_pt_amazon_type360_datm(self): except RuntimeError as e: raise AssertionError(str(e)) from e + def test_subset_data_pt_amazon_type180_datm(self): + """ + Test subset_data --create-datm for Amazon point with longitude type 180 + FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER + """ + start_year = 1986 + end_year = 1988 + sitename = "TMP" + outdir = self.temp_dir_out.name + sys.argv = [ + "subset_data", + "point", + "--lat", + "-12", + "--lon", + "-69", + "--site", + sitename, + "--create-datm", + "--datm-syr", + str(start_year), + "--datm-eyr", + str(end_year), + "--create-user-mods", + "--outdir", + outdir, + "--user-mods-dir", + self.temp_dir_umd.name, + "--overwrite", + ] + subset_data.main() + + # Loop through all the output files, making sure they exist. + daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + expected_output_files = [ + f"domain.crujra_v2.3_0.5x0.5_{sitename}_c{daystr}.nc", + ] + for year in list(range(start_year, end_year + 1)): + for forcing in ["Solr", "Prec", "TPQWL"]: + expected_output_files.append( + f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename}.{year}.nc" + ) + for file_basename in expected_output_files: + file_path = os.path.join(outdir, "datmdata", file_basename) + # The below will error if exactly one matching file isn't found + try: + find_one_file_matching_pattern(file_path) + except FileNotFoundError as e: + raise AssertionError(str(e)) from e + except RuntimeError as e: + raise AssertionError(str(e)) from e + if __name__ == "__main__": unit_testing.setup_for_tests() From 51c87410cb55fdd6c6fdc909b36f3808e217d22a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 17:19:56 -0600 Subject: [PATCH 11/19] Refactor test_sys_subset_data.py. --- python/ctsm/test/test_sys_subset_data.py | 125 ++++-------------- ...test_subset_data_pt_amazon_type360_surface | 1 - ...P_amazon_hist_16pfts_CMIP6_2000_c250617.nc | 0 ...test_subset_data_pt_surface_amazon_type360 | 1 + 4 files changed, 28 insertions(+), 99 deletions(-) delete mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface rename python/ctsm/test/testinputs/expected_result_files/{test_subset_data_pt_amazon_type180_surface => test_subset_data_pt_surface_amazon_type180}/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc (100%) create mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type360 diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index d33a00e80f..49e2c66f4e 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -37,7 +37,7 @@ def tearDown(self): self.temp_dir_out.cleanup() self.temp_dir_umd.cleanup() - def _check_result_file_matches_expected(self, expected_output_files): + def _check_result_file_matches_expected(self, expected_output_files, caller_n): """ Loop through a list of output files, making sure they match what we expect. """ @@ -49,7 +49,7 @@ def _check_result_file_matches_expected(self, expected_output_files): os.path.dirname(__file__), "testinputs", "expected_result_files", - inspect.stack()[1][3], # Name of calling function (i.e., test name) + inspect.stack()[caller_n][3], # Name of calling function (i.e., test name) basename, ) expected_file = find_one_file_matching_pattern(expected_file) @@ -112,7 +112,7 @@ def test_subset_data_reg_amazon(self): f"domain.lnd.5x5pt-amazon_navy_TMP_c{daystr}.nc", f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", ] - self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 1)) def test_subset_data_reg_infile_detect360(self): """ @@ -185,9 +185,9 @@ def test_subset_data_reg_infile_detect180_error(self): ): subset_data.main() - def test_subset_data_pt_amazon_type360_surface(self): + def _do_test_subset_data_pt_surface(self, lon): """ - Test subset_data --create-surface for Amazon point with longitude type 360 + Given a longitude, test subset_data point --create-surface """ cfg_file = os.path.join( self.inputdata_dir, @@ -203,7 +203,7 @@ def test_subset_data_pt_amazon_type360_surface(self): "--lat", "-12", "--lon", - "291", + str(lon), "--site", "TMP", "--create-domain", @@ -228,56 +228,23 @@ def test_subset_data_pt_amazon_type360_surface(self): expected_output_files = [ f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", ] - self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) - def test_subset_data_pt_amazon_type180_surface(self): + def test_subset_data_pt_surface_amazon_type360(self): """ - Test subset_data --create-surface for Amazon point with longitude type 180 + Test subset_data --create-surface for Amazon point with longitude type 360 """ - cfg_file = os.path.join( - self.inputdata_dir, - "ctsm", - "test", - "testinputs", - "subset_data_amazon.cfg", - ) - print(cfg_file) - sys.argv = [ - "subset_data", - "point", - "--lat", - "-12", - "--lon", - "-69", - "--site", - "TMP", - "--create-surface", - "--surf-year", - "2000", - "--create-user-mods", - "--outdir", - self.temp_dir_out.name, - "--user-mods-dir", - self.temp_dir_umd.name, - "--inputdata-dir", - self.inputdata_dir, - "--cfg-file", - cfg_file, - "--overwrite", - ] - subset_data.main() + self._do_test_subset_data_pt_surface(291) - # Loop through all the output files, making sure they match what we expect. - daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd - expected_output_files = [ - f"surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c{daystr}.nc", - ] - self.assertTrue(self._check_result_file_matches_expected(expected_output_files)) + def test_subset_data_pt_surface_amazon_type180(self): + """ + Test subset_data --create-surface for Amazon point with longitude type 180 + """ + self._do_test_subset_data_pt_surface(-69) - def test_subset_data_pt_amazon_type360_datm(self): + def _do_test_subset_data_pt_datm(self, lon): """ - Test subset_data --create-datm for Amazon point with longitude type 360 - FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER + Given a longitude, test subset_data point --create-datm """ start_year = 1986 end_year = 1988 @@ -289,7 +256,7 @@ def test_subset_data_pt_amazon_type360_datm(self): "--lat", "-12", "--lon", - "291", + str(lon), "--site", sitename, "--create-datm", @@ -326,57 +293,19 @@ def test_subset_data_pt_amazon_type360_datm(self): except RuntimeError as e: raise AssertionError(str(e)) from e - def test_subset_data_pt_amazon_type180_datm(self): + def test_subset_data_pt_datm_amazon_type360(self): """ - Test subset_data --create-datm for Amazon point with longitude type 180 + Test subset_data --create-datm for Amazon point with longitude type 360 FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER """ - start_year = 1986 - end_year = 1988 - sitename = "TMP" - outdir = self.temp_dir_out.name - sys.argv = [ - "subset_data", - "point", - "--lat", - "-12", - "--lon", - "-69", - "--site", - sitename, - "--create-datm", - "--datm-syr", - str(start_year), - "--datm-eyr", - str(end_year), - "--create-user-mods", - "--outdir", - outdir, - "--user-mods-dir", - self.temp_dir_umd.name, - "--overwrite", - ] - subset_data.main() + self._do_test_subset_data_pt_datm(291) - # Loop through all the output files, making sure they exist. - daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd - expected_output_files = [ - f"domain.crujra_v2.3_0.5x0.5_{sitename}_c{daystr}.nc", - ] - for year in list(range(start_year, end_year + 1)): - for forcing in ["Solr", "Prec", "TPQWL"]: - expected_output_files.append( - f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename}.{year}.nc" - ) - for file_basename in expected_output_files: - file_path = os.path.join(outdir, "datmdata", file_basename) - # The below will error if exactly one matching file isn't found - try: - find_one_file_matching_pattern(file_path) - except FileNotFoundError as e: - raise AssertionError(str(e)) from e - except RuntimeError as e: - raise AssertionError(str(e)) from e + def test_subset_data_pt_datm_amazon_type180(self): + """ + Test subset_data --create-datm for Amazon point with longitude type 180 + FOR NOW CAN ONLY BE RUN ON DERECHO/CASPER + """ + self._do_test_subset_data_pt_datm(-69) if __name__ == "__main__": diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface deleted file mode 120000 index 7bcdf69458..0000000000 --- a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type360_surface +++ /dev/null @@ -1 +0,0 @@ -test_subset_data_pt_amazon_type180_surface \ No newline at end of file diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180_surface/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc similarity index 100% rename from python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_amazon_type180_surface/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc rename to python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type180/surfdata_TMP_amazon_hist_16pfts_CMIP6_2000_c250617.nc diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type360 b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type360 new file mode 120000 index 0000000000..3a7bc5efe3 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_surface_amazon_type360 @@ -0,0 +1 @@ +test_subset_data_pt_surface_amazon_type180 \ No newline at end of file From cceb4d0f6c345430e9ab4da4480a3eaae00af96c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 17 Jun 2025 17:27:23 -0600 Subject: [PATCH 12/19] Make error handling in _do_test_subset_data_pt_datm() more robust. --- python/ctsm/test/test_sys_subset_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 49e2c66f4e..501a7869fd 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -288,9 +288,7 @@ def _do_test_subset_data_pt_datm(self, lon): # The below will error if exactly one matching file isn't found try: find_one_file_matching_pattern(file_path) - except FileNotFoundError as e: - raise AssertionError(str(e)) from e - except RuntimeError as e: + except Exception as e: raise AssertionError(str(e)) from e def test_subset_data_pt_datm_amazon_type360(self): From 761bb47b2a717c4086a3300201cd4a8a342bcfab Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 18 Jun 2025 11:35:37 -0600 Subject: [PATCH 13/19] Add subset_data Python system test for --create-landuse. --- python/ctsm/test/test_sys_subset_data.py | 59 +++++++++++++++++++ ...MP_amazon_hist_1850-1853_78pfts_c250618.nc | 3 + ...ata_TMP_amazon_hist_1850_78pfts_c250618.nc | 3 + ...test_subset_data_pt_landuse_amazon_type360 | 1 + ...x5_amazon_hist_1850-1853_78pfts_c250617.nc | 3 + .../testinputs/subset_data_amazon_1850.cfg | 14 +++++ ...ata_5x5_amazon_hist_1850_78pfts_c250617.nc | 3 + 7 files changed, 86 insertions(+) create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c250618.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/surfdata_TMP_amazon_hist_1850_78pfts_c250618.nc create mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360 create mode 100644 python/ctsm/test/testinputs/landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc create mode 100644 python/ctsm/test/testinputs/subset_data_amazon_1850.cfg create mode 100644 python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 501a7869fd..42377f5b00 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -242,6 +242,65 @@ def test_subset_data_pt_surface_amazon_type180(self): """ self._do_test_subset_data_pt_surface(-69) + def _do_test_subset_data_pt_landuse(self, lon): + """ + Given a longitude, test subset_data point --create-landuse + """ + cfg_file = os.path.join( + self.inputdata_dir, + "ctsm", + "test", + "testinputs", + "subset_data_amazon_1850.cfg", + ) + print(cfg_file) + sys.argv = [ + "subset_data", + "point", + "--lat", + "-12", + "--lon", + str(lon), + "--site", + "TMP", + "--create-domain", + "--create-surface", + "--surf-year", + "1850", + "--create-landuse", + "--create-user-mods", + "--outdir", + self.temp_dir_out.name, + "--user-mods-dir", + self.temp_dir_umd.name, + "--inputdata-dir", + self.inputdata_dir, + "--cfg-file", + cfg_file, + "--overwrite", + ] + subset_data.main() + + # Loop through all the output files, making sure they match what we expect. + daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd + expected_output_files = [ + f"surfdata_TMP_amazon_hist_1850_78pfts_c{daystr}.nc", + f"landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c{daystr}.nc", + ] + self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) + + def test_subset_data_pt_landuse_amazon_type360(self): + """ + Test subset_data --create-landuse for Amazon point with longitude type 360 + """ + self._do_test_subset_data_pt_landuse(291) + + def test_subset_data_pt_landuse_amazon_type180(self): + """ + Test subset_data --create-landuse for Amazon point with longitude type 180 + """ + self._do_test_subset_data_pt_landuse(-69) + def _do_test_subset_data_pt_datm(self, lon): """ Given a longitude, test subset_data point --create-datm diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c250618.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c250618.nc new file mode 100644 index 0000000000..d34fdf3acf --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/landuse.timeseries_TMP_amazon_hist_1850-1853_78pfts_c250618.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b063aeb04ed3a0a613608ecf88ac47efb39de7ba74bf6e33a490925540bf47fb +size 18176 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/surfdata_TMP_amazon_hist_1850_78pfts_c250618.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/surfdata_TMP_amazon_hist_1850_78pfts_c250618.nc new file mode 100644 index 0000000000..02999b6b00 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type180/surfdata_TMP_amazon_hist_1850_78pfts_c250618.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efbf02729f8741bfdfbd51d748cce31c2d90b0c9ef2f00d841d2940dea5bc144 +size 53256 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360 b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360 new file mode 120000 index 0000000000..ad4f251586 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_landuse_amazon_type360 @@ -0,0 +1 @@ +test_subset_data_pt_landuse_amazon_type180 \ No newline at end of file diff --git a/python/ctsm/test/testinputs/landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc b/python/ctsm/test/testinputs/landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc new file mode 100644 index 0000000000..9e81ad351c --- /dev/null +++ b/python/ctsm/test/testinputs/landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83b34be6da2047bb9a099346f7f5472b932ead7033fe8ab817540b99ff3117b8 +size 215248 diff --git a/python/ctsm/test/testinputs/subset_data_amazon_1850.cfg b/python/ctsm/test/testinputs/subset_data_amazon_1850.cfg new file mode 100644 index 0000000000..6b16160f48 --- /dev/null +++ b/python/ctsm/test/testinputs/subset_data_amazon_1850.cfg @@ -0,0 +1,14 @@ +[surfdat] +dir = ctsm/test/testinputs +surfdat_16pft = surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc +surfdat_78pft = surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc +mesh_dir = ctsm/test/testinputs +mesh_surf = ESMF_mesh_5x5pt_amazon_from_domain_c230308.nc + +[landuse] +dir = ctsm/test/testinputs +landuse_16pft = landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc +landuse_78pft = landuse.timeseries_5x5_amazon_hist_1850-1853_78pfts_c250617.nc + +[domain] +file = ctsm/test/testinputs/domain.lnd.5x5pt-amazon_navy.090715.nc diff --git a/python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc b/python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc new file mode 100644 index 0000000000..747c33a2b0 --- /dev/null +++ b/python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_1850_78pfts_c250617.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0795d84b3e07a9437c7e9869810b74002210f7c55349f57983c36db9990db4a +size 893512 From 7245f21ec83a26b0201809f86c02b4fc71b181b1 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 15:09:04 -0600 Subject: [PATCH 14/19] test_sys_subset_data: pt datm tests now compare vs. expected. --- python/ctsm/test/test_sys_subset_data.py | 11 +++-------- .../clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc | 3 +++ .../clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc | 3 +++ .../domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc | 3 +++ .../test_subset_data_pt_datm_amazon_type360 | 1 + 12 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc create mode 100644 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc create mode 120000 python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type360 diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 42377f5b00..2c47919dae 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -332,7 +332,7 @@ def _do_test_subset_data_pt_datm(self, lon): ] subset_data.main() - # Loop through all the output files, making sure they exist. + # Loop through all the output files, making sure they match what we expect. daystr = "[0-9][0-9][0-9][0-9][0-9][0-9]" # 6-digit day code, yymmdd expected_output_files = [ f"domain.crujra_v2.3_0.5x0.5_{sitename}_c{daystr}.nc", @@ -342,13 +342,8 @@ def _do_test_subset_data_pt_datm(self, lon): expected_output_files.append( f"clmforc.CRUJRAv2.5_0.5x0.5.{forcing}.{sitename}.{year}.nc" ) - for file_basename in expected_output_files: - file_path = os.path.join(outdir, "datmdata", file_basename) - # The below will error if exactly one matching file isn't found - try: - find_one_file_matching_pattern(file_path) - except Exception as e: - raise AssertionError(str(e)) from e + expected_output_files = [os.path.join("datmdata", x) for x in expected_output_files] + self.assertTrue(self._check_result_file_matches_expected(expected_output_files, 2)) def test_subset_data_pt_datm_amazon_type360(self): """ diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc new file mode 100644 index 0000000000..84da04d260 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1986.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e1075a199de0d85b974bd9dbd09216e460eda035b3a6652cbfc59b75829e3ee +size 13136 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc new file mode 100644 index 0000000000..f05b8eb442 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1987.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21fb1ae2b2e75336e409770988dabd80e9ee69d990e5aa63dc7008c9145a455f +size 13136 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc new file mode 100644 index 0000000000..3d521c66f4 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Prec.TMP.1988.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8e2624c686c86d5d1071ed618b564e4589731555836083cad0a1e8259b7962e +size 13136 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc new file mode 100644 index 0000000000..1d551867f0 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1986.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b4164da71cf6bdf351143b936d2ad84da0c943378cb54534ec46f03513e2d17 +size 13144 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc new file mode 100644 index 0000000000..b752309969 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1987.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93e9ab5686acc5fb7ddaf775e7f561d572a4fbecab28088b643868432e3d1ed3 +size 13144 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc new file mode 100644 index 0000000000..c3c47b61be --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.Solr.TMP.1988.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbb6d1679040959e540928b7df056a848e9a385441d725f5f84271a07c64889c +size 13144 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc new file mode 100644 index 0000000000..9be8249601 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1986.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7145768c96bdf8b3cbab234b2a09c4506916dbbc8db9fbc73282d643251ed318 +size 37324 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc new file mode 100644 index 0000000000..068a7ff28e --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1987.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e38846646d2514671bd340daa0954bf1981aa328d4923cb42044097bb77f38 +size 37324 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc new file mode 100644 index 0000000000..1b7094dbee --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.TMP.1988.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:395aa495fd3b926521cd355fd2a012cdcd07d19b7a00467fdc49dafbf80751a1 +size 37324 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc new file mode 100644 index 0000000000..c9b19f474b --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type180/datmdata/domain.crujra_v2.3_0.5x0.5_TMP_c250620.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:206ba64ca50dbd3b34e93f498eb1f526689e3a6900762f12e30c3af9b75ccb5c +size 2000 diff --git a/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type360 b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type360 new file mode 120000 index 0000000000..88385bbff2 --- /dev/null +++ b/python/ctsm/test/testinputs/expected_result_files/test_subset_data_pt_datm_amazon_type360 @@ -0,0 +1 @@ +test_subset_data_pt_datm_amazon_type180 \ No newline at end of file From 4f92b9db1bc19b2a5eae1ea804200a42b8d292c9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 15:11:36 -0600 Subject: [PATCH 15/19] _detect_lon_type() is public, so remove leading _. --- python/ctsm/longitude.py | 2 +- .../ctsm/site_and_regional/regional_case.py | 4 +-- .../site_and_regional/single_point_case.py | 4 +-- python/ctsm/subset_data.py | 8 +++--- python/ctsm/test/test_unit_longitude.py | 26 +++++++++---------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/python/ctsm/longitude.py b/python/ctsm/longitude.py index 8afa731131..fb5998524d 100644 --- a/python/ctsm/longitude.py +++ b/python/ctsm/longitude.py @@ -58,7 +58,7 @@ def _convert_lon_type_180_to_360(lon_in): return lon_out -def _detect_lon_type(lon_in): +def detect_lon_type(lon_in): """ Detect longitude type of a given numeric. If lon_in contains more than one number (as in a list or Numpy array), this function will assume all members are of the same type if (a) there is at diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index 94f6011569..ed91f3d474 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -19,7 +19,7 @@ from ctsm.utils import add_tag_to_filename from ctsm.utils import abort from ctsm.config_utils import check_lon1_lt_lon2 -from ctsm.longitude import Longitude, _detect_lon_type +from ctsm.longitude import Longitude, detect_lon_type logger = logging.getLogger(__name__) @@ -142,7 +142,7 @@ def _subset_lon_lat(self, x_dim, y_dim, f_in): # Detect longitude type (180 or 360) of input file, throwing a helpful error if it can't be # determined. - f_lon_type = _detect_lon_type(lon) + f_lon_type = detect_lon_type(lon) lon1_type = self.lon1.lon_type() lon2_type = self.lon2.lon_type() if lon1_type != lon2_type: diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index ed8b4b5562..d9a3c233aa 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -15,7 +15,7 @@ # -- import local classes for this script from ctsm.site_and_regional.base_case import BaseCase, USRDAT_DIR, DatmFiles from ctsm.utils import add_tag_to_filename, ensure_iterable -from ctsm.longitude import _detect_lon_type +from ctsm.longitude import detect_lon_type logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ def convert_plon_to_filetype_if_needed(self, lon_da): file. """ plon_in = self.plon - f_lon_type = _detect_lon_type(lon_da) + f_lon_type = detect_lon_type(lon_da) plon_type = plon_in.lon_type() if f_lon_type == plon_type: plon_out = plon_in.get(plon_type) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 81f1f703f3..b44b3fddbe 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -69,7 +69,7 @@ from ctsm.path_utils import path_to_ctsm_root from ctsm.utils import abort from ctsm.config_utils import check_lon1_lt_lon2 -from ctsm.longitude import Longitude, _detect_lon_type +from ctsm.longitude import Longitude, detect_lon_type # -- import ctsm logging flags from ctsm.ctsm_logging import ( @@ -833,10 +833,10 @@ def process_args(args): if any(lon_arg_values): if args.lon_type is None: if hasattr(args, "plon"): - args.lon_type = _detect_lon_type(args.plon) + args.lon_type = detect_lon_type(args.plon) else: - lon1_type = _detect_lon_type(args.lon1) - lon2_type = _detect_lon_type(args.lon2) + lon1_type = detect_lon_type(args.lon1) + lon2_type = detect_lon_type(args.lon2) if lon1_type != lon2_type: raise argparse.ArgumentTypeError( "--lon1 and --lon2 seem to be of different types" diff --git a/python/ctsm/test/test_unit_longitude.py b/python/ctsm/test/test_unit_longitude.py index 6bf7ec53e2..6766f90764 100644 --- a/python/ctsm/test/test_unit_longitude.py +++ b/python/ctsm/test/test_unit_longitude.py @@ -10,7 +10,7 @@ from ctsm.longitude import Longitude from ctsm.longitude import _convert_lon_type_180_to_360, _convert_lon_type_360_to_180 from ctsm.longitude import _check_lon_type_180, _check_lon_type_360 -from ctsm.longitude import _detect_lon_type +from ctsm.longitude import detect_lon_type # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -369,57 +369,57 @@ def test_lon_compare_notlon_error(self): def test_detect_lon_type_mid_180(self): """test that detect_lon_type works for an unambiguously 180 value""" - self.assertEqual(_detect_lon_type(-150), 180) + self.assertEqual(detect_lon_type(-150), 180) def test_detect_lon_type_min_180(self): """test that detect_lon_type works at -180""" - self.assertEqual(_detect_lon_type(-180), 180) + self.assertEqual(detect_lon_type(-180), 180) def test_detect_lon_type_mid_360(self): """test that detect_lon_type works for an unambiguously 360 value""" - self.assertEqual(_detect_lon_type(355), 360) + self.assertEqual(detect_lon_type(355), 360) def test_detect_lon_type_max_360(self): """test that detect_lon_type works at 360""" - self.assertEqual(_detect_lon_type(360), 360) + self.assertEqual(detect_lon_type(360), 360) def test_detect_lon_type_list_180(self): """test that detect_lon_type works for a list with just one unambiguously 180 value""" - self.assertEqual(_detect_lon_type([-150, 150]), 180) + self.assertEqual(detect_lon_type([-150, 150]), 180) def test_detect_lon_type_list_360(self): """test that detect_lon_type works for a list with just one unambiguously 360 value""" - self.assertEqual(_detect_lon_type([256, 150]), 360) + self.assertEqual(detect_lon_type([256, 150]), 360) def test_detect_lon_type_ambig(self): """test that detect_lon_type fails if ambiguous""" with self.assertRaisesRegex(ArgumentTypeError, r"Longitude\(s\) ambiguous"): - _detect_lon_type(150) + detect_lon_type(150) def test_detect_lon_type_list_ambig(self): """test that detect_lon_type fails for an ambiguous list""" with self.assertRaisesRegex(ArgumentTypeError, r"Longitude\(s\) ambiguous"): - _detect_lon_type([150, 170]) + detect_lon_type([150, 170]) def test_detect_lon_type_list_both(self): """test that detect_lon_type fails for a list with unambiguous members of both types""" with self.assertRaisesRegex(RuntimeError, r"Longitude array contains values of both types"): - _detect_lon_type([-150, 270]) + detect_lon_type([-150, 270]) def test_detect_lon_type_ambig0(self): """test that detect_lon_type fails at 0""" with self.assertRaisesRegex(ArgumentTypeError, r"Longitude\(s\) ambiguous"): - _detect_lon_type(0) + detect_lon_type(0) def test_detect_lon_type_oob_low(self): """test that detect_lon_type fails if out of bounds below min""" with self.assertRaisesRegex(ValueError, r"\(Minimum\) longitude < -180"): - _detect_lon_type(-300) + detect_lon_type(-300) def test_detect_lon_type_oob_high(self): """test that detect_lon_type fails if out of bounds above max""" with self.assertRaisesRegex(ValueError, r"\(Maximum\) longitude > 360"): - _detect_lon_type(500) + detect_lon_type(500) def test_list_as_lon(self): """ From ea011eda8d2616e0901db31c316389e326f60e23 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 15:13:45 -0600 Subject: [PATCH 16/19] Rename plon_converted to plon_float. --- .../site_and_regional/single_point_case.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index d9a3c233aa..e122b2c251 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -381,10 +381,10 @@ def create_landuse_at_point(self, indir, file, user_mods_dir): f_in = self.create_1d_coord(fluse_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) + plon_float = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) # extract gridcell closest to plon/plat - f_out = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") + f_out = f_in.sel(lsmlon=plon_float, lsmlat=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["lsmlat", "lsmlon"]) @@ -519,10 +519,10 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out f_in = self.create_1d_coord(fsurf_in, "LONGXY", "LATIXY", "lsmlon", "lsmlat") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) + plon_float = self.convert_plon_to_filetype_if_needed(f_in["lsmlon"]) # extract gridcell closest to plon/plat - f_tmp = f_in.sel(lsmlon=plon_converted, lsmlat=self.plat, method="nearest") + f_tmp = f_in.sel(lsmlon=plon_float, lsmlat=self.plat, method="nearest") # expand dimensions f_tmp = f_tmp.expand_dims(["lsmlat", "lsmlon"]).copy(deep=True) @@ -548,10 +548,10 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out # update lsmlat and lsmlon to match site specific instead of the nearest point # we do this so that if we create user_mods the PTS_LON and PTS_LAT in CIME match # the surface data coordinates - which is required - f_out["lsmlon"] = np.atleast_1d(plon_converted) + f_out["lsmlon"] = np.atleast_1d(plon_float) f_out["lsmlat"] = np.atleast_1d(self.plat) f_out["LATIXY"][:, :] = self.plat - f_out["LONGXY"][:, :] = plon_converted + f_out["LONGXY"][:, :] = plon_float # update attributes self.update_metadata(f_out) @@ -592,10 +592,10 @@ def create_datmdomain_at_point(self, datm_tuple: DatmFiles): f_in = self.create_1d_coord(fdatmdomain_in, "xc", "yc", "ni", "nj") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lon"]) + plon_float = self.convert_plon_to_filetype_if_needed(f_in["lon"]) # extract gridcell closest to plon/plat - f_out = f_in.sel(ni=plon_converted, nj=self.plat, method="nearest") + f_out = f_in.sel(ni=plon_float, nj=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["nj", "ni"]) @@ -618,10 +618,10 @@ def extract_datm_at(self, file_in, file_out): f_in = self.create_1d_coord(file_in, "LONGXY", "LATIXY", "lon", "lat") # get point longitude, converting to match file type if needed - plon_converted = self.convert_plon_to_filetype_if_needed(f_in["lon"]) + plon_float = self.convert_plon_to_filetype_if_needed(f_in["lon"]) # extract gridcell closest to plon/plat - f_out = f_in.sel(lon=plon_converted, lat=self.plat, method="nearest") + f_out = f_in.sel(lon=plon_float, lat=self.plat, method="nearest") # expand dimensions f_out = f_out.expand_dims(["lat", "lon"]) From 1244d9eaea8cb6c27ca5abdafa2ef77fc263a087 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 16:02:29 -0600 Subject: [PATCH 17/19] subset_data: Disallow --create-datm with GSWP3 data. - See [Issue #3269: subset_data --create-datm errors with GSWP3 data](https://github.com/ESCOMP/CTSM/issues/3269) - Resolves ESCOMP/CTSM#2960 --- .../generic-single-point-regional.rst | 2 +- python/ctsm/subset_data.py | 39 ++++++++++++------- python/ctsm/test/test_unit_subset_data.py | 33 +++++++++++++++- python/ctsm/test/testinputs/default_data.cfg | 15 +------ .../test/testinputs/default_data_gswp3.cfg | 30 ++++++++++++++ tools/site_and_regional/default_data_1850.cfg | 15 +------ tools/site_and_regional/default_data_2000.cfg | 15 +------ 7 files changed, 91 insertions(+), 58 deletions(-) create mode 100644 python/ctsm/test/testinputs/default_data_gswp3.cfg diff --git a/doc/source/users_guide/running-single-points/generic-single-point-regional.rst b/doc/source/users_guide/running-single-points/generic-single-point-regional.rst index 3d418b00fb..7e0b1e72fd 100644 --- a/doc/source/users_guide/running-single-points/generic-single-point-regional.rst +++ b/doc/source/users_guide/running-single-points/generic-single-point-regional.rst @@ -41,7 +41,7 @@ You can also have the script subset land-use data. See the help (``tools/site_an .. note:: This script defaults to subsetting specific surface data, land-use timeseries, and the CRUJRA2024 DATM data. It can currently only be run as-is on Derecho. If you're not on Derecho, use ``--inputdata-dir`` to specify where the top level of your CESM input data is. - Also, to subset GSWP3 instead of CRUJRA2024 DATM data, you currently need to hardwire ``datm_type = "datm_gswp3"`` (instead of the default ``"datm_crujra"``) in ``python/ctsm/subset_data.py``. + Using ``--create-datm`` with GSWP3 data is no longer supported; see `CTSM issue #3269 `_. diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index b44b3fddbe..96e2aa379c 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -629,15 +629,26 @@ def setup_files(args, defaults, cesmroot): file_dict = {"main_dir": clmforcingindir} # DATM data - # TODO Issue #2960: Make datm_type a user option at the command - # line. For reference, this option affects three .cfg files: + # For reference, this option affects three .cfg files: # tools/site_and_regional/default_data_1850.cfg # tools/site_and_regional/default_data_2000.cfg # python/ctsm/test/testinputs/default_data.cfg if args.create_datm: - datm_type = "datm_crujra" # also available: datm_type = "datm_gswp3" + datm_cfg_section = "datm" + + # Issue #3269: Changes in PR #3259 mean that --create-datm won't work with GSWP3 + settings_to_check_for_gswp3 = ["solartag", "prectag", "tpqwtag"] + for setting in settings_to_check_for_gswp3: + value = defaults.get(datm_cfg_section, setting) + if "gswp3" in value.lower(): + msg = ( + "--create-datm is no longer supported for GSWP3 data; " + "see https://github.com/ESCOMP/CTSM/issues/3269" + ) + raise NotImplementedError(msg) + dir_output_datm = "datmdata" - dir_input_datm = os.path.join(clmforcingindir, defaults.get(datm_type, "dir")) + dir_input_datm = os.path.join(clmforcingindir, defaults.get(datm_cfg_section, "dir")) if not os.path.isdir(os.path.join(args.out_dir, dir_output_datm)): os.mkdir(os.path.join(args.out_dir, dir_output_datm)) logger.info("dir_input_datm : %s", dir_input_datm) @@ -645,16 +656,16 @@ def setup_files(args, defaults, cesmroot): file_dict["datm_tuple"] = DatmFiles( dir_input_datm, dir_output_datm, - defaults.get(datm_type, "domain"), - defaults.get(datm_type, "solardir"), - defaults.get(datm_type, "precdir"), - defaults.get(datm_type, "tpqwdir"), - defaults.get(datm_type, "solartag"), - defaults.get(datm_type, "prectag"), - defaults.get(datm_type, "tpqwtag"), - defaults.get(datm_type, "solarname"), - defaults.get(datm_type, "precname"), - defaults.get(datm_type, "tpqwname"), + defaults.get(datm_cfg_section, "domain"), + defaults.get(datm_cfg_section, "solardir"), + defaults.get(datm_cfg_section, "precdir"), + defaults.get(datm_cfg_section, "tpqwdir"), + defaults.get(datm_cfg_section, "solartag"), + defaults.get(datm_cfg_section, "prectag"), + defaults.get(datm_cfg_section, "tpqwtag"), + defaults.get(datm_cfg_section, "solarname"), + defaults.get(datm_cfg_section, "precname"), + defaults.get(datm_cfg_section, "tpqwname"), ) # if the crop flag is on - we need to use a different land use and surface data file diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index eeb0a9a38a..c4ce21e959 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -7,6 +7,8 @@ """ import unittest +import tempfile +import shutil import configparser import argparse import os @@ -85,6 +87,18 @@ def setUp(self): self.defaults = configparser.ConfigParser() self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) + # Work in temporary directory + self._previous_dir = os.getcwd() + self._tempdir = tempfile.mkdtemp() + os.chdir(self._tempdir) # cd to tempdir + + def tearDown(self): + """ + Remove temporary directory + """ + os.chdir(self._previous_dir) + shutil.rmtree(self._tempdir, ignore_errors=True) + def test_inputdata_setup_files_basic(self): """ Test @@ -116,6 +130,23 @@ def test_inputdata_setup_files_inputdata_dne(self): with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) + def test_inputdata_setup_files_gswp3_error(self): + """ + Test that error is thrown if user tries to --create-datm GSWP3 + """ + cfg_file = os.path.join( + _CTSM_PYTHON, "ctsm", "test", "testinputs", "default_data_gswp3.cfg" + ) + sys.argv = ["subset_data", "point", "--create-datm", "--cfg-file", cfg_file] + self.args = self.parser.parse_args() + self.defaults = configparser.ConfigParser() + self.defaults.read(self.args.config_file) + + with self.assertRaisesRegex( + NotImplementedError, "https://github.com/ESCOMP/CTSM/issues/3269" + ): + setup_files(self.args, self.defaults, self.cesmroot) + def test_check_args_nooutput(self): """ Test that check args aborts when no-output is asked for @@ -229,7 +260,7 @@ def test_check_args_outsurfdat_fails_without_overwrite(self): for an existing dataset without the overwrite option """ outfile = os.path.join( - os.getcwd(), + _CTSM_PYTHON, "ctsm/test/testinputs/", "surfdata_1x1_mexicocityMEX_hist_16pfts_CMIP6_2000_c231103.nc", ) diff --git a/python/ctsm/test/testinputs/default_data.cfg b/python/ctsm/test/testinputs/default_data.cfg index a832d810cc..60c012561c 100644 --- a/python/ctsm/test/testinputs/default_data.cfg +++ b/python/ctsm/test/testinputs/default_data.cfg @@ -1,7 +1,7 @@ [main] clmforcingindir = /glade/campaign/cesm/cesmdata/cseg/inputdata -[datm_crujra] +[datm] dir = atm/datm7/atm_forcing.datm7.CRUJRA.0.5d.c20241231/three_stream domain = domain.crujra_v2.3_0.5x0.5.c220801.nc solardir = . @@ -14,19 +14,6 @@ solarname = CLMCRUJRA2024.Solar precname = CLMCRUJRA2024.Precip tpqwname = CLMCRUJRA2024.TPQW -[datm_gswp3] -dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 -domain = domain.lnd.360x720_gswp3.0v1.c170606.nc -solardir = Solar -precdir = Precip -tpqwdir = TPHWL -solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. -prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. -tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. -solarname = CLMGSWP3v1.Solar -precname = CLMGSWP3v1.Precip -tpqwname = CLMGSWP3v1.TPQW - [surfdat] dir = lnd/clm2/surfdata_esmf/ctsm5.3.0 surfdat_16pft = surfdata_0.9x1.25_hist_2000_16pfts_c240908.nc diff --git a/python/ctsm/test/testinputs/default_data_gswp3.cfg b/python/ctsm/test/testinputs/default_data_gswp3.cfg new file mode 100644 index 0000000000..09e1463eb2 --- /dev/null +++ b/python/ctsm/test/testinputs/default_data_gswp3.cfg @@ -0,0 +1,30 @@ +[main] +clmforcingindir = /glade/campaign/cesm/cesmdata/cseg/inputdata + +[datm] +dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +domain = domain.lnd.360x720_gswp3.0v1.c170606.nc +solardir = Solar +precdir = Precip +tpqwdir = TPHWL +solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. +prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. +tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. +solarname = CLMGSWP3v1.Solar +precname = CLMGSWP3v1.Precip +tpqwname = CLMGSWP3v1.TPQW + +[surfdat] +dir = lnd/clm2/surfdata_esmf/ctsm5.3.0 +surfdat_16pft = surfdata_0.9x1.25_hist_2000_16pfts_c240908.nc +surfdat_78pft = surfdata_0.9x1.25_hist_2000_78pfts_c240908.nc +mesh_dir = share/meshes/ +mesh_surf = fv0.9x1.25_141008_ESMFmesh.nc + +[landuse] +dir = lnd/clm2/surfdata_esmf/ctsm5.3.0 +landuse_16pft = landuse.timeseries_0.9x1.25_SSP2-4.5_1850-2100_78pfts_c240908.nc +landuse_78pft = landuse.timeseries_0.9x1.25_SSP2-4.5_1850-2100_78pfts_c240908.nc + +[domain] +file = share/domains/domain.lnd.fv0.9x1.25_gx1v7.151020.nc diff --git a/tools/site_and_regional/default_data_1850.cfg b/tools/site_and_regional/default_data_1850.cfg index 3c9f28c0a2..ce68b1debf 100644 --- a/tools/site_and_regional/default_data_1850.cfg +++ b/tools/site_and_regional/default_data_1850.cfg @@ -1,7 +1,7 @@ [main] clmforcingindir = /glade/campaign/cesm/cesmdata/inputdata -[datm_crujra] +[datm] dir = atm/datm7/atm_forcing.datm7.CRUJRA.0.5d.c20241231/three_stream domain = domain.crujra_v2.3_0.5x0.5.c220801.nc solardir = . @@ -14,19 +14,6 @@ solarname = CLMCRUJRA2024.Solar precname = CLMCRUJRA2024.Precip tpqwname = CLMCRUJRA2024.TPQW -[datm_gswp3] -dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 -domain = domain.lnd.360x720_gswp3.0v1.c170606.nc -solardir = Solar -precdir = Precip -tpqwdir = TPHWL -solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. -prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. -tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. -solarname = CLMGSWP3v1.Solar -precname = CLMGSWP3v1.Precip -tpqwname = CLMGSWP3v1.TPQW - [surfdat] dir = lnd/clm2/surfdata_esmf/ctsm5.3.0 surfdat_78pft = surfdata_0.9x1.25_hist_1850_78pfts_c240908.nc diff --git a/tools/site_and_regional/default_data_2000.cfg b/tools/site_and_regional/default_data_2000.cfg index a832d810cc..60c012561c 100644 --- a/tools/site_and_regional/default_data_2000.cfg +++ b/tools/site_and_regional/default_data_2000.cfg @@ -1,7 +1,7 @@ [main] clmforcingindir = /glade/campaign/cesm/cesmdata/cseg/inputdata -[datm_crujra] +[datm] dir = atm/datm7/atm_forcing.datm7.CRUJRA.0.5d.c20241231/three_stream domain = domain.crujra_v2.3_0.5x0.5.c220801.nc solardir = . @@ -14,19 +14,6 @@ solarname = CLMCRUJRA2024.Solar precname = CLMCRUJRA2024.Precip tpqwname = CLMCRUJRA2024.TPQW -[datm_gswp3] -dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 -domain = domain.lnd.360x720_gswp3.0v1.c170606.nc -solardir = Solar -precdir = Precip -tpqwdir = TPHWL -solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. -prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. -tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. -solarname = CLMGSWP3v1.Solar -precname = CLMGSWP3v1.Precip -tpqwname = CLMGSWP3v1.TPQW - [surfdat] dir = lnd/clm2/surfdata_esmf/ctsm5.3.0 surfdat_16pft = surfdata_0.9x1.25_hist_2000_16pfts_c240908.nc From 05a2c28ba7ca1cffbd1d210736492277cf27bd7d Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 16:23:28 -0600 Subject: [PATCH 18/19] subset_data: Generalize a comment. --- python/ctsm/subset_data.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 96e2aa379c..820391ff8b 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -629,10 +629,8 @@ def setup_files(args, defaults, cesmroot): file_dict = {"main_dir": clmforcingindir} # DATM data - # For reference, this option affects three .cfg files: - # tools/site_and_regional/default_data_1850.cfg - # tools/site_and_regional/default_data_2000.cfg - # python/ctsm/test/testinputs/default_data.cfg + # To find the affected files, from the top level of ctsm, do: + # grep "\[datm\]" $(find . -type f -name "*cfg") if args.create_datm: datm_cfg_section = "datm" From ecbe0fe13722283542aefbc0cf0146cbc8bd023c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 20 Jun 2025 16:26:12 -0600 Subject: [PATCH 19/19] test_sys_subset_data.py: Explain caller_n. --- python/ctsm/test/test_sys_subset_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ctsm/test/test_sys_subset_data.py b/python/ctsm/test/test_sys_subset_data.py index 2c47919dae..39d448cccd 100644 --- a/python/ctsm/test/test_sys_subset_data.py +++ b/python/ctsm/test/test_sys_subset_data.py @@ -40,6 +40,10 @@ def tearDown(self): def _check_result_file_matches_expected(self, expected_output_files, caller_n): """ Loop through a list of output files, making sure they match what we expect. + + caller_n should be an integer giving the number of levels above this function you need to + traverse before you hit the actual test name. If the test is calling this function directly, + caller_n = 1. If the test is calling a function that calls this function, caller_n = 2. Etc. """ all_files_present_and_match = True for basename in expected_output_files: