Merge pull request #104 from andrewdelman/ecco_access_202504

Adding v4r5 monthly mean variable list

Author: andrewdelman

ecco_access/ecco_access.py

@@ -98,7 +98,8 @@ def ecco_podaac_access(query,version='v4r4',grid=None,time_res='all',\
 
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
     
     Additional keyword arguments*:
     *This is not an exhaustive list, especially for 
@@ -181,12 +182,12 @@ def shortnames_find(query_list,version,grid,time_res):
         for query_item in query_list:
             if version == 'v4r5':
                 # see if the query is an existing dataset ID
-                if query_item not in s3_datasets_list:
-                    raise ValueError("'"+query_item+"' is not a v4r5 dataset ID.\n"\
-                                     +"Please query using the following dataset IDs:\n"\
-                                     +str(s3_datasets_list))
-                else:
+                # if not, then do a text search of the ECCO variable lists
+                if query_item in s3_datasets_list:
                     shortnames_list.append(query_item)
+                else:
+                    shortname_match = ecco_podaac_varlist_query(query_item,version,grid,time_res)
+                    shortnames_list.append(shortname_match)
             else:    
                 # see if the query is an existing NASA Earthdata ShortName
                 # if not, then do a text search of the ECCO variable lists
@@ -238,10 +239,15 @@ def shortnames_find(query_list,version,grid,time_res):
         for kwarg in list(kwargs.keys()):
             if kwarg != 'jsons_root_dir':
                 del kwargs[kwarg]
+    elif mode == 's3_open':
+        for kwarg in list(kwargs.keys()):
+            if kwarg in ['n_workers','force_redownload','show_noredownload_msg']:
+                del kwargs[kwarg]
     else:
         if 'jsons_root_dir' in kwargs.keys():
             del kwargs['jsons_root_dir']
 
+    
     # download or otherwise access granules, depending on mode
 
     if mode in ['download_ifspace','s3_get_ifspace']:
@@ -295,7 +301,8 @@ def shortnames_find(query_list,version,grid,time_res):
                                               **kwargs)
             elif mode == 's3_open':
                 granule_files[shortname] = ecco_podaac_s3_open(\
-                                              shortname,StartDate,EndDate,version,snapshot_interval)
+                                              shortname,StartDate,EndDate,version,snapshot_interval,\
+                                              **kwargs)
             elif mode == 's3_open_fsspec':
                 # granule_files will consist of mapper objects rather than URL/path or file lists
                 granule_files[shortname] = ecco_podaac_s3_open_fsspec(\
@@ -406,7 +413,8 @@ def ecco_podaac_to_xrdataset(query,version='v4r4',grid=None,time_res='all',\
 
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
     
     Additional keyword arguments*:
     *This is not an exhaustive list, especially for 

ecco_access/ecco_download.py

@@ -388,7 +388,8 @@ def ecco_podaac_download(ShortName,StartDate,EndDate,version,snapshot_interval='
 
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
     
     n_workers: int, number of workers to use in concurrent downloads. Benefits typically taper off above 5-6.
     
@@ -417,16 +418,21 @@ def ecco_podaac_download(ShortName,StartDate,EndDate,version,snapshot_interval='
 
     # set default download parent directory
     if download_root_dir==None:
-        download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        if version == 'v4r4':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        elif version == 'v4r5':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
     # define the directory where the downloaded files will be saved
     download_dir = Path(download_root_dir) / ShortName
 
-    # create the download directory
+    # create the download directory if it does not already exist
+    if isdir(download_dir) == True:
+        print(f'Download to directory {download_dir}')
+    else:
+        print(f'Creating download directory {download_dir}')
     download_dir.mkdir(exist_ok = True, parents=True)
 
-    print(f'created download directory {download_dir}')
-    
     # query CMR for granules matching the request
     urls,sizes = ecco_podaac_query(ShortName,StartDate,EndDate,version,snapshot_interval)
 
@@ -478,7 +484,8 @@ def ecco_podaac_download_diskaware(ShortNames,StartDate,EndDate,version,snapshot
     
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
 
     max_avail_frac: float, maximum fraction of remaining available disk space to use in storing current ECCO datasets.
                     If storing the datasets exceeds this fraction, an error is returned.
@@ -522,7 +529,10 @@ def ecco_podaac_download_diskaware(ShortNames,StartDate,EndDate,version,snapshot
 
     # set default download parent directory
     if download_root_dir==None:
-        download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        if version == 'v4r4':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        elif version == 'v4r5':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
     # add up total size of files that would be downloaded
     dataset_sizes = np.array([])
@@ -532,8 +542,14 @@ def ecco_podaac_download_diskaware(ShortNames,StartDate,EndDate,version,snapshot
         # get list of files
         urls,sizes = ecco_podaac_query(curr_shortname,StartDate,EndDate,version,snapshot_interval)
 
-        # create the download directory if it does not already exist
+        # define the directory where the downloaded files will be saved
         download_dir = Path(download_root_dir) / curr_shortname
+        
+        # create the download directory if it does not already exist
+        if isdir(download_dir) == True:
+            print(f'Download to directory {download_dir}')
+        else:
+            print(f'Creating download directory {download_dir}')
         download_dir.mkdir(exist_ok = True, parents=True)
 
         # compute size of current dataset
@@ -718,7 +734,8 @@ def ecco_podaac_download_subset(ShortName,StartDate=None,EndDate=None,snapshot_i
         
         download_root_dir: str, defines parent directory to download files to.
                            Files will be downloaded to directory download_root_dir/ShortName/.
-                           If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                           If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                           or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
         subset_file_id: str, identifier appended to each downloaded file to identify it as a subset.
                         Default is to not append an identifier.
 
@@ -1079,7 +1096,10 @@ def download_wrapper(url: str, url_append: str, download_dir: str, subset_file_i
 
     # set default download parent directory
     if download_root_dir==None:
-        download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        if version == 'v4r4':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        elif version == 'v4r5':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
     # define the directory where the downloaded files will be saved
     download_dir = Path(download_root_dir) / ShortName

ecco_access/ecco_s3_retrieve.py

@@ -149,8 +149,9 @@ def get_granules_ecco_bucket(StartDate: str, EndDate: str,\
         s3 = s3fs.S3FileSystem(anon=False,\
                                requester_pays=True)
         if version == 'v4r5':
+            shortname_dir = "_".join(ShortName.split("_")[2:-3])
             s3_files_all = s3.ls("s3://ecco-model-granules/netcdf/V4r5/native/mon_mean/"\
-                                 +ShortName+"/")
+                                 +shortname_dir+"/")
 
         # include only the granules in the date range given by temporal_range
         s3_files_all_dates = np.array([np.datetime64(s3_file.split("_")[-5],'M')\
@@ -552,7 +553,8 @@ def ecco_podaac_s3_get(ShortName,StartDate,EndDate,version,snapshot_interval='mo
     
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
     
     n_workers: int, number of workers to use in concurrent downloads. Benefits typically taper off above 5-6.
     
@@ -588,12 +590,19 @@ def ecco_podaac_s3_get(ShortName,StartDate,EndDate,version,snapshot_interval='mo
 
     # set default download parent directory
     if download_root_dir==None:
-        download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        if version == 'v4r4':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        elif version == 'v4r5':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
     # define the directory where the downloaded files will be saved
     download_dir = Path(download_root_dir) / ShortName
 
-    # create the download directory
+    # create the download directory if it does not already exist
+    if isdir(download_dir) == True:
+        print(f'Download to directory {download_dir}')
+    else:
+        print(f'Creating download directory {download_dir}')
     download_dir.mkdir(exist_ok = True, parents=True)
 
     # get list of files
@@ -664,7 +673,8 @@ def ecco_podaac_s3_get_diskaware(ShortNames,StartDate,EndDate,version,snapshot_i
 
     download_root_dir: str, defines parent directory to download files to.
                        Files will be downloaded to directory download_root_dir/ShortName/.
-                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/'.
+                       If not specified, parent directory defaults to '~/Downloads/ECCO_V4r4_PODAAC/',
+                       or '~/Downloads/ECCO_V4r5_PODAAC/' if version == 'v4r5'.
     
     max_avail_frac: float, maximum fraction of remaining available disk space to use in storing current ECCO datasets.
                     This determines whether the dataset files are stored on the current instance, or opened on S3.
@@ -717,7 +727,10 @@ def ecco_podaac_s3_get_diskaware(ShortNames,StartDate,EndDate,version,snapshot_i
 
     # set default download parent directory
     if download_root_dir==None:
-        download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        if version == 'v4r4':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+        elif version == 'v4r5':
+            download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
     # add up total size of files that would be downloaded
     dataset_sizes = np.array([])
@@ -729,6 +742,10 @@ def ecco_podaac_s3_get_diskaware(ShortNames,StartDate,EndDate,version,snapshot_i
 
         # create the download directory if it does not already exist
         download_dir = Path(download_root_dir) / curr_shortname
+        if isdir(download_dir) == True:
+            print(f'Download to directory {download_dir}')
+        else:
+            print(f'Creating download directory {download_dir}')
         download_dir.mkdir(exist_ok = True, parents=True)
 
         # compute size of current dataset
@@ -777,7 +794,10 @@ def ecco_podaac_s3_get_diskaware(ShortNames,StartDate,EndDate,version,snapshot_i
         for curr_shortname,s3_files_list in zip(ShortNames,s3_files_list_all):
             # set default download parent directory
             if download_root_dir==None:
-                download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+                if version == 'v4r4':
+                    download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r4_PODAAC')
+                elif version == 'v4r5':
+                    download_root_dir = join(expanduser('~'),'Downloads','ECCO_V4r5_PODAAC')
 
             # define the directory where the downloaded files will be saved
             download_dir = Path(download_root_dir) / curr_shortname

ecco_access/ecco_varlist.py

@@ -108,7 +108,7 @@ def print_varlist_query_results(query,shortnames_match,\
     for count,shortname in enumerate(shortnames_match):
         shortname_line = 'Option '+str(count+1)+': '+shortname
         if (('Note' in content_dict[shortname].keys())\
-          and ('*' in content_dict[shortname]['Note'])):
+          and (' *' in content_dict[shortname]['Note'])):
             shortname_line += ('    '+content_dict[shortname]['Note'])
         else:
             shortname_line += '    *'
@@ -126,6 +126,8 @@ def print_varlist_query_results(query,shortnames_match,\
         query_resp_str += shortname_line
         for varname,descrip in content_dict[shortname].items():
             max_descrip_len_perline = 50
+            if varname == 'Note':
+                max_descrip_len_perline += descrip_pos - 6
             if len(descrip) > max_descrip_len_perline:
                 # put line breaks in long descriptions
                 descrip_withbreaks = ''
@@ -139,10 +141,13 @@ def print_varlist_query_results(query,shortnames_match,\
                     descrip_withbreaks += curr_descrip
                     descrip_remaining = descrip_remaining[(len(curr_descrip)+1):]
                     if len(descrip_remaining) > 0:
-                        descrip_withbreaks += ('\n'+(' '*descrip_pos))
+                        if varname == 'Note':
+                            descrip_withbreaks += '\n      '
+                        else:
+                            descrip_withbreaks += ('\n'+(' '*descrip_pos))
                 descrip = descrip_withbreaks
             if varname == 'Note':
-                if '*' in descrip:
+                if ' *' in descrip:
                     continue
                 else:
                     var_line = 'Note: '+descrip+'\n'
@@ -176,7 +181,7 @@ def ecco_podaac_varlist_query(query,version,grid=None,time_res='all'):
     query: str, a text string being used to query ShortNames, variable names,
            and descriptions.
     
-    version: ('v4r4'), ECCO version to search variable lists for
+    version: ('v4r4','v4r5'), ECCO version to search variable lists for
     
     grid: ('native','latlon',None), specifies whether to query datasets with output
           on the native grid or the interpolated lat/lon grid.
@@ -194,20 +199,25 @@ def ecco_podaac_varlist_query(query,version,grid=None,time_res='all'):
     
     """
 
+    
     pass
 
 
-    if version != 'v4r4':
+    if version not in ['v4r4','v4r5']:
         raise ValueError('ECCO '+version+' is not currently available from PO.DAAC')
 
     # paths to variable list files
-    varlist_url_root = 'https://raw.githubusercontent.com/ECCO-GROUP/ECCO-v4-Python-Tutorial/master/varlist/'
-    varlist_url_ids = {'native,monthly':'v4r4_nctiles_monthly_varlist.txt',\
-                       'native,daily':'v4r4_nctiles_daily_varlist.txt',\
-                       'native,snapshot':'v4r4_nctiles_snapshots_varlist.txt',\
-                       'latlon,monthly':'v4r4_latlon_monthly_varlist.txt',\
-                       'latlon,daily':'v4r4_latlon_daily_varlist.txt',\
-                       'mixed,all':'v4r4_tseries_grid_varlist.txt'}
+    varlist_url_root = 'https://raw.githubusercontent.com/ECCO-GROUP/ECCO-v4-Python-Tutorial/master/ecco_access/varlist/'
+    if version == 'v4r4':
+        varlist_url_ids = {'native,monthly':'v4r4_nctiles_monthly_varlist.txt',\
+                           'native,daily':'v4r4_nctiles_daily_varlist.txt',\
+                           'native,snapshot':'v4r4_nctiles_snapshots_varlist.txt',\
+                           'latlon,monthly':'v4r4_latlon_monthly_varlist.txt',\
+                           'latlon,daily':'v4r4_latlon_daily_varlist.txt',\
+                           'mixed,all':'v4r4_tseries_grid_varlist.txt'}
+    elif version == 'v4r5':
+        varlist_url_ids = {'native,monthly':'v4r5_nctiles_monthly_varlist.txt'}
+    
 
     # set keys of grid types and time resolutions to search
 
@@ -245,6 +255,8 @@ def ecco_podaac_varlist_query(query,version,grid=None,time_res='all'):
     # build content dictionary containing all varlists being queried
     content_dict = {}
     for curr_key in grid_timeres_keys:
+        if curr_key not in varlist_url_ids.keys():
+            continue
         curr_content_dict,varname_pos,descrip_pos = varlist_file_parse(\
                                     varlist_url_root+varlist_url_ids[curr_key])
         content_dict = {**content_dict,**curr_content_dict}