diff --git a/argopy/stores/index/extensions.py b/argopy/stores/index/extensions.py index a93a6c555..5cf13e212 100644 --- a/argopy/stores/index/extensions.py +++ b/argopy/stores/index/extensions.py @@ -202,11 +202,11 @@ def lon(self): ---------- BOX : list, tuple, int, float, optional An index box to search Argo records for. Can be: - + - Full 6-element list: [lon_min, lon_max, lat_min, lat_max, date_min, date_max] - 2-element list: [lon_min, lon_max] - Single value: interpreted as lower bound (ge) - + ge : int or float, optional Greater or equal bound for longitude filtering (lower limit). Default: -180 @@ -249,11 +249,11 @@ def lat(self): ---------- BOX : list, tuple, int, float, optional An index box to search Argo records for. Can be: - + - Full 6-element list: [lon_min, lon_max, lat_min, lat_max, date_min, date_max] - 2-element list: [lat_min, lat_max] - Single value: interpreted as lower bound (ge) - + ge : int or float, optional Greater or equal bound for latitude filtering (lower limit). Default: -90 @@ -296,11 +296,11 @@ def date(self): ---------- BOX : list or str, optional An index box to search Argo records for. Can be: - + - Full 6-element list: [lon_min, lon_max, lat_min, lat_max, date_min, date_max] - 2-element list: [date_min, date_max] - Single date string: interpreted as day-only (profiles on that specific date) - + ge : str, optional Greater or equal bound for date filtering (lower limit). Default: '1900-01-01' @@ -456,7 +456,7 @@ def parameter_data_mode(self): def profiler_type(self): """Search index for profiler types - The list of valid types is given by IDs of `Argo reference table 8 `_. + The list of valid types is given in `Argo reference table 8 / ARGO_WMO_INST_TYPE `_. Parameters ---------- @@ -479,8 +479,8 @@ def profiler_type(self): .. code-block:: python :caption: List valid types - from argopy import ArgoNVSReferenceTables - valid_types = ArgoNVSReferenceTables().tbl(8)['altLabel'] + from argopy import ArgoReferenceTable + valid_types : list[str] = ArgoReferenceTable('ARGO_WMO_INST_TYPE').keys() See Also -------- @@ -491,12 +491,12 @@ def profiler_type(self): def profiler_label(self, profiler_label: str, nrows=None, composed=False): """Search index for profiler types with a given string in their long name - Will search for string occurrences in the preferred label of `Argo reference table 8 `_. + Will search for string occurrences in the preferred label of `Argo reference table 8/ARGO_WMO_INST_TYPE `_. Parameters ---------- profiler_label: str, list(str) - The string (not exact) to be found in profiler preferred labels. + The string (not necessarily exact) to be found in profiler preferred labels. Returns ------- @@ -514,8 +514,9 @@ def profiler_label(self, profiler_label: str, nrows=None, composed=False): .. code-block:: python :caption: List valid labels - from argopy import ArgoNVSReferenceTables - valid_labels = ArgoNVSReferenceTables().tbl(8)['prefLabel'] + from argopy import ArgoReferenceTable + df = ar.ArgoReferenceTable('ARGO_WMO_INST_TYPE').to_dataframe() + valid_labels : list[str] = list(df['long_name'].to_dict().values()) See Also -------- @@ -552,11 +553,42 @@ def composer(profiler_type): self._obj.search_type.update(namer(profiler_label)) return search_filter + @abstractmethod + def profile_qc(self, param): + """Search index for parameter profile QCs with a specific value + + Parameters + ---------- + PARAMs: dict + A dictionary with parameters as keys, and profile QC as a string or a list of strings + logical: str, default='and' + Indicate to search for all (``and``) or any (``or``) of the parameters profile QC. This operator applies + between each parameter. + + Returns + ------- + :class:`ArgoIndex` + + Examples + -------- + .. code-block:: python + + from argopy import ArgoIndex + idx = ArgoIndex(index_file='core+') + + idx.query.profile_qc({'TEMP': 'A'}) + idx.query.profile_qc({'PSAL': 'A'}) + idx.query.profile_qc({'DOXY': ['A', 'B']}) + idx.query.profile_qc({'PSAL': 'A', 'DOXY': 'A'}, logical='or') + + """ + raise NotImplementedError("Not implemented") + @abstractmethod def institution_code(self, institution_code, nrows=None, composed=False): """Search index for institution codes - The list of valid codes is given by IDs of `Argo reference table 4 `_. + The list of valid codes is given in `Argo reference table 4/DATA_CENTRE_CODES `_. Parameters ---------- @@ -580,8 +612,8 @@ def institution_code(self, institution_code, nrows=None, composed=False): .. code-block:: python :caption: List valid codes - from argopy import ArgoNVSReferenceTables - valid_codes = ArgoNVSReferenceTables().tbl(4)['altLabel'] + from argopy import ArgoReferenceTable + valid_codes : list[str] = ArgoReferenceTable('DATA_CENTRE_CODES').keys() See Also -------- @@ -593,12 +625,12 @@ def institution_code(self, institution_code, nrows=None, composed=False): def institution_name(self, institution_name: str, nrows=None, composed=False): """Search index for institutions with a given string in their long name - Will search for string occurrences in the preferred label of `Argo reference table 4 `_. + Will search for string occurrences in the preferred label of `Argo reference table 4/DATA_CENTRE_CODES `_. Parameters ---------- institution_name: str, list(str) - The string (not exact) to be found in institution preferred labels. + The string (not necessarily exact) to be found in institution preferred labels. Returns ------- @@ -617,8 +649,9 @@ def institution_name(self, institution_name: str, nrows=None, composed=False): .. code-block:: python :caption: List valid names - from argopy import ArgoNVSReferenceTables - valid_names = ArgoNVSReferenceTables().tbl(4)['prefLabel'] + from argopy import ArgoReferenceTable + df = ar.ArgoReferenceTable('DATA_CENTRE_CODES').to_dataframe() + valid_names : list[str] = list(df['long_name'].to_dict().values()) See Also -------- @@ -688,6 +721,24 @@ def dac(self, dac, nrows=None, composed=False): """ raise NotImplementedError("Not implemented") + @abstractmethod + def psal_adj(self): + """Search (detailed) index for salinity adjustment values + + Defined for for delayed mode or adjusted mode profiles only. + + - Mean of psal_adjusted – psal on the deepest 500 meters with good psal_adjusted_qc (equal to 1) + - Standard deviation of psal_adjusted – psal on the deepest 500 meters with good psal_adjusted_qc (equal to 1) + + """ + raise NotImplementedError("Not implemented") + + @abstractmethod + def n_levels(self): + """Search index profiles using the maximum number of pressure levels contained in a profile + """ + raise NotImplementedError("Not implemented") + def compose(self, query: dict, nrows=None): """Compose query with multiple search methods diff --git a/argopy/stores/index/implementations/pandas/search_engine.py b/argopy/stores/index/implementations/pandas/search_engine.py index 676ccb7d1..f4c70f077 100644 --- a/argopy/stores/index/implementations/pandas/search_engine.py +++ b/argopy/stores/index/implementations/pandas/search_engine.py @@ -2,7 +2,7 @@ import logging import pandas as pd import numpy as np -from typing import List +from typing import List, Literal, Optional from functools import lru_cache from argopy.options import OPTIONS @@ -48,7 +48,7 @@ def compute_params(param: str, obj): class SearchEngine(ArgoIndexSearchEngine): @search_s3 - def wmo(self, WMOs, nrows=None, composed=False) -> indexstore: + def wmo(self, WMOs, nrows=None, composed=False): def checker(WMOs): WMOs = check_wmo(WMOs) # Check and return a valid list of WMOs log.debug( @@ -79,7 +79,7 @@ def composer(obj, WMOs): return search_filter @search_s3 - def cyc(self, CYCs, nrows=None, composed=False) -> indexstore: + def cyc(self, CYCs, nrows=None, composed=False): def checker(CYCs): if self._obj.convention in ["ar_index_global_meta"]: raise InvalidDatasetStructure( @@ -564,3 +564,98 @@ def composer(DACs): else: self._obj.search_type.update(namer(dac)) return search_filter + + def profile_qc(self, PARAMs: dict, logical="and", nrows=None, composed=False): + def checker(PARAMs): + if "profile_temp_qc" not in self._obj.convention_columns: + raise InvalidDatasetStructure("Cannot search for profile QC in this index)") + # Validate PARAMs + [ + PARAMs.update({p: to_list(PARAMs[p])}) for p in PARAMs + ] + if not np.all( + [v in ['', ' ', '1', 'A', 'B', 'C', 'D', 'E', 'F'] for vals in PARAMs.values() for v in vals] + ): + raise ValueError("Profile QC must be a value in '', 'A', 'B', 'C', 'D', 'E', 'F'") + log.debug("Argo index searching for profile QC: %s ..." % PARAMs) + return PARAMs + + def namer(PARAMs, logical): + return {"PROFQC": (PARAMs, logical)} + + def composer(PARAMs, logical): + filt = [] + + for param in PARAMs: + qcflags = PARAMs[param] + filt.append(self._obj.index[f"profile_{param.lower()}_qc"].isin(qcflags)) + + return self._obj._reduce_a_filter_list(filt, op=logical) + + PARAMs = checker(PARAMs) + self._obj.load(nrows=self._obj._nrows_index) + search_filter = composer(PARAMs, logical) + if not composed: + self._obj.search_type = namer(PARAMs, logical) + self._obj.search_filter = search_filter + self._obj.run(nrows=nrows) + return self._obj + else: + self._obj.search_type.update(namer(PARAMs, logical)) + return search_filter + + def psal_adj( + self, + where: Literal["mean", "dev"] = "mean", + ge: Optional[float] = 0.0, + le: Optional[float] = None, + nrows=None, + composed=False, + ): + def checker(where: str, ge: Optional[float], le: Optional[float])-> [str, Optional[float], Optional[float]]: + if where.lower() not in ['mean', 'dev']: + raise ValueError(f"'{where}': The 'where' argument must be 'mean' or 'dev'.") + if "ad_psal_adjustment_mean" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for salinity adjustment mean in this index)" + ) + if "ad_psal_adjustment_deviation" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for salinity adjustment deviation in this index)" + ) + + bounds = [where.lower(), ge, le] + + if bounds[0] == 'dev' and bounds[2] is not None and bounds[2] < 0: + raise ValueError(f"Deviation lower limit must be zero or positive") + + return bounds + + def namer(bounds): + return {f"PSAL_ADJ_{bounds[0].upper()}": bounds[1:]} + + def composer(obj, bounds): + filt = [] + pname: str = ( + "ad_psal_adjustment_mean" + if bounds[0] == "mean" + else "ad_psal_adjustment_deviation" + ) + if bounds[1] is not None: + filt.append(obj.index[pname].ge(bounds[1])) + if bounds[2] is not None: + filt.append(obj.index[pname].le(bounds[2])) + + return obj._reduce_a_filter_list(filt, op="and") + + bounds = checker(where, ge, le) + self._obj.load(nrows=self._obj._nrows_index) + search_filter = composer(self._obj, bounds) + if not composed: + self._obj.search_type = namer(bounds) + self._obj.search_filter = search_filter + self._obj.run(nrows=nrows) + return self._obj + else: + self._obj.search_type.update(namer(bounds)) + return search_filter diff --git a/argopy/stores/index/implementations/pyarrow/index.py b/argopy/stores/index/implementations/pyarrow/index.py index 9dfbfc245..896c7bb29 100644 --- a/argopy/stores/index/implementations/pyarrow/index.py +++ b/argopy/stores/index/implementations/pyarrow/index.py @@ -533,7 +533,7 @@ def convert_a_date(row): ]: s = s.set_column(1, "date", new_date) - if self.convention == "ar_index_global_prof": + if self.convention in ["ar_index_global_prof", "argo_profile_detailled_index"]: s = s.set_column(7, "date_update", new_date_update) elif self.convention in [ "argo_bio-profile_index", diff --git a/argopy/stores/index/implementations/pyarrow/search_engine.py b/argopy/stores/index/implementations/pyarrow/search_engine.py index bc71c8824..53894ff1e 100644 --- a/argopy/stores/index/implementations/pyarrow/search_engine.py +++ b/argopy/stores/index/implementations/pyarrow/search_engine.py @@ -1,7 +1,7 @@ import logging import pandas as pd import numpy as np -from typing import List +from typing import List, Literal, Optional, Tuple from functools import lru_cache @@ -169,8 +169,6 @@ def checker(BOX, **kwargs): log.debug("Argo index searching for date in BOX=%s ..." % BOX) return ("date", BOX) # Return key to use for time axis - key, BOX = checker(BOX, **kwargs) - def namer(BOX): return {"DATE": BOX[4:6]} @@ -190,6 +188,7 @@ def composer(BOX, key): ) return self._obj._reduce_a_filter_list(filt, op="and") + key, BOX = checker(BOX, **kwargs) self._obj.load(nrows=self._obj._nrows_index) search_filter = composer(BOX, key) if not composed: @@ -212,8 +211,6 @@ def checker(BOX, **kwargs): log.debug("Argo index searching for latitude in BOX=%s ..." % BOX) return BOX - BOX = checker(BOX, **kwargs) - def namer(BOX): return {"LAT": BOX[2:4]} @@ -223,6 +220,7 @@ def composer(BOX): filt.append(pa.compute.less_equal(self._obj.index["latitude"], BOX[3])) return self._obj._reduce_a_filter_list(filt, op="and") + BOX = checker(BOX, **kwargs) self._obj.load(nrows=self._obj._nrows_index) search_filter = composer(BOX) if not composed: @@ -245,25 +243,40 @@ def checker(BOX, **kwargs): log.debug("Argo index searching for longitude in BOX=%s ..." % BOX) return BOX - BOX = checker(BOX, **kwargs) - def namer(BOX): return {"LON": BOX[0:2]} def composer(BOX): filt = [] - if OPTIONS['longitude_convention'] == '360': + if OPTIONS["longitude_convention"] == "360": if BOX[0] is not None: - filt.append(pc.greater_equal(self._obj.index["longitude_360"], conv_lon(BOX[0], '360'))) + filt.append( + pc.greater_equal( + self._obj.index["longitude_360"], conv_lon(BOX[0], "360") + ) + ) if BOX[1] is not None: - filt.append(pc.less_equal(self._obj.index["longitude_360"], conv_lon(BOX[1], '360'))) - elif OPTIONS['longitude_convention'] == '180': + filt.append( + pc.less_equal( + self._obj.index["longitude_360"], conv_lon(BOX[1], "360") + ) + ) + elif OPTIONS["longitude_convention"] == "180": if BOX[0] is not None: - filt.append(pc.greater_equal(self._obj.index["longitude"], conv_lon(BOX[0], '180'))) + filt.append( + pc.greater_equal( + self._obj.index["longitude"], conv_lon(BOX[0], "180") + ) + ) if BOX[1] is not None: - filt.append(pc.less_equal(self._obj.index["longitude"], conv_lon(BOX[1], '180'))) + filt.append( + pc.less_equal( + self._obj.index["longitude"], conv_lon(BOX[1], "180") + ) + ) return self._obj._reduce_a_filter_list(filt, op="and") + BOX = checker(BOX, **kwargs) self._obj.load(nrows=self._obj._nrows_index) search_filter = composer(BOX) if not composed: @@ -613,3 +626,152 @@ def composer(DACs): else: self._obj.search_type.update(namer(dac)) return search_filter + + def profile_qc(self, PARAMs: dict, logical="and", nrows=None, composed=False): + def checker(PARAMs): + if "profile_temp_qc" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for profile QC in this index)" + ) + # Validate PARAMs + [PARAMs.update({p: to_list(PARAMs[p])}) for p in PARAMs] + if not np.all( + [ + v in ["", " ", "1", "A", "B", "C", "D", "E", "F"] + for vals in PARAMs.values() + for v in vals + ] + ): + raise ValueError( + "Profile QC must be a value in '', 'A', 'B', 'C', 'D', 'E', 'F'" + ) + log.debug("Argo index searching for profile QC: %s ..." % PARAMs) + return PARAMs + + def namer(PARAMs, logical): + return {"PROFQC": (PARAMs, logical)} + + def composer(PARAMs, logical): + filt = [] + + for param in PARAMs: + qcflags = PARAMs[param] + filt.append( + pa.compute.is_in( + self._obj.index[f"profile_{param.lower()}_qc"], + pa.array(qcflags), + ) + ) + + return self._obj._reduce_a_filter_list(filt, op=logical) + + PARAMs = checker(PARAMs) + self._obj.load(nrows=self._obj._nrows_index) + search_filter = composer(PARAMs, logical) + if not composed: + self._obj.search_type = namer(PARAMs, logical) + self._obj.search_filter = search_filter + self._obj.run(nrows=nrows) + return self._obj + else: + self._obj.search_type.update(namer(PARAMs, logical)) + return search_filter + + def psal_adj( + self, + where: Literal["mean", "std"] = "mean", + ge: Optional[float] = 0.0, + le: Optional[float] = None, + nrows=None, + composed=False, + ): + def checker(where: str, ge: Optional[float], le: Optional[float])-> [str, Optional[float], Optional[float]]: + if where.lower() not in ['mean', 'std']: + raise ValueError(f"'{where}': The 'where' argument must be 'mean' or 'std'.") + if "ad_psal_adjustment_mean" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for salinity adjustment mean in this index" + ) + if "ad_psal_adjustment_deviation" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for salinity adjustment standard deviation in this index" + ) + + bounds = [where.lower(), ge, le] + + if bounds[0] == 'std' and bounds[2] is not None and bounds[2] < 0: + raise ValueError(f"Standard deviation lower limit must be zero or positive") + + return bounds + + def namer(bounds): + return {f"PSAL_ADJ_{bounds[0].upper()}": bounds[1:]} + + def composer(obj, bounds): + filt = [] + pname: str = ( + "ad_psal_adjustment_mean" + if bounds[0] == "mean" + else "ad_psal_adjustment_deviation" + ) + if bounds[1] is not None: + filt.append(pc.greater_equal(obj.index[pname], bounds[1])) + if bounds[2] is not None: + filt.append(pc.less_equal(obj.index[pname], bounds[2])) + return obj._reduce_a_filter_list(filt, op="and") + + bounds = checker(where, ge, le) + self._obj.load(nrows=self._obj._nrows_index) + search_filter = composer(self._obj, bounds) + if not composed: + self._obj.search_type = namer(bounds) + self._obj.search_filter = search_filter + self._obj.run(nrows=nrows) + return self._obj + else: + self._obj.search_type.update(namer(bounds)) + return search_filter + + def n_levels( + self, + ge: Optional[int] = None, + le: Optional[int] = None, + nrows=None, + composed=False, + ): + def checker(ge: Optional[int], le: Optional[int])-> [Optional[int], Optional[int]]: + if "n_levels" not in self._obj.convention_columns: + raise InvalidDatasetStructure( + "Cannot search for number of levels in this index)" + ) + bounds = [ge, le] + if bounds[0] is not None and bounds[0] <= 0: + raise ValueError(f"The minimum number of levels 'ge' must be positive, {bounds[0]} provided") + if bounds[1] is not None and bounds[1] <= 0: + raise ValueError(f"The maximum number of levels 'le' must be positive, {bounds[1]} provided") + if bounds[0] is not None and bounds[1] is not None and bounds[0] > bounds[1]: + raise ValueError(f"Upper bound le={bounds[1]} must be small than the lower bound ge={bounds[0]}") + return bounds + + def namer(bounds): + return {f"NLEVELS": bounds} + + def composer(obj, bounds): + filt = [] + if bounds[0] is not None: + filt.append(pc.greater_equal(obj.index['n_levels'], bounds[0])) + if bounds[1] is not None: + filt.append(pc.less_equal(obj.index['n_levels'], bounds[1])) + return obj._reduce_a_filter_list(filt, op="and") + + bounds = checker(ge, le) + self._obj.load(nrows=self._obj._nrows_index) + search_filter = composer(self._obj, bounds) + if not composed: + self._obj.search_type = namer(bounds) + self._obj.search_filter = search_filter + self._obj.run(nrows=nrows) + return self._obj + else: + self._obj.search_type.update(namer(bounds)) + return search_filter diff --git a/argopy/stores/index/spec.py b/argopy/stores/index/spec.py index 0b3b3b88e..8e9880d0d 100644 --- a/argopy/stores/index/spec.py +++ b/argopy/stores/index/spec.py @@ -60,6 +60,8 @@ class ArgoIndexStoreProto(ABC): "aux", "ar_index_global_meta", "meta", + "argo_profile_detailled_index", + "core+", ] """List of supported conventions""" @@ -101,6 +103,7 @@ def __init__( - ``bgc-s`` or ``argo_synthetic-profile_index.txt`` - ``aux`` or ``etc/argo-index/argo_aux-profile_index.txt`` - ``meta`` or ``ar_index_global_meta.txt`` + - ``core+`` or ``argo_profile_detailled_index.txt`` - a local absolute path toward a file following an Argo index convention. When using a local file, you need to set the ``convention`` followed by the file. convention: str, default: None @@ -116,6 +119,7 @@ def __init__( - ``bgc-s`` or ``argo_synthetic-profile_index`` - ``aux`` or ``argo_aux-profile_index`` - ``meta`` or ``ar_index_global_meta`` + - ``core+`` or ``argo_profile_detailled_index`` cache : bool, default: False Use cache or not. @@ -140,6 +144,8 @@ def __init__( index_file = "etc/argo-index/argo_aux-profile_index.txt" elif index_file in ["meta"]: index_file = "ar_index_global_meta.txt" + elif index_file in ["core+"]: + index_file = "etc/argo-index/argo_profile_detailled_index.txt" self.index_file = index_file # Default number of commented lines to skip at the beginning of csv index files @@ -236,6 +242,8 @@ def __init__( convention = "argo_aux-profile_index" elif convention in ["meta"]: convention = "ar_index_global_meta" + elif convention in ["core+"]: + convention = "argo_profile_detailled_index" self._convention = convention # Check if the index file exists @@ -327,7 +335,6 @@ def cname(self) -> str: Return 'full' if a search was not yet performed on the :class:`ArgoIndex` instance - This method uses the BOX, WMO, CYC keys of the index instance ``search_type`` property """ cname = "full" C = [] @@ -427,6 +434,31 @@ def cname(self) -> str: else: cname = ";".join(["DAC%s" % dac for dac in sorted(DAC)]) + elif "PROFQC" == key: + PROFQC, LOG = self.search_type["PROFQC"] + cname = ("_%s_" % LOG).join( + ["%s_%s" % (p, "".join(PROFQC[p])) for p in PROFQC] + ) + + elif "PSAL_ADJ_MEAN" == key: + log.debug(self.search_type) + ADJ = self.search_type["PSAL_ADJ_MEAN"] + cname = [] + if ADJ[0] is not None: + cname.append(f"MEAN_PSAL_ADJ>={ADJ[0]}") + if ADJ[1] is not None: + cname.append(f"MEAN_PSAL_ADJ<={ADJ[1]}") + cname = "_and_".join(cname) + + elif key == "NLEVELS": + N = self.search_type[key] + if N[0] is not None and N[1] is not None: + cname = f"n={N[0]}/{N[1]}" + elif N[0] is not None and N[1] is None: + cname = f"n>={N[0]}" + elif N[1] is not None and N[0] is None: + cname = f"n<={N[1]}" + C.append(cname) return "_and_".join(C) @@ -543,6 +575,8 @@ def convention_title(self): title = "Aux-Profile directory file of the Argo GDAC" elif self.convention in ["ar_index_global_meta", "meta"]: title = "Metadata directory file of the Argo GDAC" + elif self.convention in ["argo_profile_detailled_index", "core+"]: + title = "Detailed Profile directory file of the Argo GDAC" return title @property @@ -559,6 +593,12 @@ def convention_columns(self) -> List[str]: 'parameters', 'date_update'] elif self.convention in ["ar_index_global_meta"]: columns = ['file', 'profiler_type', 'institution', 'date_update'] + elif self.convention in ["argo_profile_detailled_index"]: + columns = ['file', 'date', 'latitude', 'longitude', 'ocean', 'profiler_type', 'institution', 'date_update', + 'profile_temp_qc', 'profile_psal_qc','profile_doxy_qc', + 'ad_psal_adjustment_mean','ad_psal_adjustment_deviation', + 'gdac_date_creation','gdac_date_update','n_levels', + ] return columns @@ -1028,6 +1068,22 @@ def _insert_header(self, originalfile): # FTP root number 2 : ftp://usgodae.org/pub/outgoing/argo/dac # GDAC node : CORIOLIS file,profiler_type,institution,date_update +""" % pd.to_datetime( + "now", utc=True + ).strftime( + "%Y%m%d%H%M%S" + ) + + elif self.convention == "argo_profile_detailled_index": + header = """# Title : Profile directory file of the Argo Global Data Assembly Center +# Description : The directory file describes all individual profile files of the argo GDAC ftp site +# Project : ARGO +# Format version : 2.2 +# Date of update : %s +# FTP root number 1 : ftp://ftp.ifremer.fr/ifremer/argo/dac +# FTP root number 2 : ftp://usgodae.usgodae.org/pub/outgoing/argo/dac +# GDAC node : CORIOLIS +file,date,latitude,longitude,ocean,profiler_type,institution,date_update,profile_temp_qc,profile_psal_qc,profile_doxy_qc,ad_psal_adjustment_mean,ad_psal_adjustment_deviation,gdac_date_creation,gdac_date_update,n_levels """ % pd.to_datetime( "now", utc=True ).strftime( diff --git a/argopy/utils/checkers.py b/argopy/utils/checkers.py index b144cb37c..6b8ba3dbb 100644 --- a/argopy/utils/checkers.py +++ b/argopy/utils/checkers.py @@ -564,6 +564,10 @@ def check_index_cols(column_names: list, convention: str = "ar_index_global_prof Metadata directory file of the Argo Global Data Assembly Center file,profiler_type,institution,date_update + argo_profile_detailled_index.txt: Detailed index of profile files + The directory file describes all individual profile files of the argo GDAC ftp site + file,date,latitude,longitude,ocean,profiler_type,institution,date_update,profile_temp_qc,profile_psal_qc,profile_doxy_qc,ad_psal_adjustment_mean,ad_psal_adjustment_deviation,gdac_date_creation,gdac_date_update,n_levels + """ # Default for 'ar_index_global_prof' ref = [ @@ -616,6 +620,13 @@ def check_index_cols(column_names: list, convention: str = "ar_index_global_prof "date_update", ] + if convention == "argo_profile_detailled_index": + ref = ['file', 'date', 'latitude', 'longitude', 'ocean', 'profiler_type', 'institution', 'date_update', + 'profile_temp_qc', 'profile_psal_qc','profile_doxy_qc', + 'ad_psal_adjustment_mean','ad_psal_adjustment_deviation', + 'gdac_date_creation','gdac_date_update','n_levels', + ] + if not is_list_equal(column_names, ref): log.debug( "Expected (convention=%s): %s, got: %s" diff --git a/docs/advanced-tools/stores/argoindex.rst b/docs/advanced-tools/stores/argoindex.rst index 314cf7e2f..417e1f498 100644 --- a/docs/advanced-tools/stores/argoindex.rst +++ b/docs/advanced-tools/stores/argoindex.rst @@ -25,33 +25,47 @@ The table below summarize the **argopy** support status of all Argo index files: * - - Index file + - File pattern - Supported - * - Profile + * - Individual Profile - ar_index_global_prof.txt + - //profiles/*_.nc - ✅ - * - Synthetic-Profile + * - Detailed Individual Profile + - //profiles/*_.nc + - argo_profile_detailled_index.txt + - ❌ + * - Individual Synthetic-Profile + - //profiles/S*_.nc - argo_synthetic-profile_index.txt - ✅ - * - Bio-Profile + * - Individual Bio-Profile + - //profiles/B*_.nc - argo_bio-profile_index.txt - ✅ + * - Auxiliary Profile + - //profiles/B*__aux.nc + - etc/argo-index/argo_aux-profile_index.txt + - ✅ * - Metadata + - //_meta.nc - ar_index_global_meta.txt - ✅ - * - Auxiliary - - etc/argo-index/argo_aux-profile_index.txt - - ✅ * - Trajectory + - //_*traj.nc - ar_index_global_traj.txt - ❌ * - Bio-Trajectory + - //_B*traj.nc - argo_bio-traj_index.txt - ❌ * - Technical + - //_tech.nc - ar_index_global_tech.txt - ❌ - * - Greylist - - ar_greylist.txt + * - Detailed Synthetic-Profile + - //_Sprof.nc + - argo_synthetic-profile_detailled_index.txt - ❌ Index files support can be added on demand. `Click here to raise an issue if you'd like to access other index files `_.