Improve variable list semantics for MIP table names

.cspell/ok-unknown-words.txt

@@ -95,6 +95,7 @@ fullvar
 geolat
 geolon
 getcwd
+getncattr
 giccm
 gridding
 hpoint
@@ -108,6 +109,7 @@ intercomparisons
 interp
 ints
 ised
+isunlimited
 isort
 ivar
 ixin
@@ -182,6 +184,7 @@ rgxs
 rstcheck
 rtype
 setuptools
+setncatts
 smth
 sosv
 spinup

CODE_STYLE.md

@@ -66,7 +66,7 @@ class MyClass(object):
     :ivar str var2: description, initial value: par2
     """
 
-    var3: ClassVar[str] = "I am a class variable"
+    var3: ClassVar[str] = 'I am a class variable'
 
     def __init__(self, par1: int, par2: int):
         self.var1 = par1 # instance variables
@@ -89,7 +89,7 @@ def func_with_return_and_optional_param(a: int, c: List[int] = [1,2]) -> Any:
     """
 
     if a > 10:
-        raise ValueError("a is more than 10")
+        raise ValueError('a is more than 10')
     return c
 
 def simple_func(foo: str):

README.md

@@ -1,29 +1,45 @@
-# `fremorizer`
+# `fremorizer` / `fremor`
 `fremorizer` CMORizes FRE output with `CMOR`. It is a `conda` package and it's documentation can be found on
 [`readthedocs`](https://fremorizer.readthedocs.io/en/latest/).
 
 [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/version.svg)](https://anaconda.org/ilaflott/fremorizer)
 [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/latest_release_date.svg)](https://anaconda.org/ilaflott/fremorizer)
 [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/latest_release_relative_date.svg)](https://anaconda.org/ilaflott/fremorizer)
 
+[![pylint](https://img.shields.io/badge/pylint-%E2%89%A59.7-brightgreen)](https://github.com/ilaflott/fremorizer/actions/workflows/pylint.yml)
+[![codecov](https://codecov.io/gh/ilaflott/fremorizer/branch/main/graph/badge.svg)](https://codecov.io/gh/ilaflott/fremorizer)
+
 [![publish_conda](https://github.com/ilaflott/fremorizer/actions/workflows/publish_conda.yml/badge.svg?branch=main)](https://github.com/ilaflott/fremorizer/actions/workflows/publish_conda.yml)
 [![readthedocs](https://app.readthedocs.org/projects/fremorizer/badge/?version=latest&style=flat)](https://fremorizer.readthedocs.io/en/latest/)
-
 [![pylint](https://github.com/ilaflott/fremorizer/actions/workflows/pylint.yml/badge.svg?branch=main)](https://github.com/ilaflott/fremorizer/actions/workflows/pylint.yml)
-[![pylint](https://img.shields.io/badge/pylint-%E2%89%A59.7-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml)
-[![codecov](https://codecov.io/gh/ilaflott/fremorizer/branch/main/graph/badge.svg)](https://codecov.io/gh/ilaflott/fremorizer)
+
+`python3.11`:[![3.11](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.11)
+
+`python3.12`:[![3.12](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.12)
+
+`python3.13`:[![3.13](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.13)
+
+`python3.14`:[![3.14](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.14)
+
 
 
 ## Background and Purpose
-`fremorizer` is a model output rewriter (CMORizer) for FRE/FMS based models and output. It was originally the `fre.cmor`
-submodule of [`NOAA-GFDL/fre-cli`](https://github.com/NOAA-GFDL/fre-cli). `fremorizer` (or `fremor` for short) is geared
-for rewriting NOAA-GFDL datasets for further quality control checks, assessments and data publishing pipelines in the
+`fremorizer` is a model output rewriter (CMORizer) for FRE/FMS based models and output. `fremorizer` (or `fremor` for short) is
+geared for standardizing NOAA-GFDL datasets for further quality control checks, assessments and data publishing pipelines in the
 context of CMIP7 using the [`CMOR`](https://cmor.llnl.gov/) library.
 
+### Relationship to `fre-cli`
+`fremorizer` was originally the `fre.cmor` submodule of [`NOAA-GFDL/fre-cli`](https://github.com/NOAA-GFDL/fre-cli) and so stands
+on the shoulders of it's contributors, retaining it's general structure and lessons learned from it. Future re-integrations back 
+into `fre-cli`, as a formal package dependency, are being assessed.
 
+### Contributors
+[![Contributors](https://contrib.rocks/image?repo=ilaflott/fremorizer)](https://github.com/ilaflott/fremorizer/graphs/contributors)
 
-## Installation / Access
+#### AI Disclaimer
+AI was heavily used in the creation of this repository, primarily `github`'s `copilot` with `Claude` (`opus4.6` `sonnet4.6`, and `haiku`), and `Gemini` and `Chat-GPT` models to a lesser extent, in agent mode. `Claude` and `Codex` agents have also contributed.
 
+## Installation / Access
 
 ### Requirements
 
@@ -167,11 +183,7 @@ To view compliance results from a workflow/CI run:
 3. Download the `wcrp-compliance-reports` artifact
 
 
-### `conda` environment tests
 
-| Python 3.11 | Python 3.12 | Python 3.13 | Python 3.14 |
-|-------------|-------------|-------------|-------------|
-| [![3.11](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.11) | [![3.12](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.12) | [![3.13](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.13) | [![3.14](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.14) |
 
 
 

SECURITY.md

@@ -0,0 +1,8 @@
+# Security Policy
+
+## Supported Versions
+
+| Version    | Supported          |
+| ---------- | ------------------ |
+| 0.9.0 <=   | :white_check_mark: |
+| 0.9.0 >    | :x:                |

docs/commands.rst

@@ -40,7 +40,7 @@ workflows are supported. Available subcommands:
 * Minimal Syntax: ``fremor run -d [indir] -l [varlist] -r [table_config] -p [exp_config] -o [outdir] [options]``
 * Required Options:
    - ``-d, --indir TEXT`` — Input directory with netCDF files
-   - ``-l, --varlist TEXT`` — Variable list dictionary mapping local to MIP variable names
+   - ``-l, --varlist TEXT`` — Variable list dictionary mapping modeler variable names to MIP table variable names
    - ``-r, --table_config TEXT`` — MIP table JSON configuration
    - ``-p, --exp_config TEXT`` — Experiment/model metadata JSON
    - ``-o, --outdir TEXT`` — Output directory prefix

docs/cookbook.rst

@@ -74,7 +74,7 @@ You will need to split the platform-target string appropriately to extract the i
 Creating Variable Lists
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Variable lists map your local variable names to MIP table variable names. Generate a variable list from a directory of netCDF files:
+Variable lists map your modeler variable names to MIP table variable names. Generate a variable list from a directory of netCDF files:
 
 .. code-block:: bash
 
@@ -86,6 +86,20 @@ This tool examines filenames to extract variable names. It assumes FRE-style nam
 (e.g., ``component.YYYYMMDD.variable.nc``). Review the generated file and edit as needed to map
 local variable names to target MIP variable names.
 
+When a modeler's variable name differs from the MIP table variable name, the variable list
+maps between them. For example, if your model produces ``sea_sfc_salinity`` but the MIP table
+expects ``sos``:
+
+.. code-block:: json
+
+   {
+       "sea_sfc_salinity": "sos"
+   }
+
+The key (``sea_sfc_salinity``) is the modeler's variable name — it must match both the filename
+and the variable name inside the netCDF file. The value (``sos``) is the MIP table variable name
+used for metadata lookups.
+
 To verify variables exist in MIP tables, search for variable definitions:
 
 .. code-block:: bash
@@ -145,7 +159,7 @@ For processing individual directories or debugging specific issues, use ``fremor
 Required arguments:
 
 * ``--indir``: Directory containing netCDF files to CMORize
-* ``--varlist``: JSON file mapping local variable names to target variable names
+* ``--varlist``: JSON file mapping modeler variable names to MIP table variable names
 * ``--table_config``: MIP table JSON file (e.g., ``CMIP6_Omon.json``)
 * ``--exp_config``: Experiment configuration JSON with metadata
 * ``--outdir``: Output directory root for CMORized files

docs/glossary.rst

@@ -27,7 +27,7 @@ Glossary
       ``source_id``, ``grid_label``, and ``nominal_resolution``. Published by the CMIP community.
 
    variable list
-      A JSON file mapping local model variable names to MIP table variable names. Generated by
+      A JSON file mapping modeler variable names to MIP table variable names. Generated by
       ``fremor varlist`` and consumed by ``fremor run``.
 
    experiment configuration

fremorizer/_version.py

@@ -3,5 +3,5 @@
 """
 
 import os
-version = os.getenv("GIT_DESCRIBE_TAG", "0.1.2.post")
+version = os.getenv("GIT_DESCRIBE_TAG", "0.9.0post")
 __version__ = version

fremorizer/cli.py

@@ -24,9 +24,10 @@
                   'matching that variable name. I.e., this string help target local_vars, not ' + \
                   'target_vars.'
 VARLIST_HELP='path pointing to a json file containing directory of key/value pairs. ' + \
-             'the keys are the \'local\' names used in the filename, and the values ' + \
-             'pointed to by those keys are strings representing the name of the variable ' + \
-             'contained in targeted files. the key and value are often the same, ' + \
+             'the keys are the modeler\'s variable names used in the filename and ' + \
+             'expected as the variable name within the targeted files. the values ' + \
+             'pointed to by those keys are strings representing the corresponding ' + \
+             'MIP table variable name. the key and value are often the same, ' + \
              'but it is not required.'
 RUN_ONE_HELP='process only one file, then exit. mostly for debugging and isolating issues.'
 DRY_RUN_HELP='don\'t call the cmor_mixer subtool, just printout what would be called and move on until natural exit'
@@ -217,11 +218,10 @@ def find(varlist, table_config_dir, opt_var_name): #uncovered
               required = False)
 def run(indir, varlist, table_config, exp_config, outdir, run_one, opt_var_name,
         grid_label, grid_desc, nom_res, start, stop, calendar):
-    # pylint: disable=unused-argument
     """
     Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing
     """
-    cmor_run_subtool(
+    result = cmor_run_subtool(
         indir = indir,
         json_var_list = varlist,
         json_table_config = table_config,
@@ -236,6 +236,8 @@ def run(indir, varlist, table_config, exp_config, outdir, run_one, opt_var_name,
         stop = stop,
         calendar_type = calendar
     )
+    if result < 0:
+        raise click.ClickException(f'cmor_run_subtool returned non-zero status: {result}')
 
 
 @fremor.command('varlist')

fremorizer/cmor_finder.py

@@ -56,6 +56,8 @@ def print_var_content(table_config_file: IO[str],
         table_name = proj_table_vars['Header'].get('table_id').split(' ')[1]
     except KeyError:
         fre_logger.warning('couldn\'t get header and table_name field')
+    except IndexError:
+        fre_logger.warning("couldn't get header and table_name, probably not a variable table")
 
     if table_name is not None:
         fre_logger.info('looking for %s data in table %s!', var_name, table_name)
@@ -182,12 +184,19 @@ def make_simple_varlist( dir_targ: str,
     # Build a deduplicated dict of variable names extracted from all filenames across
     # all datetimes. Assigning to a dict naturally deduplicates while preserving
     # first-seen insertion order (Python 3.7+).
+    # If a MIP table is provided, variables that match a MIP variable name get
+    # self-mapped (key==value). Variables NOT in the MIP table get an empty string
+    # as value, signaling they need manual mapping by the user.
     var_list: Dict[str, str] = {}
     for targetfile in all_nc_files:
         var_name=os.path.basename(targetfile).split('.')[-2]
-        if mip_vars is not None and var_name not in mip_vars:
-            continue
-        var_list[var_name] = var_name
+        if mip_vars is not None:
+            if var_name in mip_vars:
+                var_list[var_name] = var_name
+            else:
+                var_list[var_name] = ''
+        else:
+            var_list[var_name] = var_name
 
     if not var_list:
         fre_logger.warning('WARNING: no variables in target mip table found, or no matching pattern,'