fix(provenance): exclude job tables, and never let source break an insert

Both from review of #1555.

Job tables were getting `_prov`. `is_entry_table` excluded the other tiers'
prefixes one at a time -- `_`, `#`, and `__` for parts -- and `~` was not among
them, so `~~analysis`, `~jobs` and `~lineage` all passed. The column really
landed and every job row carried a payload; on a large queue that is a lot of
JSON nobody asked for. It only surfaced after `jobs.refresh()` materialises the
table, which is why a plain populate() in the first round of tests missed it.

Now matched against `Manual.tier_regexp`. Enumerating what to exclude makes
every tier added later an Entry table until someone remembers this function;
matching the tier definition inverts that, so a name is an Entry table only if
the library says it is.

A `source` that json could not render broke every insert.
`config.provenance.source` is deployment-supplied and typed `dict[str, Any]`,
so a `date` in it raised `TypeError: Object of type date is not JSON
serializable` from inside every insert into every Entry table, naming neither
provenance nor the setting responsible. Three layers close it:

- `serialize` passes `default=str`, so ordinary values a deployment would
  actually set -- dates, paths -- record correctly rather than failing;
- a validator on the field rejects what remains (non-string keys, cycles) at
  assignment, where the error belongs;
- `_attach_provenance` catches and logs, so recording where a row came from can
  never stop the row being written. That guarantee previously covered only the
  version call.

Also fixes the `datajoint.migrate.add_prov_column` reference in the settings
description -- it lives in `deploy`, and that string shows up in config help.

Regression tests fail against the previous code: 4 of them, including the
integration one that drives a real job queue.

Author: dimitri-yatsenko

src/datajoint/provenance.py

@@ -29,6 +29,7 @@
 import contextvars
 import datetime
 import json
+import re
 from typing import Any
 
 #: Name of the hidden attribute.  Hidden attributes are excluded from
@@ -92,12 +93,18 @@ def _jsonable(value):
 def is_entry_table(table_name):
     """Whether a stripped table name denotes an Entry (``dj.Manual``) table.
 
-    Entry tables carry no tier prefix.  ``#`` marks Lookup, ``_`` Imported and
-    ``__`` Computed, while a part table carries ``__`` between its master's name
-    and its own.  A part inherits its master's provenance and gets no slot of
-    its own.
+    Matched against ``Manual.tier_regexp``, the definition the rest of the
+    library uses, rather than by excluding the prefixes of the other tiers.
+    Enumerating prefixes means every tier added later is an Entry table until
+    someone remembers this function -- which is how job tables (``~``) first
+    acquired the slot.
+
+    A part table carries its master's name and ``__`` before its own, so it
+    fails the match and inherits its master's provenance, which is what we want.
     """
-    return not table_name.startswith(("_", "#")) and "__" not in table_name
+    from .user_tables import Manual
+
+    return re.fullmatch(Manual.tier_regexp, table_name) is not None
 
 
 def build_payload(connection, config=None):
@@ -141,5 +148,11 @@ def build_payload(connection, config=None):
 
 
 def serialize(payload):
-    """Render a payload for the ``json`` column."""
-    return json.dumps(payload)
+    """Render a payload for the ``json`` column.
+
+    ``default=str`` because ``config.provenance.source`` is deployment-supplied
+    and typed ``dict[str, Any]``: a ``date`` or a ``Path`` in it would otherwise
+    raise from inside every insert into every Entry table, with an error naming
+    neither provenance nor the setting that caused it.
+    """
+    return json.dumps(payload, default=str)

src/datajoint/settings.py

@@ -326,7 +326,7 @@ class ProvenanceSettings(BaseSettings):
         default=True,
         description="Add the hidden `_prov` attribute to Entry (dj.Manual) tables at declaration "
         "and fill it on insert. Tables declared while this is False never receive the column; "
-        "use datajoint.migrate.add_prov_column to add it to an existing table.",
+        "use datajoint.deploy.add_prov_column to add it to an existing table.",
     )
     source: dict[str, Any] = Field(
         default_factory=dict,
@@ -336,6 +336,24 @@ class ProvenanceSettings(BaseSettings):
         "No author supplies this at the insert call site.",
     )
 
+    @field_validator("source")
+    @classmethod
+    def _source_must_be_json_serializable(cls, value: dict) -> dict:
+        """Reject a source that cannot be recorded, at the point it is set.
+
+        Every insert into an Entry table serializes this. Without the check the
+        failure surfaces from inside an unrelated insert, naming neither
+        provenance nor the setting responsible.
+        """
+        try:
+            json.dumps(value, default=str)
+        except (TypeError, ValueError) as error:
+            raise ValueError(
+                f"provenance.source must be JSON-serializable; it is recorded on every row "
+                f"entering an Entry table. {error.__class__.__name__}: {error}"
+            ) from error
+        return value
+
 
 class Config(BaseSettings):
     """

src/datajoint/table.py

@@ -966,13 +966,23 @@ def _attach_provenance(self, rows, field_list):
         if not rows or not self.connection._config.provenance.capture:
             return
         if not self._has_prov_attribute():
-            # Declared before capture was enabled.  datajoint.migrate.add_prov_column
+            # Declared before capture was enabled.  datajoint.deploy.add_prov_column
             # adds the slot to such a table.
             return
-        payload = provenance.build_payload(self.connection, self.connection._config)
-        if payload is None:
+        try:
+            payload = provenance.build_payload(self.connection, self.connection._config)
+            value = provenance.serialize(payload) if payload is not None else None
+        except Exception as error:
+            # Recording where a row came from must never stop it being written.
+            # `source` is deployment-supplied and typed `dict[str, Any]`, so this
+            # is reachable from configuration alone; the validator on that field
+            # catches the common case at assignment, and this covers the rest.
+            logger.warning(
+                f"Provenance not recorded for insert into {self.full_table_name}: " f"{error.__class__.__name__}: {error}"
+            )
+            return
+        if value is None:
             return
-        value = provenance.serialize(payload)
         for row in rows:
             row["names"] = list(row["names"]) + [provenance.PROV_ATTRIBUTE]
             row["placeholders"] = list(row["placeholders"]) + ["%s"]

tests/integration/test_entry_provenance.py

@@ -219,3 +219,30 @@ class Legacy(dj.Manual):
     finally:
         schema.drop()
         dj.config.provenance.capture = original
+
+
+def test_job_tables_do_not_get_prov(schema_prov):
+    """Regression for the #1555 review: `~` passed the old prefix test.
+
+    The job table is only materialised by a refresh, so a plain populate() does
+    not surface this -- which is how it survived the first round of tests.
+    """
+    schema, t = schema_prov
+    t["RecordingFile"].insert1({"file_id": 11, "path": "/data/b.tif"})
+    t["Ingest"].jobs.refresh()
+    t["Ingest"].populate(reserve_jobs=True)
+
+    conn = t["Ingest"]().connection
+    tables = [r[0] for r in conn.query(f"SHOW TABLES IN `{schema.database}`").fetchall()]
+    job_tables = [name for name in tables if name.startswith("~")]
+    assert job_tables, "no job table was created; the test would pass vacuously"
+
+    for name in job_tables:
+        columns = {
+            r[0]
+            for r in conn.query(
+                "SELECT COLUMN_NAME FROM information_schema.COLUMNS " "WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s",
+                args=(schema.database, name),
+            ).fetchall()
+        }
+        assert provenance.PROV_ATTRIBUTE not in columns, f"{name} carries {provenance.PROV_ATTRIBUTE}"

tests/unit/test_provenance.py

@@ -40,6 +40,9 @@ def config():
         ("subject__detail", False),  # Part of an Entry master
         ("__analysis__unit", False),  # Part of a Computed master
         ("_ingest__row", False),  # Part of an Imported master
+        ("~jobs", False),  # job table
+        ("~~analysis", False),  # per-table job queue
+        ("~lineage", False),  # lineage table
     ],
 )
 def test_is_entry_table(table_name, expected):
@@ -125,3 +128,50 @@ def test_settings_come_from_the_environment(monkeypatch):
     settings = ProvenanceSettings()
     assert settings.capture is False
     assert settings.source == {"system": "PyRat", "endpoint": "https://x"}
+
+
+def test_entry_test_follows_the_tier_definition_not_a_prefix_list():
+    """Regression for #1555 review: `~` tables were Entry tables.
+
+    Excluding the other tiers' prefixes one by one means any tier added later is
+    an Entry table until someone remembers this function. Matching
+    `Manual.tier_regexp` inverts that: a name is an Entry table only if the
+    library says it is.
+    """
+    import re
+
+    from datajoint.user_tables import Computed, Imported, Lookup, Manual, Part
+
+    assert provenance.is_entry_table("subject")
+    for tier in (Lookup, Imported, Computed, Part):
+        sample = {Lookup: "#param", Imported: "_ingest", Computed: "__analysis", Part: "subject__detail"}[tier]
+        assert re.fullmatch(tier.tier_regexp, sample), f"{sample} is not a {tier.__name__}"
+        assert not provenance.is_entry_table(sample)
+    # The job prefix belongs to no user tier at all, which is how it slipped through.
+    assert not any(re.fullmatch(t.tier_regexp, "~~analysis") for t in (Manual, Lookup, Imported, Computed, Part))
+    assert not provenance.is_entry_table("~~analysis")
+
+
+def test_serialize_survives_a_deployment_supplied_source(config):
+    """`source` is dict[str, Any]; a date in it must not break an insert."""
+    import datetime
+    import pathlib
+
+    payload = {
+        "time": "t",
+        "source": {"when": datetime.date(2026, 1, 1), "where": pathlib.Path("/mnt/raw")},
+    }
+    rendered = json.loads(provenance.serialize(payload))
+    assert rendered["source"] == {"when": "2026-01-01", "where": "/mnt/raw"}
+
+
+def test_source_must_be_serializable_at_assignment():
+    """The error belongs where the setting is made, not inside an unrelated insert."""
+    from pydantic import ValidationError
+
+    from datajoint.settings import ProvenanceSettings
+
+    cyclic: dict = {}
+    cyclic["self"] = cyclic
+    with pytest.raises(ValidationError, match="JSON-serializable"):
+        ProvenanceSettings(source=cyclic)