diff --git a/docs/api/build_from_gguf.md b/docs/api/build_from_gguf.md index 1fa2527f9..f0aff76f1 100644 --- a/docs/api/build_from_gguf.md +++ b/docs/api/build_from_gguf.md @@ -132,6 +132,10 @@ additionally require every exact identifier to select the same implementation an in pinned `llama-vocab.cpp`; the tokenizer matrix links each promoted route to its dispatch line. This does not claim graph or runtime support. +### Fail-closed tokenizer evidence + +- `plm-1.8b-instruct-q4-k-m-tokenizer-blocker` — **GGUF/source:** `PLM-Team/PLM-1.8B-Instruct-gguf@7bec6546983bcf0d99526c943580bd49e2237445`
`PLM-1.8B-Instruct-Q4_K_M.gguf`
1,182,708,992 B
`b38570ee56ebec82a1e9ef45ab408c0d8230ececef1d7f1b267c49cff35638b8`
`PLM-Team/PLM-1.8B-Instruct@62d188c7d58843d7013d5b3ffe198db448787860`
`merges.txt` 1,671,853 B `8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5`, `special_tokens_map.json` 410 B `c83747485fba9ef20c42793b4b02b05001f214250f0d787f573df216c91047a3`, `tokenizer.json` 11,418,266 B `bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9`, `tokenizer_config.json` 1,327 B `1becffcfa09c98935043f1724d988887c618c5f6e7a249087d3ae29eb70e2a6f`, `vocab.json` 2,776,833 B `ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910`
`config.json` 934 B `91e6e13695a6de82556438667e64b60d9910269f300cd97f8c667d19e75f115e`; **closure:** architecture `plm`; pre `qwen2`
metadata `698bfa31cd069292437bc3509fea7be2445324a536d95a6e813d947c283bd989`
tokens 151,936 `f3ea8e8cf45bd58a8d5ad420306a3ccd925894cdb61a89062fd9e3a6de255a0e`; source 151,646 `696f26322524de87f49427fd1be6d1afce910574d9656c5d3f4f64064bdb83c1`
merges 151,387 `24fa2ae2a398e50784a1fff678482094af4f63e6783d35686726abacda8dc371`; source `cc098baa4a74ce5156487605aa048a34e54f0eee6a704691a738c8fb22dafdd5`
scores=0; types `0286431feb975d95a59a3f39957f8183d929295635fb627b6940627c63918bf1`
added tokens `e7b5f7013431aa26739424d92f40423f175a46f6d1fdc8453edf6005c99412f7`; chat `af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db`
normalizer `NFC`; pipeline `97c53ee89fb584b10798f44b02c60c9a8b746165a32dc34737d178fc20618a69`; **witness:** official tokenizer.json applies NFC normalization, but pinned llama.cpp qwen2 preserves decomposed Unicode; exact materialization is blocked
`e\u0301 \xe9`: llama.cpp `[68, 53839, 3958]` vs source `[963, 3958]`
corpus `0f3c77d24208f2ac0833668128cb2a00adcb7e6b4b4eedc6e4783e3ec7b41f5a`; llama.cpp oracle `8d9af256337d1a501250f9bbf4c0859a654bddd6`: 72 cases `66513168812575ccac974ecb454e916def5f4492d558c1866b8811d4f587a41d` + ## Supported GGUF architectures Reason codes are concise user-facing categories; detailed architecture audits remain in diff --git a/src/mobius/integrations/gguf/_docs.py b/src/mobius/integrations/gguf/_docs.py index 76242ade3..c1525d49e 100644 --- a/src/mobius/integrations/gguf/_docs.py +++ b/src/mobius/integrations/gguf/_docs.py @@ -38,7 +38,10 @@ from mobius.integrations.gguf._spec import GGUFArchitectureSpec, StorageRole, Support from mobius.integrations.gguf._tokenizer_alias_evidence import tokenizer_alias_evidence from mobius.integrations.gguf._tokenizer_census import tokenizer_route_census -from mobius.integrations.gguf._tokenizer_evidence import iter_tokenizer_evidence +from mobius.integrations.gguf._tokenizer_evidence import ( + iter_tokenizer_blocker_evidence, + iter_tokenizer_evidence, +) from mobius.integrations.gguf._tokenizer_registry import tokenizer_pre_policies from mobius.integrations.gguf._upstream import ( UPSTREAM_COMMIT, @@ -383,6 +386,49 @@ def _tokenizer_evidence_table() -> str: return "\n".join(rows) +def _tokenizer_blocker_evidence_table() -> str: + rows = [] + for evidence in iter_tokenizer_blocker_evidence(): + assets = ", ".join( + f"`{name}` {size:,} B `{sha256}`" + for name, size, sha256 in evidence.tokenizer_assets + ) + text, llamacpp_ids, source_ids = evidence.mismatch + identity = ( + f"`{evidence.repository}@{evidence.revision}`
" + f"`{evidence.filename}`
{evidence.size:,} B
`{evidence.lfs_sha256}`
" + f"`{evidence.tokenizer_repository}@{evidence.tokenizer_revision}`
{assets}" + f"
`{evidence.source_config_asset[0]}` " + f"{evidence.source_config_asset[1]:,} B `{evidence.source_config_asset[2]}`" + ) + closure = ( + f"architecture `{evidence.architecture}`; pre `{evidence.pre_identifier}`
" + f"metadata `{evidence.tokenizer_metadata_sha256}`
" + f"tokens {evidence.token_count:,} `{evidence.ordered_vocabulary_sha256}`; " + f"source {evidence.source_token_count:,} `{evidence.source_vocabulary_sha256}`
" + f"merges {evidence.merge_count:,} `{evidence.ordered_merges_sha256}`; " + f"source `{evidence.source_merges_sha256}`
" + f"scores={evidence.score_count}; types `{evidence.ordered_token_types_sha256}`
" + f"added tokens `{evidence.source_added_tokens_sha256}`; " + f"chat `{evidence.chat_template_sha256}`
" + f"normalizer `{evidence.source_normalizer}`; " + f"pipeline `{evidence.source_pipeline_sha256}`" + ) + witness = ( + f"{evidence.disposition}
" + f"`{text.encode('unicode_escape').decode()}`: llama.cpp `{list(llamacpp_ids)}` " + f"vs source `{list(source_ids)}`
" + f"corpus `{evidence.oracle_corpus_sha256}`; " + f"llama.cpp oracle `{evidence.llamacpp_oracle[0]}`: " + f"{evidence.llamacpp_oracle[1]} cases `{evidence.llamacpp_oracle[2]}`" + ) + rows.append( + f"- `{evidence.evidence_id}` — **GGUF/source:** {_cell(identity)}; " + f"**closure:** {_cell(closure)}; **witness:** {_cell(witness)}" + ) + return "\n".join(rows) + + def _projector_evidence_table() -> str: rows = [ "| Artifact ID | Immutable sidecar | Bytes | SHA-256 | Projector types |", @@ -507,6 +553,10 @@ def render_document() -> str: in pinned `llama-vocab.cpp`; the tokenizer matrix links each promoted route to its dispatch line. This does not claim graph or runtime support. +### Fail-closed tokenizer evidence + +{_tokenizer_blocker_evidence_table()} + ## Supported GGUF architectures Reason codes are concise user-facing categories; detailed architecture audits remain in diff --git a/src/mobius/integrations/gguf/_tokenizer_evidence.py b/src/mobius/integrations/gguf/_tokenizer_evidence.py index 6dd875b93..87715e8a7 100644 --- a/src/mobius/integrations/gguf/_tokenizer_evidence.py +++ b/src/mobius/integrations/gguf/_tokenizer_evidence.py @@ -6,9 +6,12 @@ from __future__ import annotations __all__ = [ + "GGUFTokenizerBlockerEvidence", "GGUFTokenizerEvidence", + "iter_tokenizer_blocker_evidence", "iter_tokenizer_evidence", "matching_tokenizer_evidence", + "tokenizer_blocker_evidence", "tokenizer_evidence", ] @@ -220,6 +223,134 @@ def source(self) -> GGUFTokenizerSource: ) +@dataclasses.dataclass(frozen=True, slots=True) +class GGUFTokenizerBlockerEvidence: + """Immutable evidence that a pinned artifact cannot use its official tokenizer.""" + + evidence_id: str + architecture: str + pre_identifier: str + repository: str + revision: str + filename: str + size: int + lfs_sha256: str + tensor_count: int + tensor_qtypes: tuple[tuple[str, int], ...] + tokenizer_repository: str + tokenizer_revision: str + source_config_asset: tuple[str, int, str] + tokenizer_assets: tuple[tuple[str, int, str], ...] + tokenizer_metadata_sha256: str + token_count: int + source_token_count: int + embedding_vocabulary_size: int + deterministic_padding_range: tuple[int, int] + ordered_vocabulary_sha256: str + source_vocabulary_sha256: str + merge_count: int + ordered_merges_sha256: str + source_merges_sha256: str + score_count: int + ordered_scores_sha256: str | None + ordered_token_types_sha256: str + source_added_tokens_sha256: str + source_pipeline_sha256: str + chat_template_sha256: str + source_normalizer: str + special_token_ids: tuple[tuple[str, int], ...] + oracle_corpus_sha256: str + llamacpp_oracle: tuple[str, int, str] + mismatch: tuple[str, tuple[int, ...], tuple[int, ...]] + disposition: str + + def __post_init__(self) -> None: + digests = ( + self.lfs_sha256, + self.tokenizer_metadata_sha256, + self.ordered_vocabulary_sha256, + self.source_vocabulary_sha256, + self.ordered_merges_sha256, + self.source_merges_sha256, + self.ordered_token_types_sha256, + self.source_added_tokens_sha256, + self.source_pipeline_sha256, + self.chat_template_sha256, + self.oracle_corpus_sha256, + self.llamacpp_oracle[2], + ) + if any( + re.fullmatch(r"[0-9a-f]{40}", revision) is None + for revision in (self.revision, self.tokenizer_revision, self.llamacpp_oracle[0]) + ): + raise ValueError("Tokenizer blocker revisions must be immutable 40-hex commits") + if any(re.fullmatch(r"[0-9a-f]{64}", digest) is None for digest in digests): + raise ValueError("Tokenizer blocker digests must be lowercase SHA-256") + if self.score_count < 0 or ( + (self.score_count == 0) != (self.ordered_scores_sha256 is None) + ): + raise ValueError("Tokenizer blocker score count and digest disagree") + if ( + self.ordered_scores_sha256 is not None + and re.fullmatch(r"[0-9a-f]{64}", self.ordered_scores_sha256) is None + ): + raise ValueError("Tokenizer blocker score digest must be lowercase SHA-256") + if ( + min( + self.size, + self.tensor_count, + self.token_count, + self.source_token_count, + self.embedding_vocabulary_size, + self.merge_count, + self.llamacpp_oracle[1], + ) + <= 0 + ): + raise ValueError("Tokenizer blocker counts and artifact size must be positive") + if ( + self.source_token_count > self.token_count + or self.embedding_vocabulary_size != self.token_count + or self.deterministic_padding_range + != (self.source_token_count, self.token_count - 1) + ): + raise ValueError( + "Tokenizer blocker vocabulary, padding, and embedding sizes disagree" + ) + if tuple(sorted(self.tokenizer_assets)) != self.tokenizer_assets: + raise ValueError("Tokenizer blocker assets must be sorted") + assets = (self.source_config_asset, *self.tokenizer_assets) + if self.source_config_asset[0] != "config.json" or any( + not name or size <= 0 or re.fullmatch(r"[0-9a-f]{64}", digest) is None + for name, size, digest in assets + ): + raise ValueError("Tokenizer blocker requires exact source asset identities") + if tuple(sorted(self.special_token_ids)) != self.special_token_ids: + raise ValueError("Tokenizer blocker special token IDs must be sorted") + if any( + token_id < 0 or token_id >= self.token_count + for _, token_id in self.special_token_ids + ): + raise ValueError("Tokenizer blocker special token IDs must be in the vocabulary") + if self.pre_identifier not in tokenizer_pre_policies(): + raise ValueError("Tokenizer blocker contains an unknown pre identifier") + if not self.source_normalizer: + raise ValueError("Tokenizer blocker requires explicit source normalizer semantics") + dispatch = tokenizer_alias_evidence().get(self.pre_identifier) + if dispatch is None or dispatch.source_commit != self.llamacpp_oracle[0]: + raise ValueError("Tokenizer blocker oracle is not pinned to dispatch evidence") + text, llamacpp_ids, source_ids = self.mismatch + if not text or not llamacpp_ids or not source_ids or llamacpp_ids == source_ids: + raise ValueError("Tokenizer blocker requires an exact non-empty mismatch witness") + if any( + token_id < 0 or token_id >= self.token_count + for token_id in (*llamacpp_ids, *source_ids) + ): + raise ValueError("Tokenizer blocker mismatch IDs must be in the vocabulary") + if not self.disposition: + raise ValueError("Tokenizer blocker requires a fail-closed disposition") + + _QWEN35_08B_Q4_TOKENIZER = GGUFTokenizerEvidence( evidence_id="qwen3.5-0.8b-q4-tokenizer", architecture="qwen35", @@ -986,6 +1117,90 @@ def source(self) -> GGUFTokenizerSource: ), ) +_PLM_18B_Q4_K_M_TOKENIZER_BLOCKER = GGUFTokenizerBlockerEvidence( + evidence_id="plm-1.8b-instruct-q4-k-m-tokenizer-blocker", + architecture="plm", + pre_identifier="qwen2", + repository="PLM-Team/PLM-1.8B-Instruct-gguf", + revision="7bec6546983bcf0d99526c943580bd49e2237445", + filename="PLM-1.8B-Instruct-Q4_K_M.gguf", + size=1_182_708_992, + lfs_sha256="b38570ee56ebec82a1e9ef45ab408c0d8230ececef1d7f1b267c49cff35638b8", + tensor_count=290, + tensor_qtypes=(("F32", 97), ("Q4_K", 176), ("Q6_K", 17)), + tokenizer_repository="PLM-Team/PLM-1.8B-Instruct", + tokenizer_revision="62d188c7d58843d7013d5b3ffe198db448787860", + source_config_asset=( + "config.json", + 934, + "91e6e13695a6de82556438667e64b60d9910269f300cd97f8c667d19e75f115e", + ), + tokenizer_assets=( + ( + "merges.txt", + 1_671_853, + "8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5", + ), + ( + "special_tokens_map.json", + 410, + "c83747485fba9ef20c42793b4b02b05001f214250f0d787f573df216c91047a3", + ), + ( + "tokenizer.json", + 11_418_266, + "bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9", + ), + ( + "tokenizer_config.json", + 1_327, + "1becffcfa09c98935043f1724d988887c618c5f6e7a249087d3ae29eb70e2a6f", + ), + ( + "vocab.json", + 2_776_833, + "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910", + ), + ), + tokenizer_metadata_sha256="698bfa31cd069292437bc3509fea7be2445324a536d95a6e813d947c283bd989", + token_count=151_936, + source_token_count=151_646, + embedding_vocabulary_size=151_936, + deterministic_padding_range=(151_646, 151_935), + ordered_vocabulary_sha256="f3ea8e8cf45bd58a8d5ad420306a3ccd925894cdb61a89062fd9e3a6de255a0e", + source_vocabulary_sha256="696f26322524de87f49427fd1be6d1afce910574d9656c5d3f4f64064bdb83c1", + merge_count=151_387, + ordered_merges_sha256="24fa2ae2a398e50784a1fff678482094af4f63e6783d35686726abacda8dc371", + source_merges_sha256="cc098baa4a74ce5156487605aa048a34e54f0eee6a704691a738c8fb22dafdd5", + score_count=0, + ordered_scores_sha256=None, + ordered_token_types_sha256=( + "0286431feb975d95a59a3f39957f8183d929295635fb627b6940627c63918bf1" + ), + source_added_tokens_sha256=( + "e7b5f7013431aa26739424d92f40423f175a46f6d1fdc8453edf6005c99412f7" + ), + source_pipeline_sha256="97c53ee89fb584b10798f44b02c60c9a8b746165a32dc34737d178fc20618a69", + chat_template_sha256="af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db", + source_normalizer="NFC", + special_token_ids=( + ("<|endoftext|>", 151643), + ("<|im_end|>", 151645), + ("<|im_start|>", 151644), + ), + oracle_corpus_sha256="0f3c77d24208f2ac0833668128cb2a00adcb7e6b4b4eedc6e4783e3ec7b41f5a", + llamacpp_oracle=( + "8d9af256337d1a501250f9bbf4c0859a654bddd6", + 72, + "66513168812575ccac974ecb454e916def5f4492d558c1866b8811d4f587a41d", + ), + mismatch=("é é", (68, 53839, 3958), (963, 3958)), + disposition=( + "official tokenizer.json applies NFC normalization, but pinned llama.cpp qwen2 " + "preserves decomposed Unicode; exact materialization is blocked" + ), +) + _LFM2_350M_F16_TOKENIZER = GGUFTokenizerEvidence( evidence_id="lfm2-350m-f16-tokenizer", architecture="lfm2", @@ -1083,6 +1298,10 @@ def source(self) -> GGUFTokenizerSource: } ) +_TOKENIZER_BLOCKER_EVIDENCE = MappingProxyType( + {_PLM_18B_Q4_K_M_TOKENIZER_BLOCKER.evidence_id: (_PLM_18B_Q4_K_M_TOKENIZER_BLOCKER)} +) + def tokenizer_evidence(evidence_id: str) -> GGUFTokenizerEvidence | None: """Return exact tokenizer evidence by stable ID.""" @@ -1094,6 +1313,18 @@ def iter_tokenizer_evidence() -> tuple[GGUFTokenizerEvidence, ...]: return tuple(_TOKENIZER_EVIDENCE[key] for key in sorted(_TOKENIZER_EVIDENCE)) +def tokenizer_blocker_evidence(evidence_id: str) -> GGUFTokenizerBlockerEvidence | None: + """Return exact fail-closed tokenizer evidence by stable ID.""" + return _TOKENIZER_BLOCKER_EVIDENCE.get(evidence_id) + + +def iter_tokenizer_blocker_evidence() -> tuple[GGUFTokenizerBlockerEvidence, ...]: + """Return every fail-closed tokenizer record in stable evidence-ID order.""" + return tuple( + _TOKENIZER_BLOCKER_EVIDENCE[key] for key in sorted(_TOKENIZER_BLOCKER_EVIDENCE) + ) + + def matching_tokenizer_evidence( source_path: Path, gguf_model: Any, @@ -1150,6 +1381,36 @@ def sequence_digest(key: str) -> tuple[int, str]: for token, token_id in evidence.special_token_ids ) ] + blockers = [ + evidence + for evidence in _TOKENIZER_BLOCKER_EVIDENCE.values() + if evidence.architecture == architecture + and evidence.pre_identifier == metadata.get("tokenizer.ggml.pre") + and evidence.filename == identity.filename + and evidence.size == identity.size + and evidence.lfs_sha256 == identity.sha256 + and evidence.tensor_count == identity.tensor_count + and evidence.tensor_qtypes == identity.tensor_qtypes + and evidence.tokenizer_metadata_sha256 == metadata_sha256 + and evidence.token_count == token_count + and evidence.ordered_vocabulary_sha256 == vocabulary_sha256 + and evidence.merge_count == merge_count + and evidence.ordered_merges_sha256 == merges_sha256 + and evidence.score_count == score_count + and evidence.ordered_scores_sha256 == (scores_sha256 or None) + and evidence.token_count == token_type_count + and evidence.ordered_token_types_sha256 == token_types_sha256 + and gguf_model.get_tensor_shape("token_embd.weight")[0] + == evidence.embedding_vocabulary_size + ] + if len(blockers) == 1: + blocker = blockers[0] + raise ValueError( + f"Tokenizer materialization is explicitly blocked by {blocker.evidence_id!r}: " + f"{blocker.disposition}" + ) + if blockers: + raise RuntimeError("Tokenizer blocker evidence contains duplicate artifact identities") if len(matches) != 1: raise ValueError( "No unique exact tokenizer evidence matches " diff --git a/src/mobius/integrations/gguf/_tokenizer_evidence_test.py b/src/mobius/integrations/gguf/_tokenizer_evidence_test.py index 70130ed31..6cee780d3 100644 --- a/src/mobius/integrations/gguf/_tokenizer_evidence_test.py +++ b/src/mobius/integrations/gguf/_tokenizer_evidence_test.py @@ -22,8 +22,10 @@ ) from mobius.integrations.gguf._tokenizer_census import tokenizer_route_census from mobius.integrations.gguf._tokenizer_evidence import ( + iter_tokenizer_blocker_evidence, iter_tokenizer_evidence, matching_tokenizer_evidence, + tokenizer_blocker_evidence, tokenizer_evidence, ) @@ -75,6 +77,100 @@ def test_first_tokenizer_evidence_batch_is_complete_and_artifact_scoped() -> Non assert all(record.source_config_asset[0] == "config.json" for record in records) +def test_plm_tokenizer_blocker_is_exact_and_architecture_scoped() -> None: + blockers = iter_tokenizer_blocker_evidence() + assert [record.evidence_id for record in blockers] == [ + "plm-1.8b-instruct-q4-k-m-tokenizer-blocker" + ] + blocker = blockers[0] + assert blocker.architecture == "plm" + assert blocker.pre_identifier == "qwen2" + assert blocker.revision == "7bec6546983bcf0d99526c943580bd49e2237445" + assert blocker.lfs_sha256 == ( + "b38570ee56ebec82a1e9ef45ab408c0d8230ececef1d7f1b267c49cff35638b8" + ) + assert blocker.tokenizer_revision == "62d188c7d58843d7013d5b3ffe198db448787860" + assert blocker.token_count == blocker.embedding_vocabulary_size == 151_936 + assert blocker.source_token_count == 151_646 + assert blocker.deterministic_padding_range == (151_646, 151_935) + assert blocker.merge_count == 151_387 + assert blocker.score_count == 0 + assert blocker.ordered_scores_sha256 is None + assert blocker.source_normalizer == "NFC" + assert blocker.mismatch == ("é é", (68, 53839, 3958), (963, 3958)) + assert "NFC normalization" in blocker.disposition + + +def test_plm_llamacpp_oracle_fixture_is_bound_to_fail_closed_evidence() -> None: + path = Path(__file__).parents[4] / "tests/data/gguf_plm_qwen2_tokenizer_blocker.json" + oracle = json.loads(path.read_text(encoding="utf-8")) + blocker = tokenizer_blocker_evidence("plm-1.8b-instruct-q4-k-m-tokenizer-blocker") + assert blocker is not None + assert blocker.llamacpp_oracle == ( + oracle["llamacpp_commit"], + oracle["case_count"], + oracle["ordered_results_sha256"], + ) + assert blocker.oracle_corpus_sha256 == oracle["corpus_sha256"] + assert blocker.lfs_sha256 == oracle["artifact_sha256"] + assert blocker.tokenizer_revision == oracle["tokenizer_revision"] + assert oracle["case_count"] == len(oracle["modes"]) * len(oracle["fixed_inputs"]) + assert oracle["mismatch_count"] == len(oracle["mismatch"]["modes"]) + assert blocker.source_pipeline_sha256 == oracle["source_pipeline_sha256"] + assert blocker.chat_template_sha256 == oracle["chat_template_sha256"] + assert blocker.source_normalizer == oracle["source_normalizer"] + assert blocker.mismatch == ( + oracle["mismatch"]["text"], + tuple(oracle["mismatch"]["llamacpp_ids"]), + tuple(oracle["mismatch"]["official_source_ids"]), + ) + assert oracle["source_normalizer"] == "NFC" + assert oracle["default_add_bos_matches_no_add"] + assert not oracle["scores_present"] + + +def test_plm_llamacpp_oracle_hashes_are_derived_from_committed_cases() -> None: + path = Path(__file__).parents[4] / "tests/data/gguf_plm_qwen2_tokenizer_blocker.json" + oracle = json.loads(path.read_text(encoding="utf-8")) + assert list(oracle["results"]) == oracle["modes"] + assert all( + len(oracle["results"][mode]) == len(oracle["fixed_inputs"]) for mode in oracle["modes"] + ) + results = [ + [mode, text, token_ids] + for mode in oracle["modes"] + for text, token_ids in zip( + oracle["fixed_inputs"], oracle["results"][mode], strict=True + ) + ] + assert oracle["case_count"] == len(results) + assert oracle["corpus_sha256"] == _digest([oracle["modes"], oracle["fixed_inputs"]]) + assert oracle["ordered_results_sha256"] == _digest(results) + assert oracle["mode_results_sha256"] == { + mode: _digest([result for result in results if result[0] == mode]) + for mode in oracle["modes"] + } + assert all( + isinstance(token_id, int) and token_id >= 0 + for _, _, token_ids in results + for token_id in token_ids + ) + + results_by_case = {(mode, text): tuple(token_ids) for mode, text, token_ids in results} + mismatch = oracle["mismatch"] + for mode in mismatch["modes"]: + assert results_by_case[mode, mismatch["text"]] == tuple(mismatch["llamacpp_ids"]) + for text in oracle["fixed_inputs"]: + assert ( + results_by_case["add-special/parse-special", text] + == results_by_case["no-add/parse-special", text] + ) + assert ( + results_by_case["no-add/no-parse-special", "<|im_start|>user\nHello<|im_end|>\n"] + != results_by_case["no-add/parse-special", "<|im_start|>user\nHello<|im_end|>\n"] + ) + + def test_evidence_schema_accepts_scores_instead_of_bpe_merges() -> None: evidence = tokenizer_evidence("qwen3.5-0.8b-q4-tokenizer") assert evidence is not None @@ -298,7 +394,7 @@ def test_existing_qwen25_runtime_tokenizer_evidence_remains_pinned() -> None: ) -def _digest(values: list[object]) -> str: +def _digest(values: object) -> str: payload = json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode() return hashlib.sha256(payload).hexdigest() @@ -420,3 +516,63 @@ def test_matching_evidence_fails_closed_on_compact_identity_mismatch( model, metadata_sha256=metadata_sha256, ) + + +def test_matching_plm_blocker_reports_exact_normalizer_mismatch(tmp_path, monkeypatch) -> None: + blocker = tokenizer_blocker_evidence("plm-1.8b-instruct-q4-k-m-tokenizer-blocker") + assert blocker is not None + tiny = dataclasses.replace( + blocker, + filename="tiny.gguf", + size=4, + lfs_sha256="a" * 64, + tensor_count=1, + tensor_qtypes=(("F32", 1),), + token_count=2, + source_token_count=2, + embedding_vocabulary_size=2, + deterministic_padding_range=(2, 1), + ordered_vocabulary_sha256=_digest(["a", ""]), + source_vocabulary_sha256=_digest(["a", ""]), + merge_count=1, + ordered_merges_sha256=_digest(["a "]), + source_merges_sha256=_digest([["a", ""]]), + ordered_token_types_sha256=_digest([1, 3]), + special_token_ids=(("", 1),), + mismatch=("é", (0,), (1,)), + ) + monkeypatch.setattr(_tokenizer_evidence, "_TOKENIZER_EVIDENCE", MappingProxyType({})) + monkeypatch.setattr( + _tokenizer_evidence, + "_TOKENIZER_BLOCKER_EVIDENCE", + MappingProxyType({tiny.evidence_id: tiny}), + ) + monkeypatch.setattr( + _tokenizer_evidence, + "gguf_artifact_identity", + lambda *_a, **_k: GGUFArtifactIdentity( + "plm", + "tiny.gguf", + 4, + "a" * 64, + 1, + (("F32", 1),), + ), + ) + model = SimpleNamespace( + architecture="plm", + metadata={ + "tokenizer.ggml.pre": "qwen2", + "tokenizer.ggml.tokens": ["a", ""], + "tokenizer.ggml.merges": ["a "], + "tokenizer.ggml.token_type": [1, 3], + }, + get_tensor_shape=lambda _name: (2, 4), + ) + + with pytest.raises(ValueError, match=r"explicitly blocked.*NFC normalization"): + matching_tokenizer_evidence( + tmp_path / "tiny.gguf", + model, + metadata_sha256=tiny.tokenizer_metadata_sha256, + ) diff --git a/tests/data/gguf_plm_qwen2_tokenizer_blocker.json b/tests/data/gguf_plm_qwen2_tokenizer_blocker.json new file mode 100644 index 000000000..30e6838f8 --- /dev/null +++ b/tests/data/gguf_plm_qwen2_tokenizer_blocker.json @@ -0,0 +1,149 @@ +{ + "llamacpp_commit": "8d9af256337d1a501250f9bbf4c0859a654bddd6", + "artifact_sha256": "b38570ee56ebec82a1e9ef45ab408c0d8230ececef1d7f1b267c49cff35638b8", + "tokenizer_revision": "62d188c7d58843d7013d5b3ffe198db448787860", + "modes": [ + "no-add/no-parse-special", + "no-add/parse-special", + "add-special/parse-special" + ], + "fixed_inputs": [ + "", + " ", + " ", + "\n", + "\r\n", + "\t", + " spaced text\n", + "I'm we'd they'll can't I'M", + "0 1 12 123 1234 12345 000001234567890", + "!?. ,;:—/\\[]{}()\n", + "你好，世界！", + "漢字かなカナ 한글", + "Café — κόσμος 🚀", + "é é", + "🙂🚀👩‍💻🇺🇳", + "\u0000\u0001", + "A/B\r\nC", + "<|im_start|>user\nHello<|im_end|>\n", + "<|endoftext|>", + "x<|im_start|>y<|im_end|>z", + "<|im_start|><|im_end|><|endoftext|>", + "<|im_start|>assistant\n", + " leading NBSP​zero-width ", + "Ｆｕｌｌｗｉｄｔｈ ffi" + ], + "case_count": 72, + "ordered_results_sha256": "66513168812575ccac974ecb454e916def5f4492d558c1866b8811d4f587a41d", + "mode_results_sha256": { + "no-add/no-parse-special": "e9a8359648d0e0a93503babd854ee11cfddbef07bf43d07f1f1c7e9b37011062", + "no-add/parse-special": "1a8c815b42d031e3ed9674bb02b6d88f5f2e4827aa3fd204e658364d86cc8827", + "add-special/parse-special": "53bf2a76cf18ac7b85d7e154533b89c21d823537deaa2a7dca8d5d0717e62932" + }, + "chat_template_sha256": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db", + "source_pipeline_sha256": "97c53ee89fb584b10798f44b02c60c9a8b746165a32dc34737d178fc20618a69", + "source_normalizer": "NFC", + "scores_present": false, + "bos_id": 151643, + "source_bos_token": null, + "default_add_bos_matches_no_add": true, + "mismatch_count": 3, + "mismatch": { + "text": "é é", + "llamacpp_ids": [ + 68, + 53839, + 3958 + ], + "official_source_ids": [ + 963, + 3958 + ], + "modes": [ + "no-add/no-parse-special", + "no-add/parse-special", + "add-special/parse-special" + ] + }, + "corpus_sha256": "0f3c77d24208f2ac0833668128cb2a00adcb7e6b4b4eedc6e4783e3ec7b41f5a", + "results": { + "no-add/no-parse-special": [ + [], + [220], + [256], + [198], + [319], + [197], + [220,63828,220,1467,198], + [40,2776,582,4172,807,3278,646,944,358,27603], + [15,220,16,220,16,17,220,16,17,18,220,16,17,18,19,220,16,17,18,19,20,220,15,15,15,15,15,16,17,18,19,20,21,22,23,24,15], + [0,4607,1154,26,25,2293,34319,1294,6257,741], + [108386,3837,99489,6313], + [110007,18600,128777,70393,95352,61298,83291], + [34,2577,963,1959,71638,75195,43928,43123,27554,45642,11162,248,222], + [68,53839,3958], + [145080,145836,145233,378,235,145851,145526,145754], + [188,189,221], + [32,16276,319,34], + [27,91,318,4906,91,29,872,198,9707,27,91,318,6213,91,397], + [27,91,8691,723,427,91,29], + [87,27,91,318,4906,91,29,88,27,91,318,6213,91,29,89], + [27,91,318,4906,91,1784,91,318,6213,91,1784,91,8691,723,427,91,29], + [27,91,318,4906,91,29,77091,198], + [6388,4102,33342,4592,15692,14154,9347,220], + [141161,144552,144433,144433,135116,144155,144514,144297,144455,32495,105,225] + ], + "no-add/parse-special": [ + [], + [220], + [256], + [198], + [319], + [197], + [220,63828,220,1467,198], + [40,2776,582,4172,807,3278,646,944,358,27603], + [15,220,16,220,16,17,220,16,17,18,220,16,17,18,19,220,16,17,18,19,20,220,15,15,15,15,15,16,17,18,19,20,21,22,23,24,15], + [0,4607,1154,26,25,2293,34319,1294,6257,741], + [108386,3837,99489,6313], + [110007,18600,128777,70393,95352,61298,83291], + [34,2577,963,1959,71638,75195,43928,43123,27554,45642,11162,248,222], + [68,53839,3958], + [145080,145836,145233,378,235,145851,145526,145754], + [188,189,221], + [32,16276,319,34], + [151644,872,198,9707,151645,198], + [151643], + [87,151644,88,151645,89], + [151644,151645,151643], + [151644,77091,198], + [6388,4102,33342,4592,15692,14154,9347,220], + [141161,144552,144433,144433,135116,144155,144514,144297,144455,32495,105,225] + ], + "add-special/parse-special": [ + [], + [220], + [256], + [198], + [319], + [197], + [220,63828,220,1467,198], + [40,2776,582,4172,807,3278,646,944,358,27603], + [15,220,16,220,16,17,220,16,17,18,220,16,17,18,19,220,16,17,18,19,20,220,15,15,15,15,15,16,17,18,19,20,21,22,23,24,15], + [0,4607,1154,26,25,2293,34319,1294,6257,741], + [108386,3837,99489,6313], + [110007,18600,128777,70393,95352,61298,83291], + [34,2577,963,1959,71638,75195,43928,43123,27554,45642,11162,248,222], + [68,53839,3958], + [145080,145836,145233,378,235,145851,145526,145754], + [188,189,221], + [32,16276,319,34], + [151644,872,198,9707,151645,198], + [151643], + [87,151644,88,151645,89], + [151644,151645,151643], + [151644,77091,198], + [6388,4102,33342,4592,15692,14154,9347,220], + [141161,144552,144433,144433,135116,144155,144514,144297,144455,32495,105,225] + ] + } +}