diff --git a/docs/api/build_from_gguf.md b/docs/api/build_from_gguf.md index 685edd67d..5264ea2dc 100644 --- a/docs/api/build_from_gguf.md +++ b/docs/api/build_from_gguf.md @@ -9,7 +9,7 @@ Support is capability-specific: graph import does not imply runtime packaging. | Census | Total | Closure | |---|---:|---| -| Architectures | 148 | graph verdicts: {'deferred': 46, 'rejected': 2, 'supported': 100}; importable: 99; quantized import: {'rejected': 41, 'supported': 107}; runtime: {'deferred': 142, 'rejected': 2, 'supported': 4} | +| Architectures | 148 | graph verdicts: {'deferred': 46, 'rejected': 2, 'supported': 100}; importable: 99; quantized import: {'rejected': 41, 'supported': 107}; runtime: {'deferred': 136, 'rejected': 2, 'supported': 10} | | Active stored qtypes | 25 | 24 have an import route; 1 are explicitly deferred with no route | | Serialized projector strings | 60 | {'graph-importable': 9, 'runtime-supported': 0} | | Tokenizer pre identifiers | 87 | 56 semantic groups; route dispositions: {'deferred-compiled-semantics': 45, 'deferred-pinned-artifact-evidence': 11, 'validated-pinned-source': 31} | @@ -67,13 +67,25 @@ Processor-owned `image_token_id` overrides are forwarded unchanged for `mmproj=` ## Runtime evidence +The first low-cost architecture batch promotes GPT-2, GPT-NeoX/Pythia, MPT, OLMo, +StarCoder, and StarCoder2 using 334,238,976 bytes of GGUF payload and 346,825,051 +download bytes including tokenizer assets. Every route is explicit-float only. The +network-free selection, budget, exclusions, and fail-closed candidate reasons are recorded in +`testdata/evidence/gguf_low_cost_runtime_batch.json`. + | Evidence ID | GGUF identity | Config identity | Tokenizer identity | Runtime proof | |---|---|---|---|---| +| `gpt2-q2-k-ort-genai-0.15.2` | `tensorblock/gpt2-GGUF@5b01870b15c4b2e43695d7f3f3bfb5b26106f23b`
`gpt2-Q2_K.gguf`
81,196,544 B
`4234545f917ec1df10dab4d926796a83422b68e9010d85a4c111b8b541f32892` | `openai-community/gpt2@607a30d783dfa663caf39e06633721c8d4cfcd7e` | `Xenova/gpt2@bf2c7f02e0b826c60d03af341171bde20893da66`
`special_tokens_map.json` 99 B `6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad`, `tokenizer.json` 2,107,653 B `cda20b8ca044949aa07ac4078420c80d1a57139d5f9f33700e46fb2d891e7c66`, `tokenizer_config.json` 234 B `551e26ec611d8d0c8edc3ef72e518a38418cb71f40de1347dd486a595e1557d7`
metadata `b2417176025f8500d864004b0bf93b1403dc3c52238f6628f82fb0e3c498977e` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route; source GGUF blocks are dequantized. | | `lfm2-350m-f16-ort-genai-0.15.2` | `LiquidAI/LFM2-350M-GGUF@8fdc9d526b7ed346b19257551b05816c7912ecc2`
`LFM2-350M-F16.gguf`
711,482,304 B
`379ffdcbf08147c0313f6f1ce7ff558a2bc935eda633f4b46c52347032419c42` | `LiquidAI/LFM2-350M@f37d3f5c8c5484bc01dad379a595cf4c68c4e70e` | `LiquidAI/LFM2-350M@73e3c253078a3b97c2e14b4c4665679f4d9b6d56`
`chat_template.jinja` 209 B `a805e50fed68938a076b07e2e602639611b50b1ced0e50f11eb92f1ba25be4dc`, `special_tokens_map.json` 434 B `742aefe2b7dec496e8caffdba03a75d0c1a9925d53bd3f3e0d388c96b591b6f4`, `tokenizer.json` 4,732,426 B `98cff83b4f6d7e9d8929bebc62b07e92cf1b3f99c80d16bafe8b84a75448f40b`, `tokenizer_config.json` 91,509 B `36f511115e9d8952cbc9d15d9a20dfa7ce7d1444940e5c1dc42a762020c99bf5`
metadata `e5626d605bb50bc53fdb0fbfcf374fb33dfbaa0cc698d9746ba1e9b0b7e6d07d` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; hybrid convolution and KV state prefill, replay, rollback, reorder, and 20 decode steps | +| `pythia-70m-q2-k-ort-genai-0.15.2` | `mradermacher/pythia-70m-GGUF@52d6f045404c9f93418df2a0144d20c9de34316b`
`pythia-70m.Q2_K.gguf`
38,508,192 B
`8e331c8c8016bed8ff1863b78fafe51e86b2364f32c2d7f3e201687e081cf7f7` | `EleutherAI/pythia-70m@a39f36b100fe8a5377810d56c3f4789b9c53ac42` | `EleutherAI/pythia-70m@a39f36b100fe8a5377810d56c3f4789b9c53ac42`
`special_tokens_map.json` 99 B `6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad`, `tokenizer.json` 2,113,710 B `c24618a1b3e6a38167beff1c72cffd126c3a66254347304b50547d12c5f25624`, `tokenizer_config.json` 396 B `70e38394e494931c6f773ba41e19460dd4436526b852207367f04341b4066d3f`
metadata `d5c722f646ff6462ac217da5e3514d1fa8b4a7b33aedced3daa8e7f00cc74f78` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route on the portable graph; source GGUF blocks are dequantized and the pinned tokenizer vocabulary is extended only with deterministic padding IDs present in the GGUF. | | `qwen2.5-0.5b-instruct-q8-ort-genai-0.15.2` | `Qwen/Qwen2.5-0.5B-Instruct-GGUF@9217f5db79a29953eb74d5343926648285ec7e67`
`qwen2.5-0.5b-instruct-q8_0.gguf`
675,710,816 B
`ca59ca7f13d0e15a8cfa77bd17e65d24f6844b554a7b6c12e07a5f89ff76844e` | `Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf67be50417f59c2c2f167def9a775` | `Qwen/Qwen2.5-0.5B-Instruct@a338b55dd21219a5f4da42bc11a9313d1a27d4cc`
`tokenizer.json` 7,031,645 B `c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539`, `tokenizer_config.json` 7,308 B `5214600ee45ca2f887ce2eede8910378a0111ea99d657428bcbce94778e65a92`
metadata `8fc8ef848104e931f14ae03d9581699d54813a2ff952fb7caac0654e8aa27ee3` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, replay, rollback, reorder, and 20 decode steps | | `qwen3.5-moe-0.87b-q2-k-ort-genai-0.15.2` | `Flexan/kshitijthakkar-qwen3.5-moe-0.87B-d0.8B-GGUF@a9b8adbec2cc87479c772dac1944f313b4036c26`
`qwen3.5-moe-0.87B-d0.8B.Q2_K.gguf`
626,599,552 B
`e8a84df1a50ce65cf80c2b55bba8c6e80f913679fdf9e9439f2c3b52ef3145d5` | `kshitijthakkar/qwen3.5-moe-0.87B-d0.8B@e5b5b3d7c3cc5593196902fd3c23964e891a6ea6` | `kshitijthakkar/qwen3.5-moe-0.87B-d0.8B@e5b5b3d7c3cc5593196902fd3c23964e891a6ea6`
`chat_template.jinja` 7,755 B `273d8e0e683b885071fb17e08d71e5f2a5ddfb5309756181681de4f5a1822d80`, `tokenizer.json` 12,807,982 B `5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42`, `tokenizer_config.json` 16,709 B `49e2b6e395f959f077f1e992b338919c0d4a9732fc6e613995e06557f843500c`
metadata `45302b58b2086a666a874652d0e9e1d5b4b26e786ffbaf9362a4f902eba0b10d` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; hybrid convolution, recurrent, and KV state prefill, replay, rollback, reorder, and 20 cache-threaded decode steps; Explicit-float correctness route only: source quantization is dequantized, dense MoE execution is opt-in, and the selected publisher marks this reduced checkpoint as low quality. Same-value full-logit comparison uses atol=0.35 because small backend differences can cross a routed-expert boundary; all greedy tokens match. | | `smollm-135m-f16-onnxruntime-1.29.0` | `neopolita/smollm-135m-gguf@22cca988936eafe92908e7558907c3964e10bba7`
`ggml-model-f16.gguf`
270,885,504 B
`ec8c775c16944a7e4b5251f97b3f848500dcc3e701b0d492ce9055cea42138a2` | `HuggingFaceTB/SmolLM-135M@1d461723eec654e65efdc40cf49301c89c0c92f4` | `HuggingFaceTB/SmolLM-135M@1d461723eec654e65efdc40cf49301c89c0c92f4`
`special_tokens_map.json` 831 B `e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3`, `tokenizer.json` 2,104,556 B `9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c`, `tokenizer_config.json` 3,685 B `238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8`
metadata `46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; onnx-genai 1.29.0; result=passed; full-logit; dynamic KV cache prefill plus 20 cache-threaded decode steps | | `smollm-135m-f16-ort-genai-0.15.2` | `neopolita/smollm-135m-gguf@22cca988936eafe92908e7558907c3964e10bba7`
`ggml-model-f16.gguf`
270,885,504 B
`ec8c775c16944a7e4b5251f97b3f848500dcc3e701b0d492ce9055cea42138a2` | `HuggingFaceTB/SmolLM-135M@1d461723eec654e65efdc40cf49301c89c0c92f4` | `HuggingFaceTB/SmolLM-135M@1d461723eec654e65efdc40cf49301c89c0c92f4`
`special_tokens_map.json` 831 B `e786b595b9a23148bf1630df78d9037a048ea671e48bfd3549a1e3c233742bb3`, `tokenizer.json` 2,104,556 B `9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c`, `tokenizer_config.json` 3,685 B `238ad6b60d48e471624ea70bc79e92f2611844d5016471fee8c167854bcb98e8`
metadata `46646ba36ecae43de6f9f649d217774b889e0fd405af92205319b882927493fc` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; ORT GenAI prefill plus 20 cache-threaded decode steps | +| `tiny-mpt-q2-k-ort-genai-0.15.2` | `tensorblock/tiny-mpt-random-remote-code-GGUF@c151eb3f349485ee8ae72841d7d90b9121d5baa2`
`tiny-mpt-random-remote-code-Q2_K.gguf`
8,734,304 B
`5627dcb0ff18f6f7200f83c0aed2056a6a7c86b5f2d865833e1b5f00b00e4daa` | `echarlaix/tiny-mpt-random-remote-code@85e64794e74a6fb2e71e7055c7e0188ccdd32905` | `echarlaix/tiny-mpt-random-remote-code@85e64794e74a6fb2e71e7055c7e0188ccdd32905`
`special_tokens_map.json` 99 B `6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad`, `tokenizer.json` 2,113,738 B `3cf430678137c8491ca82fb7092ee49e44ad38857fffe1e4a4a5ed860139a5b8`, `tokenizer_config.json` 237 B `7671fbb5b3d610e6e11d4f5fc78d3a7716e8846112ac7e0f72124caedf887570`
metadata `f15522d34f33354bc96d36a73ad4619925240328342d84a156855251c20d43af` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route; source GGUF blocks are dequantized. | +| `tiny-olmo-q2-k-ort-genai-0.15.2` | `tensorblock/tiny-random-olmo-GGUF@d0ee9498d082d6dc3e730b9765e6e87a9bb5d995`
`tiny-random-olmo-Q2_K.gguf`
33,860,576 B
`be1c5a22ac0e75cd5874467ffd80bcd2c8500609d3bf7ccdb5b269373e4d6da4` | `hyper-accel/tiny-random-olmo@88675ef0caa5bd10ece810c0f2a79faa7724f536` | `hyper-accel/tiny-random-olmo@88675ef0caa5bd10ece810c0f2a79faa7724f536`
`special_tokens_map.json` 293 B `a6188c1e366f8ed715e60ff39c46a8c500fc33508e2affeb23e8c547c5853193`, `tokenizer.json` 2,115,417 B `a094266ac6c4982efba277bc251349a5a6d6ad37efb39a2a90f53d8be2a40a40`, `tokenizer_config.json` 5,372 B `78a839c7851f14f9fb30e664c2b46166dc0628f2900679e5ec160656f702edff`
metadata `25efe0090ffe5a6deb777743917fe572548a0f23357f74d8bd9e969cc911fd73` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route; source GGUF blocks are dequantized. | +| `tiny-starcoder-q2-k-ort-genai-0.15.2` | `RichardErkhov/bigcode_-_tiny_starcoder_py-gguf@fa6f9fdbc134d86d78a3ff9ce08ab14ba33d4718`
`tiny_starcoder_py.Q2_K.gguf`
103,899,456 B
`aa8c2170bb9172447baba14309916cfc0d901dbffaf10f1448f4f631e10c1f41` | `bigcode/tiny_starcoder_py@8547527bef0bc927268c1653cce6948c5c242dd1` | `bigcode/tiny_starcoder_py@8547527bef0bc927268c1653cce6948c5c242dd1`
`special_tokens_map.json` 532 B `0823292e24ea07b89317e9ede9d08da2a1b6c014290c06908a7ad04f1efd6719`, `tokenizer.json` 2,057,395 B `42b5a37ba11199f024f2b8873e1ecba98da33166e16f700bf7cb2304b0a5583f`, `tokenizer_config.json` 677 B `95684c52ad9a970dbbb17576ee2237cb62902c1eff6804c7c91a4d6219a4a6d7`
metadata `23379a715b3983ce0f1559645984431bc039c16eaf6e062b7e62bceac6fa64cd` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route; source GGUF blocks are dequantized. | +| `tiny-starcoder2-q2-k-ort-genai-0.15.2` | `tensorblock/tiny-random-starcoder2-GGUF@82c9eb61d1af6ea00dff834f1ff0620144b333e8`
`tiny-random-starcoder2-Q2_K.gguf`
68,039,904 B
`ab0a4b4e79c906520808db065a00a317c8b097e2176638b34088975eada6e0ed` | `hyper-accel/tiny-random-starcoder2@193576733055d2108dc9906d0da6e0806ad9be57` | `hyper-accel/tiny-random-starcoder2@193576733055d2108dc9906d0da6e0806ad9be57`
`special_tokens_map.json` 1,300 B `0fc9ac706a35e6337b19d484bc8f866b6a0ee7ad2509b03f7305dc0838c82d2c`, `tokenizer.json` 2,060,947 B `17fa145258b20c18287f1e3bd804e074cc13333f11984a2f5a2f11c5110437aa`, `tokenizer_config.json` 7,877 B `8149d8e6b21275ad2cc346885ad92c0e9b5aa3e28a78bb5d39b7febc7e52545d`
metadata `38608a4848dc8535113cff0312858853a0c8e29a5cdc145f258e4a9d84b35113` | ONNX Runtime 1.29.0 `CPUExecutionProvider`; ort-genai 0.15.2; result=passed; full-logit; dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps; Explicit-float correctness route; source GGUF blocks are dequantized. The tiny source config declares BOS/EOS ID 50256 outside its 49152-token model vocabulary. | Runtime support above is independent from tokenizer materialization support below. @@ -98,7 +110,7 @@ remain machine-readable in `_route_census.py`; this table groups only shared nex | `dependency-or-runtime-abi-blocked` | `mtp-specialized-abi` | `mtp:bailingmoe3`, `mtp:cohere2moe`, `mtp:deepseek2`, `mtp:deepseek32`, `mtp:deepseek4`, `mtp:glm-dsa`, `mtp:mimo2`, `mtp:nemotron_h_moe`, `mtp:qwen35moe`, `mtp:qwen3next`, `mtp:step35` | specialized sidecar graph; routed/cache state ABI | | `dependency-or-runtime-abi-blocked` | `projector-runtime-abi` | `projector:resampler` | dynamic processor-to-graph media shape ABI | | `dependency-or-runtime-abi-blocked` | `tokenizer-compiled-semantics` | `tokenizer:afmoe`, `tokenizer:bloom`, `tokenizer:chameleon`, `tokenizer:codeshell`, `tokenizer:command-r`, `tokenizer:dbrx`, `tokenizer:deepseek-coder`, `tokenizer:deepseek-llm`, `tokenizer:deepseek-v3`, `tokenizer:default`, `tokenizer:exaone`, `tokenizer:exaone-moe`, `tokenizer:falcon`, `tokenizer:gpt3-finnish`, `tokenizer:granite-docling`, `tokenizer:granite-embed-multi-97m`, `tokenizer:grok-2`, `tokenizer:hunyuan`, `tokenizer:hunyuan-dense`, `tokenizer:jais`, `tokenizer:jais-2`, `tokenizer:joyai-llm`, `tokenizer:kimi-k2`, `tokenizer:laguna`, `tokenizer:megrez`, `tokenizer:mellum2`, `tokenizer:minerva-7b`, `tokenizer:minicpm5`, `tokenizer:minimax-m2`, `tokenizer:mpt`, `tokenizer:olmo`, `tokenizer:poro-chat`, `tokenizer:refact`, `tokenizer:sarvam-moe`, `tokenizer:seed-coder`, `tokenizer:smaug-bpe`, `tokenizer:solar-open`, `tokenizer:stablelm2`, `tokenizer:starcoder`, `tokenizer:superbpe`, `tokenizer:tekken`, `tokenizer:trillion`, `tokenizer:viking`, `tokenizer:whitespace`, `tokenizer:youtu` | compiled pinned llama.cpp oracle; dispatch-equivalence fixture | -| `evidence-only` | `architecture-runtime-evidence` | `architecture:apertus`, `architecture:arcee`, `architecture:arctic`, `architecture:baichuan`, `architecture:bailingmoe`, `architecture:bert`, `architecture:bitnet`, `architecture:bloom`, `architecture:chatglm`, `architecture:codeshell`, `architecture:cohere2`, `architecture:command-r`, `architecture:dbrx`, `architecture:deci`, `architecture:deepseek`, `architecture:dflash`, `architecture:dots1`, `architecture:dream`, `architecture:eagle3`, `architecture:ernie4_5`, `architecture:ernie4_5-moe`, `architecture:eurobert`, `architecture:exaone`, `architecture:falcon`, `architecture:gemma`, `architecture:gemma-embedding`, `architecture:gemma2`, `architecture:gemma3`, `architecture:gemma4`, `architecture:gpt2`, `architecture:gptneox`, `architecture:granite`, `architecture:granitemoe`, `architecture:hunyuan-dense`, `architecture:hy_v3`, `architecture:internlm2`, `architecture:jais`, `architecture:jais2`, `architecture:jina-bert-v2`, `architecture:jina-bert-v3`, `architecture:lfm2moe`, `architecture:llada`, `architecture:llada-moe`, `architecture:llama-embed`, `architecture:maincoder`, `architecture:mamba`, `architecture:mamba2`, `architecture:minicpm`, `architecture:minicpm3`, `architecture:modern-bert`, `architecture:mpt`, `architecture:muse-glimmer`, `architecture:nemotron`, `architecture:nemotron_h`, `architecture:neo-bert`, `architecture:nomic-bert`, `architecture:nomic-bert-moe`, `architecture:olmo`, `architecture:olmo2`, `architecture:olmoe`, `architecture:openelm`, `architecture:orion`, `architecture:pangu-embedded`, `architecture:phi2`, `architecture:phi3`, `architecture:phimoe`, `architecture:plamo`, `architecture:plm`, `architecture:qwen`, `architecture:qwen2moe`, `architecture:qwen2vl`, `architecture:qwen3`, `architecture:qwen35`, `architecture:qwen3moe`, `architecture:qwen3next`, `architecture:refact`, `architecture:rnd1`, `architecture:seed_oss`, `architecture:smallthinker`, `architecture:smollm3`, `architecture:stablelm`, `architecture:starcoder`, `architecture:starcoder2`, `architecture:t5`, `architecture:t5encoder`, `architecture:talkie`, `architecture:xverse` | immutable representative GGUF; full-logit prefill and cached-decode parity; deterministic generation/state evidence | +| `evidence-only` | `architecture-runtime-evidence` | `architecture:apertus`, `architecture:arcee`, `architecture:arctic`, `architecture:baichuan`, `architecture:bailingmoe`, `architecture:bert`, `architecture:bitnet`, `architecture:bloom`, `architecture:chatglm`, `architecture:codeshell`, `architecture:cohere2`, `architecture:command-r`, `architecture:dbrx`, `architecture:deci`, `architecture:deepseek`, `architecture:dflash`, `architecture:dots1`, `architecture:dream`, `architecture:eagle3`, `architecture:ernie4_5`, `architecture:ernie4_5-moe`, `architecture:eurobert`, `architecture:exaone`, `architecture:falcon`, `architecture:gemma`, `architecture:gemma-embedding`, `architecture:gemma2`, `architecture:gemma3`, `architecture:gemma4`, `architecture:granite`, `architecture:granitemoe`, `architecture:hunyuan-dense`, `architecture:hy_v3`, `architecture:internlm2`, `architecture:jais`, `architecture:jais2`, `architecture:jina-bert-v2`, `architecture:jina-bert-v3`, `architecture:lfm2moe`, `architecture:llada`, `architecture:llada-moe`, `architecture:llama-embed`, `architecture:maincoder`, `architecture:mamba`, `architecture:mamba2`, `architecture:minicpm`, `architecture:minicpm3`, `architecture:modern-bert`, `architecture:muse-glimmer`, `architecture:nemotron`, `architecture:nemotron_h`, `architecture:neo-bert`, `architecture:nomic-bert`, `architecture:nomic-bert-moe`, `architecture:olmo2`, `architecture:olmoe`, `architecture:openelm`, `architecture:orion`, `architecture:pangu-embedded`, `architecture:phi2`, `architecture:phi3`, `architecture:phimoe`, `architecture:plamo`, `architecture:plm`, `architecture:qwen`, `architecture:qwen2moe`, `architecture:qwen2vl`, `architecture:qwen3`, `architecture:qwen35`, `architecture:qwen3moe`, `architecture:qwen3next`, `architecture:refact`, `architecture:rnd1`, `architecture:seed_oss`, `architecture:smallthinker`, `architecture:smollm3`, `architecture:stablelm`, `architecture:t5`, `architecture:t5encoder`, `architecture:talkie`, `architecture:xverse` | immutable representative GGUF; full-logit prefill and cached-decode parity; deterministic generation/state evidence | | `evidence-only` | `draft-runtime-evidence` | `draft:dflash`, `draft:eagle3` | target acceptance loop; draft cache orchestration; deterministic speedup parity | | `evidence-only` | `mtp-runtime-evidence` | `mtp:hy_v3`, `mtp:qwen35` | target acceptance loop; cache-threaded draft/target parity | | `evidence-only` | `projector-runtime-evidence` | `projector:adapter`, `projector:gemma3`, `projector:gemma4v`, `projector:ldp`, `projector:ldpv2`, `projector:mlp`, `projector:muse-glimmer`, `projector:qwen2.5vl_merger`, `projector:qwen2vl_merger` | paired text target; processor boundary; deterministic multimodal package execution | @@ -110,7 +122,7 @@ remain machine-readable in `_route_census.py`; this table groups only shared nex | `intentionally-rejected` | `policy-rejections` | `projector:pockettts_gen`, `projector:qwen3tts_gen` | sidecar role must become a valid projector contract | | `intentionally-rejected` | `policy-rejections` | `mtp:bailingmoe2`, `mtp:dots3note`, `mtp:exaone-moe`, `mtp:exaone4`, `mtp:gemma4-assistant`, `mtp:glm4`, `mtp:glm4moe`, `mtp:graniteswitch`, `mtp:nemotron_h` | upstream executable ownership change | -Recent PR dependencies: #645 (merged) — authoritative tokenizer route inventory and compiled-semantics blockers; #651 (merged) — gemma4 tokenizer evidence merged into the authoritative route census; #652 (closed) — superseded by merged PR #656; #656 (merged) — qwen4exp route and complete sharded import merged, superseding PR #652; #675 (merged) — follow-up hardening for sharded GGUF evidence edge cases after PR #656. +Recent PR dependencies: #645 (merged) — authoritative tokenizer route inventory and compiled-semantics blockers; #651 (merged) — gemma4 tokenizer evidence merged into the authoritative route census; #652 (closed) — superseded by merged PR #656; #656 (merged) — qwen4exp route and complete sharded import merged, superseding PR #652; #675 (merged) — follow-up hardening for sharded GGUF evidence edge cases after PR #656; #672 (merged) — Qwen3.5 MoE runtime evidence is complete; exclude it from follow-up evidence batches; #674 (merged) — Nemotron-H MoE blocker evidence is complete; exclude it from follow-up evidence batches; #677 (merged) — the text architecture cohort is merged into the authoritative graph-route census; #678 (merged) — the generic projector cohort is merged into the authoritative projector census; #679 (merged) — Hunyuan V3 graph and MTP routes are merged into the authoritative route census; #680 (merged) — the MoE architecture cohort is merged into the authoritative graph-route census. ## Tokenizer evidence @@ -203,9 +215,9 @@ Reason codes are concise user-facing categories; detailed architecture audits re | `glm4` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — GLM4 serializes complete fused-FFN trailing blocks and NextN tensors, but the pinned loader skips appended blocks; GLM-OCR converter transforms also permute Q/K for M-RoPE. | | `glm4moe` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — GLM4-MoE serializes biased attention and periodic dense/routed expert trailing blocks with mandatory router bias, but the pinned loader skips them. | | `gpt-oss` | — | none (fails before config extraction) | not claimed | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — The pinned GPT-OSS converter splits interleaved gate/up expert rows and repacks checkpoint block+scale tensors into expert-major MXFP4 values. | -| `gpt2` | — | model=`gpt2`; tensor=`gpt2` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Quantization preservation is rejected because canonical GPT-2 GGUF projections must be transposed into graph order, and the current packed route cannot transpose values together with their scales and zero-points. | +| `gpt2` | — | model=`gpt2`; tensor=`gpt2` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=rejected | FLOAT_IMPORT_ONLY — Runtime support is restricted to the exact evidenced GPT-2 Q2_K artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2. | | `gptj` | — | none (fails before config extraction) | no-loader | config=rejected; tensor_map=rejected; graph=rejected; runtime=rejected; quantized_import=supported | CONFIG_REJECTED — The pinned census reserves gptj but llama.cpp has no model loader for it. | -| `gptneox` | — | model=`gpt_neox`; module=`gguf_legacy`; tensor=`legacy_layernorm` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — The admitted subset requires parallel residual MHA. | +| `gptneox` | — | model=`gpt_neox`; module=`gguf_legacy`; tensor=`legacy_layernorm` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=rejected | FLOAT_IMPORT_ONLY — Runtime support is restricted to the exact evidenced Pythia-70M Q2_K artifact, explicit-float portable graph, pinned tokenizer, and ORT GenAI 0.15.2. | | `granite` | — | model=`granite`; tensor=`llama`+`diffusion_fused_qkv`+`moe_extras` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Exact float and quantization-preserving import covers the pinned dense-or-MoE union, fused or split QKV, optional projection biases, ungated shared experts, and Granite scaling. | | `granite_swa` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — Granite SWA requires attention sinks, a complete interleaved sliding-window schedule, residual/logit scaling, fused routed gate-up experts, and optional fused shared experts/deep-stack injection. | | `granitehybrid` | — | model=`granitemoehybrid`; tensor=`granitehybrid` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Exact mixed attention/Mamba2 scheduling, architecture-wide dense or routed MoE feed-forward selection, optional shared experts, Granite scaling, value-preserving float expert fusion, and strict pinned tensor closure are supported. | @@ -246,7 +258,7 @@ Reason codes are concise user-facing categories; detailed architecture audits re | `mistral3` | — | none (fails before config extraction) | exact-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — The pinned Mistral3 loader selects dense or routed-expert text blocks from metadata and applies architecture-specific output temperature scaling. | | `mistral4` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — Mistral4 has no NextN metadata or MTP graph; it inherits Mistral3's conditional dense/MoE tensor loader and overrides graph construction. | | `modern-bert` | — | model=`modernbert`; tensor=`modern_bert` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact pinned tensor closure, encoder-only task dispatch, and synthetic ORT execution are covered, but no pinned real GGUF artifact has passed independent embedding parity. | -| `mpt` | — | model=`mpt`; module=`gguf_legacy`; tensor=`legacy_layernorm` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — The admitted subset rejects learned positions, Q/K norms, KQV clipping, AWQ activation scales, and inconsistent optional bias families. | +| `mpt` | — | model=`mpt`; module=`gguf_legacy`; tensor=`legacy_layernorm` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=rejected | FLOAT_IMPORT_ONLY — Runtime support is restricted to the exact evidenced tiny MPT Q2_K artifact, explicit-float portable graph, pinned tokenizer, and ORT GenAI 0.15.2. | | `muse-glimmer` | `muse_glimmer` | model=`muse_glimmer_text`; tensor=`llama`+`muse_glimmer_extras`; mmproj=`muse_glimmer` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `nanbeige` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — Nanbeige reuses physical layer weights across a configurable logical loop count, optionally normalizes between loops, and allocates a distinct KV slot for every logical occurrence. | | `nemotron` | — | model=`nemotron`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | @@ -255,7 +267,7 @@ Reason codes are concise user-facing categories; detailed architecture audits re | `neo-bert` | — | model=`neobert`; module=`neo_bert_gguf`; tensor=`neo_bert` | exact-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Config extraction, exact pinned tensor closure, encoder-only task dispatch, and synthetic ORT execution are covered, but no pinned real GGUF artifact has passed independent embedding parity. | | `nomic-bert` | — | model=`nomic_bert`; module=`nomic_bert_gguf`; tensor=`nomic_bert` | exact-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Config extraction, exact pinned tensor closure, encoder-only task dispatch, and synthetic ORT execution are covered, but no pinned real GGUF artifact has passed independent embedding parity. | | `nomic-bert-moe` | — | model=`nomic_bert_moe`; module=`nomic_bert_moe_gguf`; tensor=`nomic_bert`+`nomic_bert_moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Config extraction, exact pinned tensor closure, encoder-only task dispatch, and synthetic ORT execution are covered, but no pinned real GGUF artifact has passed independent embedding parity. | -| `olmo` | — | model=`olmo`; tensor=`olmo` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | +| `olmo` | — | model=`olmo`; tensor=`olmo` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | EVIDENCED_SCOPE — Runtime support is restricted to the exact evidenced tiny OLMo Q2_K artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2. | | `olmo2` | — | model=`olmo2`; tensor=`llama`+`olmo2_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `olmoe` | — | model=`olmoe`; tensor=`llama`+`moe_qk_norm_extras`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `openelm` | — | model=`openelm`; module=`gguf_legacy`; tensor=`legacy_layernorm`+`exact_legacy_gguf_extras` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Quantization preservation is rejected because every OpenELM layer stores fused QKV rows that must be split into per-layer Q/K/V graph projections. | @@ -274,7 +286,7 @@ Reason codes are concise user-facing categories; detailed architecture audits re | `qwen2` | — | model=`qwen2`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | EVIDENCED_SCOPE — Runtime support is restricted to the official Qwen2.5-0.5B-Instruct Q8_0 artifact, pinned CPU import route, exact tokenizer revision, and ORT GenAI 0.15.2 evidence. | | `qwen2moe` | `qwen2_moe` | model=`qwen2_moe`; tensor=`llama`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `qwen2vl` | — | model=`qwen2_vl_text`; tensor=`llama`; mmproj=`qwen_vl` | exact-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Text and paired Qwen2/Qwen2.5-VL projector graph import are supported for the exact split-QKV llama.cpp artifacts, but downstream multimodal runtime execution has not been evidenced. | -| `qwen3` | — | model=`qwen3`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | +| `qwen3` | — | model=`qwen3`; tensor=`llama`+`moe_qk_norm_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `qwen35` | — | model=`qwen3_5_text`; tensor=`llama`+`qwen35_hybrid_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact pinned tensor-name closure, GGUF value transforms, and synthetic recurrent-state execution are covered, but no representative real-weight GGUF has yet passed independent full-logit parity and deterministic multi-token stateful ORT generation. | | `qwen35moe` | — | model=`qwen3_5_moe`; tensor=`llama`+`moe_extras`+`qwen35_hybrid_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | EVIDENCED_SCOPE — Runtime support is restricted to the pinned Q2_K reduced checkpoint's explicit-float CPU route, exact tokenizer revision, hybrid KV/convolution/recurrent-state evidence, and ORT GenAI 0.15.2. | | `qwen3moe` | `qwen3_moe` | model=`qwen3_moe`; tensor=`llama`+`moe_qk_norm_extras`+`moe_extras` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | @@ -292,8 +304,8 @@ Reason codes are concise user-facing categories; detailed architecture audits re | `smallthinker` | — | model=`smallthinker_gguf`; module=`smallthinker_gguf`; tensor=`smallthinker` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — The exact float-import graph owns SmallThinker's pre-norm router, ReGLU experts, metadata-selected sigmoid/softmax gate, and per-layer SWA/NoPE schedule. | | `smollm3` | — | model=`smollm3`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | | `stablelm` | — | model=`stablelm`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | -| `starcoder` | — | model=`gpt_bigcode`; tensor=`starcoder` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=rejected | RUNTIME_EVIDENCE_PENDING / FLOAT_IMPORT_ONLY — Quantization preservation is rejected because StarCoder stores one fused biased MQA projection that must be split for the graph. | -| `starcoder2` | — | model=`starcoder2`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Config extraction, exact tensor-name closure, and a full synthetic GGUF graph build are covered, but no representative real-weight GGUF has yet passed ORT parity or generation validation. | +| `starcoder` | — | model=`gpt_bigcode`; tensor=`starcoder` | audited-direct-loader-conditional-union | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=rejected | FLOAT_IMPORT_ONLY — Runtime support is restricted to the exact evidenced tiny StarCoder Q2_K artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2. | +| `starcoder2` | — | model=`starcoder2`; tensor=`llama` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=supported; quantized_import=supported | EVIDENCED_SCOPE — Runtime support is restricted to the exact evidenced tiny StarCoder2 Q2_K artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2. | | `step35` | — | none (fails before config extraction) | audited-direct-loader-conditional-union | config=deferred; tensor_map=deferred; graph=deferred; runtime=deferred; quantized_import=supported | CONFIG_DEFERRED — Step3.5 executes one or more interleaved-SWA NextN heads with optional gates, routed/shared experts, centered-norm transforms, per-layer head geometry, and dedicated cache offsets. | | `t5` | — | model=`t5`; tensor=`t5` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Graph import is covered, but no independent full-logit and generation parity run has yet validated a pinned real T5 GGUF runtime package. | | `t5encoder` | — | model=`t5encoder`; tensor=`t5` | not claimed | config=supported; tensor_map=supported; graph=supported; runtime=deferred; quantized_import=supported | RUNTIME_EVIDENCE_PENDING — Encoder hidden-state import is covered, but the pinned real artifact lacks independent provenance and full hidden-state parity evidence. | diff --git a/docs/cli_reference.md b/docs/cli_reference.md index 11678618d..91512bb05 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -170,10 +170,10 @@ and global cache indices, so dense, MoE, tied-weight, quantized, and unknown architecture names do not need a runtime registry entry. Architecture-specific types remain only where the runtime selects different -behavior. `lfm2` uses its legacy convolution-cache implementation. `gpt2` selects -`Gpt_Model`, but Mobius's separate rank-4 key/value cache ABI does not match that -runtime's rank-5 combined-cache contract, so config generation currently fails -closed. `phi3`, `phimoe`, and `phi3small` retain their names only when their config selects +behavior. `lfm2` uses its legacy convolution-cache implementation. `gpt2` uses the +generic decoder because Mobius exports separate rank-4 key/value caches rather than +the specialized `Gpt_Model` rank-5 combined-cache contract. `phi3`, `phimoe`, and +`phi3small` retain their names only when their config selects LongRoPE, because the released generator uses those names to recompute caches when generation crosses the short-context threshold. Ordinary Phi-3-family graphs use `decoder`. Multimodal, audio, encoder-decoder, special-position-ID, @@ -197,10 +197,11 @@ dense cache. The deferred state-manifest work is tracked by Each export also writes `runtime_compatibility.json`. Generic decoder metadata records the minimum runtime version and the latest stable release exercised by Mobius (0.15.2); it never emits the unreleased `decoder.state_groups` field. -Generic config availability does not promote a GGUF runtime verdict: the only -runtime-supported GGUF route remains the exact pinned SmolLM F16/CPU package, while -SmolLM2 remains rejected because its GGUF padding-token metadata conflicts with the -official pinned tokenizer. +Generic config availability does not promote a GGUF runtime verdict. Runtime +admission is restricted to exact artifacts named by the generated GGUF evidence +matrix, which now includes GPT-2 and other independently verified decoder routes in +addition to SmolLM. SmolLM2 remains rejected because its GGUF padding-token metadata +conflicts with the official pinned tokenizer. #### Example diff --git a/src/mobius/integrations/gguf/_arch_registry.py b/src/mobius/integrations/gguf/_arch_registry.py index e260af984..103db5b51 100644 --- a/src/mobius/integrations/gguf/_arch_registry.py +++ b/src/mobius/integrations/gguf/_arch_registry.py @@ -659,7 +659,8 @@ GGUFArchitectureSpec( gguf_arch="qwen3", model_type="qwen3", - tensor_map_recipe=("llama",), + tensor_map_recipe=("llama", "moe_qk_norm_extras"), + config_postprocessor="qwen3", runtime=Support.DEFERRED, reason=_RUNTIME_VALIDATION_PENDING, ), @@ -1042,11 +1043,13 @@ model_type="gpt2", tensor_map_recipe=("gpt2",), tensor_processor="gpt2", - runtime=Support.DEFERRED, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("gpt2-q2-k-ort-genai-0.15.2",), quantized_import=Support.REJECTED, reason=( - _RUNTIME_VALIDATION_PENDING - + " Quantization preservation is rejected because canonical GPT-2 GGUF " + "Runtime support is restricted to the exact evidenced GPT-2 Q2_K artifact, " + "explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2. " + "Quantization preservation is rejected because canonical GPT-2 GGUF " "projections must be transposed into graph order, and the current packed " "route cannot transpose values together with their scales and zero-points. " "Use keep_quantized=False for explicit float import." @@ -1610,8 +1613,13 @@ gguf_arch="starcoder2", model_type="starcoder2", tensor_map_recipe=("llama",), - runtime=Support.DEFERRED, - reason=_RUNTIME_VALIDATION_PENDING, + config_postprocessor="starcoder2", + runtime=Support.SUPPORTED, + runtime_evidence_ids=("tiny-starcoder2-q2-k-ort-genai-0.15.2",), + reason=( + "Runtime support is restricted to the exact evidenced tiny StarCoder2 Q2_K " + "artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2." + ), ), GGUFArchitectureSpec( gguf_arch="stablelm", @@ -1638,8 +1646,12 @@ required_metadata=("attention.layer_norm_epsilon",), tensor_processor="llama", llama_qk_permute=True, - runtime=Support.DEFERRED, - reason=_RUNTIME_VALIDATION_PENDING, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("tiny-olmo-q2-k-ort-genai-0.15.2",), + reason=( + "Runtime support is restricted to the exact evidenced tiny OLMo Q2_K artifact, " + "explicit-float CPU import route, pinned tokenizer, and ORT GenAI 0.15.2." + ), ), GGUFArchitectureSpec( gguf_arch="olmo2", @@ -2174,11 +2186,13 @@ tensor_map_recipe=("legacy_layernorm",), config_postprocessor="exact_legacy_gguf", required_metadata=("attention.layer_norm_epsilon", "attention.max_alibi_bias"), - runtime=Support.DEFERRED, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("tiny-mpt-q2-k-ort-genai-0.15.2",), quantized_import=Support.REJECTED, reason=( - _RUNTIME_VALIDATION_PENDING - + " The admitted subset rejects learned positions, Q/K norms, KQV clipping, " + "Runtime support is restricted to the exact evidenced tiny MPT Q2_K artifact, " + "explicit-float portable graph, pinned tokenizer, and ORT GenAI 0.15.2. " + "The admitted subset rejects learned positions, Q/K norms, KQV clipping, " "AWQ activation scales, and inconsistent optional bias families. Quantization " "preservation is rejected because fused QKV must be split." ), @@ -2190,11 +2204,13 @@ tensor_map_recipe=("legacy_layernorm",), config_postprocessor="exact_legacy_gguf", required_metadata=("attention.layer_norm_epsilon", "use_parallel_residual"), - runtime=Support.DEFERRED, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("pythia-70m-q2-k-ort-genai-0.15.2",), quantized_import=Support.REJECTED, reason=( - _RUNTIME_VALIDATION_PENDING - + " The admitted subset requires parallel residual MHA. Quantization " + "Runtime support is restricted to the exact evidenced Pythia-70M Q2_K artifact, " + "explicit-float portable graph, pinned tokenizer, and ORT GenAI 0.15.2. " + "The admitted subset requires parallel residual MHA. Quantization " "preservation is rejected because fused QKV rows must be split." ), ), @@ -2336,11 +2352,13 @@ tensor_map_recipe=("starcoder",), config_postprocessor="conventional_legacy", required_metadata=("attention.layer_norm_epsilon",), - runtime=Support.DEFERRED, + runtime=Support.SUPPORTED, + runtime_evidence_ids=("tiny-starcoder-q2-k-ort-genai-0.15.2",), quantized_import=Support.REJECTED, reason=( - _RUNTIME_VALIDATION_PENDING - + " Quantization preservation is rejected because StarCoder stores one fused " + "Runtime support is restricted to the exact evidenced tiny StarCoder Q2_K " + "artifact, explicit-float CPU import route, pinned tokenizer, and ORT GenAI " + "0.15.2. Quantization preservation is rejected because StarCoder stores one fused " "biased MQA projection that must be split for the graph." ), ), diff --git a/src/mobius/integrations/gguf/_config_mapping.py b/src/mobius/integrations/gguf/_config_mapping.py index da0aea79d..043f6f34b 100644 --- a/src/mobius/integrations/gguf/_config_mapping.py +++ b/src/mobius/integrations/gguf/_config_mapping.py @@ -1533,6 +1533,27 @@ def _moe_postprocess( return dataclasses.replace(config, **updates) +def _qwen3_postprocess( + config: ArchitectureConfig, + metadata: dict[str, Any], + model: Any = None, +) -> ArchitectureConfig: + """Apply Qwen3's per-head Q/K normalization omitted by GGUF metadata.""" + return dataclasses.replace(config, attn_qk_norm=True, attn_qk_norm_full=False) + + +def _starcoder2_postprocess( + config: ArchitectureConfig, + metadata: dict[str, Any], + model: Any = None, +) -> ArchitectureConfig: + """Restore StarCoder2's architecture-owned uniform sliding window.""" + del metadata, model + if config.sliding_window is not None: + return config + return dataclasses.replace(config, sliding_window=4096) + + def _dbrx_postprocess( config: ArchitectureConfig, metadata: dict[str, Any], @@ -4716,6 +4737,8 @@ def _qwen4exp_postprocess( "phimoe": _phimoe_postprocess, "pangu_embedded": _pangu_embedded_postprocess, "dense_sliding": _dense_sliding_postprocess, + "qwen3": _qwen3_postprocess, + "starcoder2": _starcoder2_postprocess, "gemma2": _gemma2_postprocess, "baichuan": _baichuan_postprocess, "chatglm": _chatglm_postprocess, @@ -4764,15 +4787,17 @@ def _default_activation(model_type: str) -> str: # Most modern models use SiLU/Swish if model_type == "arcee": return "relu2" + if model_type in {"gpt2", "starcoder2"}: + # These architectures use tanh-approximate GELU by default. Their GGUF + # metadata omits that architecture-owned choice, so exact GELU is not equivalent. + return "gelu_pytorch_tanh" gelu_models = { "bert", "bloom", - "gpt2", "gpt_bigcode", "jais2", "kclgpt", "modernbert", - "starcoder2", "t5", } if model_type in gelu_models: diff --git a/src/mobius/integrations/gguf/_config_mapping_test.py b/src/mobius/integrations/gguf/_config_mapping_test.py index bb0c5a426..1bd90d11f 100644 --- a/src/mobius/integrations/gguf/_config_mapping_test.py +++ b/src/mobius/integrations/gguf/_config_mapping_test.py @@ -1382,12 +1382,18 @@ def test_gemma_uses_gelu_pytorch_tanh(self, model_type: str) -> None: assert _default_activation(model_type) == "gelu_pytorch_tanh" - @pytest.mark.parametrize("model_type", ["gpt2", "bloom", "starcoder2", "t5"]) + @pytest.mark.parametrize("model_type", ["bloom", "t5"]) def test_gelu_models(self, model_type: str) -> None: from mobius.integrations.gguf._config_mapping import _default_activation assert _default_activation(model_type) == "gelu" + @pytest.mark.parametrize("model_type", ["gpt2", "starcoder2"]) + def test_architecture_default_uses_tanh_approximate_gelu(self, model_type: str) -> None: + from mobius.integrations.gguf._config_mapping import _default_activation + + assert _default_activation(model_type) == "gelu_pytorch_tanh" + @pytest.mark.parametrize("model_type", ["llama", "qwen2", "mistral"]) def test_silu_default(self, model_type: str) -> None: from mobius.integrations.gguf._config_mapping import _default_activation @@ -1395,6 +1401,48 @@ def test_silu_default(self, model_type: str) -> None: assert _default_activation(model_type) == "silu" +def test_qwen3_qk_norm_weights_have_graph_consumers() -> None: + from mobius.integrations.gguf._config_mapping import gguf_to_config + + tensor_names = [ + "token_embd.weight", + "output.weight", + *[ + f"blk.{layer}.{name}.weight" + for layer in range(2) + for name in ("attn_q_norm", "attn_k_norm") + ], + ] + config = gguf_to_config(_FakeDenseGGUF("qwen3", _dense_metadata("qwen3"), tensor_names)) + + assert config.attn_qk_norm is True + assert config.attn_qk_norm_full is False + module = registry.get(config.model_type)(config) + for layer in module.model.layers: + assert layer.self_attn.q_norm is not None + assert layer.self_attn.k_norm is not None + + +@pytest.mark.parametrize( + ("metadata_window", "expected_window"), + [(None, 4096), (2048, 2048)], +) +def test_starcoder2_restores_architecture_sliding_window( + metadata_window: int | None, + expected_window: int, +) -> None: + from mobius.integrations.gguf._config_mapping import gguf_to_config + + metadata = _dense_metadata("starcoder2") + if metadata_window is not None: + metadata["starcoder2.attention.sliding_window"] = metadata_window + config = gguf_to_config(_FakeDenseGGUF("starcoder2", metadata, [])) + + assert config.sliding_window == expected_window + module = registry.get(config.model_type)(config) + assert module.model._sliding_window == expected_window + + class TestQwen35MtpBlockExclusion: """Qwen3.5/3.8 GGUF ``block_count`` includes trailing MTP (nextn) blocks. diff --git a/src/mobius/integrations/gguf/_conventional_decoder_test.py b/src/mobius/integrations/gguf/_conventional_decoder_test.py index 26f8be2a6..ad2b5534a 100644 --- a/src/mobius/integrations/gguf/_conventional_decoder_test.py +++ b/src/mobius/integrations/gguf/_conventional_decoder_test.py @@ -199,7 +199,10 @@ def test_conventional_decoder_support_and_runtime_verdicts_are_explicit() -> Non for architecture in _ARCHITECTURES: spec = get_arch_spec(architecture) assert spec.is_importable - assert spec.runtime is Support.DEFERRED + expected_runtime = ( + Support.SUPPORTED if architecture == "starcoder" else Support.DEFERRED + ) + assert spec.runtime is expected_runtime expected = ( Support.SUPPORTED if architecture in supported_quantized else Support.REJECTED ) diff --git a/src/mobius/integrations/gguf/_docs.py b/src/mobius/integrations/gguf/_docs.py index 430f8e02f..543c77568 100644 --- a/src/mobius/integrations/gguf/_docs.py +++ b/src/mobius/integrations/gguf/_docs.py @@ -647,6 +647,12 @@ def render_document() -> str: ## Runtime evidence +The first low-cost architecture batch promotes GPT-2, GPT-NeoX/Pythia, MPT, OLMo, +StarCoder, and StarCoder2 using 334,238,976 bytes of GGUF payload and 346,825,051 +download bytes including tokenizer assets. Every route is explicit-float only. The +network-free selection, budget, exclusions, and fail-closed candidate reasons are recorded in +`testdata/evidence/gguf_low_cost_runtime_batch.json`. + {_runtime_evidence_table()} Runtime support above is independent from tokenizer materialization support below. diff --git a/src/mobius/integrations/gguf/_docs_test.py b/src/mobius/integrations/gguf/_docs_test.py index 89d8938f9..664ce398a 100644 --- a/src/mobius/integrations/gguf/_docs_test.py +++ b/src/mobius/integrations/gguf/_docs_test.py @@ -39,7 +39,7 @@ def test_document_is_exact_generator_output() -> None: def test_document_is_concise_and_reason_coded() -> None: document = Path("docs/api/build_from_gguf.md").read_text(encoding="utf-8") - assert len(document.splitlines()) < 545 + assert len(document.splitlines()) < 560 assert "RUNTIME_EVIDENCE_PENDING" in document assert "qwen3.5-0.8b-q4-tokenizer" in document assert "qwen2.5-0.5b-instruct-q8-tokenizer" in document @@ -143,6 +143,12 @@ def test_runtime_support_requires_structured_evidence() -> None: ("qwen2", ("qwen2.5-0.5b-instruct-q8-ort-genai-0.15.2",)), ("lfm2", ("lfm2-350m-f16-ort-genai-0.15.2",)), ("qwen35moe", ("qwen3.5-moe-0.87b-q2-k-ort-genai-0.15.2",)), + ("gpt2", ("gpt2-q2-k-ort-genai-0.15.2",)), + ("starcoder2", ("tiny-starcoder2-q2-k-ort-genai-0.15.2",)), + ("olmo", ("tiny-olmo-q2-k-ort-genai-0.15.2",)), + ("mpt", ("tiny-mpt-q2-k-ort-genai-0.15.2",)), + ("gptneox", ("pythia-70m-q2-k-ort-genai-0.15.2",)), + ("starcoder", ("tiny-starcoder-q2-k-ort-genai-0.15.2",)), ] pins = {pin.artifact_id for pin in MMPROJ_ARTIFACT_PINS} diff --git a/src/mobius/integrations/gguf/_exact_legacy_decoder_test.py b/src/mobius/integrations/gguf/_exact_legacy_decoder_test.py index 7bf33a872..1c6078458 100644 --- a/src/mobius/integrations/gguf/_exact_legacy_decoder_test.py +++ b/src/mobius/integrations/gguf/_exact_legacy_decoder_test.py @@ -154,7 +154,10 @@ def test_exact_legacy_config_tensor_and_graph_closure(architecture: str) -> None "openelm": "openelm", }[architecture] ) - assert spec.runtime is Support.DEFERRED + expected_runtime = ( + Support.SUPPORTED if architecture in {"gptneox", "mpt"} else Support.DEFERRED + ) + assert spec.runtime is expected_runtime if architecture == "ernie4_5": assert config.rope_interleave diff --git a/src/mobius/integrations/gguf/_quant_capabilities_test.py b/src/mobius/integrations/gguf/_quant_capabilities_test.py index 2d4e38055..fb4726f6b 100644 --- a/src/mobius/integrations/gguf/_quant_capabilities_test.py +++ b/src/mobius/integrations/gguf/_quant_capabilities_test.py @@ -167,10 +167,10 @@ def test_selected_real_artifacts_stay_within_global_budget() -> None: assert isinstance(policy, dict) assert isinstance(artifacts, list) assert isinstance(lossy, list) - assert len(artifacts) == 4 + assert len(artifacts) == 10 assert len(lossy) == 1 selected = sum(int(record["size"]) for record in [*artifacts, *lossy]) - assert selected == 2_390_132_320 + assert selected == 2_724_371_296 assert selected == policy["selected_artifact_bytes"] assert selected <= policy["max_selected_artifact_bytes"] assert lossy[0]["lfs_sha256"] == ( diff --git a/src/mobius/integrations/gguf/_route_census.py b/src/mobius/integrations/gguf/_route_census.py index a2974ad1c..2fc4c4dc4 100644 --- a/src/mobius/integrations/gguf/_route_census.py +++ b/src/mobius/integrations/gguf/_route_census.py @@ -86,6 +86,42 @@ class RecentPRDependency: "merged", "follow-up hardening for sharded GGUF evidence edge cases after PR #656", ), + RecentPRDependency( + 672, + "Prove Qwen3.5 MoE GGUF explicit-float runtime route", + "merged", + "Qwen3.5 MoE runtime evidence is complete; exclude it from follow-up evidence batches", + ), + RecentPRDependency( + 674, + "Keep Nemotron-H GGUF runtime fail-closed with pinned evidence", + "merged", + "Nemotron-H MoE blocker evidence is complete; exclude it from follow-up evidence batches", + ), + RecentPRDependency( + 677, + "Promote first GGUF text architecture cohort", + "merged", + "the text architecture cohort is merged into the authoritative graph-route census", + ), + RecentPRDependency( + 678, + "Promote generic GGUF projector cohort", + "merged", + "the generic projector cohort is merged into the authoritative projector census", + ), + RecentPRDependency( + 679, + "Add exact Hunyuan V3 GGUF graph support", + "merged", + "Hunyuan V3 graph and MTP routes are merged into the authoritative route census", + ), + RecentPRDependency( + 680, + "Promote first GGUF MoE architecture cohort", + "merged", + "the MoE architecture cohort is merged into the authoritative graph-route census", + ), ) # Reviewed dispositions for routes that cannot advance until Mobius owns a new diff --git a/src/mobius/integrations/gguf/_route_census_test.py b/src/mobius/integrations/gguf/_route_census_test.py index bb65297c4..4d2a1afd7 100644 --- a/src/mobius/integrations/gguf/_route_census_test.py +++ b/src/mobius/integrations/gguf/_route_census_test.py @@ -56,7 +56,7 @@ def test_every_route_has_one_actionable_classification() -> None: assert {item.category for item in items} == allowed assert all(item.batch and item.dependencies and item.reason.strip() for item in items) assert Counter(item.kind for item in items) == { - "architecture": 144, + "architecture": 138, "projector": 60, "tokenizer": 56, "mtp": 22, @@ -64,7 +64,7 @@ def test_every_route_has_one_actionable_classification() -> None: } assert Counter(item.category for item in items) == { "dependency-or-runtime-abi-blocked": 100, - "evidence-only": 107, + "evidence-only": 101, "immediately-implementable": 53, "intentionally-rejected": 19, "artifact-unavailable": 5, @@ -118,6 +118,12 @@ def test_recent_pr_reconciliation_is_explicit() -> None: (652, "closed"), (656, "merged"), (675, "merged"), + (672, "merged"), + (674, "merged"), + (677, "merged"), + (678, "merged"), + (679, "merged"), + (680, "merged"), ] diff --git a/src/mobius/integrations/gguf/_runtime_evidence.py b/src/mobius/integrations/gguf/_runtime_evidence.py index 69027ede6..4862c8a6a 100644 --- a/src/mobius/integrations/gguf/_runtime_evidence.py +++ b/src/mobius/integrations/gguf/_runtime_evidence.py @@ -198,12 +198,13 @@ def _is_hex(value: str) -> bool: _SMOLLM_F16_ROUTE = ( '{"architecture":"llama","config_sha256":' - '"d3f3f2abf531abde55e04a52b5c892c93943b7e260160c796c490618a2e84886",' + '"9f917f4a59c907325a069735b9a5d07177f3d665b5e812c10b626ecf1c94708e",' '"execution_provider":"cpu","model_type":"llama","module_type":"llama",' '"preserve_quantization":false,"registry_import":{"config_key_map":null,' '"config_postprocessor":null,"llama_qk_permute":true,"offset_norm":false,' '"required_metadata":[],"rope_interleave":false,"tensor_processor":"llama",' - '"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,' + '"v_head_reorder":false,"vlm_builder":"generic_projector"},"route_schema":1,' + '"static_cache":false,' '"task":{"class":"builtins.str","state":"text-generation"},' '"tensor_map_recipe":["llama"]}' ) @@ -330,10 +331,10 @@ def _is_hex(value: str) -> bool: ), tensor_count=291, tensor_qtypes=(("F32", 121), ("Q8_0", 170)), - import_route='{"architecture":"qwen2","config_sha256":"f7391f2aac9a7617c1c10e397e91b6f31b80bb3c5f338966b46e7d3935246500","execution_provider":"cpu","model_type":"qwen2","module_type":"qwen2","preserve_quantization":true,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":false,"required_metadata":[],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["llama"]}', + import_route='{"architecture":"qwen2","config_sha256":"d2b0c54fdd96c5122a344240557f6b35aafa65a0e8f8981158683c089559b29a","execution_provider":"cpu","model_type":"qwen2","module_type":"qwen2","preserve_quantization":true,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":false,"required_metadata":[],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["llama"]}', source_fidelity=True, storage_quantized=True, - target_storage_format="native GGUF block storage", + target_storage_format="INT8 affine block-32", compute_mode="runtime-dependent native custom op or inline standard-ONNX fallback", graph_files=("model.onnx", "model.onnx.data", "quantization_report.json"), graph_sha256="240e5e374803c94efdb17eee39c09b0d3e9aed10b6d8b4e1c92e39918ea2155e", @@ -395,7 +396,7 @@ def _is_hex(value: str) -> bool: ), tensor_count=148, tensor_qtypes=(("F16", 93), ("F32", 55)), - import_route='{"architecture":"lfm2","config_sha256":"c961bba579ea33a2472a7c5d3f469c76f1f8c7aae8440a7eaa86bc6e878a42f4","execution_provider":"cpu","model_type":"lfm2","module_type":"lfm2","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":false,"required_metadata":["attention.head_count_kv","attention.layer_norm_rms_epsilon","shortconv.l_cache"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"hybrid-text-generation"},"tensor_map_recipe":["lfm2"]}', + import_route='{"architecture":"lfm2","config_sha256":"e7746a202f3679ac05311fbc7a25414e4c106008fc3bd257001e97a8d7cab575","execution_provider":"cpu","model_type":"lfm2","module_type":"lfm2","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":false,"required_metadata":["attention.head_count_kv","attention.layer_norm_rms_epsilon","shortconv.l_cache"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"hybrid-text-generation"},"tensor_map_recipe":["lfm2"]}', source_fidelity=True, storage_quantized=False, target_storage_format="float", @@ -467,7 +468,7 @@ def _is_hex(value: str) -> bool: ("Q6_K", 1), ("Q8_0", 12), ), - import_route='{"architecture":"qwen35moe","config_sha256":"f61aa39910c1b2ef27b9f8c707ef17dc7d16eff6fd97fae33d3a2eb367bf76f9","execution_provider":"cpu","model_type":"qwen3_5_moe","module_type":"qwen3_5_moe","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":true,"required_metadata":["attention.layer_norm_rms_epsilon","expert_count","expert_used_count","rope.dimension_sections","ssm.conv_kernel","ssm.group_count","ssm.inner_size","ssm.state_size","ssm.time_step_rank"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":true,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"hybrid-text-generation"},"tensor_map_recipe":["llama","moe_extras","qwen35_hybrid_extras"]}', + import_route='{"architecture":"qwen35moe","config_sha256":"48019eea50654171acb6b87074d3c3a212bdf156aa6045c221067732bf25af91","execution_provider":"cpu","model_type":"qwen3_5_moe","module_type":"qwen3_5_moe","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":true,"required_metadata":["attention.layer_norm_rms_epsilon","expert_count","expert_used_count","rope.dimension_sections","ssm.conv_kernel","ssm.group_count","ssm.inner_size","ssm.state_size","ssm.time_step_rank"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":true,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"hybrid-text-generation"},"tensor_map_recipe":["llama","moe_extras","qwen35_hybrid_extras"]}', source_fidelity=False, storage_quantized=False, target_storage_format="float", @@ -505,6 +506,347 @@ def _is_hex(value: str) -> bool: ), ) +_LOW_COST_GRAPH_FILES = ("model.onnx", "model.onnx.data", "quantization_report.json") +_LOW_COST_RUNTIME_PACKAGE_FILES = ( + "genai_config.json", + "gguf_tokenizer_manifest.json", + "model.onnx", + "model.onnx.data", + "quantization_report.json", + "runtime_compatibility.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", +) +_LOW_COST_STATEFUL_SEMANTICS = ( + "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps" +) + +_GPT2_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="gpt2-q2-k-ort-genai-0.15.2", + architecture="gpt2", + repository="tensorblock/gpt2-GGUF", + revision="5b01870b15c4b2e43695d7f3f3bfb5b26106f23b", + filename="gpt2-Q2_K.gguf", + size=81_196_544, + lfs_sha256="4234545f917ec1df10dab4d926796a83422b68e9010d85a4c111b8b541f32892", + config_repository="openai-community/gpt2", + config_revision="607a30d783dfa663caf39e06633721c8d4cfcd7e", + tokenizer_repository="Xenova/gpt2", + tokenizer_revision="bf2c7f02e0b826c60d03af341171bde20893da66", + tokenizer_metadata_sha256="b2417176025f8500d864004b0bf93b1403dc3c52238f6628f82fb0e3c498977e", + tokenizer_assets=( + ( + "special_tokens_map.json", + 99, + "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad", + ), + ( + "tokenizer.json", + 2_107_653, + "cda20b8ca044949aa07ac4078420c80d1a57139d5f9f33700e46fb2d891e7c66", + ), + ( + "tokenizer_config.json", + 234, + "551e26ec611d8d0c8edc3ef72e518a38418cb71f40de1347dd486a595e1557d7", + ), + ), + tensor_count=149, + tensor_qtypes=(("F32", 99), ("Q2_K", 25), ("Q3_K", 24), ("Q6_K", 1)), + import_route='{"architecture":"gpt2","config_sha256":"1e0363dbc5f3427d873aeb46c27d5713ae0668f0e7cf695b5dde00bdd4d390a6","execution_provider":"cpu","model_type":"gpt2","module_type":"gpt2","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":null,"llama_qk_permute":false,"offset_norm":false,"required_metadata":[],"rope_interleave":false,"tensor_processor":"gpt2","v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["gpt2"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="ae121ccf53782ffdc530cf96329e5eacabfc0baf5fcdbfc71925e6edc4ad1423", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="7bfca0bcb5d5aacf39738306c15d0c27db7f7c5810fd6259609fd0d2370becde", + parity_test="test_promoted_gguf_full_runtime_evidence[gpt2-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[gpt2-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations="Explicit-float correctness route; source GGUF blocks are dequantized.", +) + +_GPTNEOX_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="pythia-70m-q2-k-ort-genai-0.15.2", + architecture="gptneox", + repository="mradermacher/pythia-70m-GGUF", + revision="52d6f045404c9f93418df2a0144d20c9de34316b", + filename="pythia-70m.Q2_K.gguf", + size=38_508_192, + lfs_sha256="8e331c8c8016bed8ff1863b78fafe51e86b2364f32c2d7f3e201687e081cf7f7", + config_repository="EleutherAI/pythia-70m", + config_revision="a39f36b100fe8a5377810d56c3f4789b9c53ac42", + tokenizer_repository="EleutherAI/pythia-70m", + tokenizer_revision="a39f36b100fe8a5377810d56c3f4789b9c53ac42", + tokenizer_metadata_sha256="d5c722f646ff6462ac217da5e3514d1fa8b4a7b33aedced3daa8e7f00cc74f78", + tokenizer_assets=( + ( + "special_tokens_map.json", + 99, + "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad", + ), + ( + "tokenizer.json", + 2_113_710, + "c24618a1b3e6a38167beff1c72cffd126c3a66254347304b50547d12c5f25624", + ), + ( + "tokenizer_config.json", + 396, + "70e38394e494931c6f773ba41e19460dd4436526b852207367f04341b4066d3f", + ), + ), + tensor_count=76, + tensor_qtypes=(("F32", 50), ("Q2_K", 13), ("Q3_K", 12), ("Q6_K", 1)), + import_route='{"architecture":"gptneox","config_sha256":"19607e074d570fe92a572204c482e7fde81dd3f7d6d29758fd36dc4a89ba6153","execution_provider":"default","model_type":"gpt_neox","module_type":"gguf_legacy","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":"exact_legacy_gguf","llama_qk_permute":false,"offset_norm":false,"required_metadata":["attention.layer_norm_epsilon","use_parallel_residual"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["legacy_layernorm"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="115a5b47ae04263465e8063661381c183c0b77c870ce3c3dfc339387fbbf9c2f", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="798cc7d612214b032e6354d75c282d75119b3649d603a85d49643c815c24ca77", + parity_test="test_promoted_gguf_full_runtime_evidence[pythia-70m-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[pythia-70m-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations=( + "Explicit-float correctness route on the portable graph; source GGUF blocks are " + "dequantized and the pinned tokenizer vocabulary is extended only with deterministic " + "padding IDs present in the GGUF." + ), +) + +_MPT_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="tiny-mpt-q2-k-ort-genai-0.15.2", + architecture="mpt", + repository="tensorblock/tiny-mpt-random-remote-code-GGUF", + revision="c151eb3f349485ee8ae72841d7d90b9121d5baa2", + filename="tiny-mpt-random-remote-code-Q2_K.gguf", + size=8_734_304, + lfs_sha256="5627dcb0ff18f6f7200f83c0aed2056a6a7c86b5f2d865833e1b5f00b00e4daa", + config_repository="echarlaix/tiny-mpt-random-remote-code", + config_revision="85e64794e74a6fb2e71e7055c7e0188ccdd32905", + tokenizer_repository="echarlaix/tiny-mpt-random-remote-code", + tokenizer_revision="85e64794e74a6fb2e71e7055c7e0188ccdd32905", + tokenizer_metadata_sha256="f15522d34f33354bc96d36a73ad4619925240328342d84a156855251c20d43af", + tokenizer_assets=( + ( + "special_tokens_map.json", + 99, + "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad", + ), + ( + "tokenizer.json", + 2_113_738, + "3cf430678137c8491ca82fb7092ee49e44ad38857fffe1e4a4a5ed860139a5b8", + ), + ( + "tokenizer_config.json", + 237, + "7671fbb5b3d610e6e11d4f5fc78d3a7716e8846112ac7e0f72124caedf887570", + ), + ), + tensor_count=8, + tensor_qtypes=(("F32", 3), ("IQ4_NL", 3), ("Q3_K", 1), ("Q8_0", 1)), + import_route='{"architecture":"mpt","config_sha256":"7239b094edec93d3b29e2ffc53b08be450cb7c8536dd93120c4db618d47cd4f3","execution_provider":"default","model_type":"mpt","module_type":"gguf_legacy","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":"exact_legacy_gguf","llama_qk_permute":false,"offset_norm":false,"required_metadata":["attention.layer_norm_epsilon","attention.max_alibi_bias"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["legacy_layernorm"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="5daf516f886110f257105a4122cba9fbdb920fab92bedf057a4f67899e513c8a", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="20dd912c318d8115af5a5e6e92672624c00120455346ff5c9294e9dc1a1cc234", + parity_test="test_promoted_gguf_full_runtime_evidence[tiny-mpt-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[tiny-mpt-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations="Explicit-float correctness route; source GGUF blocks are dequantized.", +) + +_OLMO_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="tiny-olmo-q2-k-ort-genai-0.15.2", + architecture="olmo", + repository="tensorblock/tiny-random-olmo-GGUF", + revision="d0ee9498d082d6dc3e730b9765e6e87a9bb5d995", + filename="tiny-random-olmo-Q2_K.gguf", + size=33_860_576, + lfs_sha256="be1c5a22ac0e75cd5874467ffd80bcd2c8500609d3bf7ccdb5b269373e4d6da4", + config_repository="hyper-accel/tiny-random-olmo", + config_revision="88675ef0caa5bd10ece810c0f2a79faa7724f536", + tokenizer_repository="hyper-accel/tiny-random-olmo", + tokenizer_revision="88675ef0caa5bd10ece810c0f2a79faa7724f536", + tokenizer_metadata_sha256="25efe0090ffe5a6deb777743917fe572548a0f23357f74d8bd9e969cc911fd73", + tokenizer_assets=( + ( + "special_tokens_map.json", + 293, + "a6188c1e366f8ed715e60ff39c46a8c500fc33508e2affeb23e8c547c5853193", + ), + ( + "tokenizer.json", + 2_115_417, + "a094266ac6c4982efba277bc251349a5a6d6ad37efb39a2a90f53d8be2a40a40", + ), + ( + "tokenizer_config.json", + 5_372, + "78a839c7851f14f9fb30e664c2b46166dc0628f2900679e5ec160656f702edff", + ), + ), + tensor_count=16, + tensor_qtypes=(("IQ4_NL", 2), ("Q2_K", 9), ("Q3_K", 4), ("Q6_K", 1)), + import_route='{"architecture":"olmo","config_sha256":"c02542bb6b7c076f641ff0724409471deefb0782eab37dff076e80746a59652f","execution_provider":"cpu","model_type":"olmo","module_type":"olmo","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":"olmo","llama_qk_permute":true,"offset_norm":false,"required_metadata":["attention.layer_norm_epsilon"],"rope_interleave":false,"tensor_processor":"llama","v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["olmo"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="eb67fb20ae4257af04db8b1554c56ae6b3f66340f2b6ff97dd55cbd14c7bce5b", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="4f2817a0af1b58922f3b82d455844664948dbc67f0ffe6af576baedf2eac3835", + parity_test="test_promoted_gguf_full_runtime_evidence[tiny-olmo-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[tiny-olmo-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations="Explicit-float correctness route; source GGUF blocks are dequantized.", +) + +_STARCODER_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="tiny-starcoder-q2-k-ort-genai-0.15.2", + architecture="starcoder", + repository="RichardErkhov/bigcode_-_tiny_starcoder_py-gguf", + revision="fa6f9fdbc134d86d78a3ff9ce08ab14ba33d4718", + filename="tiny_starcoder_py.Q2_K.gguf", + size=103_899_456, + lfs_sha256="aa8c2170bb9172447baba14309916cfc0d901dbffaf10f1448f4f631e10c1f41", + config_repository="bigcode/tiny_starcoder_py", + config_revision="8547527bef0bc927268c1653cce6948c5c242dd1", + tokenizer_repository="bigcode/tiny_starcoder_py", + tokenizer_revision="8547527bef0bc927268c1653cce6948c5c242dd1", + tokenizer_metadata_sha256="23379a715b3983ce0f1559645984431bc039c16eaf6e062b7e62bceac6fa64cd", + tokenizer_assets=( + ( + "special_tokens_map.json", + 532, + "0823292e24ea07b89317e9ede9d08da2a1b6c014290c06908a7ad04f1efd6719", + ), + ( + "tokenizer.json", + 2_057_395, + "42b5a37ba11199f024f2b8873e1ecba98da33166e16f700bf7cb2304b0a5583f", + ), + ( + "tokenizer_config.json", + 677, + "95684c52ad9a970dbbb17576ee2237cb62902c1eff6804c7c91a4d6219a4a6d7", + ), + ), + tensor_count=244, + tensor_qtypes=(("F32", 163), ("Q2_K", 40), ("Q3_K", 40), ("Q6_K", 1)), + import_route='{"architecture":"starcoder","config_sha256":"45f95ad88af3f5378bdfacd633f1717012e1f3699bbe880af4f6d29fd79f2d4f","execution_provider":"cpu","model_type":"gpt_bigcode","module_type":"gpt_bigcode","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":"conventional_legacy","llama_qk_permute":false,"offset_norm":false,"required_metadata":["attention.layer_norm_epsilon"],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["starcoder"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="ab4edee4c001e3df8f4bce477447bea1c0360221163e01dd1cb03e89737107bd", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="ecbbae525e12d250c3eaa782b9eec14dba2c09fb9dbde29a8331e741e7676171", + parity_test="test_promoted_gguf_full_runtime_evidence[tiny-starcoder-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[tiny-starcoder-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations="Explicit-float correctness route; source GGUF blocks are dequantized.", +) + +_STARCODER2_Q2_K_ORT_GENAI = GGUFRuntimeEvidence( + evidence_id="tiny-starcoder2-q2-k-ort-genai-0.15.2", + architecture="starcoder2", + repository="tensorblock/tiny-random-starcoder2-GGUF", + revision="82c9eb61d1af6ea00dff834f1ff0620144b333e8", + filename="tiny-random-starcoder2-Q2_K.gguf", + size=68_039_904, + lfs_sha256="ab0a4b4e79c906520808db065a00a317c8b097e2176638b34088975eada6e0ed", + config_repository="hyper-accel/tiny-random-starcoder2", + config_revision="193576733055d2108dc9906d0da6e0806ad9be57", + tokenizer_repository="hyper-accel/tiny-random-starcoder2", + tokenizer_revision="193576733055d2108dc9906d0da6e0806ad9be57", + tokenizer_metadata_sha256="38608a4848dc8535113cff0312858853a0c8e29a5cdc145f258e4a9d84b35113", + tokenizer_assets=( + ( + "special_tokens_map.json", + 1_300, + "0fc9ac706a35e6337b19d484bc8f866b6a0ee7ad2509b03f7305dc0838c82d2c", + ), + ( + "tokenizer.json", + 2_060_947, + "17fa145258b20c18287f1e3bd804e074cc13333f11984a2f5a2f11c5110437aa", + ), + ( + "tokenizer_config.json", + 7_877, + "8149d8e6b21275ad2cc346885ad92c0e9b5aa3e28a78bb5d39b7febc7e52545d", + ), + ), + tensor_count=36, + tensor_qtypes=( + ("F32", 22), + ("Q2_K", 7), + ("Q3_K", 4), + ("Q4_K", 2), + ("Q6_K", 1), + ), + import_route='{"architecture":"starcoder2","config_sha256":"25eaa4f5baa4e4725930298aaa921804d1e6f202737a3b10ec32070c4acfcb71","execution_provider":"cpu","model_type":"starcoder2","module_type":"starcoder2","preserve_quantization":false,"registry_import":{"config_key_map":null,"config_postprocessor":"starcoder2","llama_qk_permute":false,"offset_norm":false,"required_metadata":[],"rope_interleave":false,"tensor_processor":null,"v_head_reorder":false,"vlm_builder":null},"route_schema":1,"static_cache":false,"task":{"class":"builtins.str","state":"text-generation"},"tensor_map_recipe":["llama"]}', + source_fidelity=False, + storage_quantized=False, + target_storage_format="float", + compute_mode="float operators", + graph_files=_LOW_COST_GRAPH_FILES, + graph_sha256="34c3e58ac9d95b5d1097393782d41a942ac9b6188e6d57adff21212030897a23", + runtime_package_files=_LOW_COST_RUNTIME_PACKAGE_FILES, + runtime_package_sha256="4a9ab2edf80d8cf778fccd56d306cb56dfda5bb190068faf04c78ac5d180f564", + parity_test="test_promoted_gguf_full_runtime_evidence[tiny-starcoder2-q2-k]", + parity_kind="full-logit", + deterministic_test="test_promoted_gguf_full_runtime_evidence[tiny-starcoder2-q2-k]", + stateful_semantics=_LOW_COST_STATEFUL_SEMANTICS, + execution_provider="CPUExecutionProvider", + onnxruntime_version="1.29.0", + runtime="ort-genai", + runtime_version="0.15.2", + limitations=( + "Explicit-float correctness route; source GGUF blocks are dequantized. The tiny " + "source config declares BOS/EOS ID 50256 outside its 49152-token model vocabulary." + ), +) + _RUNTIME_EVIDENCE: MappingProxyType[str, GGUFRuntimeEvidence] = MappingProxyType( { record.evidence_id: record @@ -514,6 +856,12 @@ def _is_hex(value: str) -> bool: _QWEN35MOE_087B_Q2_K_ORT_GENAI, _SMOLLM_F16_ONNX_RUNTIME, _SMOLLM_F16_ORT_GENAI, + _GPT2_Q2_K_ORT_GENAI, + _GPTNEOX_Q2_K_ORT_GENAI, + _MPT_Q2_K_ORT_GENAI, + _OLMO_Q2_K_ORT_GENAI, + _STARCODER_Q2_K_ORT_GENAI, + _STARCODER2_Q2_K_ORT_GENAI, ) } ) diff --git a/src/mobius/integrations/gguf/_runtime_evidence_test.py b/src/mobius/integrations/gguf/_runtime_evidence_test.py index db39a1df8..5ebd4b500 100644 --- a/src/mobius/integrations/gguf/_runtime_evidence_test.py +++ b/src/mobius/integrations/gguf/_runtime_evidence_test.py @@ -6,9 +6,11 @@ from __future__ import annotations import hashlib +import json import os from collections import Counter from dataclasses import replace +from pathlib import Path from types import MappingProxyType, SimpleNamespace import onnx_ir as ir @@ -17,6 +19,7 @@ from mobius._builder import build_from_module from mobius._configs import NemotronHConfig from mobius.integrations.gguf import _runtime_evidence +from mobius.integrations.gguf._arch_registry import get_arch_spec from mobius.integrations.gguf._reader import _descriptor_identity from mobius.integrations.gguf._runtime_blocker_evidence import ( iter_runtime_blocker_evidence, @@ -134,6 +137,43 @@ def test_runtime_evidence_rejects_non_hex_tokenizer_metadata_digest() -> None: replace(_record(b"pinned-gguf"), tokenizer_metadata_sha256="g" * 64) +def test_low_cost_runtime_batch_manifest_is_closed_and_within_budget() -> None: + manifest = json.loads( + Path("testdata/evidence/gguf_low_cost_runtime_batch.json").read_text(encoding="utf-8") + ) + selected = manifest["selected"] + assert 6 <= len(selected) <= 10 + assert len({item["architecture"] for item in selected}) == len(selected) + assert len({item["evidence_id"] for item in selected}) == len(selected) + assert all(item["storage_mode"] == "explicit-float" for item in selected) + + artifact_bytes = sum(item["artifact_size"] for item in selected) + tokenizer_bytes = sum(item["tokenizer_payload_size"] for item in selected) + assert manifest["totals"] == { + "artifact_bytes": artifact_bytes, + "tokenizer_bytes": tokenizer_bytes, + "download_bytes": artifact_bytes + tokenizer_bytes, + } + assert manifest["totals"]["download_bytes"] <= manifest["policy"]["maximum_download_bytes"] + + for item in selected: + evidence = _runtime_evidence.runtime_evidence(item["evidence_id"]) + assert evidence is not None + assert evidence.architecture == item["architecture"] + assert evidence.size == item["artifact_size"] + assert evidence.lfs_sha256 == item["artifact_sha256"] + assert ( + sum(size for _, size, _ in evidence.tokenizer_assets) + == item["tokenizer_payload_size"] + ) + spec = get_arch_spec(item["architecture"]) + assert item["evidence_id"] in spec.runtime_evidence_ids + + assert all(item["result"] == "blocked" for item in manifest["fail_closed"]) + assert all(item["reason_code"] for item in manifest["fail_closed"]) + assert {item.get("pull_request") for item in manifest["excluded"]} >= {672, 674} + + def test_nemotron_h_runtime_blocker_is_pinned_without_support_claim() -> None: records = iter_runtime_blocker_evidence() assert len(records) == 1 diff --git a/src/mobius/integrations/gguf/_runtime_package.py b/src/mobius/integrations/gguf/_runtime_package.py index f4b49bb91..842f90079 100644 --- a/src/mobius/integrations/gguf/_runtime_package.py +++ b/src/mobius/integrations/gguf/_runtime_package.py @@ -285,16 +285,21 @@ def write_gguf_runtime_package( from mobius.integrations.ort_genai import write_ort_genai_config execution_provider = getattr(pkg, "gguf_execution_provider", None) - if execution_provider not in {"cpu", "cuda", "dml"}: + if execution_provider not in {"default", "cpu", "cuda", "dml"}: raise ValueError( "ORT GenAI runtime packaging requires an explicit evidenced execution " - "provider: cpu, cuda, or dml." + "provider: default, cpu, cuda, or dml." ) + # The portable/default graph intentionally keeps standard ONNX operators. + # ORT GenAI still needs a concrete provider for session construction. + runtime_execution_provider = ( + "cpu" if execution_provider == "default" else execution_provider + ) artifacts.update( write_ort_genai_config( pkg, str(stage), - ep=execution_provider, + ep=runtime_execution_provider, runtime_version=runtime_version, ) ) diff --git a/src/mobius/integrations/gguf/_runtime_package_test.py b/src/mobius/integrations/gguf/_runtime_package_test.py index 562962142..537059880 100644 --- a/src/mobius/integrations/gguf/_runtime_package_test.py +++ b/src/mobius/integrations/gguf/_runtime_package_test.py @@ -170,6 +170,42 @@ def test_atomically_emits_graph_tokenizer_and_runtime_config(self, tmp_path): assert Path(artifacts["inference_metadata"]) == out / "inference_metadata.yaml" assert not list(tmp_path.glob(".out.*.tmp")) + def test_portable_graph_targets_ort_genai_cpu(self, tmp_path): + pkg = _FakePackage() + pkg.gguf_execution_provider = "default" + out = tmp_path / "out" + with ( + mock.patch( + "mobius.integrations.gguf._runtime_package.open_gguf_model", + return_value=SimpleNamespace( + metadata={}, + architecture="llama", + source_matches_path=lambda: True, + ), + ), + mock.patch( + "mobius.integrations.gguf._runtime_package.inspect_gguf_tokenizer", + return_value=_materialized(), + ), + mock.patch( + "mobius.integrations.gguf._runtime_package.materialize_gguf_tokenizer", + side_effect=_write_tokenizer, + ), + mock.patch( + "mobius.integrations.ort_genai.write_ort_genai_config", + side_effect=_write_config, + ) as write_config, + ): + _write_runtime( + pkg, + tmp_path / "m.gguf", + out, + runtime="ort-genai", + runtime_version="0.15.2", + ) + + assert write_config.call_args.kwargs["ep"] == "cpu" + def test_failure_after_graph_save_removes_staging_and_publishes_nothing(self, tmp_path): pkg = _FakePackage() out = tmp_path / "out" diff --git a/src/mobius/integrations/gguf/_tensor_mapping.py b/src/mobius/integrations/gguf/_tensor_mapping.py index 04fc7421b..f6dbdb7eb 100644 --- a/src/mobius/integrations/gguf/_tensor_mapping.py +++ b/src/mobius/integrations/gguf/_tensor_mapping.py @@ -916,7 +916,7 @@ "blk.{bid}.ffn_down_shexp": ("model.layers.{bid}.mlp.shared_expert.down_proj"), } -# Per-head/full-projection Q/K norms used by Qwen3-MoE and OLMoE. The +# Per-head/full-projection Q/K norms used by dense Qwen3, Qwen3-MoE, and OLMoE. The # ArchitectureConfig selects the logical width; the GGUF tensor family is the # same for both representations. _MOE_QK_NORM_EXTRAS: dict[str, str] = { diff --git a/src/mobius/integrations/gguf/_tensor_mapping_test.py b/src/mobius/integrations/gguf/_tensor_mapping_test.py index de972977c..819ce6e63 100644 --- a/src/mobius/integrations/gguf/_tensor_mapping_test.py +++ b/src/mobius/integrations/gguf/_tensor_mapping_test.py @@ -155,6 +155,13 @@ def test_llama_family_aliases(self, arch: str) -> None: result = map_gguf_to_hf_names("blk.0.attn_q.weight", arch) assert result == "model.layers.0.self_attn.q_proj.weight" + @pytest.mark.parametrize("kind", ["q", "k"]) + def test_qwen3_maps_attention_norms(self, kind: str) -> None: + assert ( + map_gguf_to_hf_names(f"blk.0.attn_{kind}_norm.weight", "qwen3") + == f"model.layers.0.self_attn.{kind}_norm.weight" + ) + @pytest.mark.parametrize( ("architecture", "gguf_name", "expected"), [ diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index c882fbeb8..7e45f7b55 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -145,7 +145,6 @@ def _revision_kwargs(revision: str | None) -> dict[str, str]: # described by the ordinary decoder graph ABI. All other compatible, single- # model decoder packages use ORT GenAI's released generic DecoderOnly_Model. _ARCHITECTURE_SPECIFIC_TEXT_TYPES = { - "gpt2": "gpt2", "lfm2": "lfm2", "lfm2_vl": "lfm2", } @@ -295,7 +294,9 @@ def _select_ort_model_type( retained only for LongRoPE cache recomputation after the short-context threshold. Standalone Qwen3.5 text configs also normalize to ``decoder``: their specialized names dispatch to the same ``DecoderOnly_Model`` and are - not available in the latest released ORT GenAI. + not available in the latest released ORT GenAI. Mobius GPT-2 graphs also + normalize to ``decoder`` because they expose the generic separate-key/value + cache ABI rather than ``Gpt_Model``'s rank-5 combined cache. Multimodal and encoder-decoder packages retain their architecture-specific type because those values select distinct runtime pipelines and position-ID diff --git a/src/mobius/integrations/ort_genai/auto_export_test.py b/src/mobius/integrations/ort_genai/auto_export_test.py index 5fd7786b3..21b49aaff 100644 --- a/src/mobius/integrations/ort_genai/auto_export_test.py +++ b/src/mobius/integrations/ort_genai/auto_export_test.py @@ -259,13 +259,12 @@ def test_multimodal_qwen35_preserves_parent_vlm_runtime_type( def test_decoder_only_preserves_specialized_hf_fallback(self): assert ( - _select_ort_model_type("not_a_real_type", "gpt2", is_decoder_only=True) == "gpt2" + _select_ort_model_type("not_a_real_type", "lfm2", is_decoder_only=True) == "lfm2" ) @pytest.mark.parametrize( ("model_type", "expected"), [ - ("gpt2", "gpt2"), ("lfm2", "lfm2"), ("lfm2_vl", "lfm2"), ], @@ -3732,10 +3731,10 @@ def test_generic_decoder_rejects_pre_014_runtime(tmp_path): assert not (tmp_path / "genai_config.json").exists() -def test_gpt2_specialized_runtime_rejects_separate_cache_graph(tmp_path): - with pytest.raises(ValueError, match="rank-5 combined KV-cache"): - write_ort_genai_config(_make_fake_llm_pkg("gpt2"), str(tmp_path)) - assert not (tmp_path / "genai_config.json").exists() +def test_gpt2_separate_cache_graph_uses_generic_decoder(tmp_path): + write_ort_genai_config(_make_fake_llm_pkg("gpt2"), str(tmp_path)) + generated = json.loads((tmp_path / "genai_config.json").read_text()) + assert generated["model"]["type"] == "decoder" @pytest.mark.parametrize("model_type", ["phi3", "phi3small", "phimoe"]) diff --git a/testdata/cases/causal-lm/gpt2-q2-k.yaml b/testdata/cases/causal-lm/gpt2-q2-k.yaml new file mode 100644 index 000000000..ba12ca6c4 --- /dev/null +++ b/testdata/cases/causal-lm/gpt2-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "openai-community/gpt2" +model_type: "gpt2" +revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "tensorblock/gpt2-GGUF" + revision: "5b01870b15c4b2e43695d7f3f3bfb5b26106f23b" + filename: "gpt2-Q2_K.gguf" + size: 81196544 + lfs_sha256: "4234545f917ec1df10dab4d926796a83422b68e9010d85a4c111b8b541f32892" + tensor_count: 149 + tensor_qtypes: {F32: 99, Q2_K: 25, Q3_K: 24, Q6_K: 1} + execution_provider: "cpu" + config_sha256: "1e0363dbc5f3427d873aeb46c27d5713ae0668f0e7cf695b5dde00bdd4d390a6" + preserve_quantization: false + tokenizer: + repository: "Xenova/gpt2" + revision: "bf2c7f02e0b826c60d03af341171bde20893da66" + metadata_sha256: "b2417176025f8500d864004b0bf93b1403dc3c52238f6628f82fb0e3c498977e" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 99, sha256: "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad"} + - {filename: "tokenizer.json", size: 2107653, sha256: "cda20b8ca044949aa07ac4078420c80d1a57139d5f9f33700e46fb2d891e7c66"} + - {filename: "tokenizer_config.json", size: 234, sha256: "551e26ec611d8d0c8edc3ef72e518a38418cb71f40de1347dd486a595e1557d7"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "gpt2-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 84000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Exact explicit-float GGUF evidence; no native/source-faithful quantized runtime claim." diff --git a/testdata/cases/causal-lm/lfm2-350m-f16.yaml b/testdata/cases/causal-lm/lfm2-350m-f16.yaml index 76a9ed799..754bf80ed 100644 --- a/testdata/cases/causal-lm/lfm2-350m-f16.yaml +++ b/testdata/cases/causal-lm/lfm2-350m-f16.yaml @@ -15,7 +15,7 @@ gguf: F16: 93 F32: 55 execution_provider: "cpu" - config_sha256: "c961bba579ea33a2472a7c5d3f469c76f1f8c7aae8440a7eaa86bc6e878a42f4" + config_sha256: "e7746a202f3679ac05311fbc7a25414e4c106008fc3bd257001e97a8d7cab575" preserve_quantization: false keep_quantized: true tokenizer: diff --git a/testdata/cases/causal-lm/pythia-70m-q2-k.yaml b/testdata/cases/causal-lm/pythia-70m-q2-k.yaml new file mode 100644 index 000000000..226950bc5 --- /dev/null +++ b/testdata/cases/causal-lm/pythia-70m-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "EleutherAI/pythia-70m" +model_type: "gpt_neox" +revision: "a39f36b100fe8a5377810d56c3f4789b9c53ac42" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "mradermacher/pythia-70m-GGUF" + revision: "52d6f045404c9f93418df2a0144d20c9de34316b" + filename: "pythia-70m.Q2_K.gguf" + size: 38508192 + lfs_sha256: "8e331c8c8016bed8ff1863b78fafe51e86b2364f32c2d7f3e201687e081cf7f7" + tensor_count: 76 + tensor_qtypes: {F32: 50, Q2_K: 13, Q3_K: 12, Q6_K: 1} + execution_provider: "default" + config_sha256: "19607e074d570fe92a572204c482e7fde81dd3f7d6d29758fd36dc4a89ba6153" + preserve_quantization: false + tokenizer: + repository: "EleutherAI/pythia-70m" + revision: "a39f36b100fe8a5377810d56c3f4789b9c53ac42" + metadata_sha256: "d5c722f646ff6462ac217da5e3514d1fa8b4a7b33aedced3daa8e7f00cc74f78" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 99, sha256: "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad"} + - {filename: "tokenizer.json", size: 2113710, sha256: "c24618a1b3e6a38167beff1c72cffd126c3a66254347304b50547d12c5f25624"} + - {filename: "tokenizer_config.json", size: 396, sha256: "70e38394e494931c6f773ba41e19460dd4436526b852207367f04341b4066d3f"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "pythia-70m-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 41000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Portable standard-ONNX graph packaged for ORT GenAI CPU; explicit-float only." diff --git a/testdata/cases/causal-lm/qwen2.5-0.5b-instruct-q8.yaml b/testdata/cases/causal-lm/qwen2.5-0.5b-instruct-q8.yaml index ba913cac2..156caad06 100644 --- a/testdata/cases/causal-lm/qwen2.5-0.5b-instruct-q8.yaml +++ b/testdata/cases/causal-lm/qwen2.5-0.5b-instruct-q8.yaml @@ -15,7 +15,7 @@ gguf: F32: 121 Q8_0: 170 execution_provider: "cpu" - config_sha256: "f7391f2aac9a7617c1c10e397e91b6f31b80bb3c5f338966b46e7d3935246500" + config_sha256: "d2b0c54fdd96c5122a344240557f6b35aafa65a0e8f8981158683c089559b29a" preserve_quantization: true keep_quantized: true tokenizer: diff --git a/testdata/cases/causal-lm/qwen3.5-moe-0.87b-q2-k.yaml b/testdata/cases/causal-lm/qwen3.5-moe-0.87b-q2-k.yaml index 30969c242..973dd49da 100644 --- a/testdata/cases/causal-lm/qwen3.5-moe-0.87b-q2-k.yaml +++ b/testdata/cases/causal-lm/qwen3.5-moe-0.87b-q2-k.yaml @@ -19,7 +19,7 @@ gguf: Q6_K: 1 Q8_0: 12 execution_provider: "cpu" - config_sha256: "f61aa39910c1b2ef27b9f8c707ef17dc7d16eff6fd97fae33d3a2eb367bf76f9" + config_sha256: "48019eea50654171acb6b87074d3c3a212bdf156aa6045c221067732bf25af91" preserve_quantization: false keep_quantized: false tokenizer: diff --git a/testdata/cases/causal-lm/tiny-mpt-q2-k.yaml b/testdata/cases/causal-lm/tiny-mpt-q2-k.yaml new file mode 100644 index 000000000..54ef08c24 --- /dev/null +++ b/testdata/cases/causal-lm/tiny-mpt-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "echarlaix/tiny-mpt-random-remote-code" +model_type: "mpt" +revision: "85e64794e74a6fb2e71e7055c7e0188ccdd32905" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "tensorblock/tiny-mpt-random-remote-code-GGUF" + revision: "c151eb3f349485ee8ae72841d7d90b9121d5baa2" + filename: "tiny-mpt-random-remote-code-Q2_K.gguf" + size: 8734304 + lfs_sha256: "5627dcb0ff18f6f7200f83c0aed2056a6a7c86b5f2d865833e1b5f00b00e4daa" + tensor_count: 8 + tensor_qtypes: {F32: 3, IQ4_NL: 3, Q3_K: 1, Q8_0: 1} + execution_provider: "default" + config_sha256: "7239b094edec93d3b29e2ffc53b08be450cb7c8536dd93120c4db618d47cd4f3" + preserve_quantization: false + tokenizer: + repository: "echarlaix/tiny-mpt-random-remote-code" + revision: "85e64794e74a6fb2e71e7055c7e0188ccdd32905" + metadata_sha256: "f15522d34f33354bc96d36a73ad4619925240328342d84a156855251c20d43af" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 99, sha256: "6f50ab5a5a509a1c309d6171f339b196a900dc9c99ad0408ff23bb615fdae7ad"} + - {filename: "tokenizer.json", size: 2113738, sha256: "3cf430678137c8491ca82fb7092ee49e44ad38857fffe1e4a4a5ed860139a5b8"} + - {filename: "tokenizer_config.json", size: 237, sha256: "7671fbb5b3d610e6e11d4f5fc78d3a7716e8846112ac7e0f72124caedf887570"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "tiny-mpt-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 11000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Portable standard-ONNX graph packaged for ORT GenAI CPU; explicit-float only." diff --git a/testdata/cases/causal-lm/tiny-olmo-q2-k.yaml b/testdata/cases/causal-lm/tiny-olmo-q2-k.yaml new file mode 100644 index 000000000..eb9dfcf8d --- /dev/null +++ b/testdata/cases/causal-lm/tiny-olmo-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "hyper-accel/tiny-random-olmo" +model_type: "olmo" +revision: "88675ef0caa5bd10ece810c0f2a79faa7724f536" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "tensorblock/tiny-random-olmo-GGUF" + revision: "d0ee9498d082d6dc3e730b9765e6e87a9bb5d995" + filename: "tiny-random-olmo-Q2_K.gguf" + size: 33860576 + lfs_sha256: "be1c5a22ac0e75cd5874467ffd80bcd2c8500609d3bf7ccdb5b269373e4d6da4" + tensor_count: 16 + tensor_qtypes: {IQ4_NL: 2, Q2_K: 9, Q3_K: 4, Q6_K: 1} + execution_provider: "cpu" + config_sha256: "c02542bb6b7c076f641ff0724409471deefb0782eab37dff076e80746a59652f" + preserve_quantization: false + tokenizer: + repository: "hyper-accel/tiny-random-olmo" + revision: "88675ef0caa5bd10ece810c0f2a79faa7724f536" + metadata_sha256: "25efe0090ffe5a6deb777743917fe572548a0f23357f74d8bd9e969cc911fd73" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 293, sha256: "a6188c1e366f8ed715e60ff39c46a8c500fc33508e2affeb23e8c547c5853193"} + - {filename: "tokenizer.json", size: 2115417, sha256: "a094266ac6c4982efba277bc251349a5a6d6ad37efb39a2a90f53d8be2a40a40"} + - {filename: "tokenizer_config.json", size: 5372, sha256: "78a839c7851f14f9fb30e664c2b46166dc0628f2900679e5ec160656f702edff"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "tiny-olmo-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 37000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Independent same-value Transformers reference; explicit-float runtime evidence only." diff --git a/testdata/cases/causal-lm/tiny-starcoder-q2-k.yaml b/testdata/cases/causal-lm/tiny-starcoder-q2-k.yaml new file mode 100644 index 000000000..89d3f4ff8 --- /dev/null +++ b/testdata/cases/causal-lm/tiny-starcoder-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "bigcode/tiny_starcoder_py" +model_type: "gpt_bigcode" +revision: "8547527bef0bc927268c1653cce6948c5c242dd1" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "RichardErkhov/bigcode_-_tiny_starcoder_py-gguf" + revision: "fa6f9fdbc134d86d78a3ff9ce08ab14ba33d4718" + filename: "tiny_starcoder_py.Q2_K.gguf" + size: 103899456 + lfs_sha256: "aa8c2170bb9172447baba14309916cfc0d901dbffaf10f1448f4f631e10c1f41" + tensor_count: 244 + tensor_qtypes: {F32: 163, Q2_K: 40, Q3_K: 40, Q6_K: 1} + execution_provider: "cpu" + config_sha256: "45f95ad88af3f5378bdfacd633f1717012e1f3699bbe880af4f6d29fd79f2d4f" + preserve_quantization: false + tokenizer: + repository: "bigcode/tiny_starcoder_py" + revision: "8547527bef0bc927268c1653cce6948c5c242dd1" + metadata_sha256: "23379a715b3983ce0f1559645984431bc039c16eaf6e062b7e62bceac6fa64cd" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 532, sha256: "0823292e24ea07b89317e9ede9d08da2a1b6c014290c06908a7ad04f1efd6719"} + - {filename: "tokenizer.json", size: 2057395, sha256: "42b5a37ba11199f024f2b8873e1ecba98da33166e16f700bf7cb2304b0a5583f"} + - {filename: "tokenizer_config.json", size: 677, sha256: "95684c52ad9a970dbbb17576ee2237cb62902c1eff6804c7c91a4d6219a4a6d7"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "tiny-starcoder-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 107000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Independent same-value Transformers reference; explicit-float runtime evidence only." diff --git a/testdata/cases/causal-lm/tiny-starcoder2-q2-k.yaml b/testdata/cases/causal-lm/tiny-starcoder2-q2-k.yaml new file mode 100644 index 000000000..512d1ce52 --- /dev/null +++ b/testdata/cases/causal-lm/tiny-starcoder2-q2-k.yaml @@ -0,0 +1,47 @@ +model_id: "hyper-accel/tiny-random-starcoder2" +model_type: "starcoder2" +revision: "193576733055d2108dc9906d0da6e0806ad9be57" +task_type: "text-generation" +dtype: "float32" + +gguf: + repository: "tensorblock/tiny-random-starcoder2-GGUF" + revision: "82c9eb61d1af6ea00dff834f1ff0620144b333e8" + filename: "tiny-random-starcoder2-Q2_K.gguf" + size: 68039904 + lfs_sha256: "ab0a4b4e79c906520808db065a00a317c8b097e2176638b34088975eada6e0ed" + tensor_count: 36 + tensor_qtypes: {F32: 22, Q2_K: 7, Q3_K: 4, Q4_K: 2, Q6_K: 1} + execution_provider: "cpu" + config_sha256: "25eaa4f5baa4e4725930298aaa921804d1e6f202737a3b10ec32070c4acfcb71" + preserve_quantization: false + tokenizer: + repository: "hyper-accel/tiny-random-starcoder2" + revision: "193576733055d2108dc9906d0da6e0806ad9be57" + metadata_sha256: "38608a4848dc8535113cff0312858853a0c8e29a5cdc145f258e4a9d84b35113" + identity_status: "exact" + assets: + - {filename: "special_tokens_map.json", size: 1300, sha256: "0fc9ac706a35e6337b19d484bc8f866b6a0ee7ad2509b03f7305dc0838c82d2c"} + - {filename: "tokenizer.json", size: 2060947, sha256: "17fa145258b20c18287f1e3bd804e074cc13333f11984a2f5a2f11c5110437aa"} + - {filename: "tokenizer_config.json", size: 7877, sha256: "8149d8e6b21275ad2cc346885ad92c0e9b5aa3e28a78bb5d39b7febc7e52545d"} + +ort_genai: + tier: "real" + runtime_versions: ["0.15.2"] + model_type: "decoder" + runtime_evidence_id: "tiny-starcoder2-q2-k-ort-genai-0.15.2" + execution_provider: "cpu" + max_download_bytes: 71000000 + released_capabilities: + "0.15.2": {generic_decoder: true, state_groups: false} + +inputs: + prompts: ["The capital of France is"] + +level: "L4+L5" +generation: + max_new_tokens: 20 + do_sample: false + exact_match: true + +notes: "Tiny source BOS/EOS IDs exceed its model vocabulary; runtime evidence is explicit-float." diff --git a/testdata/evidence/gguf_low_cost_runtime_batch.json b/testdata/evidence/gguf_low_cost_runtime_batch.json new file mode 100644 index 000000000..8f0023ac5 --- /dev/null +++ b/testdata/evidence/gguf_low_cost_runtime_batch.json @@ -0,0 +1,137 @@ +{ + "schema_version": 1, + "policy": { + "maximum_download_bytes": 17179869184, + "selected_architecture_minimum": 6, + "selected_architecture_maximum": 10, + "production_revision_policy": "revision=None remains unpinned; arbitrary explicit revisions are forwarded", + "runtime": "ort-genai", + "runtime_version": "0.15.2", + "onnxruntime_version": "1.29.0" + }, + "selected": [ + { + "architecture": "gpt2", + "evidence_id": "gpt2-q2-k-ort-genai-0.15.2", + "artifact_size": 81196544, + "artifact_sha256": "4234545f917ec1df10dab4d926796a83422b68e9010d85a4c111b8b541f32892", + "tokenizer_payload_size": 2107986, + "storage_mode": "explicit-float" + }, + { + "architecture": "gptneox", + "evidence_id": "pythia-70m-q2-k-ort-genai-0.15.2", + "artifact_size": 38508192, + "artifact_sha256": "8e331c8c8016bed8ff1863b78fafe51e86b2364f32c2d7f3e201687e081cf7f7", + "tokenizer_payload_size": 2114205, + "storage_mode": "explicit-float" + }, + { + "architecture": "mpt", + "evidence_id": "tiny-mpt-q2-k-ort-genai-0.15.2", + "artifact_size": 8734304, + "artifact_sha256": "5627dcb0ff18f6f7200f83c0aed2056a6a7c86b5f2d865833e1b5f00b00e4daa", + "tokenizer_payload_size": 2114074, + "storage_mode": "explicit-float" + }, + { + "architecture": "olmo", + "evidence_id": "tiny-olmo-q2-k-ort-genai-0.15.2", + "artifact_size": 33860576, + "artifact_sha256": "be1c5a22ac0e75cd5874467ffd80bcd2c8500609d3bf7ccdb5b269373e4d6da4", + "tokenizer_payload_size": 2121082, + "storage_mode": "explicit-float" + }, + { + "architecture": "starcoder", + "evidence_id": "tiny-starcoder-q2-k-ort-genai-0.15.2", + "artifact_size": 103899456, + "artifact_sha256": "aa8c2170bb9172447baba14309916cfc0d901dbffaf10f1448f4f631e10c1f41", + "tokenizer_payload_size": 2058604, + "storage_mode": "explicit-float" + }, + { + "architecture": "starcoder2", + "evidence_id": "tiny-starcoder2-q2-k-ort-genai-0.15.2", + "artifact_size": 68039904, + "artifact_sha256": "ab0a4b4e79c906520808db065a00a317c8b097e2176638b34088975eada6e0ed", + "tokenizer_payload_size": 2070124, + "storage_mode": "explicit-float" + } + ], + "totals": { + "artifact_bytes": 334238976, + "tokenizer_bytes": 12586075, + "download_bytes": 346825051 + }, + "fail_closed": [ + { + "architecture": "qwen3", + "reason_code": "TOKENIZER_CHAT_TEMPLATE_IDENTITY_MISMATCH", + "result": "blocked", + "detail": "All 310 tensors close after Q/K norm mapping, but the immutable source tokenizer chat template differs from the GGUF template." + }, + { + "architecture": "bloom", + "reason_code": "FULL_LOGIT_PARITY_DIVERGENCE", + "result": "blocked", + "detail": "Cache replay remains within 0.000214, but same-value full-sequence logits diverge by up to 6.731415." + }, + { + "architecture": "stablelm", + "reason_code": "TENSOR_SHAPE_CLOSURE_FAILED", + "result": "blocked", + "detail": "The replacement artifact does not satisfy exact graph weight-shape closure." + }, + { + "architecture": "gemma", + "reason_code": "TOKENIZER_COMPILED_SEMANTICS_UNAVAILABLE", + "result": "blocked", + "detail": "The representative artifact declares tokenizer.ggml.pre semantics that the pinned tokenizer loader cannot prove." + }, + { + "architecture": "gemma2", + "reason_code": "TOKENIZER_COMPILED_SEMANTICS_UNAVAILABLE", + "result": "blocked", + "detail": "The representative artifact declares tokenizer.ggml.pre semantics that the pinned tokenizer loader cannot prove." + }, + { + "architecture": "gemma3", + "reason_code": "TOKENIZER_COMPILED_SEMANTICS_UNAVAILABLE", + "result": "blocked", + "detail": "The representative artifact declares tokenizer.ggml.pre semantics that the pinned tokenizer loader cannot prove." + }, + { + "architecture": "phi3", + "reason_code": "TOKENIZER_COMPILED_SEMANTICS_UNAVAILABLE", + "result": "blocked", + "detail": "The representative artifact declares tokenizer.ggml.pre semantics that the pinned tokenizer loader cannot prove." + }, + { + "architecture": "falcon", + "reason_code": "INVALID_GGUF_HEADER", + "result": "blocked", + "detail": "The smallest official candidate is not a readable GGUF artifact." + } + ], + "excluded": [ + { + "route": "qwen35moe", + "reason_code": "RECENTLY_HANDLED", + "pull_request": 672 + }, + { + "route": "nemotron_h_moe", + "reason_code": "RECENTLY_HANDLED", + "pull_request": 674 + }, + { + "route_class": "tokenizer-only", + "reason_code": "OUT_OF_SCOPE" + }, + { + "route_class": "abi-blocked", + "reason_code": "OUT_OF_SCOPE" + } + ] +} diff --git a/testdata/evidence/gguf_quantization_capabilities.json b/testdata/evidence/gguf_quantization_capabilities.json index 091362ae3..c82f92b57 100644 --- a/testdata/evidence/gguf_quantization_capabilities.json +++ b/testdata/evidence/gguf_quantization_capabilities.json @@ -28,7 +28,7 @@ "max_selected_artifact_bytes": 17179869184, "preserved_definition": "Only byte-identical native blocks or exact affine repacks preserve source represented values. Dequantize/requantize is never preserved.", "runtime_definition": "Runtime support requires immutable same-artifact full-logit parity plus deterministic prefill/decode/replay/rollback/reorder evidence.", - "selected_artifact_bytes": 2390132320, + "selected_artifact_bytes": 2724371296, "target_storage_definition": "Lossy affine normalization may still produce supported packed target storage while source_fidelity is false." }, "qtypes": [ @@ -2595,7 +2595,7 @@ "evidence_id": "qwen3.5-moe-0.87b-q2-k-ort-genai-0.15.2", "execution_provider": "CPUExecutionProvider", "graph_sha256": "8c1aa1075cee03ffd5ce5bbd283ee88b2466e6c1d645f0a41e542230951d6f09", - "import_route": "{\"architecture\":\"qwen35moe\",\"config_sha256\":\"f61aa39910c1b2ef27b9f8c707ef17dc7d16eff6fd97fae33d3a2eb367bf76f9\",\"execution_provider\":\"cpu\",\"model_type\":\"qwen3_5_moe\",\"module_type\":\"qwen3_5_moe\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":true,\"required_metadata\":[\"attention.layer_norm_rms_epsilon\",\"expert_count\",\"expert_used_count\",\"rope.dimension_sections\",\"ssm.conv_kernel\",\"ssm.group_count\",\"ssm.inner_size\",\"ssm.state_size\",\"ssm.time_step_rank\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":true,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"hybrid-text-generation\"},\"tensor_map_recipe\":[\"llama\",\"moe_extras\",\"qwen35_hybrid_extras\"]}", + "import_route": "{\"architecture\":\"qwen35moe\",\"config_sha256\":\"48019eea50654171acb6b87074d3c3a212bdf156aa6045c221067732bf25af91\",\"execution_provider\":\"cpu\",\"model_type\":\"qwen3_5_moe\",\"module_type\":\"qwen3_5_moe\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":true,\"required_metadata\":[\"attention.layer_norm_rms_epsilon\",\"expert_count\",\"expert_used_count\",\"rope.dimension_sections\",\"ssm.conv_kernel\",\"ssm.group_count\",\"ssm.inner_size\",\"ssm.state_size\",\"ssm.time_step_rank\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":true,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"hybrid-text-generation\"},\"tensor_map_recipe\":[\"llama\",\"moe_extras\",\"qwen35_hybrid_extras\"]}", "limitations": "Explicit-float correctness route only: source quantization is dequantized, dense MoE execution is opt-in, and the selected publisher marks this reduced checkpoint as low quality. Same-value full-logit comparison uses atol=0.35 because small backend differences can cross a routed-expert boundary; all greedy tokens match.", "onnxruntime_version": "1.29.0", "parity_kind": "full-logit", @@ -2635,7 +2635,7 @@ "evidence_id": "lfm2-350m-f16-ort-genai-0.15.2", "execution_provider": "CPUExecutionProvider", "graph_sha256": "27e4ebb4c0c8b6c01ee57fa7825f34c5ddadc8ca5dc0c75d989e4507d4dcdfdb", - "import_route": "{\"architecture\":\"lfm2\",\"config_sha256\":\"c961bba579ea33a2472a7c5d3f469c76f1f8c7aae8440a7eaa86bc6e878a42f4\",\"execution_provider\":\"cpu\",\"model_type\":\"lfm2\",\"module_type\":\"lfm2\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[\"attention.head_count_kv\",\"attention.layer_norm_rms_epsilon\",\"shortconv.l_cache\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"hybrid-text-generation\"},\"tensor_map_recipe\":[\"lfm2\"]}", + "import_route": "{\"architecture\":\"lfm2\",\"config_sha256\":\"e7746a202f3679ac05311fbc7a25414e4c106008fc3bd257001e97a8d7cab575\",\"execution_provider\":\"cpu\",\"model_type\":\"lfm2\",\"module_type\":\"lfm2\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[\"attention.head_count_kv\",\"attention.layer_norm_rms_epsilon\",\"shortconv.l_cache\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"hybrid-text-generation\"},\"tensor_map_recipe\":[\"lfm2\"]}", "limitations": null, "onnxruntime_version": "1.29.0", "parity_kind": "full-logit", @@ -2671,7 +2671,7 @@ "evidence_id": "qwen2.5-0.5b-instruct-q8-ort-genai-0.15.2", "execution_provider": "CPUExecutionProvider", "graph_sha256": "240e5e374803c94efdb17eee39c09b0d3e9aed10b6d8b4e1c92e39918ea2155e", - "import_route": "{\"architecture\":\"qwen2\",\"config_sha256\":\"f7391f2aac9a7617c1c10e397e91b6f31b80bb3c5f338966b46e7d3935246500\",\"execution_provider\":\"cpu\",\"model_type\":\"qwen2\",\"module_type\":\"qwen2\",\"preserve_quantization\":true,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", + "import_route": "{\"architecture\":\"qwen2\",\"config_sha256\":\"d2b0c54fdd96c5122a344240557f6b35aafa65a0e8f8981158683c089559b29a\",\"execution_provider\":\"cpu\",\"model_type\":\"qwen2\",\"module_type\":\"qwen2\",\"preserve_quantization\":true,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", "limitations": null, "onnxruntime_version": "1.29.0", "parity_kind": "full-logit", @@ -2681,7 +2681,7 @@ "source_fidelity": true, "stateful_semantics": "dynamic KV cache prefill, replay, rollback, reorder, and 20 decode steps", "storage_quantized": true, - "target_storage_format": "native GGUF block storage" + "target_storage_format": "INT8 affine block-32" } ], "size": 675710816, @@ -2690,6 +2690,82 @@ "Q8_0": 170 } }, + { + "evidence_ids": [ + "tiny-starcoder-q2-k-ort-genai-0.15.2" + ], + "filename": "tiny_starcoder_py.Q2_K.gguf", + "lfs_sha256": "aa8c2170bb9172447baba14309916cfc0d901dbffaf10f1448f4f631e10c1f41", + "repository": "RichardErkhov/bigcode_-_tiny_starcoder_py-gguf", + "revision": "fa6f9fdbc134d86d78a3ff9ce08ab14ba33d4718", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[tiny-starcoder-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "tiny-starcoder-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "ab4edee4c001e3df8f4bce477447bea1c0360221163e01dd1cb03e89737107bd", + "import_route": "{\"architecture\":\"starcoder\",\"config_sha256\":\"45f95ad88af3f5378bdfacd633f1717012e1f3699bbe880af4f6d29fd79f2d4f\",\"execution_provider\":\"cpu\",\"model_type\":\"gpt_bigcode\",\"module_type\":\"gpt_bigcode\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":\"conventional_legacy\",\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[\"attention.layer_norm_epsilon\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"starcoder\"]}", + "limitations": "Explicit-float correctness route; source GGUF blocks are dequantized.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[tiny-starcoder-q2-k]", + "result": "passed", + "runtime_package_sha256": "ecbbae525e12d250c3eaa782b9eec14dba2c09fb9dbde29a8331e741e7676171", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 103899456, + "tensor_qtypes": { + "F32": 163, + "Q2_K": 40, + "Q3_K": 40, + "Q6_K": 1 + } + }, + { + "evidence_ids": [ + "pythia-70m-q2-k-ort-genai-0.15.2" + ], + "filename": "pythia-70m.Q2_K.gguf", + "lfs_sha256": "8e331c8c8016bed8ff1863b78fafe51e86b2364f32c2d7f3e201687e081cf7f7", + "repository": "mradermacher/pythia-70m-GGUF", + "revision": "52d6f045404c9f93418df2a0144d20c9de34316b", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[pythia-70m-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "pythia-70m-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "115a5b47ae04263465e8063661381c183c0b77c870ce3c3dfc339387fbbf9c2f", + "import_route": "{\"architecture\":\"gptneox\",\"config_sha256\":\"19607e074d570fe92a572204c482e7fde81dd3f7d6d29758fd36dc4a89ba6153\",\"execution_provider\":\"default\",\"model_type\":\"gpt_neox\",\"module_type\":\"gguf_legacy\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":\"exact_legacy_gguf\",\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[\"attention.layer_norm_epsilon\",\"use_parallel_residual\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"legacy_layernorm\"]}", + "limitations": "Explicit-float correctness route on the portable graph; source GGUF blocks are dequantized and the pinned tokenizer vocabulary is extended only with deterministic padding IDs present in the GGUF.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[pythia-70m-q2-k]", + "result": "passed", + "runtime_package_sha256": "798cc7d612214b032e6354d75c282d75119b3649d603a85d49643c815c24ca77", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 38508192, + "tensor_qtypes": { + "F32": 50, + "Q2_K": 13, + "Q3_K": 12, + "Q6_K": 1 + } + }, { "evidence_ids": [ "smollm-135m-f16-onnxruntime-1.29.0", @@ -2708,7 +2784,7 @@ "evidence_id": "smollm-135m-f16-onnxruntime-1.29.0", "execution_provider": "CPUExecutionProvider", "graph_sha256": "4b608b099fb17471f342c925c20173f297abd0f8456c9e96a11b1d044272d1ad", - "import_route": "{\"architecture\":\"llama\",\"config_sha256\":\"d3f3f2abf531abde55e04a52b5c892c93943b7e260160c796c490618a2e84886\",\"execution_provider\":\"cpu\",\"model_type\":\"llama\",\"module_type\":\"llama\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":true,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":\"llama\",\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", + "import_route": "{\"architecture\":\"llama\",\"config_sha256\":\"9f917f4a59c907325a069735b9a5d07177f3d665b5e812c10b626ecf1c94708e\",\"execution_provider\":\"cpu\",\"model_type\":\"llama\",\"module_type\":\"llama\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":true,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":\"llama\",\"v_head_reorder\":false,\"vlm_builder\":\"generic_projector\"},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", "limitations": null, "onnxruntime_version": "1.29.0", "parity_kind": "full-logit", @@ -2728,7 +2804,7 @@ "evidence_id": "smollm-135m-f16-ort-genai-0.15.2", "execution_provider": "CPUExecutionProvider", "graph_sha256": "4b608b099fb17471f342c925c20173f297abd0f8456c9e96a11b1d044272d1ad", - "import_route": "{\"architecture\":\"llama\",\"config_sha256\":\"d3f3f2abf531abde55e04a52b5c892c93943b7e260160c796c490618a2e84886\",\"execution_provider\":\"cpu\",\"model_type\":\"llama\",\"module_type\":\"llama\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":true,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":\"llama\",\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", + "import_route": "{\"architecture\":\"llama\",\"config_sha256\":\"9f917f4a59c907325a069735b9a5d07177f3d665b5e812c10b626ecf1c94708e\",\"execution_provider\":\"cpu\",\"model_type\":\"llama\",\"module_type\":\"llama\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":true,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":\"llama\",\"v_head_reorder\":false,\"vlm_builder\":\"generic_projector\"},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", "limitations": null, "onnxruntime_version": "1.29.0", "parity_kind": "full-logit", @@ -2746,6 +2822,159 @@ "F16": 211, "F32": 61 } + }, + { + "evidence_ids": [ + "gpt2-q2-k-ort-genai-0.15.2" + ], + "filename": "gpt2-Q2_K.gguf", + "lfs_sha256": "4234545f917ec1df10dab4d926796a83422b68e9010d85a4c111b8b541f32892", + "repository": "tensorblock/gpt2-GGUF", + "revision": "5b01870b15c4b2e43695d7f3f3bfb5b26106f23b", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[gpt2-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "gpt2-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "ae121ccf53782ffdc530cf96329e5eacabfc0baf5fcdbfc71925e6edc4ad1423", + "import_route": "{\"architecture\":\"gpt2\",\"config_sha256\":\"1e0363dbc5f3427d873aeb46c27d5713ae0668f0e7cf695b5dde00bdd4d390a6\",\"execution_provider\":\"cpu\",\"model_type\":\"gpt2\",\"module_type\":\"gpt2\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":null,\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":\"gpt2\",\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"gpt2\"]}", + "limitations": "Explicit-float correctness route; source GGUF blocks are dequantized.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[gpt2-q2-k]", + "result": "passed", + "runtime_package_sha256": "7bfca0bcb5d5aacf39738306c15d0c27db7f7c5810fd6259609fd0d2370becde", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 81196544, + "tensor_qtypes": { + "F32": 99, + "Q2_K": 25, + "Q3_K": 24, + "Q6_K": 1 + } + }, + { + "evidence_ids": [ + "tiny-mpt-q2-k-ort-genai-0.15.2" + ], + "filename": "tiny-mpt-random-remote-code-Q2_K.gguf", + "lfs_sha256": "5627dcb0ff18f6f7200f83c0aed2056a6a7c86b5f2d865833e1b5f00b00e4daa", + "repository": "tensorblock/tiny-mpt-random-remote-code-GGUF", + "revision": "c151eb3f349485ee8ae72841d7d90b9121d5baa2", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[tiny-mpt-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "tiny-mpt-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "5daf516f886110f257105a4122cba9fbdb920fab92bedf057a4f67899e513c8a", + "import_route": "{\"architecture\":\"mpt\",\"config_sha256\":\"7239b094edec93d3b29e2ffc53b08be450cb7c8536dd93120c4db618d47cd4f3\",\"execution_provider\":\"default\",\"model_type\":\"mpt\",\"module_type\":\"gguf_legacy\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":\"exact_legacy_gguf\",\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[\"attention.layer_norm_epsilon\",\"attention.max_alibi_bias\"],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"legacy_layernorm\"]}", + "limitations": "Explicit-float correctness route; source GGUF blocks are dequantized.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[tiny-mpt-q2-k]", + "result": "passed", + "runtime_package_sha256": "20dd912c318d8115af5a5e6e92672624c00120455346ff5c9294e9dc1a1cc234", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 8734304, + "tensor_qtypes": { + "F32": 3, + "IQ4_NL": 3, + "Q3_K": 1, + "Q8_0": 1 + } + }, + { + "evidence_ids": [ + "tiny-olmo-q2-k-ort-genai-0.15.2" + ], + "filename": "tiny-random-olmo-Q2_K.gguf", + "lfs_sha256": "be1c5a22ac0e75cd5874467ffd80bcd2c8500609d3bf7ccdb5b269373e4d6da4", + "repository": "tensorblock/tiny-random-olmo-GGUF", + "revision": "d0ee9498d082d6dc3e730b9765e6e87a9bb5d995", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[tiny-olmo-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "tiny-olmo-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "eb67fb20ae4257af04db8b1554c56ae6b3f66340f2b6ff97dd55cbd14c7bce5b", + "import_route": "{\"architecture\":\"olmo\",\"config_sha256\":\"c02542bb6b7c076f641ff0724409471deefb0782eab37dff076e80746a59652f\",\"execution_provider\":\"cpu\",\"model_type\":\"olmo\",\"module_type\":\"olmo\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":\"olmo\",\"llama_qk_permute\":true,\"offset_norm\":false,\"required_metadata\":[\"attention.layer_norm_epsilon\"],\"rope_interleave\":false,\"tensor_processor\":\"llama\",\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"olmo\"]}", + "limitations": "Explicit-float correctness route; source GGUF blocks are dequantized.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[tiny-olmo-q2-k]", + "result": "passed", + "runtime_package_sha256": "4f2817a0af1b58922f3b82d455844664948dbc67f0ffe6af576baedf2eac3835", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 33860576, + "tensor_qtypes": { + "IQ4_NL": 2, + "Q2_K": 9, + "Q3_K": 4, + "Q6_K": 1 + } + }, + { + "evidence_ids": [ + "tiny-starcoder2-q2-k-ort-genai-0.15.2" + ], + "filename": "tiny-random-starcoder2-Q2_K.gguf", + "lfs_sha256": "ab0a4b4e79c906520808db065a00a317c8b097e2176638b34088975eada6e0ed", + "repository": "tensorblock/tiny-random-starcoder2-GGUF", + "revision": "82c9eb61d1af6ea00dff834f1ff0620144b333e8", + "runtime_results": [ + { + "compute_mode": "float operators", + "deterministic_test": "test_promoted_gguf_full_runtime_evidence[tiny-starcoder2-q2-k]", + "downstream_runtime": "ort-genai", + "downstream_runtime_version": "0.15.2", + "evidence_id": "tiny-starcoder2-q2-k-ort-genai-0.15.2", + "execution_provider": "CPUExecutionProvider", + "graph_sha256": "34c3e58ac9d95b5d1097393782d41a942ac9b6188e6d57adff21212030897a23", + "import_route": "{\"architecture\":\"starcoder2\",\"config_sha256\":\"25eaa4f5baa4e4725930298aaa921804d1e6f202737a3b10ec32070c4acfcb71\",\"execution_provider\":\"cpu\",\"model_type\":\"starcoder2\",\"module_type\":\"starcoder2\",\"preserve_quantization\":false,\"registry_import\":{\"config_key_map\":null,\"config_postprocessor\":\"starcoder2\",\"llama_qk_permute\":false,\"offset_norm\":false,\"required_metadata\":[],\"rope_interleave\":false,\"tensor_processor\":null,\"v_head_reorder\":false,\"vlm_builder\":null},\"route_schema\":1,\"static_cache\":false,\"task\":{\"class\":\"builtins.str\",\"state\":\"text-generation\"},\"tensor_map_recipe\":[\"llama\"]}", + "limitations": "Explicit-float correctness route; source GGUF blocks are dequantized. The tiny source config declares BOS/EOS ID 50256 outside its 49152-token model vocabulary.", + "onnxruntime_version": "1.29.0", + "parity_kind": "full-logit", + "parity_test": "test_promoted_gguf_full_runtime_evidence[tiny-starcoder2-q2-k]", + "result": "passed", + "runtime_package_sha256": "4a9ab2edf80d8cf778fccd56d306cb56dfda5bb190068faf04c78ac5d180f564", + "source_fidelity": false, + "stateful_semantics": "dynamic KV cache prefill, full-sequence replay, rollback, reorder, and 20 decode steps", + "storage_quantized": false, + "target_storage_format": "float" + } + ], + "size": 68039904, + "tensor_qtypes": { + "F32": 22, + "Q2_K": 7, + "Q3_K": 4, + "Q4_K": 2, + "Q6_K": 1 + } } ], "transform_evidence": { diff --git a/tests/gguf_small_model_runtime_integration_test.py b/tests/gguf_small_model_runtime_integration_test.py index 1cf1a4322..8a6ce53a1 100644 --- a/tests/gguf_small_model_runtime_integration_test.py +++ b/tests/gguf_small_model_runtime_integration_test.py @@ -19,6 +19,7 @@ import os import shutil from collections import Counter +from contextlib import ExitStack from dataclasses import dataclass from importlib.metadata import packages_distributions, version from pathlib import Path @@ -31,7 +32,16 @@ import torch import yaml from huggingface_hub import get_hf_file_metadata, hf_hub_download, hf_hub_url -from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer +from transformers import ( + AutoConfig, + AutoModelForCausalLM, + AutoTokenizer, + DynamicCache, + GPTBigCodeForCausalLM, + GPTNeoXForCausalLM, + MptConfig, + MptForCausalLM, +) from mobius import ModelPackage, build_from_gguf from mobius.__main__ import main @@ -47,6 +57,7 @@ ) from mobius.integrations.gguf._reader import GGUFModel from mobius.integrations.gguf._runtime_evidence import ( + GGUFRuntimeEvidence, gguf_graph_package_identity, runtime_evidence, ) @@ -86,7 +97,7 @@ class _RuntimeCase: reference_revision="1d461723eec654e65efdc40cf49301c89c0c92f4", prompt="Once upon a time,", tensor_qtypes={"F16": 211, "F32": 61}, - config_sha256="d3f3f2abf531abde55e04a52b5c892c93943b7e260160c796c490618a2e84886", + config_sha256="9f917f4a59c907325a069735b9a5d07177f3d665b5e812c10b626ecf1c94708e", generated_tokens=( 665, 436, @@ -144,7 +155,7 @@ class _RuntimeCase: reference_revision="12fd25f77366fa6b3b4b768ec3050bf629380bac", prompt="Here is my poem:", tensor_qtypes={"F16": 211, "F32": 61}, - config_sha256="f9ceb816433d5aa32918761944be76ecdb090df8cce7cdb64d5d8f9186f7117f", + config_sha256="089d097c1394a74ee386c1d6e2651474bee77d38a8bc0b5d813594c1fede3054", generated_tokens=( 198, 198, @@ -204,7 +215,7 @@ class _RuntimeCase: reference_revision="12fd25f77366fa6b3b4b768ec3050bf629380bac", prompt="Here is my poem:", tensor_qtypes={"F32": 61, "Q4_K": 16, "Q5_0": 166, "Q6_K": 14, "Q8_0": 15}, - config_sha256="f9ceb816433d5aa32918761944be76ecdb090df8cce7cdb64d5d8f9186f7117f", + config_sha256="089d097c1394a74ee386c1d6e2651474bee77d38a8bc0b5d813594c1fede3054", generated_tokens=(198, 198, 18, 504, 2388, 13685, 284, 5208, 28, 198), tokenizer_repository="HuggingFaceTB/SmolLM2-135M-Instruct", tokenizer_revision="12fd25f77366fa6b3b4b768ec3050bf629380bac", @@ -239,6 +250,9 @@ class _PromotedRuntimeCase: prompt: str generated_tokens: tuple[int, ...] atol: float + cache_atol: float + execution_provider: str = "cpu" + reference_kind: str = "direct" dequantize: bool = False release: bool = False allow_dense_moe: bool = False @@ -246,6 +260,192 @@ class _PromotedRuntimeCase: _PROMOTED_RUNTIME_CASES = ( + _PromotedRuntimeCase( + name="gpt2-q2-k", + evidence_id="gpt2-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 262, + 717, + 284, + 307, + 262, + 717, + 284, + 307, + 262, + 6342, + 286, + 262, + 1578, + 1829, + 13, + 198, + 198, + 464, + 1578, + 1829, + ), + atol=1e-4, + cache_atol=1e-4, + dequantize=True, + ), + _PromotedRuntimeCase( + name="pythia-70m-q2-k", + evidence_id="pythia-70m-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 253, + 187, + 2097, + 273, + 253, + 987, + 273, + 697, + 987, + 273, + 187, + 187, + 510, + 187, + 187, + 187, + 10, + 187, + 187, + 187, + ), + atol=0.015, + cache_atol=0.012, + execution_provider="default", + reference_kind="gptneox", + dequantize=True, + ), + _PromotedRuntimeCase( + name="tiny-mpt-q2-k", + evidence_id="tiny-mpt-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 16322, + 29093, + 23684, + 45373, + 18311, + 20725, + 47672, + 43521, + 33057, + 20725, + 47672, + 43521, + 33057, + 18888, + 31622, + 2306, + 16644, + 15903, + 4251, + 33057, + ), + atol=1e-5, + cache_atol=1e-6, + execution_provider="default", + reference_kind="mpt", + dequantize=True, + ), + _PromotedRuntimeCase( + name="tiny-olmo-q2-k", + evidence_id="tiny-olmo-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 34423, + 35007, + 32269, + 13481, + 30755, + 16833, + 3137, + 4002, + 34423, + 25900, + 31879, + 15216, + 4390, + 33762, + 4390, + 33762, + 4390, + 4390, + 4390, + 4390, + ), + atol=1e-5, + cache_atol=2e-6, + reference_kind="olmo", + dequantize=True, + ), + _PromotedRuntimeCase( + name="tiny-starcoder-q2-k", + evidence_id="tiny-starcoder-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 225, + 35, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + 34, + ), + atol=5e-5, + cache_atol=3e-5, + reference_kind="starcoder", + dequantize=True, + ), + _PromotedRuntimeCase( + name="tiny-starcoder2-q2-k", + evidence_id="tiny-starcoder2-q2-k-ort-genai-0.15.2", + prompt="The capital of France is", + generated_tokens=( + 21284, + 47755, + 47755, + 10004, + 10004, + 10004, + 10004, + 10004, + 34798, + 34798, + 12046, + 12046, + 12046, + 12046, + 16273, + 16273, + 16273, + 13231, + 16273, + 13231, + ), + atol=1e-5, + cache_atol=2e-6, + dequantize=True, + ), _PromotedRuntimeCase( name="qwen2.5-0.5b-instruct-q8", evidence_id="qwen2.5-0.5b-instruct-q8-ort-genai-0.15.2", @@ -273,6 +473,7 @@ class _PromotedRuntimeCase: 5109, ), atol=4e-4, + cache_atol=4e-4, ), _PromotedRuntimeCase( name="lfm2-350m-f16", @@ -301,6 +502,7 @@ class _PromotedRuntimeCase: 4600, ), atol=3e-4, + cache_atol=3e-4, ), _PromotedRuntimeCase( name="qwen3.5-moe-0.87b-q2-k", @@ -329,6 +531,7 @@ class _PromotedRuntimeCase: 198, ), atol=0.35, + cache_atol=0.35, dequantize=True, release=True, allow_dense_moe=True, @@ -502,6 +705,181 @@ def _load_qwen35moe_same_value_reference( return reference.to(dtype=torch.float32).eval() +def _load_low_cost_same_value_reference( + gguf_path: Path, + evidence: GGUFRuntimeEvidence, + reference_kind: str, +) -> torch.nn.Module: + """Load the exact dequantized GGUF values into an independent Transformers graph.""" + if reference_kind == "direct": + config = AutoConfig.from_pretrained( + evidence.config_repository, + revision=evidence.config_revision, + ) + return AutoModelForCausalLM.from_pretrained( + evidence.repository, + revision=evidence.revision, + gguf_file=evidence.filename, + config=config, + dtype=torch.float32, + ).eval() + + hf_config = ( + MptConfig.from_pretrained( + evidence.config_repository, + revision=evidence.config_revision, + ) + if reference_kind == "mpt" + else AutoConfig.from_pretrained( + evidence.config_repository, + revision=evidence.config_revision, + ) + ) + from gguf import GGMLQuantizationType, GGUFReader, dequantize + + # The oracle reads and maps upstream GGUF tensors directly. It deliberately + # shares no Mobius tensor loader, name mapping, value processor, or normalizer. + source: dict[str, torch.Tensor] = {} + for tensor in GGUFReader(gguf_path).tensors: + shape = tuple(int(dimension) for dimension in reversed(tensor.shape)) + if tensor.tensor_type in (GGMLQuantizationType.F32, GGMLQuantizationType.F16): + value = tensor.data.reshape(shape) + else: + value = dequantize(tensor.data, tensor.tensor_type).reshape(shape) + source[tensor.name] = torch.from_numpy(np.array(value, copy=True)).float() + + def qkv_to_gptneox(value: torch.Tensor) -> torch.Tensor: + q, k, v = value.chunk(3, dim=0) + heads = int(hf_config.num_attention_heads) + head_dim = q.shape[0] // heads + tail = q.shape[1:] + return ( + torch.stack( + ( + q.reshape(heads, head_dim, *tail), + k.reshape(heads, head_dim, *tail), + v.reshape(heads, head_dim, *tail), + ), + dim=1, + ) + .reshape(value.shape) + .contiguous() + ) + + def undo_llama_qk_permutation(value: torch.Tensor, heads: int) -> torch.Tensor: + head_half = value.shape[0] // heads // 2 + return ( + value.reshape(heads, head_half, 2, *value.shape[1:]) + .swapaxes(1, 2) + .reshape(value.shape) + ) + + renamed: dict[str, torch.Tensor] = {} + global_names = { + "gptneox": { + "token_embd.weight": "gpt_neox.embed_in.weight", + "output_norm.weight": "gpt_neox.final_layer_norm.weight", + "output_norm.bias": "gpt_neox.final_layer_norm.bias", + "output.weight": "lm_head.weight", + }, + "mpt": { + "token_embd.weight": "transformer.wte.weight", + "output_norm.weight": "transformer.norm_f.weight", + }, + "olmo": { + "token_embd.weight": "model.embed_tokens.weight", + "output.weight": "lm_head.weight", + }, + "starcoder": { + "token_embd.weight": "transformer.wte.weight", + "position_embd.weight": "transformer.wpe.weight", + "output_norm.weight": "transformer.ln_f.weight", + "output_norm.bias": "transformer.ln_f.bias", + }, + }[reference_kind] + layer_names = { + "gptneox": { + "attn_norm": "input_layernorm", + "attn_qkv": "attention.query_key_value", + "attn_output": "attention.dense", + "ffn_norm": "post_attention_layernorm", + "ffn_up": "mlp.dense_h_to_4h", + "ffn_down": "mlp.dense_4h_to_h", + }, + "mpt": { + "attn_norm": "norm_1", + "attn_qkv": "attn.Wqkv", + "attn_output": "attn.out_proj", + "ffn_norm": "norm_2", + "ffn_up": "ffn.up_proj", + "ffn_down": "ffn.down_proj", + }, + "olmo": { + "attn_q": "self_attn.q_proj", + "attn_k": "self_attn.k_proj", + "attn_v": "self_attn.v_proj", + "attn_output": "self_attn.o_proj", + "ffn_gate": "mlp.gate_proj", + "ffn_up": "mlp.up_proj", + "ffn_down": "mlp.down_proj", + }, + "starcoder": { + "attn_norm": "ln_1", + "attn_qkv": "attn.c_attn", + "attn_output": "attn.c_proj", + "ffn_norm": "ln_2", + "ffn_up": "mlp.c_fc", + "ffn_down": "mlp.c_proj", + }, + }[reference_kind] + layer_prefix = { + "gptneox": "gpt_neox.layers", + "mpt": "transformer.blocks", + "olmo": "model.layers", + "starcoder": "transformer.h", + }[reference_kind] + for name, value in source.items(): + target = global_names.get(name) + if target is None: + parts = name.split(".") + assert parts[0] == "blk" and len(parts) == 4, name + target_stem = layer_names[parts[2]] + target = f"{layer_prefix}.{int(parts[1])}.{target_stem}.{parts[3]}" + if reference_kind == "gptneox" and parts[2] == "attn_qkv": + value = qkv_to_gptneox(value) + elif reference_kind == "olmo" and parts[2] in {"attn_q", "attn_k"}: + heads = int( + hf_config.num_attention_heads + if parts[2] == "attn_q" + else hf_config.num_key_value_heads + ) + value = undo_llama_qk_permutation(value, heads) + renamed[target] = value + assert len(renamed) == len(source) + + if reference_kind == "gptneox": + reference = GPTNeoXForCausalLM(hf_config) + reference.load_state_dict(renamed, strict=True) + elif reference_kind == "starcoder": + reference = GPTBigCodeForCausalLM(hf_config) + missing, unexpected = reference.load_state_dict(renamed, strict=False) + assert missing == ["lm_head.weight"] + assert not unexpected + reference.tie_weights() + elif reference_kind == "mpt": + reference = MptForCausalLM(hf_config) + missing, unexpected = reference.load_state_dict(renamed, strict=False) + assert missing == ["lm_head.weight"] + assert not unexpected + reference.tie_weights() + elif reference_kind == "olmo": + reference = AutoModelForCausalLM.from_config(hf_config) + reference.load_state_dict(renamed, strict=True) + else: + raise AssertionError(f"Unknown same-value reference kind: {reference_kind}") + return reference.to(dtype=torch.float32).eval() + + def _run_promoted_ort( session: ort.InferenceSession, input_ids: np.ndarray, @@ -526,6 +904,74 @@ def _run_promoted_ort( return dict(zip(output_names, session.run(output_names, feeds), strict=True)) +def _assert_starcoder2_long_context_sliding_window( + session: ort.InferenceSession, + reference: torch.nn.Module, + token_id: int, + *, + atol: float, +) -> None: + """Prove the exact route applies StarCoder2's window past position 4096.""" + window = int(reference.config.sliding_window) + assert window == 4096 + past_length = window + num_layers = int(reference.config.num_hidden_layers) + num_kv_heads = int(reference.config.num_key_value_heads) + head_dim = int(reference.config.hidden_size // reference.config.num_attention_heads) + + state: dict[str, np.ndarray] = {} + cache_data: list[tuple[torch.Tensor, torch.Tensor]] = [] + for layer_idx in range(num_layers): + key = np.zeros((1, num_kv_heads, past_length, head_dim), dtype=np.float32) + value = np.zeros_like(key) + # Only the oldest value is nonzero. Sliding attention must exclude it at + # position 4096; full causal attention still observes it. + value[:, :, 0, :] = 1024.0 + state[f"past_key_values.{layer_idx}.key"] = key + state[f"past_key_values.{layer_idx}.value"] = value + cache_data.append((torch.from_numpy(key.copy()), torch.from_numpy(value.copy()))) + + input_ids = np.asarray([[token_id]], dtype=np.int64) + ort_logits = _run_promoted_ort(session, input_ids, state, past_length)["logits"] + attention_mask = torch.ones((1, past_length + 1), dtype=torch.int64) + position_ids = torch.asarray([[past_length]], dtype=torch.int64) + + def reference_logits(sliding_window: int | None) -> np.ndarray: + reference.config.sliding_window = sliding_window + cache = DynamicCache( + ((key.clone(), value.clone()) for key, value in cache_data), + config=reference.config, + ) + with torch.no_grad(): + return ( + reference( + torch.from_numpy(input_ids), + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=cache, + use_cache=False, + ) + .logits.numpy() + .copy() + ) + + try: + expected_logits = reference_logits(window) + np.testing.assert_allclose(ort_logits, expected_logits, rtol=1e-4, atol=atol) + + full_causal_logits = reference_logits(None) + assert np.max(np.abs(ort_logits - full_causal_logits)) > 1e-3 + with pytest.raises(AssertionError): + np.testing.assert_allclose( + ort_logits, + full_causal_logits, + rtol=1e-4, + atol=atol, + ) + finally: + reference.config.sliding_window = window + + def _assert_replay_rollback_and_reorder( session: ort.InferenceSession, prompt_ids: np.ndarray, @@ -1164,7 +1610,7 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None "--dtype", "f32", "--execution-provider", - "cpu", + case.execution_provider, "--runtime", "ort-genai", "--runtime-version", @@ -1228,6 +1674,13 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None packaged_tokenizer = AutoTokenizer.from_pretrained(output_dir, local_files_only=True) prompt_ids = tokenizer(case.prompt, return_tensors="pt").input_ids assert packaged_tokenizer(case.prompt).input_ids == prompt_ids.tolist()[0] + source_vocab = tokenizer.get_vocab() + packaged_vocab = packaged_tokenizer.get_vocab() + assert all(packaged_vocab[token] == token_id for token, token_id in source_vocab.items()) + extra_vocab = packaged_vocab.keys() - source_vocab.keys() + assert all(token == f"[PAD{packaged_vocab[token]}]" for token in extra_vocab) + assert packaged_tokenizer.all_special_ids == tokenizer.all_special_ids + assert packaged_tokenizer.model_input_names == tokenizer.model_input_names if evidence.architecture == "qwen35moe": reference = _load_qwen35moe_same_value_reference( gguf_path, @@ -1235,34 +1688,29 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None revision=evidence.config_revision, ) else: - reference = AutoModelForCausalLM.from_pretrained( - evidence.repository, - revision=evidence.revision, - gguf_file=evidence.filename, - dtype=torch.float32, - ).eval() + with ExitStack() as reference_guard: + if case.reference_kind in {"gptneox", "mpt", "olmo", "starcoder"}: + for helper in ( + "mobius.integrations.gguf._builder._load_dequantized_state_dict", + "mobius.integrations.gguf._builder._normalize_gguf_weights", + "mobius.integrations.gguf._tensor_processors.process_tensors", + ): + reference_guard.enter_context( + mock.patch( + helper, + side_effect=AssertionError( + "same-artifact reference reused the production weight path" + ), + ) + ) + reference = _load_low_cost_same_value_reference( + gguf_path, + evidence, + case.reference_kind, + ) input_ids = prompt_ids.numpy() - reference_logits: list[np.ndarray] = [] with torch.no_grad(): - reference_output = reference(prompt_ids, use_cache=True) - reference_logits.append(reference_output.logits.numpy().copy()) - reference_state = reference_output.past_key_values - reference_generated: list[int] = [] - for _ in case.generated_tokens: - token = int(reference_output.logits[0, -1].argmax()) - reference_generated.append(token) - token_ids = torch.tensor([[token]], dtype=torch.int64) - with torch.no_grad(): - reference_output = reference( - token_ids, - past_key_values=reference_state, - use_cache=True, - ) - reference_state = reference_output.past_key_values - reference_logits.append(reference_output.logits.numpy().copy()) - assert reference_generated == list(case.generated_tokens) - del reference, reference_output, reference_state - gc.collect() + reference_logits = reference(prompt_ids, use_cache=False).logits.numpy() session = ort.InferenceSession( str(output_dir / "model.onnx"), providers=["CPUExecutionProvider"] @@ -1275,29 +1723,81 @@ def capture_save(package: ModelPackage, *args: object, **kwargs: object) -> None ) np.testing.assert_allclose( ort_output["logits"], - reference_logits[0], + reference_logits, rtol=1e-4, atol=case.atol, ) + if evidence.architecture in {"gpt2", "starcoder2"}: + mutated_config = AutoConfig.from_pretrained( + evidence.config_repository, + revision=evidence.config_revision, + ) + if evidence.architecture == "gpt2": + mutated_config.activation_function = "gelu" + else: + mutated_config.hidden_act = "gelu" + mutated = AutoModelForCausalLM.from_config(mutated_config).eval() + mutated.load_state_dict(reference.state_dict(), assign=True, strict=True) + with torch.no_grad(): + mutated_logits = mutated(prompt_ids, use_cache=False).logits.numpy() + with pytest.raises(AssertionError): + np.testing.assert_allclose( + ort_output["logits"], + mutated_logits, + rtol=1e-4, + atol=case.atol, + ) + del mutated, mutated_logits + + if evidence.architecture == "starcoder2": + _assert_starcoder2_long_context_sliding_window( + session, + reference, + int(prompt_ids[0, -1]), + atol=case.atol, + ) state = _next_cache(ort_output) generated: list[int] = [] + reference_generated: list[int] = [] + full_ids = input_ids.copy() for step in range(len(case.generated_tokens)): token = int(ort_output["logits"][0, -1].argmax()) generated.append(token) + reference_generated.append(int(reference_logits[0, -1].argmax())) token_ids = np.asarray([[token]], dtype=np.int64) + full_ids = np.concatenate((full_ids, token_ids), axis=1) ort_output = _run_promoted_ort(session, token_ids, state, input_ids.shape[1] + step) state = _next_cache(ort_output) + with torch.no_grad(): + reference_logits = ( + reference(torch.from_numpy(full_ids), use_cache=False) + .logits[:, -1:, :] + .numpy() + ) + full_ort_logits = _run_promoted_ort( + session, + full_ids, + _empty_promoted_state(session, batch_size=1), + 0, + )["logits"][:, -1:, :] np.testing.assert_allclose( - ort_output["logits"], - reference_logits[step + 1], + full_ort_logits, + reference_logits, rtol=1e-4, atol=case.atol, ) + np.testing.assert_allclose( + ort_output["logits"], + full_ort_logits, + rtol=1e-4, + atol=case.cache_atol, + ) assert len(generated) == len(case.generated_tokens) assert generated == list(case.generated_tokens) + assert reference_generated == list(case.generated_tokens) _assert_replay_rollback_and_reorder(session, input_ids) - del session, ort_output, state, reference_logits + del session, ort_output, state, reference, reference_logits gc.collect() compatibility = json.loads(