diff --git a/src/mobius/integrations/onnx_genai/__init__.py b/src/mobius/integrations/onnx_genai/__init__.py index a7f719cec..fc82c6e59 100644 --- a/src/mobius/integrations/onnx_genai/__init__.py +++ b/src/mobius/integrations/onnx_genai/__init__.py @@ -71,6 +71,7 @@ PackageFacts, SpecialTokenFact, SpecialTokenRole, + TokenFacts, TokenizerArtifact, TokenizerFacts, build_tokenizer_facts, @@ -119,6 +120,7 @@ "SchedulerConfig", "SpecialTokenFact", "SpecialTokenRole", + "TokenFacts", "TokenizerArtifact", "TokenizerFacts", "add_policy_components_to_workflow", diff --git a/src/mobius/integrations/onnx_genai/_schema/inference_metadata.schema.json b/src/mobius/integrations/onnx_genai/_schema/inference_metadata.schema.json index 91861c8f5..48fbb616b 100644 --- a/src/mobius/integrations/onnx_genai/_schema/inference_metadata.schema.json +++ b/src/mobius/integrations/onnx_genai/_schema/inference_metadata.schema.json @@ -2234,7 +2234,7 @@ "type": "null" } ], - "description": "Exact tokenizer, vocabulary, and special-token facts." + "description": "Exact tokenizer and vocabulary facts." } }, "type": "object" @@ -2621,7 +2621,12 @@ }, { "const": "eos_token_ids", - "description": "Token ids that end generation.\n\nA set, not a single id: a model may end a turn with one token and a\nmessage with another, and both must stop. Declaring it here is what lets\nthe runtime's stop policy read a package's EOS from the package instead\nof rediscovering it from tokenizer side-files it may not ship.", + "description": "Token ids that end generation.\n\nThe request value overrides\n`package.tokenizer.special_tokens.eos_token_id`. It never carries an\nauthored package default.", + "type": "string" + }, + { + "const": "eos_token_lengths", + "description": "Number of valid entries in each row of a padded EOS-id tensor.", "type": "string" }, { @@ -2976,28 +2981,6 @@ }, "type": "object" }, - "SpecialTokenFact": { - "additionalProperties": false, - "description": "One special token, pinned by id and exact surface bytes.", - "properties": { - "content": { - "description": "Exact UTF-8 surface form of the token.", - "minLength": 1, - "type": "string" - }, - "id": { - "description": "Vocabulary id of the token.", - "format": "uint32", - "minimum": 0, - "type": "integer" - } - }, - "required": [ - "id", - "content" - ], - "type": "object" - }, "SpeculationSafety": { "description": "Whether an effect may be executed inside a speculative region.", "oneOf": [ @@ -3993,6 +3976,94 @@ ], "type": "object" }, + "TokenFacts": { + "additionalProperties": false, + "description": "Numeric model and control-token facts.\n\nThese ids are model/package facts. Token spellings, added-token maps, and\nchat templates remain in tokenizer assets and are not repeated here.", + "properties": { + "audio_token_id": { + "description": "Prompt placeholder replaced by audio features.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "bos_token_id": { + "description": "Beginning-of-sequence token.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "decoder_start_token_id": { + "description": "First token fed to an encoder-decoder's autoregressive decoder.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "eos_token_id": { + "description": "Every token id that terminates package-default autoregressive generation.", + "items": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "type": "array" + }, + "image_token_id": { + "description": "Prompt placeholder replaced by image features.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "pad_token_id": { + "description": "Padding token used by package-authored tensor contracts.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "sep_token_id": { + "description": "Separator token used by sequence-pair models.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "video_token_id": { + "description": "Prompt placeholder replaced by video features.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vision_start_token_id": { + "description": "Token that opens a vision segment in a multimodal prompt.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "TokenizerArtifact": { "additionalProperties": false, "description": "One package-relative tokenizer artifact.", @@ -4015,7 +4086,10 @@ "algorithm": { "description": "Tokenizer algorithm identifier, e.g. `bpe`, `unigram`, `wordpiece`.", "minLength": 1, - "type": "string" + "type": [ + "string", + "null" + ] }, "artifacts": { "description": "Package-relative tokenizer artifacts.", @@ -4031,23 +4105,26 @@ "type": "boolean" }, "special_tokens": { - "additionalProperties": { - "$ref": "#/$defs/SpecialTokenFact" - }, - "description": "Special tokens by semantic role, e.g. `bos`, `eos`, `pad`.", - "type": "object" + "anyOf": [ + { + "$ref": "#/$defs/TokenFacts" + }, + { + "type": "null" + } + ], + "description": "Numeric model and control-token facts for this tokenizer vocabulary.\n\nToken strings, added-token mappings, and chat templates remain in the\ntokenizer assets. Request EOS inputs may override these defaults, but\nworkflow literals and termination components do not own another copy." }, "vocab_size": { "description": "Number of entries in the vocabulary, including added tokens.", "format": "uint", "minimum": 1, - "type": "integer" + "type": [ + "integer", + "null" + ] } }, - "required": [ - "algorithm", - "vocab_size" - ], "type": "object" }, "VisionOutputBinding": { diff --git a/src/mobius/integrations/onnx_genai/auto_export_test.py b/src/mobius/integrations/onnx_genai/auto_export_test.py index 9a776a896..f54e5c3b0 100644 --- a/src/mobius/integrations/onnx_genai/auto_export_test.py +++ b/src/mobius/integrations/onnx_genai/auto_export_test.py @@ -244,11 +244,11 @@ def test_dispatch_decoder(tmp_path): } assert application_inputs == { "request.prompt_lengths", - "request.eos_ids", - "request.eos_lengths", "request.row_max_iterations", "request.rng_counter", } + assert workflow["inputs"]["request.eos_ids"]["role"]["role"] == "eos_token_ids" + assert workflow["inputs"]["request.eos_lengths"]["role"]["role"] == "eos_token_lengths" assert workflow["inputs"]["request.prompt_lengths"]["default"] == -1 assert [node["component"] for node in workflow["steps"][0]["setup"]] == [ "decoder_state_initializer", diff --git a/src/mobius/integrations/onnx_genai/package_facts.py b/src/mobius/integrations/onnx_genai/package_facts.py index 74e64542a..53ec00097 100644 --- a/src/mobius/integrations/onnx_genai/package_facts.py +++ b/src/mobius/integrations/onnx_genai/package_facts.py @@ -6,28 +6,28 @@ A published package carries neural graphs and the workflow that drives them, but a request arrives as text and media. Turning that request into the token stream the graphs consume requires the vocabulary contract the package was built -against: which algorithm produced the ids, how many ids exist, and which id -plays each semantic role. onnx-genai calls that section ``package.tokenizer`` +against: which algorithm produced the ids, how many ids exist, and which numeric ids +play each execution-relevant role. onnx-genai calls that section ``package.tokenizer`` (``PackageFacts``/``TokenizerFacts`` in ``onnx-genai-metadata``), and it is the only place a front end looks for those facts. -Two of those roles are load-bearing rather than informational: +Two of those facts are load-bearing rather than informational: -``eos`` +``eos_token_id`` The workflow's termination policy already compares generated ids against a stop id. Publishing the same id under a role means a caller can render and trim a transcript without re-deriving it from a side file. -``image_placeholder`` +``image_token_id`` The prompt token whose position an image's features replace. A multimodal package that omits it declares no place in the token stream for its own image features, so an attached image is preprocessed and then dropped. -Every value here is read from the package's own artifacts. The algorithm, -vocabulary size and byte-level flag come from the packaged tokenizer definition; -each special token pairs a config-declared id with the surface form that id has -in that same vocabulary. A role the package does not declare is simply absent, -because a guessed special token is worse than a missing one. +Every value here is read from the package's own artifacts or resolved config. +The algorithm, vocabulary size and byte-level flag come from the packaged +tokenizer definition; numeric token IDs come from the package's runtime config. +Token spellings and chat templates remain authoritative in tokenizer assets, +so execution metadata never carries a second, potentially stale vocabulary. Nothing in this module knows a model name or a literal token id: architecture defaults belong to the config adapters under :mod:`mobius._configs`, and reach @@ -47,9 +47,8 @@ _LOGGER = logging.getLogger(__name__) -#: Semantic role naming the prompt token that stands for one whole image. -#: Fixed by the onnx-genai runtime, which looks this role up by name. -IMAGE_PLACEHOLDER_ROLE: Final = "image_placeholder" +#: Schema field naming the prompt token that stands for one whole image. +IMAGE_PLACEHOLDER_ROLE: Final = "image_token_id" @dataclasses.dataclass(frozen=True) @@ -63,21 +62,24 @@ class SpecialTokenRole: name: str fields: tuple[str, ...] + multiple: bool = False #: Roles every text-producing package can state about its own vocabulary. TEXT_TOKEN_ROLES: Final[tuple[SpecialTokenRole, ...]] = ( - SpecialTokenRole("bos", ("bos_token_id",)), - SpecialTokenRole("eos", ("eos_token_id",)), - SpecialTokenRole("pad", ("pad_token_id",)), - SpecialTokenRole("unk", ("unk_token_id",)), + SpecialTokenRole("pad_token_id", ("pad_token_id",)), + SpecialTokenRole("bos_token_id", ("bos_token_id",)), + SpecialTokenRole("eos_token_id", ("eos_token_id",), multiple=True), + SpecialTokenRole("sep_token_id", ("sep_token_id",)), + SpecialTokenRole("decoder_start_token_id", ("decoder_start_token_id",)), ) #: Roles a package adds when a modality's features replace a prompt token. MEDIA_TOKEN_ROLES: Final[tuple[SpecialTokenRole, ...]] = ( SpecialTokenRole(IMAGE_PLACEHOLDER_ROLE, ("image_token_id",)), - SpecialTokenRole("audio_placeholder", ("audio_token_id",)), - SpecialTokenRole("video_placeholder", ("video_token_id",)), + SpecialTokenRole("video_token_id", ("video_token_id",)), + SpecialTokenRole("audio_token_id", ("audio_token_id",)), + SpecialTokenRole("vision_start_token_id", ("vision_start_token_id",)), ) #: Nested config objects searched after the root for a role's id. A composite @@ -116,7 +118,11 @@ class SpecialTokenRole: @dataclasses.dataclass(frozen=True) class SpecialTokenFact: - """One special token, pinned by id and exact surface bytes.""" + """Compatibility value for callers of the former text-bearing API. + + New metadata uses :class:`TokenFacts`; token content remains in tokenizer + assets. Keeping this value exported avoids breaking existing imports. + """ id: int content: str @@ -125,6 +131,19 @@ def to_metadata(self) -> dict[str, Any]: return {"id": self.id, "content": self.content} +@dataclasses.dataclass(frozen=True) +class TokenFacts: + """Numeric model and control-token facts owned by the package.""" + + values: Mapping[str, int | tuple[int, ...]] = dataclasses.field(default_factory=dict) + + def to_metadata(self) -> dict[str, Any]: + return { + name: list(value) if isinstance(value, tuple) else value + for name, value in self.values.items() + } + + @dataclasses.dataclass(frozen=True) class TokenizerArtifact: """One package-relative tokenizer artifact.""" @@ -139,30 +158,27 @@ def to_metadata(self) -> dict[str, Any]: class TokenizerFacts: """Tokenizer facts and package-relative artifacts. - Mirrors onnx-genai's ``TokenizerFacts``. ``algorithm`` and ``vocab_size`` - are required by that contract; the rest are omitted when empty rather than - published as defaults. + Mirrors onnx-genai's ``TokenizerFacts``. ``algorithm`` and ``vocab_size`` + are omitted when no tokenizer definition is available, while numeric token + facts can still describe the package's execution contract. """ - algorithm: str - vocab_size: int + algorithm: str | None = None + vocab_size: int | None = None byte_level: bool = False artifacts: tuple[TokenizerArtifact, ...] = () - special_tokens: Mapping[str, SpecialTokenFact] = dataclasses.field(default_factory=dict) + special_tokens: TokenFacts = dataclasses.field(default_factory=TokenFacts) def to_metadata(self) -> dict[str, Any]: - facts: dict[str, Any] = { - "algorithm": self.algorithm, - "vocab_size": self.vocab_size, - "byte_level": self.byte_level, - } + facts: dict[str, Any] = {"byte_level": self.byte_level} + if self.algorithm is not None: + facts["algorithm"] = self.algorithm + if self.vocab_size is not None: + facts["vocab_size"] = self.vocab_size if self.artifacts: facts["artifacts"] = [artifact.to_metadata() for artifact in self.artifacts] - if self.special_tokens: - facts["special_tokens"] = { - role: token.to_metadata() - for role, token in sorted(self.special_tokens.items()) - } + if self.special_tokens.values: + facts["special_tokens"] = self.special_tokens.to_metadata() return facts @@ -433,14 +449,23 @@ def _config_value(config: Any, field: str) -> Any: def _token_id(value: Any) -> int | None: - """Coerce a declared token id, taking the first of a stop-id list.""" - if isinstance(value, (list, tuple)): - value = value[0] if value else None + """Coerce one declared token id.""" if isinstance(value, bool) or not isinstance(value, int): return None return value if value >= 0 else None +def _token_ids(value: Any) -> tuple[int, ...]: + """Coerce an ordered token-id set without silently dropping multi-EOS.""" + values = value if isinstance(value, (list, tuple)) else (value,) + result: list[int] = [] + for candidate in values: + token_id = _token_id(candidate) + if token_id is not None and token_id not in result: + result.append(token_id) + return tuple(result) + + _MISSING: Final = object() @@ -491,41 +516,39 @@ def source_declared_roles( return declared -def build_special_tokens( - definition: TokenizerDefinition, +def build_token_facts( config: Any, roles: Iterable[SpecialTokenRole], *, declared: Mapping[str, Any] | None = None, -) -> dict[str, SpecialTokenFact]: - """Resolve declared roles against the packaged vocabulary. +) -> TokenFacts: + """Resolve numeric package token facts. ``declared`` overrides the config for roles the emitting workflow has already resolved for itself, so a document never states one id for its termination policy and a different one for the same role here. - A role survives only when both halves of the fact are known: an id the - package declares *and* the exact surface bytes that id has in this - vocabulary. An id with no surface form is a stale config field pointing - outside the shipped vocabulary, and publishing it would let a front end - splice a token the tokenizer cannot render. + Multi-valued roles preserve their declared order. Scalar roles take exactly + one non-negative integer. Token spellings remain in tokenizer assets. """ declared = declared or {} - special_tokens: dict[str, SpecialTokenFact] = {} + values: dict[str, int | tuple[int, ...]] = {} for role in roles: - token_id = _token_id(declared.get(role.name)) - if token_id is None: + value = declared.get(role.name) + if value is None: for field in role.fields: - token_id = _token_id(_config_value(config, field)) - if token_id is not None: + value = _config_value(config, field) + if value is not None: break - if token_id is None: + if role.multiple: + token_ids = _token_ids(value) + if token_ids: + values[role.name] = token_ids continue - content = definition.surface_forms.get(token_id) - if not content: - continue - special_tokens[role.name] = SpecialTokenFact(id=token_id, content=content) - return special_tokens + token_id = _token_id(value) + if token_id is not None: + values[role.name] = token_id + return TokenFacts(values) def build_tokenizer_facts( @@ -560,8 +583,11 @@ def build_tokenizer_facts( address nothing, so the vocabulary is the width a caller can actually render. """ definition = read_tokenizer_definition(package_dir) or read_tokenizer_definition(source) - if definition is None or definition.vocab_size <= 0: - return None + special_tokens = build_token_facts( + config, + roles, + declared=source_declared_roles(source, roles), + ) artifacts = () if package_dir is not None: artifacts = tuple( @@ -569,17 +595,14 @@ def build_tokenizer_facts( for name in TOKENIZER_ARTIFACT_NAMES if os.path.isfile(os.path.join(package_dir, name)) ) + if definition is None and not artifacts and not special_tokens.values: + return None return TokenizerFacts( - algorithm=definition.algorithm, - vocab_size=definition.vocab_size, - byte_level=definition.byte_level, + algorithm=definition.algorithm if definition is not None else None, + vocab_size=definition.vocab_size if definition is not None else None, + byte_level=definition.byte_level if definition is not None else False, artifacts=artifacts, - special_tokens=build_special_tokens( - definition, - config, - roles, - declared=source_declared_roles(source, roles), - ), + special_tokens=special_tokens, ) @@ -602,4 +625,5 @@ def attach_package_facts( tokenizer = build_tokenizer_facts(source, config, roles=roles, package_dir=package_dir) if tokenizer is None: return + metadata["schema_version"] = "v1.2" metadata.setdefault("package", {})["tokenizer"] = tokenizer.to_metadata() diff --git a/src/mobius/integrations/onnx_genai/package_facts_test.py b/src/mobius/integrations/onnx_genai/package_facts_test.py index d327b7df9..876879806 100644 --- a/src/mobius/integrations/onnx_genai/package_facts_test.py +++ b/src/mobius/integrations/onnx_genai/package_facts_test.py @@ -23,6 +23,7 @@ from mobius._configs import MMSConfig from mobius._model_package import ModelPackage +from mobius.integrations.onnx_genai import SpecialTokenFact from mobius.integrations.onnx_genai.auto_export import write_onnx_genai_config from mobius.integrations.onnx_genai.auto_export_test import _vlm_package, _VlmCfg from mobius.integrations.onnx_genai.inference_metadata_test import ( @@ -81,6 +82,7 @@ class _MediaCfg(_VlmCfg): """A vision config that can also declare a video placeholder.""" video_token_id: int | None = None + audio_token_id: int | None = None def _write_tokenizer( @@ -165,6 +167,12 @@ def test_an_unreadable_tokenizer_states_nothing(self, tmp_path): class TestDecoderPackageFacts: """A text package states the vocabulary the ids it emits belong to.""" + def test_legacy_special_token_fact_remains_importable(self): + assert SpecialTokenFact(7, "").to_metadata() == { + "id": 7, + "content": "", + } + def test_facts_survive_into_package_metadata(self, tmp_path): _write_tokenizer(tmp_path) metadata = build_decoder_workflow_metadata( @@ -175,10 +183,11 @@ def test_facts_survive_into_package_metadata(self, tmp_path): assert tokenizer["vocab_size"] == _TEXT_VOCAB_SIZE assert tokenizer["byte_level"] is True assert tokenizer["special_tokens"] == { - "bos": {"id": _BOS_ID, "content": ""}, - "eos": {"id": _EOS_ID, "content": ""}, - "pad": {"id": _PAD_ID, "content": ""}, + "pad_token_id": _PAD_ID, + "bos_token_id": _BOS_ID, + "eos_token_id": [_EOS_ID], } + assert metadata["schema_version"] == "v1.2" _validate(metadata) def test_the_stated_width_is_the_vocabulary_not_the_logits_row(self, tmp_path): @@ -205,8 +214,8 @@ def test_pipeline_workflow_remains_the_sole_representation(self, tmp_path): def test_the_stated_stop_token_is_the_one_the_loop_terminates_on(self, tmp_path): # A repackaged checkpoint may be retuned without its config being # rewritten, so genai_config.json outranks the config. Both the - # termination policy and the published role must follow it, or one - # document names two different stop tokens. + # package facts must follow it, while the workflow only exposes the + # request override consumed by termination. _write_tokenizer(tmp_path, added={_IMAGE_ID: "<|endoftext|>"}) (tmp_path / "genai_config.json").write_text( json.dumps({"model": {"eos_token_id": _IMAGE_ID}}), encoding="utf-8" @@ -215,29 +224,49 @@ def test_the_stated_stop_token_is_the_one_the_loop_terminates_on(self, tmp_path) _decoder_package(_TextCfg()), _TextCfg(), source=str(tmp_path) ) workflow = metadata["pipeline"]["workflow"] - assert metadata["package"]["tokenizer"]["special_tokens"]["eos"] == { - "id": _IMAGE_ID, - "content": "<|endoftext|>", - } - assert workflow["inputs"]["package.eos_ids"]["default"] == _IMAGE_ID + assert metadata["package"]["tokenizer"]["special_tokens"]["eos_token_id"] == [ + _IMAGE_ID + ] + assert "package.eos_ids" not in workflow["inputs"] + assert workflow["inputs"]["request.eos_ids"]["role"]["role"] == "eos_token_ids" + assert "default" not in workflow["inputs"]["request.eos_ids"] - def test_a_role_the_vocabulary_cannot_render_is_omitted(self, tmp_path): - # A config field pointing outside the shipped vocabulary is stale. A - # front end that spliced that id would emit a token the tokenizer - # cannot render, so the role is dropped rather than half-stated. + def test_multi_eos_order_is_preserved_without_a_workflow_literal(self, tmp_path): + _write_tokenizer(tmp_path, added={_IMAGE_ID: "<|im_end|>"}) + (tmp_path / "genai_config.json").write_text( + json.dumps({"model": {"eos_token_id": [_EOS_ID, _IMAGE_ID]}}), + encoding="utf-8", + ) + + metadata = build_decoder_workflow_metadata( + _decoder_package(_TextCfg()), _TextCfg(), source=str(tmp_path) + ) + + assert metadata["package"]["tokenizer"]["special_tokens"]["eos_token_id"] == [ + _EOS_ID, + _IMAGE_ID, + ] + assert "package.eos_ids" not in metadata["pipeline"]["workflow"]["inputs"] + + def test_numeric_facts_do_not_duplicate_tokenizer_surface_forms(self, tmp_path): + # Token spellings belong to tokenizer assets. The package still states + # its execution-relevant numeric fact without copying vocabulary text. _write_tokenizer(tmp_path) metadata = build_decoder_workflow_metadata( _decoder_package(_TextCfg()), _TextCfg(bos_token_id=9_999), source=str(tmp_path), ) - assert "bos" not in metadata["package"]["tokenizer"]["special_tokens"] + assert metadata["package"]["tokenizer"]["special_tokens"]["bos_token_id"] == 9_999 - def test_a_package_without_a_readable_tokenizer_states_no_facts(self, tmp_path): + def test_a_package_without_a_readable_tokenizer_still_states_numeric_facts(self, tmp_path): metadata = build_decoder_workflow_metadata( _decoder_package(_TextCfg()), _TextCfg(), source=str(tmp_path) ) - assert "package" not in metadata + tokenizer = metadata["package"]["tokenizer"] + assert "algorithm" not in tokenizer + assert "vocab_size" not in tokenizer + assert tokenizer["special_tokens"]["eos_token_id"] == [_EOS_ID] def test_artifacts_name_only_files_the_package_contains(self, tmp_path): source = tmp_path / "source" @@ -311,13 +340,10 @@ def _materialize_tokenizer(output_dir, source, *, revision=None): metadata = yaml.safe_load(Path(artifacts["inference_metadata"]).read_text()) tokenizer = metadata["package"]["tokenizer"] assert tokenizer["vocab_size"] == _IMAGE_ID + 1 - assert tokenizer["special_tokens"]["eos"] == { - "id": _IMAGE_ID, - "content": "<|endoftext|>", - } + assert tokenizer["special_tokens"]["eos_token_id"] == [_IMAGE_ID] assert tokenizer["artifacts"] == [{"location": "tokenizer.json"}] workflow = metadata["pipeline"]["workflow"] - assert workflow["inputs"]["package.eos_ids"]["default"] == _IMAGE_ID + assert "package.eos_ids" not in workflow["inputs"] class TestMultimodalPackageFacts: @@ -338,11 +364,8 @@ def _metadata(self, source, **overrides): def test_image_placeholder_role_is_published(self, source): tokenizer = self._metadata(source)["package"]["tokenizer"] - assert tokenizer["special_tokens"][IMAGE_PLACEHOLDER_ROLE] == { - "id": _IMAGE_ID, - "content": "", - } - assert tokenizer["special_tokens"]["eos"] == {"id": _EOS_ID, "content": ""} + assert tokenizer["special_tokens"][IMAGE_PLACEHOLDER_ROLE] == _IMAGE_ID + assert tokenizer["special_tokens"]["eos_token_id"] == [_EOS_ID] def test_image_input_routing_survives_alongside_the_facts(self, source): metadata = self._metadata(source) @@ -374,15 +397,15 @@ def test_image_input_routing_survives_alongside_the_facts(self, source): def test_a_declared_video_placeholder_is_published_too(self, source): tokenizer = self._metadata(source, video_token_id=_VIDEO_ID)["package"]["tokenizer"] - assert tokenizer["special_tokens"]["video_placeholder"] == { - "id": _VIDEO_ID, - "content": "