diff --git a/.gitignore b/.gitignore index fb6507ee..1ee9c3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,15 @@ .venv/ __pycache__/ *.pyc +*.egg-info/ +dist/ +build/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Inspect AI eval logs +logs/ # Jupyter Notebook .ipynb_checkpoints/ diff --git a/sdks/python/src/learning_commons_evaluators/__init__.py b/sdks/python/src/learning_commons_evaluators/__init__.py index 6169df43..2ec1b989 100644 --- a/sdks/python/src/learning_commons_evaluators/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/__init__.py @@ -59,6 +59,11 @@ TextInputField, ) from learning_commons_evaluators.schemas.config import EvaluationSettings, LLMProvider +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.conventionality import ( ConventionalityEvaluationSettings, ConventionalityOutput, @@ -163,6 +168,9 @@ "create_config_telemetry_with_full_input", "create_logger", "create_silent_logger", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "get_logger", "wrap_provider_error", ] diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 9954c987..70d2b412 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import json as _json +import re import time from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable @@ -32,6 +34,11 @@ EvaluationInput, EvaluationResult, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -50,6 +57,75 @@ ParsedT = TypeVar("ParsedT", bound=BaseModel) +def _parse_json_output( + raw: str, + parser_output_type: type[ParsedT], + json_dict_normalizer: Callable[[dict], dict] | None, + provider_type: Any, + model: str, +) -> ParsedT: + """Parse a raw JSON string into ``parser_output_type``, wrapping errors consistently. + + Shared by both the protocol path and (for the normalizer branch) the LangChain path + so that error handling is symmetric and normaliser logic lives in one place. + """ + raw = _strip_json_fences(raw) + try: + if json_dict_normalizer is not None: + parsed_dict = _json.loads(raw) + if not isinstance(parsed_dict, dict): + raise OutputValidationError( + "Model output is not a JSON object", + provider=provider_type, + model=model, + ) + try: + normalized = json_dict_normalizer(parsed_dict) + except (TypeError, ValueError) as norm_err: + raise OutputValidationError( + "Model output could not be normalized before validation", + provider=provider_type, + model=model, + ) from norm_err + return parser_output_type.model_validate(normalized) + return parser_output_type.model_validate_json(raw) + except PydanticValidationError as e: + raise OutputValidationError( + provider=provider_type, + model=model, + validation_errors=sanitize_pydantic_errors(e.errors()), + ) from e + except OutputValidationError: + raise + except Exception as e: + raise OutputValidationError(provider=provider_type, model=model) from e + + +def _strip_json_fences(text: str) -> str: + """Strip markdown code fences and extract the first valid JSON object or array. + + Uses ``json.JSONDecoder.raw_decode`` to locate the first balanced JSON structure + and discard any surrounding prose or trailing text. This correctly handles: + - Markdown-fenced responses: ````json\\n{...}\\n``` `` + - Prose-prefixed responses: ``Here is the result:\\n{...}`` + - Trailing-prose responses: ``{...} Here is my reasoning...`` + """ + text = text.strip() + text = re.sub(r"^```(?:json)?\s*\n?", "", text) + text = re.sub(r"\n?```\s*$", "", text) + text = text.strip() + # Find the first { or [ and use raw_decode to extract the complete JSON structure, + # correctly discarding any trailing prose or a second JSON object in the response. + start = next((i for i, ch in enumerate(text) if ch in ("{", "[")), -1) + if start != -1: + try: + _, end = _json.JSONDecoder().raw_decode(text, start) + return text[start:end] + except _json.JSONDecodeError: + pass + return text + + class BaseEvaluator(ABC, Generic[InputT, OutputT, SettingsT]): """ Abstract base class for all evaluators. @@ -74,9 +150,11 @@ def __init__( self, config: EvaluatorConfig, *, + llm_provider: LLMGeneratorProtocol | None = None, default_evaluation_settings: SettingsT | None = None, ) -> None: self.config = config + self._llm_provider = llm_provider if default_evaluation_settings is not None: self.default_evaluation_settings = default_evaluation_settings # TODO: validate config @@ -312,6 +390,14 @@ async def execute_prompt_chain_step( Parsed instance of ``parser_output_type`` when it is a model class; plain ``str`` when ``parser_output_type`` is omitted or ``None``. + Note: + **Execution path**: when ``self._llm_provider`` is set (injected at + construction via ``BaseEvaluator.__init__``), the *protocol path* is taken + — the LangChain template is formatted to extract system/human strings, the + injected provider is called directly, and JSON is parsed via Pydantic. + When ``self._llm_provider`` is ``None`` (default), the *LangChain path* + is taken and ``create_provider()`` is called internally. + Raises: ConfigurationError: No provider config for ``prompt_settings.provider_type``. OutputValidationError: The LLM response didn't satisfy the expected @@ -330,6 +416,91 @@ async def execute_prompt_chain_step( # Populated after a successful LLM invoke so we can attach usage even if parsing fails. token_usage: TokenUsage | None = None + if self._llm_provider is not None: + # ── Protocol path ───────────────────────────────────────────── + provider = self._llm_provider + + async def _run_via_provider() -> BaseModel | str: + nonlocal token_usage + try: + # Inside try: missing template variables become EvaluatorErrors, + # not bare KeyErrors — consistent with the LangChain path's error contract. + formatted = await template.aformat_messages(**chain_inputs) + system_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "system"), + "", + ) + human_str = next( + (str(m.content) for m in formatted if getattr(m, "type", "") == "human"), + "", + ) + if not human_str: + raise ValueError( + f"Template for step '{step_name}' produced no human message. " + 'Ensure the template contains at least one ("human", ...) turn.' + ) + if not system_str: + self.config.logger.debug( + "No system message in template for step '%s'; " + "passing empty string to adapter.", + step_name, + ) + response: LLMResponse = await provider.generate( + system=system_str, + human=human_str, + config=GenerateConfig( + temperature=prompt_settings.temperature, + model=prompt_settings.model, + ), + ) + except EvaluatorError: + raise + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: + raise wrap_provider_error( + e, + provider=prompt_settings.provider_type, + model=prompt_settings.model, + ) from e + if response.input_tokens is not None or response.output_tokens is not None: + token_usage = TokenUsage( + provider_type=prompt_settings.provider_type, + model=response.model, + input_tokens=response.input_tokens or 0, + output_tokens=response.output_tokens or 0, + ) + if parser_output_type is None: + return response.content + return _parse_json_output( + response.content, + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + response.model, + ) + + try: + return await self.execute_step( + step_name, + evaluation_metadata, + _run_via_provider, + extras={ + PROMPT_STEP_EXTRA_PROMPT_SETTINGS: prompt_settings_to_extras_value( + prompt_settings + ), + }, + ) + finally: + if token_usage is not None: + self.update_total_token_usage(token_usage, evaluation_metadata) + step = evaluation_metadata.step_details.get(step_name) + if step is not None: + step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE] = token_usage.model_dump( + mode="json" + ) + + # ── LangChain path (default, unchanged) ─────────────────────────── async def _run_chain() -> BaseModel | str: nonlocal token_usage try: @@ -342,29 +513,13 @@ async def _run_chain() -> BaseModel | str: from langchain_core.output_parsers.json import JsonOutputParser if json_dict_normalizer is not None: - loose = JsonOutputParser() - parsed_dict = await loose.ainvoke(ai_message) - if not isinstance(parsed_dict, dict): - # JSON parsed cleanly but the top-level value isn't an object - # (e.g. the LLM returned a JSON array or scalar). That's an - # output-shape failure, not a parse failure — surface it as - # OutputValidationError so callers can treat it consistently - # with schema-mismatch errors, and avoid the TypeError that - # ``dict(parsed_dict)`` would raise on a non-dict. - raise OutputValidationError( - "Model output is not a JSON object", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) - try: - normalized = json_dict_normalizer(parsed_dict) - except (TypeError, ValueError) as norm_err: - raise OutputValidationError( - "Model output could not be normalized before validation", - provider=prompt_settings.provider_type, - model=prompt_settings.model, - ) from norm_err - return parser_output_type.model_validate(normalized) + return _parse_json_output( + str(ai_message.content), + parser_output_type, + json_dict_normalizer, + prompt_settings.provider_type, + prompt_settings.model, + ) parser = JsonOutputParser(pydantic_object=parser_output_type) raw = await parser.ainvoke(ai_message) diff --git a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py index 27caf004..bb41b809 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/__init__.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/__init__.py @@ -33,6 +33,11 @@ InputSpec, TextInputSpec, ) +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) from learning_commons_evaluators.schemas.metadata import ( PROMPT_STEP_EXTRA_PROMPT_SETTINGS, PROMPT_STEP_EXTRA_TOKEN_USAGE, @@ -81,5 +86,8 @@ "TextInputField", "TokenUsage", "InputValidationError", + "GenerateConfig", + "LLMGeneratorProtocol", + "LLMResponse", "prompt_settings_to_extras_value", ] diff --git a/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py new file mode 100644 index 00000000..0ccf489b --- /dev/null +++ b/sdks/python/src/learning_commons_evaluators/schemas/llm_provider.py @@ -0,0 +1,133 @@ +"""LLM provider protocol and associated types for framework-agnostic model injection. + +These types define the interface that evaluation frameworks (Inspect AI, Braintrust, +Arize/Phoenix, Langfuse, etc.) implement to provide their own model execution to the SDK. + +``LLMGeneratorProtocol`` is a structural protocol (``typing.Protocol``) — integration +packages do not need to import or inherit from it. Any class with the correct ``generate`` +signature satisfies the protocol automatically for static type checkers. + +Response fields are aligned with OpenTelemetry GenAI semantic conventions: +https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import NamedTuple, Protocol + + +class LLMResponse(NamedTuple): + """Structured response from an LLM generation call. + + A ``NamedTuple`` — immutable, constructible by keyword or position, and + iterable for adapters that need to destructure the result. + + Fields are aligned with OpenTelemetry GenAI semantic conventions so that + observability adapters (Arize/Phoenix, Langfuse) can populate their spans + without additional parsing. + + Required fields (``content``, ``model``) must always be populated. + Optional fields should be populated whenever the underlying provider returns them. + """ + + content: str + """The model's text response.""" + + model: str + """The model that generated the response (``gen_ai.response.model``).""" + + input_tokens: int | None = None + """Number of input/prompt tokens consumed (``gen_ai.usage.input_tokens``).""" + + output_tokens: int | None = None + """Number of output/completion tokens generated (``gen_ai.usage.output_tokens``).""" + + +@dataclass +class GenerateConfig: + """Configuration for a single LLM generation call. + + All fields are optional. Adapters should apply whatever the underlying + provider supports and ignore the rest. + """ + + temperature: float | None = None + """Sampling temperature. 0.0 for deterministic output (recommended for evals).""" + + max_tokens: int | None = None + """Maximum number of tokens to generate.""" + + model: str | None = None + """Model identifier to request from the provider (e.g. ``"claude-opus-4-8"``). + + Adapters should use this when set and ignore it otherwise — the contract is + identical to all other ``GenerateConfig`` fields. When ``None`` (default), + the adapter uses whatever model it was constructed with. + + Populated from ``PromptSettings.model`` on the protocol path so that + adapter authors can inspect which model the evaluator expects without + reaching into ``prompt_settings`` directly. + """ + + +class LLMGeneratorProtocol(Protocol): + """Structural protocol for LLM generation adapters. + + Implement this protocol in an integration package to allow the SDK to call + your framework's model system. + + No import of this class is required in the implementing package. Structural + conformance (correct ``generate`` signature) is sufficient for static type + checkers. + + **Lifecycle**: if your adapter holds a connection pool or HTTP session, add + ``async def aclose(self) -> None`` and call it when done. The protocol does + not include ``aclose`` so that stateless adapters remain fully conformant + without boilerplate. Callers that want to support teardown should use + ``hasattr(adapter, "aclose")``. + + Example:: + + from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMResponse, + ) + + class MyFrameworkAdapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + response = await my_framework.call(system, human) + return LLMResponse( + content=response.text, + model=response.model_name, + input_tokens=response.usage.input, + output_tokens=response.usage.output, + ) + """ + + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + """Generate a response from the LLM. + + Args: + system: The system prompt. + human: The human/user prompt. + config: Optional generation configuration. Adapters apply whatever + fields the underlying provider supports and ignore the rest. + + Returns: + ``LLMResponse`` with at minimum ``content`` and ``model`` populated. + Populate optional fields (token counts) whenever the provider returns them. + """ + ... diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index f595af9c..863d36e0 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -857,3 +857,295 @@ def passthrough(d: dict) -> dict: assert "JSON object" in str(exc_info.value) assert exc_info.value.provider is LLMProvider.GOOGLE assert exc_info.value.model == "gemini-2.0-flash" + + +# --------------------------------------------------------------------------- +# execute_prompt_chain_step — protocol path (llm_provider injected) +# --------------------------------------------------------------------------- + + +from learning_commons_evaluators.schemas.llm_provider import LLMResponse # noqa: E402 + + +def _make_adapter( + content: str, + model: str = "test-model", + input_tokens: int | None = 10, + output_tokens: int | None = 5, +) -> AsyncMock: + """Minimal mock that satisfies LLMGeneratorProtocol.generate().""" + adapter = AsyncMock() + adapter.generate = AsyncMock( + return_value=LLMResponse( + content=content, model=model, input_tokens=input_tokens, output_tokens=output_tokens + ) + ) + return adapter + + +_PROTO_SETTINGS = PromptSettings( + provider_type=LLMProvider.ANTHROPIC, + model="claude-opus-4-8", + temperature=0.0, +) + +_PROTO_TEMPLATE = ChatPromptTemplate.from_messages( + [("system", "You are a grader."), ("human", "{input}")] +) + + +class TestExecutePromptChainStepProtocolPath: + """Protocol path: llm_provider injected — LangChain provider is never called.""" + + def _ev(self, adapter: AsyncMock) -> _StubEvaluator: + return _StubEvaluator(create_config_no_telemetry(), llm_provider=adapter) + + async def test_returns_raw_string_when_parser_type_is_none(self, evaluation_metadata): + ev = self._ev(_make_adapter("plain prose")) + out = await ev.execute_prompt_chain_step( + step_name="raw", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=None, + ) + assert out == "plain prose" + + async def test_parses_clean_json(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(result, _ChainOutput) + assert result.label == "ok" + assert result.score == 7 + + async def test_strips_markdown_fences(self, evaluation_metadata): + fenced = f"```json\n{_CHAIN_JSON}\n```" + ev = self._ev(_make_adapter(fenced)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_trailing_prose(self, evaluation_metadata): + with_prose = f"{_CHAIN_JSON}\n\nHere is my reasoning for this score." + ev = self._ev(_make_adapter(with_prose)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_strips_leading_prose(self, evaluation_metadata): + with_prefix = f"Here is the result:\n{_CHAIN_JSON}" + ev = self._ev(_make_adapter(with_prefix)) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert result.label == "ok" + + async def test_json_dict_normalizer_path(self, evaluation_metadata): + class _Out(BaseModel): + n: int + doubled: int + + ev = self._ev(_make_adapter('{"n": 3}')) + result = await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: {**d, "doubled": d["n"] * 2}, + ) + assert result.n == 3 + assert result.doubled == 6 + + async def test_non_dict_json_in_normalizer_path_raises_output_validation_error( + self, evaluation_metadata + ): + class _Out(BaseModel): + n: int + + ev = self._ev(_make_adapter('["not", "an", "object"]')) + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_Out, + json_dict_normalizer=lambda d: d, + ) + assert "JSON object" in str(exc_info.value) + + async def test_malformed_json_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter("not json at all")) + with pytest.raises(OutputValidationError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_schema_mismatch_raises_output_validation_error(self, evaluation_metadata): + ev = self._ev(_make_adapter('{"label": "only"}')) # missing required `score` + with pytest.raises(OutputValidationError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, PydanticValidationError) + + async def test_token_usage_recorded_in_step_extras_and_total(self, evaluation_metadata): + ev = self._ev( + _make_adapter(_CHAIN_JSON, model="claude-opus-4-8", input_tokens=42, output_tokens=17) + ) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + step = evaluation_metadata.step_details["main"] + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["input_tokens"] == 42 + assert step.extras[PROMPT_STEP_EXTRA_TOKEN_USAGE]["output_tokens"] == 17 + assert evaluation_metadata.total_token_usage[LLMProvider.ANTHROPIC].input_tokens == 42 + + async def test_token_usage_absent_when_llm_response_has_none_tokens(self, evaluation_metadata): + ev = self._ev(_make_adapter(_CHAIN_JSON, input_tokens=None, output_tokens=None)) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert not evaluation_metadata.total_token_usage + + async def test_provider_error_wrapped_as_api_error(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=RuntimeError("network timeout")) + ev = self._ev(adapter) + with pytest.raises(APIError) as exc_info: + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + async def test_evaluator_error_from_provider_reraises_unchanged(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=EvaluatorError("already wrapped")) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError, match="already wrapped"): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_keyboard_interrupt_from_provider_propagates(self, evaluation_metadata): + adapter = AsyncMock() + adapter.generate = AsyncMock(side_effect=KeyboardInterrupt) + ev = self._ev(adapter) + with pytest.raises(KeyboardInterrupt): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + + async def test_adapter_called_with_formatted_system_and_human(self, evaluation_metadata): + """Template formatting actually reaches the adapter with the correct strings.""" + from unittest.mock import ANY + + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with( + system="You are a grader.", + human="Hello", + config=ANY, + ) + + async def test_human_only_template_passes_empty_system(self, evaluation_metadata): + """Templates with no system turn pass empty string to the adapter without error.""" + from unittest.mock import ANY + + human_only = ChatPromptTemplate.from_messages([("human", "{input}")]) + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=human_only, + chain_inputs={"input": "Hello"}, + parser_output_type=_ChainOutput, + ) + adapter.generate.assert_awaited_once_with(system="", human="Hello", config=ANY) + + async def test_template_with_missing_variable_raises_evaluator_error(self, evaluation_metadata): + """A missing template variable becomes an EvaluatorError, not a bare KeyError.""" + adapter = _make_adapter(_CHAIN_JSON) + ev = self._ev(adapter) + with pytest.raises(EvaluatorError): + await ev.execute_prompt_chain_step( + step_name="main", + prompt_settings=_PROTO_SETTINGS, + evaluation_metadata=evaluation_metadata, + template=_PROTO_TEMPLATE, + chain_inputs={}, # missing required "input" variable + parser_output_type=_ChainOutput, + ) diff --git a/sdks/python/tests/schemas/test_llm_provider.py b/sdks/python/tests/schemas/test_llm_provider.py new file mode 100644 index 00000000..b0782097 --- /dev/null +++ b/sdks/python/tests/schemas/test_llm_provider.py @@ -0,0 +1,105 @@ +"""Tests for LLMGeneratorProtocol, LLMResponse, and GenerateConfig.""" + +from __future__ import annotations + +import learning_commons_evaluators +from learning_commons_evaluators.schemas.llm_provider import ( + GenerateConfig, + LLMGeneratorProtocol, + LLMResponse, +) + + +class TestLLMResponse: + def test_required_fields(self): + r = LLMResponse(content="hello", model="anthropic/claude-opus-4-8") + assert r.content == "hello" + assert r.model == "anthropic/claude-opus-4-8" + assert r.input_tokens is None + assert r.output_tokens is None + + def test_optional_token_fields(self): + r = LLMResponse(content="text", model="test", input_tokens=100, output_tokens=50) + assert r.input_tokens == 100 + assert r.output_tokens == 50 + + +class TestGenerateConfig: + def test_defaults(self): + c = GenerateConfig() + assert c.temperature is None + assert c.max_tokens is None + + def test_with_values(self): + c = GenerateConfig(temperature=0.0, max_tokens=512) + assert c.temperature == 0.0 + assert c.max_tokens == 512 + + +class TestLLMGeneratorProtocol: + async def test_generate_returns_llm_response(self): + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + return LLMResponse(content="hi", model="m", input_tokens=5, output_tokens=2) + + result = await Adapter().generate(system="sys", human="hello") + assert result.content == "hi" + assert result.model == "m" + assert result.input_tokens == 5 + + async def test_generate_config_passed_through(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + cfg = GenerateConfig(temperature=0.0, max_tokens=256) + await Adapter().generate(system="sys", human="hello", config=cfg) + assert received[0] is cfg + assert received[0].temperature == 0.0 + + async def test_generate_config_none_by_default(self): + received: list[GenerateConfig | None] = [] + + class Adapter: + async def generate( + self, *, system: str, human: str, config: GenerateConfig | None = None + ) -> LLMResponse: + received.append(config) + return LLMResponse(content="", model="test") + + await Adapter().generate(system="sys", human="hello") + assert received[0] is None + + def test_structural_conformance(self): + """Static type checkers validate this assignment — no subclassing required. + + LLMGeneratorProtocol is not @runtime_checkable; isinstance() is not available. + Conformance is enforced at type-check time (mypy/pyright) by the annotation below. + """ + + class Adapter: + async def generate( + self, + *, + system: str, + human: str, + config: GenerateConfig | None = None, + ) -> LLMResponse: + return LLMResponse(content="", model="test") + + # mypy/pyright validate this structurally + _adapter: LLMGeneratorProtocol = Adapter() + assert _adapter is not None # runtime no-op; static check is the value + + +def test_exported_from_package(): + assert "GenerateConfig" in learning_commons_evaluators.__all__ + assert "LLMGeneratorProtocol" in learning_commons_evaluators.__all__ + assert "LLMResponse" in learning_commons_evaluators.__all__