From 612a1f8ba139c5ced49095d2ac4b7b0808628d07 Mon Sep 17 00:00:00 2001 From: Fredrick Sisenda Date: Tue, 19 May 2026 21:21:49 -0600 Subject: [PATCH 1/4] feat(python-sdk): validate config --- sdks/python/README.md | 36 ++++++++- .../evaluators/base.py | 12 ++- .../schemas/config.py | 35 ++++++++ sdks/python/tests/conftest.py | 23 +++++- .../contract_tests/test_conventionality.py | 11 +-- .../tests/contract_tests/test_vocabulary.py | 16 ++-- sdks/python/tests/evaluators/test_base.py | 74 +++++++++++++++++ .../tests/evaluators/test_conventionality.py | 20 ++--- .../tests/evaluators/test_vocabulary.py | 73 +++++++---------- sdks/python/tests/schemas/test_config.py | 81 +++++++++++++++++++ 10 files changed, 304 insertions(+), 77 deletions(-) diff --git a/sdks/python/README.md b/sdks/python/README.md index 0b26e378..7fd9c93b 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -230,6 +230,31 @@ config = create_config( ) ``` +### Provider config validation + +When you construct an evaluator (or call `evaluate` / `evaluate_sync`), the SDK checks +that every `LLMProvider` referenced by a `PromptSettings` field on the active +`EvaluationSettings` has a matching provider config on `EvaluatorConfig`. For example, +the vocabulary evaluator’s defaults use both Google and OpenAI, so both +`google_llm_provider_config` and `openai_llm_provider_config` must be set; conventionality +only needs Google. + +Validation runs: + +- In `BaseEvaluator.__init__` against the resolved default evaluation settings (constructor + override or the subclass class attribute). +- At the start of each `evaluate()` call against the settings used for that run (including + per-call `evaluation_settings` overrides). + +If a required provider is missing, construction or evaluation raises `ConfigurationError` +with the same message used at LLM call time (for example, +`Google provider config is not set on EvaluatorConfig`). You can also call +`config.validate_supports_evaluation_settings(settings)` directly before constructing an +evaluator. + +Only providers actually used in the settings object are required — you do not need to +configure every provider on every evaluator. + ### Per-instance default evaluation settings Every `BaseEvaluator` subclass defines **class-level** `default_evaluation_settings` @@ -261,6 +286,11 @@ result = evaluator.evaluate_sync(input) result = evaluator.evaluate_sync(input, evaluation_settings=other_settings) ``` +If `other_settings` references a provider that is not on `config`, `evaluate_sync` raises +`ConfigurationError` before any LLM call. The same applies when you pass +`default_evaluation_settings` at construction: every provider in those settings must be +configured on `config`. + If you omit `default_evaluation_settings` at construction, attribute lookup uses the subclass class attribute, same as before. Whenever you call `evaluate_sync()` or `await evaluator.evaluate(...)` without `evaluation_settings`, the SDK uses @@ -304,7 +334,7 @@ config = create_config_no_telemetry(logger=create_silent_logger()) ```python from learning_commons_evaluators import ( - ConfigurationError, # Missing/invalid config + ConfigurationError, # Missing provider config for evaluation settings, invalid setup ValidationError, # Invalid input AuthenticationError, # Invalid API keys (401/403) RateLimitError, # Rate limit exceeded (429) - has retry_after @@ -371,6 +401,10 @@ class MyEvaluator(BaseEvaluator[MyInput, EvaluationResult, MySettings]): If you override `__init__` on the subclass, accept the same keyword-only argument and forward it: `super().__init__(config, default_evaluation_settings=default_evaluation_settings)`. +Declare each prompt step as a `PromptSettings` field on your settings model (typically named +`prompt_settings_*`). The base class uses those fields to determine which provider configs +must be present on `EvaluatorConfig`. + ## License MIT diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 2ec69c05..3463bc30 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -60,6 +60,10 @@ class BaseEvaluator(ABC, Generic[InputT, OutputT, SettingsT]): Pass ``default_evaluation_settings`` at construction to override the class-level defaults for that instance (used when :meth:`evaluate` is called without ``evaluation_settings``). + + Raises: + ConfigurationError: Default evaluation settings require an LLM provider + that is not configured on ``config``. """ config: EvaluatorConfig @@ -75,7 +79,12 @@ def __init__( self.config = config if default_evaluation_settings is not None: self.default_evaluation_settings = default_evaluation_settings - # TODO: validate config + settings_for_validation = ( + default_evaluation_settings + if default_evaluation_settings is not None + else self.__class__.default_evaluation_settings + ) + config.validate_supports_evaluation_settings(settings_for_validation) async def evaluate( self, @@ -115,6 +124,7 @@ async def evaluate( """ if evaluation_settings is None: evaluation_settings = self.default_evaluation_settings.model_copy(deep=True) + self.config.validate_supports_evaluation_settings(evaluation_settings) start = time.perf_counter() evaluation_metadata = EvaluationMetadata( evaluator_metadata=self.metadata, diff --git a/sdks/python/src/learning_commons_evaluators/schemas/config.py b/sdks/python/src/learning_commons_evaluators/schemas/config.py index 8d1d035c..f5710d12 100644 --- a/sdks/python/src/learning_commons_evaluators/schemas/config.py +++ b/sdks/python/src/learning_commons_evaluators/schemas/config.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict from learning_commons_evaluators.logger import Logger, get_logger +from learning_commons_evaluators.schemas.errors import ConfigurationError # --- LLM provider configs (for LLM calls in prompt steps) --- @@ -75,6 +76,29 @@ class EvaluationSettings(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) +_PROVIDER_CONFIG_ATTR: dict[LLMProvider, str] = { + LLMProvider.GOOGLE: "google_llm_provider_config", + LLMProvider.OPENAI: "openai_llm_provider_config", + LLMProvider.ANTHROPIC: "anthropic_llm_provider_config", +} + +_PROVIDER_MISSING_MESSAGE: dict[LLMProvider, str] = { + LLMProvider.GOOGLE: "Google provider config is not set on EvaluatorConfig", + LLMProvider.OPENAI: "OpenAI provider config is not set on EvaluatorConfig", + LLMProvider.ANTHROPIC: "Anthropic provider config is not set on EvaluatorConfig", +} + + +def _required_llm_providers(settings: EvaluationSettings) -> set[LLMProvider]: + """Collect LLM providers referenced by PromptSettings fields on settings.""" + providers: set[LLMProvider] = set() + for name in type(settings).model_fields: + value = getattr(settings, name) + if isinstance(value, PromptSettings): + providers.add(value.provider_type) + return providers + + @dataclass(frozen=True) class TelemetryConfig: """Config for telemetry.""" @@ -106,6 +130,17 @@ class EvaluatorConfig: logger: Logger = field(default_factory=get_logger) telemetry: TelemetryConfig = field(default_factory=TelemetryConfig) + def validate_supports_evaluation_settings(self, settings: EvaluationSettings) -> None: + """Raise ConfigurationError if settings require an LLM provider not configured on self.""" + required = _required_llm_providers(settings) + missing_messages = [ + _PROVIDER_MISSING_MESSAGE[provider] + for provider in sorted(required, key=lambda p: p.value) + if getattr(self, _PROVIDER_CONFIG_ATTR[provider]) is None + ] + if missing_messages: + raise ConfigurationError("; ".join(missing_messages)) + def create_config( *, diff --git a/sdks/python/tests/conftest.py b/sdks/python/tests/conftest.py index 34951ece..03e16191 100644 --- a/sdks/python/tests/conftest.py +++ b/sdks/python/tests/conftest.py @@ -3,7 +3,11 @@ import pytest from learning_commons_evaluators import create_config_no_telemetry -from learning_commons_evaluators.schemas.config import EvaluationSettings +from learning_commons_evaluators.schemas.config import ( + EvaluationSettings, + GoogleLLMProviderConfig, + OpenAILLMProviderConfig, +) from learning_commons_evaluators.schemas.metadata import ( EvaluationMetadata, EvaluatorMaturity, @@ -42,3 +46,20 @@ def evaluation_metadata(evaluator_metadata): def config(): """EvaluatorConfig with no telemetry, suitable for unit tests.""" return create_config_no_telemetry() + + +@pytest.fixture +def config_with_google(): + """EvaluatorConfig with Google provider set (conventionality and similar evaluators).""" + return create_config_no_telemetry( + google_llm_provider_config=GoogleLLMProviderConfig(api_key="test-google-key"), + ) + + +@pytest.fixture +def config_with_google_and_openai(): + """EvaluatorConfig with Google and OpenAI providers set (vocabulary evaluator).""" + return create_config_no_telemetry( + google_llm_provider_config=GoogleLLMProviderConfig(api_key="test-google-key"), + openai_llm_provider_config=OpenAILLMProviderConfig(api_key="test-openai-key"), + ) diff --git a/sdks/python/tests/contract_tests/test_conventionality.py b/sdks/python/tests/contract_tests/test_conventionality.py index 128f772c..ab2170ca 100644 --- a/sdks/python/tests/contract_tests/test_conventionality.py +++ b/sdks/python/tests/contract_tests/test_conventionality.py @@ -23,11 +23,7 @@ copy. """ -from learning_commons_evaluators import ( - ConventionalityEvaluationInput, - ConventionalityEvaluator, - create_config_no_telemetry, -) +from learning_commons_evaluators import ConventionalityEvaluationInput, ConventionalityEvaluator from learning_commons_evaluators.schemas.metadata import Status from .conventionality import ( @@ -38,7 +34,7 @@ class TestConventionalityContract: - def test_turnip_grade4(self) -> None: + def test_turnip_grade4(self, config_with_google) -> None: """Turnip classroom narrative, grade 4. Verifies: @@ -49,8 +45,7 @@ def test_turnip_grade4(self) -> None: """ case = load_conventionality_turnip_case() - config = create_config_no_telemetry() - evaluator = ConventionalityEvaluator(config) + evaluator = ConventionalityEvaluator(config_with_google) inp = ConventionalityEvaluationInput( text=case.input["text"], grade=case.input["grade"], diff --git a/sdks/python/tests/contract_tests/test_vocabulary.py b/sdks/python/tests/contract_tests/test_vocabulary.py index bfcd88da..2a8cc581 100644 --- a/sdks/python/tests/contract_tests/test_vocabulary.py +++ b/sdks/python/tests/contract_tests/test_vocabulary.py @@ -32,11 +32,7 @@ the ``user_prompt`` and ``llm_response`` fields need to be populated. """ -from learning_commons_evaluators import ( - VocabularyEvaluationInput, - VocabularyEvaluator, - create_config_no_telemetry, -) +from learning_commons_evaluators import VocabularyEvaluationInput, VocabularyEvaluator from learning_commons_evaluators.schemas.metadata import Status from .harness import ContractTestHarness @@ -49,7 +45,7 @@ class TestVocabularyContractGrades34: - def test_marco_polo_grade3(self) -> None: + def test_marco_polo_grade3(self, config_with_google_and_openai) -> None: """Marco Polo passage, grade 3 — grades 3–4 Gemini path. Verifies: @@ -60,8 +56,7 @@ def test_marco_polo_grade3(self) -> None: """ case = load_vocabulary_grade34_case() - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput( text=case.input["text"], grade=case.input["grade"], @@ -94,7 +89,7 @@ def test_marco_polo_grade3(self) -> None: class TestVocabularyContractOtherGrades: - def test_hurricanes_grade7(self) -> None: + def test_hurricanes_grade7(self, config_with_google_and_openai) -> None: """Hurricane formation passage, grade 7 — grades 5–12 GPT path. Verifies: @@ -105,8 +100,7 @@ def test_hurricanes_grade7(self) -> None: """ case = load_vocabulary_other_grades_case() - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput( text=case.input["text"], grade=case.input["grade"], diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index 43d123b8..476ee83f 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -35,8 +35,10 @@ from learning_commons_evaluators.schemas.common_inputs import GradeInputField, TextInputField from learning_commons_evaluators.schemas.config import ( EvaluationSettings, + GoogleLLMProviderConfig, LLMProvider, PromptSettings, + create_config_no_telemetry, ) from learning_commons_evaluators.schemas.errors import APIError, EvaluatorError, ValidationError from learning_commons_evaluators.schemas.input_specs import GradeInputSpec, TextInputSpec @@ -97,6 +99,10 @@ class _StubSettings(EvaluationSettings): marker: int = 0 +class _StubSettingsWithPrompt(EvaluationSettings): + prompt_settings_main: PromptSettings + + def _stub_input() -> TextComplexityEvaluationInput: return TextComplexityEvaluationInput( text=TextInputField(spec=TextInputSpec(name="text"), value="hello world"), @@ -155,6 +161,74 @@ def test_omitted_constructor_default_falls_back_to_class_attribute(self, config) ev = _StubEvaluator(config) assert ev.default_evaluation_settings is _StubEvaluator.default_evaluation_settings + def test_init_raises_when_default_settings_need_missing_provider(self, config): + settings = _StubSettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + + class _EvaluatorWithPromptSettings( + BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + ): + metadata = _StubEvaluator.metadata + default_evaluation_settings = settings + + async def evaluate_impl(self, input, evaluation_settings, evaluation_metadata): + raise NotImplementedError + + with pytest.raises(ConfigurationError, match="Google provider config is not set"): + _EvaluatorWithPromptSettings(config) + + def test_init_passes_when_required_provider_configured(self): + config = create_config_no_telemetry( + google_llm_provider_config=GoogleLLMProviderConfig(api_key="test-key"), + ) + settings = _StubSettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + + class _EvaluatorWithPromptSettings( + BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + ): + metadata = _StubEvaluator.metadata + default_evaluation_settings = settings + + async def evaluate_impl(self, input, evaluation_settings, evaluation_metadata): + raise NotImplementedError + + _EvaluatorWithPromptSettings(config) + + +class TestEvaluateConfigValidation: + def test_evaluate_raises_when_override_settings_need_missing_provider(self, config): + settings = _StubSettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + + class _EvaluatorWithPromptSettings( + BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + ): + metadata = _StubEvaluator.metadata + default_evaluation_settings = _StubSettings() + + async def evaluate_impl(self, input, evaluation_settings, evaluation_metadata): + raise NotImplementedError + + ev = _EvaluatorWithPromptSettings(config) + with pytest.raises(ConfigurationError, match="Google provider config is not set"): + ev.evaluate_sync(_stub_input(), evaluation_settings=settings) + # --------------------------------------------------------------------------- # evaluate() diff --git a/sdks/python/tests/evaluators/test_conventionality.py b/sdks/python/tests/evaluators/test_conventionality.py index ca0b2ef2..83cfa46b 100644 --- a/sdks/python/tests/evaluators/test_conventionality.py +++ b/sdks/python/tests/evaluators/test_conventionality.py @@ -4,11 +4,7 @@ import pytest -from learning_commons_evaluators import ( - ConventionalityEvaluationInput, - ConventionalityEvaluator, - create_config_no_telemetry, -) +from learning_commons_evaluators import ConventionalityEvaluationInput, ConventionalityEvaluator from learning_commons_evaluators.schemas.conventionality import ConventionalityOutput from learning_commons_evaluators.schemas.errors import ConfigurationError from learning_commons_evaluators.schemas.metadata import Status @@ -33,9 +29,8 @@ def _make_mock_output(): class TestConventionalityEvaluator: - def test_evaluate_returns_evaluation_result(self): - config = create_config_no_telemetry() - evaluator = ConventionalityEvaluator(config) + def test_evaluate_returns_evaluation_result(self, config_with_google): + evaluator = ConventionalityEvaluator(config_with_google) inp = ConventionalityEvaluationInput(text=_SAMPLE_TEXT, grade=5) with patch.object(evaluator, "execute_prompt_chain_step", return_value=_make_mock_output()): result = evaluator.evaluate_sync(inp) @@ -45,7 +40,7 @@ def test_evaluate_returns_evaluation_result(self): assert result.metadata.status == Status.succeeded assert result.metadata.evaluator_metadata.id == "conventionality" - def test_evaluate_with_explicit_settings(self): + def test_evaluate_with_explicit_settings(self, config_with_google): from learning_commons_evaluators.schemas.config import ( LLMProvider, PromptSettings, @@ -54,8 +49,7 @@ def test_evaluate_with_explicit_settings(self): ConventionalityEvaluationSettings, ) - config = create_config_no_telemetry() - evaluator = ConventionalityEvaluator(config) + evaluator = ConventionalityEvaluator(config_with_google) settings = ConventionalityEvaluationSettings( prompt_settings_step_conventionality_evaluation=PromptSettings( provider_type=LLMProvider.GOOGLE, @@ -68,8 +62,8 @@ def test_evaluate_with_explicit_settings(self): result = evaluator.evaluate_sync(inp, evaluation_settings=settings) assert result.metadata.status == Status.succeeded - def test_metadata_and_default_settings(self): - evaluator = ConventionalityEvaluator(create_config_no_telemetry()) + def test_metadata_and_default_settings(self, config_with_google): + evaluator = ConventionalityEvaluator(config_with_google) assert evaluator.metadata.id == "conventionality" assert evaluator.metadata.version == "0.1" assert evaluator.default_evaluation_settings is not None diff --git a/sdks/python/tests/evaluators/test_vocabulary.py b/sdks/python/tests/evaluators/test_vocabulary.py index 9a7d5d92..79b5ac94 100644 --- a/sdks/python/tests/evaluators/test_vocabulary.py +++ b/sdks/python/tests/evaluators/test_vocabulary.py @@ -4,11 +4,7 @@ import pytest -from learning_commons_evaluators import ( - VocabularyEvaluationInput, - VocabularyEvaluator, - create_config_no_telemetry, -) +from learning_commons_evaluators import VocabularyEvaluationInput, VocabularyEvaluator from learning_commons_evaluators.schemas.errors import ConfigurationError, ValidationError from learning_commons_evaluators.schemas.metadata import Status from learning_commons_evaluators.schemas.vocabulary import ( @@ -74,9 +70,8 @@ def _patch_steps(evaluator, bk_return, vocab_return): class TestVocabularyEvaluatorGrades34: - def test_evaluate_grade_3_returns_result(self): - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + def test_evaluate_grade_3_returns_result(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=3) with _patch_steps(evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_grades34_output()): result = evaluator.evaluate_sync(inp) @@ -86,9 +81,8 @@ def test_evaluate_grade_3_returns_result(self): assert result.metadata.status == Status.succeeded assert "tier_2_words" in result.explanation.details - def test_evaluate_grade_4_returns_result(self): - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + def test_evaluate_grade_4_returns_result(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=4) with _patch_steps( evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_grades34_output("very_complex") @@ -97,10 +91,9 @@ def test_evaluate_grade_4_returns_result(self): assert result.answer.score == "very_complex" - def test_grades34_score_with_spaces_is_normalised(self): + def test_grades34_score_with_spaces_is_normalised(self, config_with_google_and_openai): """The grades 3–4 prompt may return "slightly complex" (spaces); normalise to underscores.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=3) # The evaluator calls .lower().replace(" ", "_") before from_score(), # so we feed a space-separated label and assert it survives the path. @@ -111,9 +104,8 @@ def test_grades34_score_with_spaces_is_normalised(self): assert result.answer.score == "slightly_complex" - def test_evaluate_grades34_explanation_has_word_breakdown(self): - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + def test_evaluate_grades34_explanation_has_word_breakdown(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=3) with _patch_steps(evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_grades34_output()): result = evaluator.evaluate_sync(inp) @@ -138,10 +130,11 @@ class TestVocabularyEvaluatorOtherGrades: (4, "exceedingly_complex"), ], ) - def test_all_complexity_scores_map_correctly(self, score_label, expected_score): + def test_all_complexity_scores_map_correctly( + self, score_label, expected_score, config_with_google_and_openai + ): """Each complexity label (passed as convenience int 1–4) maps to the right score.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=7) with _patch_steps( evaluator, @@ -152,9 +145,8 @@ def test_all_complexity_scores_map_correctly(self, score_label, expected_score): assert result.answer.score == expected_score - def test_evaluate_grade_12_returns_result(self): - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + def test_evaluate_grade_12_returns_result(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=12) with _patch_steps(evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_other_grades_output(1)): result = evaluator.evaluate_sync(inp) @@ -162,10 +154,9 @@ def test_evaluate_grade_12_returns_result(self): assert result.metadata.status == Status.succeeded assert result.answer.score == "slightly_complex" - def test_other_grades_explanation_includes_word_breakdown(self): + def test_other_grades_explanation_includes_word_breakdown(self, config_with_google_and_openai): """Grades 5–12 mirror the notebook: word lists live in ``explanation.details``.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=8) with _patch_steps(evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_other_grades_output(2)): result = evaluator.evaluate_sync(inp) @@ -193,10 +184,9 @@ def test_other_grades_legacy_string_digit_answer(self): ) assert parsed.complexity_score == "Moderately Complex" - def test_other_grades_unexpected_digit_answer_raises(self): + def test_other_grades_unexpected_digit_answer_raises(self, config_with_google_and_openai): """Out-of-range rubric digit normalizes to a bare string; ``from_score`` rejects it.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=7) # Same ``complexity_score`` as ``normalize_complexity_output({"answer": 9, ...})``. unexpected = VocabularyComplexityOutput( @@ -253,19 +243,19 @@ def test_allowed_grades_set_from_toml(self): assert set(inp.grade.spec.allowed_grades) == frozenset(range(3, 13)) @pytest.mark.parametrize("unsupported_grade", [0, 1, 2]) - def test_unsupported_grade_raises_via_framework(self, unsupported_grade): + def test_unsupported_grade_raises_via_framework( + self, unsupported_grade, config_with_google_and_openai + ): """BaseEvaluator.evaluate_sync() calls input.validate(), which catches the bad grade.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=unsupported_grade) # The base evaluator catches the ValidationError, sets status=failed, then re-raises. with pytest.raises(ValidationError): evaluator.evaluate_sync(inp) - def test_unsupported_grade_sets_status_failed(self): + def test_unsupported_grade_sets_status_failed(self, config_with_google_and_openai): """Metadata status is set to failed when grade validation fails.""" - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=2) with pytest.raises(ValidationError): evaluator.evaluate_sync(inp) @@ -275,21 +265,20 @@ def test_unsupported_grade_sets_status_failed(self): class TestVocabularyEvaluatorMetadata: - def test_evaluator_metadata(self): - evaluator = VocabularyEvaluator(create_config_no_telemetry()) + def test_evaluator_metadata(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) assert evaluator.metadata.id == "vocabulary" assert evaluator.metadata.version == "0.1" - def test_default_settings_has_all_prompt_steps(self): - evaluator = VocabularyEvaluator(create_config_no_telemetry()) + def test_default_settings_has_all_prompt_steps(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) settings = evaluator.default_evaluation_settings assert settings.prompt_settings_step_background_knowledge is not None assert settings.prompt_settings_step_vocab_grades_3_4 is not None assert settings.prompt_settings_step_vocab_other_grades is not None - def test_evaluate_succeeds_and_records_metadata(self): - config = create_config_no_telemetry() - evaluator = VocabularyEvaluator(config) + def test_evaluate_succeeds_and_records_metadata(self, config_with_google_and_openai): + evaluator = VocabularyEvaluator(config_with_google_and_openai) inp = VocabularyEvaluationInput(text=_SAMPLE_TEXT, grade=5) with _patch_steps(evaluator, _MOCK_BACKGROUND_KNOWLEDGE, _make_other_grades_output(2)): result = evaluator.evaluate_sync(inp) diff --git a/sdks/python/tests/schemas/test_config.py b/sdks/python/tests/schemas/test_config.py index 92942f4b..3018d804 100644 --- a/sdks/python/tests/schemas/test_config.py +++ b/sdks/python/tests/schemas/test_config.py @@ -7,14 +7,17 @@ from learning_commons_evaluators.logger import SDK_LOGGER_NAME, get_logger from learning_commons_evaluators.schemas.config import ( AnthropicLLMProviderConfig, + EvaluationSettings, GoogleLLMProviderConfig, LLMProvider, OpenAILLMProviderConfig, + PromptSettings, TelemetryConfig, create_config, create_config_no_telemetry, create_config_telemetry_with_full_input, ) +from learning_commons_evaluators.schemas.errors import ConfigurationError class TestLLMProvider: @@ -82,3 +85,81 @@ def test_config_is_frozen(self): config.telemetry = TelemetryConfig( telemetry_partner_id="x", send_full_input_with_telemetry=False ) + + +class _SettingsWithPrompt(EvaluationSettings): + prompt_settings_main: PromptSettings + + +class _SettingsWithTwoPrompts(EvaluationSettings): + prompt_settings_a: PromptSettings + prompt_settings_b: PromptSettings + + +class TestValidateSupportsEvaluationSettings: + def test_no_prompt_settings_is_no_op(self): + config = create_config_no_telemetry() + config.validate_supports_evaluation_settings(EvaluationSettings()) + + def test_missing_google_raises(self): + config = create_config_no_telemetry() + settings = _SettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + with pytest.raises(ConfigurationError, match="Google provider config is not set"): + config.validate_supports_evaluation_settings(settings) + + def test_google_configured_passes(self): + config = create_config_no_telemetry( + google_llm_provider_config=GoogleLLMProviderConfig(api_key="gk"), + ) + settings = _SettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + config.validate_supports_evaluation_settings(settings) + + def test_duplicate_provider_in_settings_requires_one_config(self): + config = create_config_no_telemetry( + openai_llm_provider_config=OpenAILLMProviderConfig(api_key="ok"), + ) + settings = _SettingsWithTwoPrompts( + prompt_settings_a=PromptSettings( + provider_type=LLMProvider.OPENAI, + model="gpt-4o", + temperature=0.0, + ), + prompt_settings_b=PromptSettings( + provider_type=LLMProvider.OPENAI, + model="gpt-4.1", + temperature=0.0, + ), + ) + config.validate_supports_evaluation_settings(settings) + + def test_multiple_missing_providers_lists_all(self): + config = create_config_no_telemetry() + settings = _SettingsWithTwoPrompts( + prompt_settings_a=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ), + prompt_settings_b=PromptSettings( + provider_type=LLMProvider.OPENAI, + model="gpt-4o", + temperature=0.0, + ), + ) + with pytest.raises(ConfigurationError) as exc_info: + config.validate_supports_evaluation_settings(settings) + msg = str(exc_info.value) + assert "Google provider config is not set" in msg + assert "OpenAI provider config is not set" in msg From fb2882e03e17bec15ac18b8fbbf48965d0b3a8c4 Mon Sep 17 00:00:00 2001 From: Fredrick Sisenda Date: Tue, 19 May 2026 21:27:12 -0600 Subject: [PATCH 2/4] style: linting --- sdks/python/tests/evaluators/test_base.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index 476ee83f..d5e59b68 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -38,7 +38,6 @@ GoogleLLMProviderConfig, LLMProvider, PromptSettings, - create_config_no_telemetry, ) from learning_commons_evaluators.schemas.errors import APIError, EvaluatorError, ValidationError from learning_commons_evaluators.schemas.input_specs import GradeInputSpec, TextInputSpec @@ -171,7 +170,9 @@ def test_init_raises_when_default_settings_need_missing_provider(self, config): ) class _EvaluatorWithPromptSettings( - BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + BaseEvaluator[ + TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt + ] ): metadata = _StubEvaluator.metadata default_evaluation_settings = settings @@ -195,7 +196,9 @@ def test_init_passes_when_required_provider_configured(self): ) class _EvaluatorWithPromptSettings( - BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + BaseEvaluator[ + TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt + ] ): metadata = _StubEvaluator.metadata default_evaluation_settings = settings @@ -217,7 +220,9 @@ def test_evaluate_raises_when_override_settings_need_missing_provider(self, conf ) class _EvaluatorWithPromptSettings( - BaseEvaluator[TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt] + BaseEvaluator[ + TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt + ] ): metadata = _StubEvaluator.metadata default_evaluation_settings = _StubSettings() From 38f8c893cfb3bfe0af301fc7c188022806e50472 Mon Sep 17 00:00:00 2001 From: Fred Sisenda Date: Tue, 19 May 2026 21:53:33 -0600 Subject: [PATCH 3/4] chore: address PR comments --- .../evaluators/base.py | 2 +- sdks/python/tests/evaluators/test_base.py | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 3463bc30..73678745 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -124,7 +124,6 @@ async def evaluate( """ if evaluation_settings is None: evaluation_settings = self.default_evaluation_settings.model_copy(deep=True) - self.config.validate_supports_evaluation_settings(evaluation_settings) start = time.perf_counter() evaluation_metadata = EvaluationMetadata( evaluator_metadata=self.metadata, @@ -136,6 +135,7 @@ async def evaluate( extra={"evaluation_metadata": evaluation_metadata}, ) try: + self.config.validate_supports_evaluation_settings(evaluation_settings) input.validate() result = await self.evaluate_impl(input, evaluation_settings, evaluation_metadata) evaluation_metadata.status = Status.succeeded diff --git a/sdks/python/tests/evaluators/test_base.py b/sdks/python/tests/evaluators/test_base.py index d5e59b68..1e79a117 100644 --- a/sdks/python/tests/evaluators/test_base.py +++ b/sdks/python/tests/evaluators/test_base.py @@ -355,6 +355,48 @@ def emit(self, record: logging.LogRecord) -> None: assert captured[-1].status == Status.failed assert captured[-1].error_details + def test_configuration_failure_emits_end_log_with_failed_status(self, config): + captured: list = [] + + class _Capture(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + meta = getattr(record, "evaluation_metadata", None) + if meta is not None and record.getMessage() == "evaluation end": + captured.append(meta) + + settings = _StubSettingsWithPrompt( + prompt_settings_main=PromptSettings( + provider_type=LLMProvider.GOOGLE, + model="gemini-2.0-flash", + temperature=0.0, + ) + ) + + class _EvaluatorWithPromptSettings( + BaseEvaluator[ + TextComplexityEvaluationInput, TextComplexityResult, _StubSettingsWithPrompt + ] + ): + metadata = _StubEvaluator.metadata + default_evaluation_settings = _StubSettings() + + async def evaluate_impl(self, input, evaluation_settings, evaluation_metadata): + raise NotImplementedError + + ev = _EvaluatorWithPromptSettings(config) + h = _Capture() + ev.config.logger.addHandler(h) + ev.config.logger.setLevel(logging.INFO) + try: + with pytest.raises(ConfigurationError, match="Google provider config is not set"): + ev.evaluate_sync(_stub_input(), evaluation_settings=settings) + finally: + ev.config.logger.removeHandler(h) + + assert captured + assert captured[-1].status == Status.failed + assert "Google provider config" in captured[-1].error_details + # --------------------------------------------------------------------------- # update_total_token_usage From 8d6d7f89895eaddc29ac255ecca056aba46f473f Mon Sep 17 00:00:00 2001 From: Fredrick Sisenda Date: Tue, 16 Jun 2026 10:42:36 -0700 Subject: [PATCH 4/4] chore: updating tests --- .../test_grade_level_appropriateness.py | 6 ++---- .../test_grade_level_appropriateness.py | 15 ++++++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/sdks/python/tests/contract_tests/test_grade_level_appropriateness.py b/sdks/python/tests/contract_tests/test_grade_level_appropriateness.py index 2f495210..e2e9bcc2 100644 --- a/sdks/python/tests/contract_tests/test_grade_level_appropriateness.py +++ b/sdks/python/tests/contract_tests/test_grade_level_appropriateness.py @@ -26,7 +26,6 @@ from learning_commons_evaluators import ( GradeLevelAppropriatenessEvaluationInput, GradeLevelAppropriatenessEvaluator, - create_config_no_telemetry, ) from learning_commons_evaluators.schemas.metadata import Status @@ -38,7 +37,7 @@ class TestGradeLevelAppropriatenessContract: - def test_trees_passage(self) -> None: + def test_trees_passage(self, config_with_google) -> None: """Simple trees informational passage. Verifies: @@ -49,8 +48,7 @@ def test_trees_passage(self) -> None: """ case = load_gla_trees_case() - config = create_config_no_telemetry() - evaluator = GradeLevelAppropriatenessEvaluator(config) + evaluator = GradeLevelAppropriatenessEvaluator(config_with_google) inp = GradeLevelAppropriatenessEvaluationInput( text=case.input["text"], ) diff --git a/sdks/python/tests/evaluators/test_grade_level_appropriateness.py b/sdks/python/tests/evaluators/test_grade_level_appropriateness.py index 7326cb7b..f66b2ff1 100644 --- a/sdks/python/tests/evaluators/test_grade_level_appropriateness.py +++ b/sdks/python/tests/evaluators/test_grade_level_appropriateness.py @@ -7,7 +7,6 @@ from learning_commons_evaluators import ( GradeLevelAppropriatenessEvaluationInput, GradeLevelAppropriatenessEvaluator, - create_config_no_telemetry, ) from learning_commons_evaluators.schemas.errors import ConfigurationError from learning_commons_evaluators.schemas.grade_level_appropriateness import ( @@ -39,9 +38,8 @@ def _make_mock_output(): class TestGradeLevelAppropriatenessEvaluator: - def test_evaluate_returns_evaluation_result(self): - config = create_config_no_telemetry() - evaluator = GradeLevelAppropriatenessEvaluator(config) + def test_evaluate_returns_evaluation_result(self, config_with_google): + evaluator = GradeLevelAppropriatenessEvaluator(config_with_google) inp = GradeLevelAppropriatenessEvaluationInput(text=_SAMPLE_TEXT) with patch.object(evaluator, "execute_prompt_chain_step", return_value=_make_mock_output()): result = evaluator.evaluate_sync(inp) @@ -53,7 +51,7 @@ def test_evaluate_returns_evaluation_result(self): assert result.metadata.status == Status.succeeded assert result.metadata.evaluator_metadata.id == "grade-level-appropriateness" - def test_evaluate_with_explicit_settings(self): + def test_evaluate_with_explicit_settings(self, config_with_google): from learning_commons_evaluators.schemas.config import ( LLMProvider, PromptSettings, @@ -62,8 +60,7 @@ def test_evaluate_with_explicit_settings(self): GradeLevelAppropriatenessEvaluationSettings, ) - config = create_config_no_telemetry() - evaluator = GradeLevelAppropriatenessEvaluator(config) + evaluator = GradeLevelAppropriatenessEvaluator(config_with_google) settings = GradeLevelAppropriatenessEvaluationSettings( prompt_settings_step_gla_evaluation=PromptSettings( provider_type=LLMProvider.GOOGLE, @@ -76,8 +73,8 @@ def test_evaluate_with_explicit_settings(self): result = evaluator.evaluate_sync(inp, evaluation_settings=settings) assert result.metadata.status == Status.succeeded - def test_metadata_and_default_settings(self): - evaluator = GradeLevelAppropriatenessEvaluator(create_config_no_telemetry()) + def test_metadata_and_default_settings(self, config_with_google): + evaluator = GradeLevelAppropriatenessEvaluator(config_with_google) assert evaluator.metadata.id == "grade-level-appropriateness" assert evaluator.metadata.version == "0.1" assert evaluator.default_evaluation_settings is not None