Skip to content

refactor(SimplifyEvaluatorCreation): simplify evaluator creation #1245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akshaylive merged 1 commit into from
Feb 2, 2026
+21 −152
Merged

refactor(SimplifyEvaluatorCreation): simplify evaluator creation #1245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "uipath"
version = "2.6.28"
version = "2.6.29"
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
readme = { file = "README.md", content-type = "text/markdown" }
requires-python = ">=3.11"
Expand Down
169 changes: 19 additions & 150 deletions src/uipath/_cli/_evals/_evaluator_factory.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,20 @@

logger = logging.getLogger(__name__)

EVALUATOR_SCHEMA_TO_EVALUATOR_CLASS = {
ContainsEvaluatorConfig: ContainsEvaluator,
ExactMatchEvaluatorConfig: ExactMatchEvaluator,
JsonSimilarityEvaluatorConfig: JsonSimilarityEvaluator,
LLMJudgeOutputEvaluatorConfig: LLMJudgeOutputEvaluator,
LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig: LLMJudgeStrictJSONSimilarityOutputEvaluator,
LLMJudgeTrajectoryEvaluatorConfig: LLMJudgeTrajectoryEvaluator,
LLMJudgeTrajectorySimulationEvaluatorConfig: LLMJudgeTrajectorySimulationEvaluator,
ToolCallArgsEvaluatorConfig: ToolCallArgsEvaluator,
ToolCallCountEvaluatorConfig: ToolCallCountEvaluator,
ToolCallOrderEvaluatorConfig: ToolCallOrderEvaluator,
ToolCallOutputEvaluatorConfig: ToolCallOutputEvaluator,
}


class EvaluatorFactory:
"""Factory class for creating evaluator instances based on configuration."""
Expand Down Expand Up @@ -137,50 +151,15 @@ def _create_evaluator_internal(
data, file_path, class_name, evaluators_dir
)

# use built-in evaluators
config: BaseEvaluatorConfig[Any] = TypeAdapter(EvaluatorConfig).validate_python(
data
)
match config:
case ContainsEvaluatorConfig():
return EvaluatorFactory._create_contains_evaluator(data)
case ExactMatchEvaluatorConfig():
return EvaluatorFactory._create_exact_match_evaluator(data)
case JsonSimilarityEvaluatorConfig():
return EvaluatorFactory._create_json_similarity_evaluator(data)
case LLMJudgeOutputEvaluatorConfig():
return EvaluatorFactory._create_llm_judge_output_evaluator(data)
case LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig():
return EvaluatorFactory._create_llm_judge_strict_json_similarity_output_evaluator(
data
)
case LLMJudgeTrajectoryEvaluatorConfig():
return EvaluatorFactory._create_trajectory_evaluator(data)
case ToolCallArgsEvaluatorConfig():
return EvaluatorFactory._create_tool_call_args_evaluator(data)
case ToolCallCountEvaluatorConfig():
return EvaluatorFactory._create_tool_call_count_evaluator(data)
case ToolCallOrderEvaluatorConfig():
return EvaluatorFactory._create_tool_call_order_evaluator(data)
case ToolCallOutputEvaluatorConfig():
return EvaluatorFactory._create_tool_call_output_evaluator(data)
case LLMJudgeTrajectorySimulationEvaluatorConfig():
return (
EvaluatorFactory._create_llm_judge_simulation_trajectory_evaluator(
data
)
)
case _:
raise ValueError(f"Unknown evaluator configuration: {config}")

@staticmethod
def _create_contains_evaluator(data: dict[str, Any]) -> ContainsEvaluator:
evaluator_id = data.get("id")
if not evaluator_id or not isinstance(evaluator_id, str):
Copy link
Collaborator Author

@akshaylive akshaylive Feb 1, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check is redundant as evaluators have "id" as a required str field

raise ValueError("Evaluator 'id' must be a non-empty string")
return TypeAdapter(ContainsEvaluator).validate_python(
evaluator_class = EVALUATOR_SCHEMA_TO_EVALUATOR_CLASS.get(type(config))
if not evaluator_class:
raise ValueError(f"Unknown evaluator configuration: {config}")
return TypeAdapter(evaluator_class).validate_python(
{
"id": evaluator_id,
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)
Expand Down Expand Up @@ -270,116 +249,6 @@ def _create_coded_evaluator_internal(
}
)

@staticmethod
def _create_exact_match_evaluator(
data: dict[str, Any],
) -> ExactMatchEvaluator:
return TypeAdapter(ExactMatchEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_json_similarity_evaluator(
data: dict[str, Any],
) -> JsonSimilarityEvaluator:
return TypeAdapter(JsonSimilarityEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_llm_judge_output_evaluator(
data: dict[str, Any],
) -> LLMJudgeOutputEvaluator:
return TypeAdapter(LLMJudgeOutputEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_llm_judge_strict_json_similarity_output_evaluator(
data: dict[str, Any],
) -> LLMJudgeStrictJSONSimilarityOutputEvaluator:
return TypeAdapter(LLMJudgeStrictJSONSimilarityOutputEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_trajectory_evaluator(
data: dict[str, Any],
) -> LLMJudgeTrajectoryEvaluator:
return TypeAdapter(LLMJudgeTrajectoryEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_tool_call_args_evaluator(
data: dict[str, Any],
) -> ToolCallArgsEvaluator:
return TypeAdapter(ToolCallArgsEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_tool_call_count_evaluator(
data: dict[str, Any],
) -> ToolCallCountEvaluator:
return TypeAdapter(ToolCallCountEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_tool_call_order_evaluator(
data: dict[str, Any],
) -> ToolCallOrderEvaluator:
return TypeAdapter(ToolCallOrderEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_tool_call_output_evaluator(
data: dict[str, Any],
) -> ToolCallOutputEvaluator:
return TypeAdapter(ToolCallOutputEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_llm_judge_simulation_trajectory_evaluator(
data: dict[str, Any],
) -> LLMJudgeTrajectorySimulationEvaluator:
return TypeAdapter(LLMJudgeTrajectorySimulationEvaluator).validate_python(
{
"id": data.get("id"),
"config": EvaluatorFactory._prepare_evaluator_config(data),
}
)

@staticmethod
def _create_legacy_evaluator_internal(
data: dict[str, Any],
Expand Down
2 changes: 1 addition & 1 deletion uv.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.