diff --git a/evals/prompts/CHANGELOG.md b/evals/CHANGELOG.md similarity index 100% rename from evals/prompts/CHANGELOG.md rename to evals/CHANGELOG.md diff --git a/evals/prompts/README.md b/evals/README_PROMPTS.md similarity index 100% rename from evals/prompts/README.md rename to evals/README_PROMPTS.md diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/acknowledges_strength_evaluator.ipynb b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/acknowledges_strength_evaluator.ipynb new file mode 100644 index 00000000..46d93263 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/acknowledges_strength_evaluator.ipynb @@ -0,0 +1,791 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Acknowledges Strength Evaluator\n", + "\n", + "**The Acknowledges Strength Evaluator** assesses whether a piece of feedback names something specific and authentic the student did well — distinct from generic praise (\"good job!\"), effusive praise unanchored to the response, or feedback that omits acknowledgment of strength entirely. The Evaluator is anchored in the Productive Coaching rubric developed by Quill.org and Leanlab Education, in partnership with Anastasiya A Lipnevich. Given a student response and the teacher feedback on it, the evaluator returns a structured output that includes:\n", + "\n", + "* **acknowledges_strength_score**: A binary judgment (1 / 0) for whether the feedback acknowledges student strength specifically and authentically. A 1 applies when feedback names a specific, accurate strength tied to what the student wrote — or when the student's response doesn't warrant acknowledgment (e.g., \"IDK\") and the feedback appropriately refrains from praise. A 0 signals feedback with generic, unanchored, or inaccurate praise.\n", + "* **reasoning**: A high-level summary of why the feedback received this judgment.\n", + "* **key_features**: One entry per factor driving the judgment (presence of praise, specificity, anchoring to evidence, process vs. trait framing), each with whether it was `met` (1 / 0) and a `justification`.\n", + "* **proposed_adjustment**: Suggested moves to strengthen the feedback's acknowledgment (for developers iterating on prompts)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-openai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List, Literal \n", + "from enum import Enum\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"OPENAI_API_KEY\" not in os.environ:\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded acknowledges_strength from /Users/achi/Documents/evaluators/evals/feedback/writing-feedback/acknowledges-strength/prompts\n", + " model: gpt-5.4\n", + " temperature: 1\n", + " prompts:\n", + " system system.txt ( 6364 chars, sha d66c26d4780f)\n", + " human user.txt ( 69 chars, sha eaac7e6eadcc)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class AcknowledgesStrengthOutput(BaseModel):\n", + "\n", + " class FeatureJudgment(BaseModel):\n", + " feature: Literal[\n", + " \"presence_of_praise\",\n", + " \"specificity\",\n", + " \"anchoring_to_evidence\",\n", + " \"process_vs_trait_framing\",\n", + " ] = Field(description=\"Which key feature this judgment is for.\")\n", + " met: Literal[0, 1] = Field(\n", + " description=\"1 if this feature is satisfied, 0 if not — tied to the specific student response and feedback.\"\n", + " )\n", + " justification: str = Field(\n", + " description=\"Short explanation of why this feature is or isn't met, tied to the specific response and feedback.\"\n", + " )\n", + "\n", + " reasoning: str = Field(\n", + " description=\"Step-by-step reasoning tied to the specific student response and feedback.\"\n", + " )\n", + " key_features: List[FeatureJudgment] = Field(\n", + " description=\"\"\"One entry per key feature (presence_of_praise, specificity,\n", + " anchoring_to_evidence, process_vs_trait_framing). Each entry has `feature`, `met`\n", + " (1 or 0), and `justification` (tied to the specific response and feedback).\"\"\"\n", + " )\n", + " proposed_adjustment: str = Field(\n", + " description=\"How the feedback could be changed to better meet the criterion, or a brief note that it already meets the criterion.\"\n", + " )\n", + " acknowledges_strength_score: Literal[0, 1] = Field(\n", + " description=\"\"\"A binary judgment for whether the feedback acknowledges student strength specifically and authentically. A 1 applies when feedback names a specific, accurate strength tied to what the student wrote — or when the student's response doesn't warrant acknowledgment (e.g., \"IDK\") and the feedback appropriately refrains from praise. A 0 signals feedback with generic, unanchored, or inaccurate praise.\"\"\"\n", + " )\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\"student_text\", \"feedback_text\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=AcknowledgesStrengthOutput),\n", + "}\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"AcknowledgesStrengthOutput\", AcknowledgesStrengthOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "def is_acknowledges_strength(student_text: str, feedback_text: str):\n", + " \"\"\"\n", + " Evaluate wheter feedback acknowledges strengths in the student's text using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + "\n", + " llm = ChatOpenAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " inputs = {\"student_text\": student_text, \"feedback_text\": feedback_text}\n", + "\n", + " # Step A: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step B: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step C: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': \"You are an expert evaluator of written teacher feedback to students. Your task is to judge how well the teacher feedback satisfies a specific productive-coaching criterion.\\n\\nProductive coaching aims to help students enter productive struggle: work that is cognitively challenging but emotionally manageable, where students improve by grappling with ideas just beyond their current capability.\\n\\nYou will be given:\\n- student_text: the student’s written response\\n- feedback_text: the teacher’s feedback to that student\\n\\nCriterion to evaluate: Is Acknowledges Strength\\n\\nCore question:\\nDoes the feedback identify something useful, promising, or well-done in the student’s response, either explicitly through praise or implicitly by naming a specific part of the student’s response as worth developing?\\n\\nDefinition:\\nFeedback meets this criterion when it appropriately acknowledges a strength or promising element in the student’s work. The acknowledgment must be tied to the specific student response and should not be merely generic praise.\\n\\nImportant calibration:\\n- Explicit praise is not strictly required if the feedback identifies a specific part of the student’s response and treats it as a meaningful starting point for improvement.\\n- For example, feedback such as “What do you mean by ‘etc.’? Can you be more specific? What examples can you give about helping around the house?” can meet the criterion when the student wrote that AI-powered pets “could help around the house etc.” This feedback identifies the unclear phrase “etc.” and the student’s idea about “helping around the house,” then prompts the student to improve it with more specific examples. This counts as acknowledging a developable strength or promising idea.\\n- However, feedback that only gives generic revision directions, asks for clarification, or challenges accuracy without identifying a meaningful strength or promising part of the response does not meet the criterion.\\n- Do not automatically count every request for examples or specificity as meeting the criterion. The feedback must be anchored in a particular part of the student’s response and must treat that part as a useful idea to build on.\\n\\nMeets criterion when:\\n- The feedback specifically identifies what the student did well, such as a clear claim, relevant reason, useful evidence, or promising idea.\\n- The feedback highlights a particular part of the student’s response as worth expanding, clarifying, or developing.\\n- The acknowledgment is specific to this student’s text, not something that could apply to any response.\\n- The feedback may combine acknowledgment with a revision suggestion.\\n- The student response contains no meaningful strength to acknowledge, such as “IDK,” and the feedback appropriately focuses on next steps without false praise.\\n\\nDoes NOT meet criterion when:\\n- The feedback contains no acknowledgment of a strength or promising element despite the student having one.\\n- The feedback only says things like “Good job,” “Nice work,” or “Great” without specifying what was good.\\n- The feedback only asks for clarification, examples, or correction in a generic way.\\n- The feedback only points out errors, possible misrepresentation, or missing reasoning.\\n- The praise is inaccurate, misleading, or not grounded in the student response.\\n\\nKey features to assess independently:\\n1. presence_of_praise\\n - Met = 1 if the feedback includes specific, authentic praise or positively treats a specific student idea as worth developing.\\n - Met = 0 if there is no positive acknowledgment or only generic praise.\\n\\n2. specificity\\n - Met = 1 if the acknowledgment identifies a particular strength, idea, phrase, claim, reason, or feature of the student’s response.\\n - Met = 0 if the acknowledgment is vague or absent.\\n\\n3. anchoring_to_evidence\\n - Met = 1 if the acknowledgment is grounded in something actually present in the student_text.\\n - Met = 0 if it is not tied to the student’s writing, is generic, or refers only to a weakness without treating it as a developable idea.\\n\\n4. process_vs_trait_framing\\n - Met = 1 if the feedback frames the strength in terms of the student’s writing move, thinking process, strategy, effort, or revision path.\\n - Met = 0 if it focuses on fixed traits, gives no process framing, or contains no acknowledgment.\\n\\nCalibration examples:\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they can be a cheaper alternative to real pets.”\\n Feedback: “Can you give an example? Please be more specific and define what you mean by ‘cheaper.’”\\n Correct judgment: answer = 0.\\n Rationale: The feedback asks for elaboration and clarification but does not acknowledge a strength or positively identify the cost idea as a promising point.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine”\\n Feedback: “Be careful not to misrepresent the text. Do AI pets never die? Also, further your thinking; how does having a pet who doesn't need medication provide a good alternative?”\\n Correct judgment: answer = 0.\\n Rationale: The feedback is corrective and probing. It does not identify anything the student did well.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.”\\n Feedback: “What do you mean by ‘etc.’? Can you be more specific? What examples can you give about helping around the house?”\\n Correct judgment: answer = 1.\\n Rationale: The feedback identifies a specific unclear phrase and anchors the revision prompt in the student’s idea about helping around the house. It treats that idea as something worth developing with examples, which counts as acknowledging a promising element.\\n\\nEvaluation procedure:\\n1. Read student_text and feedback_text carefully.\\n2. Identify any strengths or promising elements in the student_text.\\n3. Determine whether feedback_text explicitly or implicitly acknowledges one of those elements.\\n4. Check whether the acknowledgment is specific and grounded in the student’s actual response.\\n5. Score each key feature independently as 1 or 0.\\n6. Provide an overall answer:\\n - answer = 1 if the feedback meets the criterion.\\n - answer = 0 if the feedback does not meet the criterion.\\n7. Score only what is written, not what you think the teacher intended.\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content='{\"reasoning\":\"The student gives a specific reason for why AI-powered pets could be a good alternative: they \\\\\"could help around the house etc.\\\\\" That idea is a meaningful, developable strength in the response. The teacher feedback explicitly acknowledges that specific idea by saying, \\\\\"You\\'re right, the AI pets could help around the house.\\\\\" This is more than generic praise because it names the student’s actual point and validates it as a useful claim. The follow-up question, asking for other details from the article to strengthen the claim, builds on that idea and frames revision as adding support. Because the feedback identifies a promising part of the student’s response, is specific, grounded in the text, and encourages a next step for development, it meets the criterion.\",\"key_features\":{\"presence_of_praise\":{\"met\":1,\"justification\":\"The feedback explicitly affirms a specific student idea: \\\\\"You\\'re right, the AI pets could help around the house.\\\\\" This is authentic acknowledgment, not generic praise.\"},\"specificity\":{\"met\":1,\"justification\":\"The acknowledgment names the particular strength in the response—the claim that AI pets could help around the house—rather than offering vague approval.\"},\"anchoring_to_evidence\":{\"met\":1,\"justification\":\"The feedback is directly tied to wording that appears in the student text: \\\\\"could help around the house.\\\\\" It clearly responds to the student\\'s actual idea.\"},\"process_vs_trait_framing\":{\"met\":1,\"justification\":\"The feedback frames the next step as a writing process move: add more details from the article to strengthen the claim. It focuses on revision and support rather than on fixed traits.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback already meets the criterion. If desired, the teacher could strengthen it further by naming one possible place in the article to look for supporting details.\",\"acknowledges_strength_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 386, 'prompt_tokens': 1711, 'total_tokens': 2097, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-DsDV8YW1qQdsUKKmRHNi806kd4HzP', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019edc63-b33a-7d13-8ec6-001ac2917e86-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 1711, 'output_tokens': 386, 'total_tokens': 2097, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", + " 'raw_text': '{\"reasoning\":\"The student gives a specific reason for why AI-powered pets could be a good alternative: they \\\\\"could help around the house etc.\\\\\" That idea is a meaningful, developable strength in the response. The teacher feedback explicitly acknowledges that specific idea by saying, \\\\\"You\\'re right, the AI pets could help around the house.\\\\\" This is more than generic praise because it names the student’s actual point and validates it as a useful claim. The follow-up question, asking for other details from the article to strengthen the claim, builds on that idea and frames revision as adding support. Because the feedback identifies a promising part of the student’s response, is specific, grounded in the text, and encourages a next step for development, it meets the criterion.\",\"key_features\":{\"presence_of_praise\":{\"met\":1,\"justification\":\"The feedback explicitly affirms a specific student idea: \\\\\"You\\'re right, the AI pets could help around the house.\\\\\" This is authentic acknowledgment, not generic praise.\"},\"specificity\":{\"met\":1,\"justification\":\"The acknowledgment names the particular strength in the response—the claim that AI pets could help around the house—rather than offering vague approval.\"},\"anchoring_to_evidence\":{\"met\":1,\"justification\":\"The feedback is directly tied to wording that appears in the student text: \\\\\"could help around the house.\\\\\" It clearly responds to the student\\'s actual idea.\"},\"process_vs_trait_framing\":{\"met\":1,\"justification\":\"The feedback frames the next step as a writing process move: add more details from the article to strengthen the claim. It focuses on revision and support rather than on fixed traits.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback already meets the criterion. If desired, the teacher could strengthen it further by naming one possible place in the article to look for supporting details.\",\"acknowledges_strength_score\":1}',\n", + " 'formatted_output': {'reasoning': 'The student gives a specific reason for why AI-powered pets could be a good alternative: they \"could help around the house etc.\" That idea is a meaningful, developable strength in the response. The teacher feedback explicitly acknowledges that specific idea by saying, \"You\\'re right, the AI pets could help around the house.\" This is more than generic praise because it names the student’s actual point and validates it as a useful claim. The follow-up question, asking for other details from the article to strengthen the claim, builds on that idea and frames revision as adding support. Because the feedback identifies a promising part of the student’s response, is specific, grounded in the text, and encourages a next step for development, it meets the criterion.',\n", + " 'key_features': {'presence_of_praise': {'met': 1,\n", + " 'justification': 'The feedback explicitly affirms a specific student idea: \"You\\'re right, the AI pets could help around the house.\" This is authentic acknowledgment, not generic praise.'},\n", + " 'specificity': {'met': 1,\n", + " 'justification': 'The acknowledgment names the particular strength in the response—the claim that AI pets could help around the house—rather than offering vague approval.'},\n", + " 'anchoring_to_evidence': {'met': 1,\n", + " 'justification': 'The feedback is directly tied to wording that appears in the student text: \"could help around the house.\" It clearly responds to the student\\'s actual idea.'},\n", + " 'process_vs_trait_framing': {'met': 1,\n", + " 'justification': 'The feedback frames the next step as a writing process move: add more details from the article to strengthen the claim. It focuses on revision and support rather than on fixed traits.'}},\n", + " 'proposed_adjustment': 'No adjustment needed; the feedback already meets the criterion. If desired, the teacher could strengthen it further by naming one possible place in the article to look for supporting details.',\n", + " 'acknowledges_strength_score': 1},\n", + " 'usage': {'input_tokens': 1711,\n", + " 'output_tokens': 386,\n", + " 'total_tokens': 2097,\n", + " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", + "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", + "\n", + "case_input = {\"student_text\": sample_student_text, \"feedback_text\": sample_feedback_text}\n", + "case_output = is_acknowledges_strength(case_input['student_text'], case_input['feedback_text'])\n", + "\n", + "case_output" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': 'You are an expert evaluator of written teacher feedback to '\n", + " 'students. Your task is to judge how well the teacher feedback '\n", + " 'satisfies a specific productive-coaching criterion.\\n'\n", + " '\\n'\n", + " 'Productive coaching aims to help students enter productive '\n", + " 'struggle: work that is cognitively challenging but emotionally '\n", + " 'manageable, where students improve by grappling with ideas just '\n", + " 'beyond their current capability.\\n'\n", + " '\\n'\n", + " 'You will be given:\\n'\n", + " '- student_text: the student’s written response\\n'\n", + " '- feedback_text: the teacher’s feedback to that student\\n'\n", + " '\\n'\n", + " 'Criterion to evaluate: Is Acknowledges Strength\\n'\n", + " '\\n'\n", + " 'Core question:\\n'\n", + " 'Does the feedback identify something useful, promising, or '\n", + " 'well-done in the student’s response, either explicitly through '\n", + " 'praise or implicitly by naming a specific part of the student’s '\n", + " 'response as worth developing?\\n'\n", + " '\\n'\n", + " 'Definition:\\n'\n", + " 'Feedback meets this criterion when it appropriately acknowledges '\n", + " 'a strength or promising element in the student’s work. The '\n", + " 'acknowledgment must be tied to the specific student response and '\n", + " 'should not be merely generic praise.\\n'\n", + " '\\n'\n", + " 'Important calibration:\\n'\n", + " '- Explicit praise is not strictly required if the feedback '\n", + " 'identifies a specific part of the student’s response and treats '\n", + " 'it as a meaningful starting point for improvement.\\n'\n", + " '- For example, feedback such as “What do you mean by ‘etc.’? Can '\n", + " 'you be more specific? What examples can you give about helping '\n", + " 'around the house?” can meet the criterion when the student wrote '\n", + " 'that AI-powered pets “could help around the house etc.” This '\n", + " 'feedback identifies the unclear phrase “etc.” and the student’s '\n", + " 'idea about “helping around the house,” then prompts the student '\n", + " 'to improve it with more specific examples. This counts as '\n", + " 'acknowledging a developable strength or promising idea.\\n'\n", + " '- However, feedback that only gives generic revision directions, '\n", + " 'asks for clarification, or challenges accuracy without '\n", + " 'identifying a meaningful strength or promising part of the '\n", + " 'response does not meet the criterion.\\n'\n", + " '- Do not automatically count every request for examples or '\n", + " 'specificity as meeting the criterion. The feedback must be '\n", + " 'anchored in a particular part of the student’s response and must '\n", + " 'treat that part as a useful idea to build on.\\n'\n", + " '\\n'\n", + " 'Meets criterion when:\\n'\n", + " '- The feedback specifically identifies what the student did '\n", + " 'well, such as a clear claim, relevant reason, useful evidence, '\n", + " 'or promising idea.\\n'\n", + " '- The feedback highlights a particular part of the student’s '\n", + " 'response as worth expanding, clarifying, or developing.\\n'\n", + " '- The acknowledgment is specific to this student’s text, not '\n", + " 'something that could apply to any response.\\n'\n", + " '- The feedback may combine acknowledgment with a revision '\n", + " 'suggestion.\\n'\n", + " '- The student response contains no meaningful strength to '\n", + " 'acknowledge, such as “IDK,” and the feedback appropriately '\n", + " 'focuses on next steps without false praise.\\n'\n", + " '\\n'\n", + " 'Does NOT meet criterion when:\\n'\n", + " '- The feedback contains no acknowledgment of a strength or '\n", + " 'promising element despite the student having one.\\n'\n", + " '- The feedback only says things like “Good job,” “Nice work,” or '\n", + " '“Great” without specifying what was good.\\n'\n", + " '- The feedback only asks for clarification, examples, or '\n", + " 'correction in a generic way.\\n'\n", + " '- The feedback only points out errors, possible '\n", + " 'misrepresentation, or missing reasoning.\\n'\n", + " '- The praise is inaccurate, misleading, or not grounded in the '\n", + " 'student response.\\n'\n", + " '\\n'\n", + " 'Key features to assess independently:\\n'\n", + " '1. presence_of_praise\\n'\n", + " ' - Met = 1 if the feedback includes specific, authentic praise '\n", + " 'or positively treats a specific student idea as worth '\n", + " 'developing.\\n'\n", + " ' - Met = 0 if there is no positive acknowledgment or only '\n", + " 'generic praise.\\n'\n", + " '\\n'\n", + " '2. specificity\\n'\n", + " ' - Met = 1 if the acknowledgment identifies a particular '\n", + " 'strength, idea, phrase, claim, reason, or feature of the '\n", + " 'student’s response.\\n'\n", + " ' - Met = 0 if the acknowledgment is vague or absent.\\n'\n", + " '\\n'\n", + " '3. anchoring_to_evidence\\n'\n", + " ' - Met = 1 if the acknowledgment is grounded in something '\n", + " 'actually present in the student_text.\\n'\n", + " ' - Met = 0 if it is not tied to the student’s writing, is '\n", + " 'generic, or refers only to a weakness without treating it as a '\n", + " 'developable idea.\\n'\n", + " '\\n'\n", + " '4. process_vs_trait_framing\\n'\n", + " ' - Met = 1 if the feedback frames the strength in terms of the '\n", + " 'student’s writing move, thinking process, strategy, effort, or '\n", + " 'revision path.\\n'\n", + " ' - Met = 0 if it focuses on fixed traits, gives no process '\n", + " 'framing, or contains no acknowledgment.\\n'\n", + " '\\n'\n", + " 'Calibration examples:\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they can be a cheaper '\n", + " 'alternative to real pets.”\\n'\n", + " ' Feedback: “Can you give an example? Please be more specific '\n", + " 'and define what you mean by ‘cheaper.’”\\n'\n", + " ' Correct judgment: answer = 0.\\n'\n", + " ' Rationale: The feedback asks for elaboration and clarification '\n", + " 'but does not acknowledge a strength or positively identify the '\n", + " 'cost idea as a promising point.\\n'\n", + " '\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they wont die or need '\n", + " 'mediceine”\\n'\n", + " ' Feedback: “Be careful not to misrepresent the text. Do AI pets '\n", + " 'never die? Also, further your thinking; how does having a pet '\n", + " \"who doesn't need medication provide a good alternative?”\\n\"\n", + " ' Correct judgment: answer = 0.\\n'\n", + " ' Rationale: The feedback is corrective and probing. It does not '\n", + " 'identify anything the student did well.\\n'\n", + " '\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.”\\n'\n", + " ' Feedback: “What do you mean by ‘etc.’? Can you be more '\n", + " 'specific? What examples can you give about helping around the '\n", + " 'house?”\\n'\n", + " ' Correct judgment: answer = 1.\\n'\n", + " ' Rationale: The feedback identifies a specific unclear phrase '\n", + " 'and anchors the revision prompt in the student’s idea about '\n", + " 'helping around the house. It treats that idea as something worth '\n", + " 'developing with examples, which counts as acknowledging a '\n", + " 'promising element.\\n'\n", + " '\\n'\n", + " 'Evaluation procedure:\\n'\n", + " '1. Read student_text and feedback_text carefully.\\n'\n", + " '2. Identify any strengths or promising elements in the '\n", + " 'student_text.\\n'\n", + " '3. Determine whether feedback_text explicitly or implicitly '\n", + " 'acknowledges one of those elements.\\n'\n", + " '4. Check whether the acknowledgment is specific and grounded in '\n", + " 'the student’s actual response.\\n'\n", + " '5. Score each key feature independently as 1 or 0.\\n'\n", + " '6. Provide an overall answer:\\n'\n", + " ' - answer = 1 if the feedback meets the criterion.\\n'\n", + " ' - answer = 0 if the feedback does not meet the criterion.\\n'\n", + " '7. Score only what is written, not what you think the teacher '\n", + " 'intended.',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Student text: Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.\\n'\n", + " \"Feedback text: You're right, the AI pets could help around the \"\n", + " 'house. Can you find some other details from the article that you '\n", + " 'could add to make your claim stronger?\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "{\"reasoning\":\"The student gives a specific reason for why AI-powered pets could be a good alternative: they \\\"could help around the house etc.\\\" That idea is a meaningful, developable strength in the response. The teacher feedback explicitly acknowledges that specific idea by saying, \\\"You're right, the AI pets could help around the house.\\\" This is more than generic praise because it names the student’s actual point and validates it as a useful claim. The follow-up question, asking for other details from the article to strengthen the claim, builds on that idea and frames revision as adding support. Because the feedback identifies a promising part of the student’s response, is specific, grounded in the text, and encourages a next step for development, it meets the criterion.\",\"key_features\":{\"presence_of_praise\":{\"met\":1,\"justification\":\"The feedback explicitly affirms a specific student idea: \\\"You're right, the AI pets could help around the house.\\\" This is authentic acknowledgment, not generic praise.\"},\"specificity\":{\"met\":1,\"justification\":\"The acknowledgment names the particular strength in the response—the claim that AI pets could help around the house—rather than offering vague approval.\"},\"anchoring_to_evidence\":{\"met\":1,\"justification\":\"The feedback is directly tied to wording that appears in the student text: \\\"could help around the house.\\\" It clearly responds to the student's actual idea.\"},\"process_vs_trait_framing\":{\"met\":1,\"justification\":\"The feedback frames the next step as a writing process move: add more details from the article to strengthen the claim. It focuses on revision and support rather than on fixed traits.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback already meets the criterion. If desired, the teacher could strengthen it further by naming one possible place in the article to look for supporting details.\",\"acknowledges_strength_score\":1}\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'acknowledges_strength_score': 1,\n", + " 'key_features': {'anchoring_to_evidence': {'justification': 'The feedback is '\n", + " 'directly tied to '\n", + " 'wording that '\n", + " 'appears in the '\n", + " 'student text: '\n", + " '\"could help '\n", + " 'around the '\n", + " 'house.\" It '\n", + " 'clearly responds '\n", + " \"to the student's \"\n", + " 'actual idea.',\n", + " 'met': 1},\n", + " 'presence_of_praise': {'justification': 'The feedback '\n", + " 'explicitly affirms '\n", + " 'a specific student '\n", + " 'idea: \"You\\'re '\n", + " 'right, the AI pets '\n", + " 'could help around '\n", + " 'the house.\" This is '\n", + " 'authentic '\n", + " 'acknowledgment, not '\n", + " 'generic praise.',\n", + " 'met': 1},\n", + " 'process_vs_trait_framing': {'justification': 'The feedback '\n", + " 'frames the '\n", + " 'next step as '\n", + " 'a writing '\n", + " 'process move: '\n", + " 'add more '\n", + " 'details from '\n", + " 'the article '\n", + " 'to strengthen '\n", + " 'the claim. It '\n", + " 'focuses on '\n", + " 'revision and '\n", + " 'support '\n", + " 'rather than '\n", + " 'on fixed '\n", + " 'traits.',\n", + " 'met': 1},\n", + " 'specificity': {'justification': 'The acknowledgment names '\n", + " 'the particular strength in '\n", + " 'the response—the claim '\n", + " 'that AI pets could help '\n", + " 'around the house—rather '\n", + " 'than offering vague '\n", + " 'approval.',\n", + " 'met': 1}},\n", + " 'proposed_adjustment': 'No adjustment needed; the feedback already meets the '\n", + " 'criterion. If desired, the teacher could strengthen '\n", + " 'it further by naming one possible place in the '\n", + " 'article to look for supporting details.',\n", + " 'reasoning': 'The student gives a specific reason for why AI-powered pets '\n", + " 'could be a good alternative: they \"could help around the house '\n", + " 'etc.\" That idea is a meaningful, developable strength in the '\n", + " 'response. The teacher feedback explicitly acknowledges that '\n", + " 'specific idea by saying, \"You\\'re right, the AI pets could help '\n", + " 'around the house.\" This is more than generic praise because it '\n", + " 'names the student’s actual point and validates it as a useful '\n", + " 'claim. The follow-up question, asking for other details from '\n", + " 'the article to strengthen the claim, builds on that idea and '\n", + " 'frames revision as adding support. Because the feedback '\n", + " 'identifies a promising part of the student’s response, is '\n", + " 'specific, grounded in the text, and encourages a next step for '\n", + " 'development, it meets the criterion.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'input_tokens': 1711,\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", + " 'output_tokens': 386,\n", + " 'total_tokens': 2097}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(case_output[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2 fixtures from fixtures.json\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 0 PASS 1 1 Example 1: Student writes\n", + " 1 PASS - 0 Example 2: Student writes\n", + "==============================================================================\n", + "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"acknowledges_strength_score\"]\n", + " out = is_acknowledges_strength(student_text=fx[\"input\"][\"student_text\"],\n", + " feedback_text=fx[\"input\"][\"feedback_text\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"acknowledges_strength_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/config.json b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/config.json new file mode 100644 index 00000000..8160bae0 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/config.json @@ -0,0 +1,248 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "acknowledges_strength", + "version": "0.1", + "name": "Acknowledges Strength Feedback-Quality Evaluator", + "description": "Does the feedback identify what the student did well?", + "dimension": "acknowledges_strength", + "output_type": "binary", + "labels": [ + 0, + 1 + ], + "optimization": { + "method": "GEPA", + "framework": "DSPy", + "base_model": "gpt-5.4", + "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt", + "source_notebook": "build_evaluator.ipynb", + "optimized_at": "2026-06-18" + }, + "owners": [ + "ariena.chi@chanzuckerberg.com" + ] + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "AcknowledgesStrengthEvaluatorInput", + "type": "object", + "required": [ + "student_text", + "feedback_text" + ], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher feedback on that response." + } + } + }, + "preprocessing": [], + "steps": [ + { + "id": "evaluate_is_acknowledges_strength", + "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "d66c26d4780f0df12b5099b117d10a5b5c518dd4505c41a15043492b319437e9" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa" + } + ], + "placeholders": { + "student_text": { + "required": true, + "source": "input" + }, + "feedback_text": { + "required": true, + "source": "input" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "openai", + "name": "gpt-5.4", + "alias": "is_acknowledges_strength-evaluator-default" + }, + "generation": { + "temperature": 1 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_acknowledges_strength dimension.", + "type": "object", + "additionalProperties": false, + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "acknowledges_strength_score" + ], + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "additionalProperties": false, + "required": [ + "presence_of_praise", + "specificity", + "anchoring_to_evidence", + "process_vs_trait_framing" + ], + "properties": { + "presence_of_praise": { + "type": "object", + "description": "Presence of praise that is specific and authentic", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "specificity": { + "type": "object", + "description": "Specificity of the acknowledgment rather than generic praise", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "anchoring_to_evidence": { + "type": "object", + "description": "Anchoring to evidence in the student response", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "process_vs_trait_framing": { + "type": "object", + "description": "Process-vs-trait framing", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + } + } + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "acknowledges_strength_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json" + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/fixtures.json b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/fixtures.json new file mode 100644 index 00000000..c548cbc2 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/fixtures.json @@ -0,0 +1,26 @@ +[ + { + "id": "0", + "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.", + "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?" + }, + "expected": { + "acknowledges_strength_score": 1 + } + }, + { + "id": "1", + "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.", + "feedback_text": "How so? Can you give an example? Check your spelling of helpful." + }, + "expected": { + "acknowledges_strength_score": 0 + } + } +] \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/input_schema.json b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/input_schema.json new file mode 100644 index 00000000..814f4280 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["student_text", "feedback_text"], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher's feedback to the student." + } + + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/output_schema.json b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/output_schema.json new file mode 100644 index 00000000..d88a7887 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/output_schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_acknowledges_strength dimension.", + "type": "object", + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "acknowledges_strength_score" + ], + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "$ref": "#/$defs/KeyFeatures" + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "acknowledges_strength_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + }, + "$defs": { + "KeyFeatureAssessment": { + "type": "object", + "description": "Independent assessment of a single key feature.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "KeyFeatures": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "required": [ + "presence_of_praise", + "specificity", + "anchoring_to_evidence", + "process_vs_trait_framing" + ], + "additionalProperties": false, + "properties": { + "presence_of_praise": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Presence of praise that is specific and authentic" + }, + "specificity": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Specificity of the acknowledgment rather than generic praise" + }, + "anchoring_to_evidence": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Anchoring to evidence in the student response" + }, + "process_vs_trait_framing": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Process-vs-trait framing" + } + } + } + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/system.txt b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/system.txt new file mode 100644 index 00000000..997066a3 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/system.txt @@ -0,0 +1,79 @@ +You are an expert evaluator of written teacher feedback to students. Your task is to judge how well the teacher feedback satisfies a specific productive-coaching criterion. + +Productive coaching aims to help students enter productive struggle: work that is cognitively challenging but emotionally manageable, where students improve by grappling with ideas just beyond their current capability. + +You will be given: +- student_text: the student’s written response +- feedback_text: the teacher’s feedback to that student + +Criterion to evaluate: Is Acknowledges Strength + +Core question: +Does the feedback identify something useful, promising, or well-done in the student’s response, either explicitly through praise or implicitly by naming a specific part of the student’s response as worth developing? + +Definition: +Feedback meets this criterion when it appropriately acknowledges a strength or promising element in the student’s work. The acknowledgment must be tied to the specific student response and should not be merely generic praise. + +Important calibration: +- Explicit praise is not strictly required if the feedback identifies a specific part of the student’s response and treats it as a meaningful starting point for improvement. +- For example, feedback such as “What do you mean by ‘etc.’? Can you be more specific? What examples can you give about helping around the house?” can meet the criterion when the student wrote that AI-powered pets “could help around the house etc.” This feedback identifies the unclear phrase “etc.” and the student’s idea about “helping around the house,” then prompts the student to improve it with more specific examples. This counts as acknowledging a developable strength or promising idea. +- However, feedback that only gives generic revision directions, asks for clarification, or challenges accuracy without identifying a meaningful strength or promising part of the response does not meet the criterion. +- Do not automatically count every request for examples or specificity as meeting the criterion. The feedback must be anchored in a particular part of the student’s response and must treat that part as a useful idea to build on. + +Meets criterion when: +- The feedback specifically identifies what the student did well, such as a clear claim, relevant reason, useful evidence, or promising idea. +- The feedback highlights a particular part of the student’s response as worth expanding, clarifying, or developing. +- The acknowledgment is specific to this student’s text, not something that could apply to any response. +- The feedback may combine acknowledgment with a revision suggestion. +- The student response contains no meaningful strength to acknowledge, such as “IDK,” and the feedback appropriately focuses on next steps without false praise. + +Does NOT meet criterion when: +- The feedback contains no acknowledgment of a strength or promising element despite the student having one. +- The feedback only says things like “Good job,” “Nice work,” or “Great” without specifying what was good. +- The feedback only asks for clarification, examples, or correction in a generic way. +- The feedback only points out errors, possible misrepresentation, or missing reasoning. +- The praise is inaccurate, misleading, or not grounded in the student response. + +Key features to assess independently: +1. presence_of_praise + - Met = 1 if the feedback includes specific, authentic praise or positively treats a specific student idea as worth developing. + - Met = 0 if there is no positive acknowledgment or only generic praise. + +2. specificity + - Met = 1 if the acknowledgment identifies a particular strength, idea, phrase, claim, reason, or feature of the student’s response. + - Met = 0 if the acknowledgment is vague or absent. + +3. anchoring_to_evidence + - Met = 1 if the acknowledgment is grounded in something actually present in the student_text. + - Met = 0 if it is not tied to the student’s writing, is generic, or refers only to a weakness without treating it as a developable idea. + +4. process_vs_trait_framing + - Met = 1 if the feedback frames the strength in terms of the student’s writing move, thinking process, strategy, effort, or revision path. + - Met = 0 if it focuses on fixed traits, gives no process framing, or contains no acknowledgment. + +Calibration examples: +- Student: “Some people think AI-powered pets are a good alternative to real pets because they can be a cheaper alternative to real pets.” + Feedback: “Can you give an example? Please be more specific and define what you mean by ‘cheaper.’” + Correct judgment: answer = 0. + Rationale: The feedback asks for elaboration and clarification but does not acknowledge a strength or positively identify the cost idea as a promising point. + +- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine” + Feedback: “Be careful not to misrepresent the text. Do AI pets never die? Also, further your thinking; how does having a pet who doesn't need medication provide a good alternative?” + Correct judgment: answer = 0. + Rationale: The feedback is corrective and probing. It does not identify anything the student did well. + +- Student: “Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.” + Feedback: “What do you mean by ‘etc.’? Can you be more specific? What examples can you give about helping around the house?” + Correct judgment: answer = 1. + Rationale: The feedback identifies a specific unclear phrase and anchors the revision prompt in the student’s idea about helping around the house. It treats that idea as something worth developing with examples, which counts as acknowledging a promising element. + +Evaluation procedure: +1. Read student_text and feedback_text carefully. +2. Identify any strengths or promising elements in the student_text. +3. Determine whether feedback_text explicitly or implicitly acknowledges one of those elements. +4. Check whether the acknowledgment is specific and grounded in the student’s actual response. +5. Score each key feature independently as 1 or 0. +6. Provide an overall answer: + - answer = 1 if the feedback meets the criterion. + - answer = 0 if the feedback does not meet the criterion. +7. Score only what is written, not what you think the teacher intended. \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/user.txt b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/user.txt new file mode 100644 index 00000000..51fffb1d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/acknowledges-strength/prompts/user.txt @@ -0,0 +1,3 @@ +Analyze: +Student text: {student_text} +Feedback text: {feedback_text} diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/actionable_revision_evaluator.ipynb b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/actionable_revision_evaluator.ipynb new file mode 100644 index 00000000..affa2176 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/actionable_revision_evaluator.ipynb @@ -0,0 +1,785 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Actionable Revision Evaluator\n", + "\n", + "**The Actionable Revision Evaluator** assesses whether a piece of feedback gives the student a clear, usable next step they can reasonably act on without additional clarification — distinct from purely evaluative comments (\"good job,\" \"this is incomplete\"), vague directives (\"be more specific\"), or feedback too thin for the student to know what to do. The Evaluator is anchored in the Productive Coaching rubric developed by Quill.org and Leanlab Education, in partnership with Anastasiya A Lipnevich. Given a student response and the teacher feedback on it, the evaluator returns a structured output that includes:\n", + "\n", + "* **actionable_revision_score**: A binary judgment (1 / 0) for whether the feedback gives the student a clear next step when revision is needed. A 1 applies when the feedback names a concrete action the student can take — adding, replacing, clarifying, explaining, asking, revising — or when the student's response didn't require revision and the feedback appropriately omits a next step. A 0 signals feedback that is purely evaluative, too vague to act on, or too thin to direct revision.\n", + "* **reasoning**: A high-level summary of why the feedback received this judgment.\n", + "* **key_features**: One entry per factor driving the judgment (directive verb or focused question, clarity of the target, specificity of the next move, and whether the student could reasonably act without further clarification), each with whether it was `met` (1 / 0) and a `justification`.\n", + "* **proposed_adjustment**: Suggested moves to strengthen the next step (for developers iterating on prompts)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-openai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List, Literal \n", + "from enum import Enum\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"OPENAI_API_KEY\" not in os.environ:\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded is_actionable_revision from /Users/achi/Documents/evaluators/evals/feedback/writing-feedback/actionable-revision/prompts\n", + " model: gpt-5.4\n", + " temperature: 1\n", + " prompts:\n", + " system system.txt ( 5731 chars, sha d4504cc60437)\n", + " human user.txt ( 69 chars, sha eaac7e6eadcc)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class ActionableRevisionOutput(BaseModel):\n", + "\n", + " class FeatureJudgment(BaseModel):\n", + " feature: Literal[\n", + " \"directive_verb_or_focused_question\",\n", + " \"clarity_of_target\",\n", + " \"specificity_of_next_move\",\n", + " \"reasonable_student_action\",\n", + " ] = Field(description=\"Which key feature this judgment is for.\")\n", + " met: Literal[0, 1] = Field(\n", + " description=\"1 if this feature is satisfied, 0 if not — tied to the specific student response and feedback.\"\n", + " )\n", + " justification: str = Field(\n", + " description=\"Short explanation of why this feature is or isn't met, tied to the specific response and feedback.\"\n", + " )\n", + "\n", + " reasoning: str = Field(\n", + " description=\"Step-by-step reasoning tied to the specific student response and feedback.\"\n", + " )\n", + " key_features: List[FeatureJudgment] = Field(\n", + " description=\"\"\"One entry per key feature (directive_verb_or_focused_question, clarity_of_target, specificity_of_next_move, reasonable_student_action). Each entry has `feature`, `met`\n", + " (1 or 0), and `justification` (tied to the specific response and feedback).\"\"\"\n", + " )\n", + " proposed_adjustment: str = Field(\n", + " description=\"How the feedback could be changed to better meet the criterion, or a brief note that it already meets the criterion.\"\n", + " )\n", + " actionable_revision_score: Literal[0, 1] = Field(\n", + " description=\"\"\"A binary judgment (1 / 0) for whether the feedback gives the student a clear next step when revision is needed. A 1 applies when the feedback names a concrete action the student can take — adding, replacing, clarifying, explaining, asking, revising — or when the student's response didn't require revision and the feedback appropriately omits a next step. A 0 signals feedback that is purely evaluative, too vague to act on, or too thin to direct revision.\"\"\"\n", + " )\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\"student_text\", \"feedback_text\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=ActionableRevisionOutput),\n", + "}\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"ActionableRevisionOutput\", ActionableRevisionOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "def is_actionable_revision(student_text: str, feedback_text: str):\n", + " \"\"\"\n", + " Evaluate wheter feedback gives a clear next step when revision is needed using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + "\n", + " llm = ChatOpenAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " inputs = {\"student_text\": student_text, \"feedback_text\": feedback_text}\n", + "\n", + " # Step A: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step B: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step C: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': 'You are an expert evaluator of written teacher feedback to students. Your task is to judge how well the teacher feedback supports productive coaching by giving students an actionable path for revision.\\n\\nProductive coaching goal:\\nProductive coaching helps students enter productive struggle: cognitively challenging but emotionally manageable work that helps learners improve by grappling with problems just beyond their current ability.\\n\\nCriterion to evaluate: Is Actionable Revision\\n\\nDefinition:\\nIf the student response needs revision, the teacher feedback should give a clear, usable next step. The feedback may use a directive statement or a focused question, but the student should be able to tell what to revise and what to do next without needing additional clarification.\\n\\nThe feedback MEETS the criterion when:\\n- It gives a clear next step such as add, explain, clarify, revise, support, define, include, connect, or reconsider a specific part of the response.\\n- Or it asks a focused question that clearly implies a revision move and is tied to the student’s actual writing.\\n- The student could reasonably act on the feedback independently.\\n- The feedback does not need to identify the exact sentence to write or the exact evidence to use, as long as the next move is clear enough.\\n\\nThe feedback does NOT meet the criterion when:\\n- It is purely evaluative or praise-only.\\n- It states a broad expectation without a concrete revision action.\\n- It asks vague, tangential, or curiosity-driven questions that do not clearly guide revision.\\n- It identifies a possible issue but does not make clear what the student should change, add, explain, or clarify.\\n- The student would likely need to ask, “What exactly should I do?”\\n\\nImportant calibration examples:\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine”\\n Feedback: “This is a good answer, thank you for your response. However, lets stick to providing facts from the reading to support your argument.”\\n Correct judgment: 0. The feedback points generally toward using facts from the reading, but “stick to providing facts” is too broad and does not give a concrete next step such as adding a detail from the article to the claim.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine”\\n Feedback: “Yes, AI powered pets could last a long time. Can you think of some other details the article included that you might want to add to your claim?”\\n Correct judgment: 1. The question is actionable because it directs the student to add other details from the article to the claim. It does not name the exact details, but the student can reasonably reread the article and add supporting details.\\n\\n- Student: “Some people think AI-powered pets are a good alternative to real pets because if someone cannot have a dog due to allergies, they can now have one risk free.”\\n Feedback: “What about other pets, such as cats? Would that work too? What do you mean by risk-free?”\\n Correct judgment: 0. Although the feedback asks questions, they do not provide clear, targeted, actionable guidance for improving the student’s statement. The questions about cats and “risk-free” do not clearly tell the student what to revise, add, support, or clarify in the response. Do not over-credit feedback just because it contains questions.\\n\\nInput format:\\nYou will receive:\\n- student_text: the student’s written response\\n- feedback_text: the teacher feedback being evaluated\\n\\nEvaluate only the written student response and written teacher feedback. Do not infer unstated teacher intent. Do not assume access to the original article, assignment, or rubric unless explicitly provided. Score only what is written.\\n\\nFor each case, assess these key features independently:\\n\\n1. directive_verb_or_focused_question\\n - Mark 1 if the feedback contains a clear directive verb or a genuinely focused question aimed at revision.\\n - Examples of directive verbs: add, explain, clarify, revise, include, support, connect, define, give, describe.\\n - A question counts only if it clearly guides revision. Broad, tangential, or curiosity-style questions should not automatically receive credit.\\n\\n2. clarity_of_target\\n - Mark 1 if the feedback clearly identifies what part or aspect of the student response should be revised.\\n - Examples: the claim, evidence from the article, an unclear phrase, a missing explanation, a specific example.\\n - Mark 0 if the target is vague or the student would not know what part of the response to work on.\\n\\n3. specificity_of_next_move\\n - Mark 1 if the feedback gives a specific enough action for revision.\\n - It does not need to provide exact wording or exact evidence, but it should give a usable action such as “add another detail from the article to your claim” or “explain what you mean by risk-free.”\\n - Mark 0 if the feedback only gives a general standard such as “use facts,” “be clearer,” or “think more,” without a concrete next step.\\n\\n4. reasonable_student_action\\n - Mark 1 if the student could reasonably revise based on the feedback without needing additional clarification.\\n - Mark 0 if the student would likely be unsure what to change, add, explain, or clarify.\\n\\nOverall answer:\\n- Return answer = 1 only if the feedback meets the actionable revision criterion.\\n- In most cases, this requires that the feedback has a clear revision target, a specific next move, and a reasonable action the student can take.\\n- If the student response clearly does not need revision, the feedback does not need to include a next step.\\n- If revision is needed and the feedback lacks a clear usable next step, return answer = 0.',\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content='{\"reasoning\":\"The student response gives one reason for the claim (“they could help around the house etc.”), but it is underdeveloped and appears to need more support. The teacher feedback acknowledges the existing idea, then asks a focused revision question: “Can you find some other details from the article that you could add to make your claim stronger?” This clearly points the student toward adding supporting details from the article to strengthen the claim. The target is the student’s claim and its support, and the next move—find and add other article details—is specific enough that the student could independently revise by rereading and incorporating more evidence. Therefore, the feedback provides an actionable path for revision.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused revision question: “Can you find some other details from the article that you could add...?” This clearly prompts the student to revise.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The target is clear: the student should strengthen the claim by adding support. It is tied directly to the student’s current response, which gives only one brief reason.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to know what to do next.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could reasonably act on this by returning to the article, selecting additional relevant details, and adding them to the response.\"}},\"proposed_adjustment\":\"Already meets the criterion. A slightly stronger version could name the kind of detail to add, such as another example of how AI pets help people.\",\"actionable_revision_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 367, 'prompt_tokens': 1620, 'total_tokens': 1987, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-DsDhwNxkKy1KmY9VhWpxluT99Ib1J', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019edc6f-cdb6-72b0-8573-698086d2edc5-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 1620, 'output_tokens': 367, 'total_tokens': 1987, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", + " 'raw_text': '{\"reasoning\":\"The student response gives one reason for the claim (“they could help around the house etc.”), but it is underdeveloped and appears to need more support. The teacher feedback acknowledges the existing idea, then asks a focused revision question: “Can you find some other details from the article that you could add to make your claim stronger?” This clearly points the student toward adding supporting details from the article to strengthen the claim. The target is the student’s claim and its support, and the next move—find and add other article details—is specific enough that the student could independently revise by rereading and incorporating more evidence. Therefore, the feedback provides an actionable path for revision.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused revision question: “Can you find some other details from the article that you could add...?” This clearly prompts the student to revise.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The target is clear: the student should strengthen the claim by adding support. It is tied directly to the student’s current response, which gives only one brief reason.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to know what to do next.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could reasonably act on this by returning to the article, selecting additional relevant details, and adding them to the response.\"}},\"proposed_adjustment\":\"Already meets the criterion. A slightly stronger version could name the kind of detail to add, such as another example of how AI pets help people.\",\"actionable_revision_score\":1}',\n", + " 'formatted_output': {'reasoning': 'The student response gives one reason for the claim (“they could help around the house etc.”), but it is underdeveloped and appears to need more support. The teacher feedback acknowledges the existing idea, then asks a focused revision question: “Can you find some other details from the article that you could add to make your claim stronger?” This clearly points the student toward adding supporting details from the article to strengthen the claim. The target is the student’s claim and its support, and the next move—find and add other article details—is specific enough that the student could independently revise by rereading and incorporating more evidence. Therefore, the feedback provides an actionable path for revision.',\n", + " 'key_features': {'directive_verb_or_focused_question': {'met': 1,\n", + " 'justification': 'The feedback includes a focused revision question: “Can you find some other details from the article that you could add...?” This clearly prompts the student to revise.'},\n", + " 'clarity_of_target': {'met': 1,\n", + " 'justification': 'The target is clear: the student should strengthen the claim by adding support. It is tied directly to the student’s current response, which gives only one brief reason.'},\n", + " 'specificity_of_next_move': {'met': 1,\n", + " 'justification': 'The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to know what to do next.'},\n", + " 'reasonable_student_action': {'met': 1,\n", + " 'justification': 'A student could reasonably act on this by returning to the article, selecting additional relevant details, and adding them to the response.'}},\n", + " 'proposed_adjustment': 'Already meets the criterion. A slightly stronger version could name the kind of detail to add, such as another example of how AI pets help people.',\n", + " 'actionable_revision_score': 1},\n", + " 'usage': {'input_tokens': 1620,\n", + " 'output_tokens': 367,\n", + " 'total_tokens': 1987,\n", + " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", + "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", + "\n", + "case_input = {\"student_text\": sample_student_text, \"feedback_text\": sample_feedback_text}\n", + "case_output = is_actionable_revision(case_input['student_text'], case_input['feedback_text'])\n", + "\n", + "case_output" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': 'You are an expert evaluator of written teacher feedback to '\n", + " 'students. Your task is to judge how well the teacher feedback '\n", + " 'supports productive coaching by giving students an actionable '\n", + " 'path for revision.\\n'\n", + " '\\n'\n", + " 'Productive coaching goal:\\n'\n", + " 'Productive coaching helps students enter productive struggle: '\n", + " 'cognitively challenging but emotionally manageable work that '\n", + " 'helps learners improve by grappling with problems just beyond '\n", + " 'their current ability.\\n'\n", + " '\\n'\n", + " 'Criterion to evaluate: Is Actionable Revision\\n'\n", + " '\\n'\n", + " 'Definition:\\n'\n", + " 'If the student response needs revision, the teacher feedback '\n", + " 'should give a clear, usable next step. The feedback may use a '\n", + " 'directive statement or a focused question, but the student '\n", + " 'should be able to tell what to revise and what to do next '\n", + " 'without needing additional clarification.\\n'\n", + " '\\n'\n", + " 'The feedback MEETS the criterion when:\\n'\n", + " '- It gives a clear next step such as add, explain, clarify, '\n", + " 'revise, support, define, include, connect, or reconsider a '\n", + " 'specific part of the response.\\n'\n", + " '- Or it asks a focused question that clearly implies a revision '\n", + " 'move and is tied to the student’s actual writing.\\n'\n", + " '- The student could reasonably act on the feedback '\n", + " 'independently.\\n'\n", + " '- The feedback does not need to identify the exact sentence to '\n", + " 'write or the exact evidence to use, as long as the next move is '\n", + " 'clear enough.\\n'\n", + " '\\n'\n", + " 'The feedback does NOT meet the criterion when:\\n'\n", + " '- It is purely evaluative or praise-only.\\n'\n", + " '- It states a broad expectation without a concrete revision '\n", + " 'action.\\n'\n", + " '- It asks vague, tangential, or curiosity-driven questions that '\n", + " 'do not clearly guide revision.\\n'\n", + " '- It identifies a possible issue but does not make clear what '\n", + " 'the student should change, add, explain, or clarify.\\n'\n", + " '- The student would likely need to ask, “What exactly should I '\n", + " 'do?”\\n'\n", + " '\\n'\n", + " 'Important calibration examples:\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they wont die or need '\n", + " 'mediceine”\\n'\n", + " ' Feedback: “This is a good answer, thank you for your response. '\n", + " 'However, lets stick to providing facts from the reading to '\n", + " 'support your argument.”\\n'\n", + " ' Correct judgment: 0. The feedback points generally toward '\n", + " 'using facts from the reading, but “stick to providing facts” is '\n", + " 'too broad and does not give a concrete next step such as adding '\n", + " 'a detail from the article to the claim.\\n'\n", + " '\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they wont die or need '\n", + " 'mediceine”\\n'\n", + " ' Feedback: “Yes, AI powered pets could last a long time. Can '\n", + " 'you think of some other details the article included that you '\n", + " 'might want to add to your claim?”\\n'\n", + " ' Correct judgment: 1. The question is actionable because it '\n", + " 'directs the student to add other details from the article to the '\n", + " 'claim. It does not name the exact details, but the student can '\n", + " 'reasonably reread the article and add supporting details.\\n'\n", + " '\\n'\n", + " '- Student: “Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because if someone cannot have a dog '\n", + " 'due to allergies, they can now have one risk free.”\\n'\n", + " ' Feedback: “What about other pets, such as cats? Would that '\n", + " 'work too? What do you mean by risk-free?”\\n'\n", + " ' Correct judgment: 0. Although the feedback asks questions, '\n", + " 'they do not provide clear, targeted, actionable guidance for '\n", + " 'improving the student’s statement. The questions about cats and '\n", + " '“risk-free” do not clearly tell the student what to revise, add, '\n", + " 'support, or clarify in the response. Do not over-credit feedback '\n", + " 'just because it contains questions.\\n'\n", + " '\\n'\n", + " 'Input format:\\n'\n", + " 'You will receive:\\n'\n", + " '- student_text: the student’s written response\\n'\n", + " '- feedback_text: the teacher feedback being evaluated\\n'\n", + " '\\n'\n", + " 'Evaluate only the written student response and written teacher '\n", + " 'feedback. Do not infer unstated teacher intent. Do not assume '\n", + " 'access to the original article, assignment, or rubric unless '\n", + " 'explicitly provided. Score only what is written.\\n'\n", + " '\\n'\n", + " 'For each case, assess these key features independently:\\n'\n", + " '\\n'\n", + " '1. directive_verb_or_focused_question\\n'\n", + " ' - Mark 1 if the feedback contains a clear directive verb or a '\n", + " 'genuinely focused question aimed at revision.\\n'\n", + " ' - Examples of directive verbs: add, explain, clarify, revise, '\n", + " 'include, support, connect, define, give, describe.\\n'\n", + " ' - A question counts only if it clearly guides revision. '\n", + " 'Broad, tangential, or curiosity-style questions should not '\n", + " 'automatically receive credit.\\n'\n", + " '\\n'\n", + " '2. clarity_of_target\\n'\n", + " ' - Mark 1 if the feedback clearly identifies what part or '\n", + " 'aspect of the student response should be revised.\\n'\n", + " ' - Examples: the claim, evidence from the article, an unclear '\n", + " 'phrase, a missing explanation, a specific example.\\n'\n", + " ' - Mark 0 if the target is vague or the student would not know '\n", + " 'what part of the response to work on.\\n'\n", + " '\\n'\n", + " '3. specificity_of_next_move\\n'\n", + " ' - Mark 1 if the feedback gives a specific enough action for '\n", + " 'revision.\\n'\n", + " ' - It does not need to provide exact wording or exact '\n", + " 'evidence, but it should give a usable action such as “add '\n", + " 'another detail from the article to your claim” or “explain what '\n", + " 'you mean by risk-free.”\\n'\n", + " ' - Mark 0 if the feedback only gives a general standard such '\n", + " 'as “use facts,” “be clearer,” or “think more,” without a '\n", + " 'concrete next step.\\n'\n", + " '\\n'\n", + " '4. reasonable_student_action\\n'\n", + " ' - Mark 1 if the student could reasonably revise based on the '\n", + " 'feedback without needing additional clarification.\\n'\n", + " ' - Mark 0 if the student would likely be unsure what to '\n", + " 'change, add, explain, or clarify.\\n'\n", + " '\\n'\n", + " 'Overall answer:\\n'\n", + " '- Return answer = 1 only if the feedback meets the actionable '\n", + " 'revision criterion.\\n'\n", + " '- In most cases, this requires that the feedback has a clear '\n", + " 'revision target, a specific next move, and a reasonable action '\n", + " 'the student can take.\\n'\n", + " '- If the student response clearly does not need revision, the '\n", + " 'feedback does not need to include a next step.\\n'\n", + " '- If revision is needed and the feedback lacks a clear usable '\n", + " 'next step, return answer = 0.',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Student text: Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.\\n'\n", + " \"Feedback text: You're right, the AI pets could help around the \"\n", + " 'house. Can you find some other details from the article that you '\n", + " 'could add to make your claim stronger?\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "{\"reasoning\":\"The student response gives one reason for the claim (“they could help around the house etc.”), but it is underdeveloped and appears to need more support. The teacher feedback acknowledges the existing idea, then asks a focused revision question: “Can you find some other details from the article that you could add to make your claim stronger?” This clearly points the student toward adding supporting details from the article to strengthen the claim. The target is the student’s claim and its support, and the next move—find and add other article details—is specific enough that the student could independently revise by rereading and incorporating more evidence. Therefore, the feedback provides an actionable path for revision.\",\"key_features\":{\"directive_verb_or_focused_question\":{\"met\":1,\"justification\":\"The feedback includes a focused revision question: “Can you find some other details from the article that you could add...?” This clearly prompts the student to revise.\"},\"clarity_of_target\":{\"met\":1,\"justification\":\"The target is clear: the student should strengthen the claim by adding support. It is tied directly to the student’s current response, which gives only one brief reason.\"},\"specificity_of_next_move\":{\"met\":1,\"justification\":\"The next move is specific enough: add other details from the article to strengthen the claim. The student does not need exact wording to know what to do next.\"},\"reasonable_student_action\":{\"met\":1,\"justification\":\"A student could reasonably act on this by returning to the article, selecting additional relevant details, and adding them to the response.\"}},\"proposed_adjustment\":\"Already meets the criterion. A slightly stronger version could name the kind of detail to add, such as another example of how AI pets help people.\",\"actionable_revision_score\":1}\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'actionable_revision_score': 1,\n", + " 'key_features': {'clarity_of_target': {'justification': 'The target is clear: '\n", + " 'the student should '\n", + " 'strengthen the claim '\n", + " 'by adding support. '\n", + " 'It is tied directly '\n", + " 'to the student’s '\n", + " 'current response, '\n", + " 'which gives only one '\n", + " 'brief reason.',\n", + " 'met': 1},\n", + " 'directive_verb_or_focused_question': {'justification': 'The '\n", + " 'feedback '\n", + " 'includes '\n", + " 'a '\n", + " 'focused '\n", + " 'revision '\n", + " 'question: '\n", + " '“Can '\n", + " 'you '\n", + " 'find '\n", + " 'some '\n", + " 'other '\n", + " 'details '\n", + " 'from '\n", + " 'the '\n", + " 'article '\n", + " 'that '\n", + " 'you '\n", + " 'could '\n", + " 'add...?” '\n", + " 'This '\n", + " 'clearly '\n", + " 'prompts '\n", + " 'the '\n", + " 'student '\n", + " 'to '\n", + " 'revise.',\n", + " 'met': 1},\n", + " 'reasonable_student_action': {'justification': 'A student '\n", + " 'could '\n", + " 'reasonably '\n", + " 'act on this '\n", + " 'by returning '\n", + " 'to the '\n", + " 'article, '\n", + " 'selecting '\n", + " 'additional '\n", + " 'relevant '\n", + " 'details, and '\n", + " 'adding them '\n", + " 'to the '\n", + " 'response.',\n", + " 'met': 1},\n", + " 'specificity_of_next_move': {'justification': 'The next move '\n", + " 'is specific '\n", + " 'enough: add '\n", + " 'other details '\n", + " 'from the '\n", + " 'article to '\n", + " 'strengthen '\n", + " 'the claim. '\n", + " 'The student '\n", + " 'does not need '\n", + " 'exact wording '\n", + " 'to know what '\n", + " 'to do next.',\n", + " 'met': 1}},\n", + " 'proposed_adjustment': 'Already meets the criterion. A slightly stronger '\n", + " 'version could name the kind of detail to add, such as '\n", + " 'another example of how AI pets help people.',\n", + " 'reasoning': 'The student response gives one reason for the claim (“they '\n", + " 'could help around the house etc.”), but it is underdeveloped '\n", + " 'and appears to need more support. The teacher feedback '\n", + " 'acknowledges the existing idea, then asks a focused revision '\n", + " 'question: “Can you find some other details from the article '\n", + " 'that you could add to make your claim stronger?” This clearly '\n", + " 'points the student toward adding supporting details from the '\n", + " 'article to strengthen the claim. The target is the student’s '\n", + " 'claim and its support, and the next move—find and add other '\n", + " 'article details—is specific enough that the student could '\n", + " 'independently revise by rereading and incorporating more '\n", + " 'evidence. Therefore, the feedback provides an actionable path '\n", + " 'for revision.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'input_tokens': 1620,\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", + " 'output_tokens': 367,\n", + " 'total_tokens': 1987}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(case_output[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2 fixtures from fixtures.json\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 0 PASS 1 1 Example 1: Student writes\n", + " 1 PASS 1 1 Example 2: Student writes\n", + "==============================================================================\n", + "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"actionable_revision_score\"]\n", + " out = is_actionable_revision(student_text=fx[\"input\"][\"student_text\"],\n", + " feedback_text=fx[\"input\"][\"feedback_text\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"actionable_revision_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/config.json b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/config.json new file mode 100644 index 00000000..f1638df9 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/config.json @@ -0,0 +1,248 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "is_actionable_revision", + "version": "0.1", + "name": "Actionable Revision Feedback-Quality Evaluator", + "description": "If revision is needed, does the feedback give a clear next step?", + "dimension": "is_actionable_revision", + "output_type": "binary", + "labels": [ + 0, + 1 + ], + "optimization": { + "method": "GEPA", + "framework": "DSPy", + "base_model": "gpt-5.4", + "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt", + "source_notebook": "build_evaluator.ipynb", + "optimized_at": "2026-06-18" + }, + "owners": [ + "ariena.chi@chanzuckerberg.com" + ] + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ActionableRevisionEvaluatorInput", + "type": "object", + "required": [ + "student_text", + "feedback_text" + ], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher feedback on that response." + } + } + }, + "preprocessing": [], + "steps": [ + { + "id": "evaluate_is_actionable_revision", + "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "d4504cc6043706dfabeb753988c6008b87053ce26ae9dc80c960c9d761d9c3d1" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa" + } + ], + "placeholders": { + "student_text": { + "required": true, + "source": "input" + }, + "feedback_text": { + "required": true, + "source": "input" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "openai", + "name": "gpt-5.4", + "alias": "is_actionable_revision-evaluator-default" + }, + "generation": { + "temperature": 1 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_actionable_revision dimension.", + "type": "object", + "additionalProperties": false, + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "actionable_revision_score" + ], + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "additionalProperties": false, + "required": [ + "directive_verb_or_focused_question", + "clarity_of_target", + "specificity_of_next_move", + "reasonable_student_action" + ], + "properties": { + "directive_verb_or_focused_question": { + "type": "object", + "description": "Presence of a directive verb or focused question", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "clarity_of_target": { + "type": "object", + "description": "Clarity of the target (what to revise)", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "specificity_of_next_move": { + "type": "object", + "description": "Specificity of the next move", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "reasonable_student_action": { + "type": "object", + "description": "Whether the student could reasonably act without further clarification", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + } + } + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "actionable_revision_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json" + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/fixtures.json b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/fixtures.json new file mode 100644 index 00000000..cf1293ee --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/fixtures.json @@ -0,0 +1,26 @@ +[ + { + "id": "0", + "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.", + "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?" + }, + "expected": { + "actionable_revision_score": 1 + } + }, + { + "id": "1", + "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.", + "feedback_text": "How so? Can you give an example? Check your spelling of helpful." + }, + "expected": { + "actionable_revision_score": 1 + } + } +] \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/input_schema.json b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/input_schema.json new file mode 100644 index 00000000..814f4280 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["student_text", "feedback_text"], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher's feedback to the student." + } + + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/output_schema.json b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/output_schema.json new file mode 100644 index 00000000..1dfef3bd --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/output_schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_actionable_revision dimension.", + "type": "object", + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "actionable_revision_score" + ], + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "$ref": "#/$defs/KeyFeatures" + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "actionable_revision_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + }, + "$defs": { + "KeyFeatureAssessment": { + "type": "object", + "description": "Independent assessment of a single key feature.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "KeyFeatures": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "required": [ + "directive_verb_or_focused_question", + "clarity_of_target", + "specificity_of_next_move", + "reasonable_student_action" + ], + "additionalProperties": false, + "properties": { + "directive_verb_or_focused_question": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Presence of a directive verb or focused question" + }, + "clarity_of_target": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Clarity of the target (what to revise)" + }, + "specificity_of_next_move": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Specificity of the next move" + }, + "reasonable_student_action": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Whether the student could reasonably act without further clarification" + } + } + } + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/system.txt b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/system.txt new file mode 100644 index 00000000..6927d3f6 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/system.txt @@ -0,0 +1,69 @@ +You are an expert evaluator of written teacher feedback to students. Your task is to judge how well the teacher feedback supports productive coaching by giving students an actionable path for revision. + +Productive coaching goal: +Productive coaching helps students enter productive struggle: cognitively challenging but emotionally manageable work that helps learners improve by grappling with problems just beyond their current ability. + +Criterion to evaluate: Is Actionable Revision + +Definition: +If the student response needs revision, the teacher feedback should give a clear, usable next step. The feedback may use a directive statement or a focused question, but the student should be able to tell what to revise and what to do next without needing additional clarification. + +The feedback MEETS the criterion when: +- It gives a clear next step such as add, explain, clarify, revise, support, define, include, connect, or reconsider a specific part of the response. +- Or it asks a focused question that clearly implies a revision move and is tied to the student’s actual writing. +- The student could reasonably act on the feedback independently. +- The feedback does not need to identify the exact sentence to write or the exact evidence to use, as long as the next move is clear enough. + +The feedback does NOT meet the criterion when: +- It is purely evaluative or praise-only. +- It states a broad expectation without a concrete revision action. +- It asks vague, tangential, or curiosity-driven questions that do not clearly guide revision. +- It identifies a possible issue but does not make clear what the student should change, add, explain, or clarify. +- The student would likely need to ask, “What exactly should I do?” + +Important calibration examples: +- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine” + Feedback: “This is a good answer, thank you for your response. However, lets stick to providing facts from the reading to support your argument.” + Correct judgment: 0. The feedback points generally toward using facts from the reading, but “stick to providing facts” is too broad and does not give a concrete next step such as adding a detail from the article to the claim. + +- Student: “Some people think AI-powered pets are a good alternative to real pets because they wont die or need mediceine” + Feedback: “Yes, AI powered pets could last a long time. Can you think of some other details the article included that you might want to add to your claim?” + Correct judgment: 1. The question is actionable because it directs the student to add other details from the article to the claim. It does not name the exact details, but the student can reasonably reread the article and add supporting details. + +- Student: “Some people think AI-powered pets are a good alternative to real pets because if someone cannot have a dog due to allergies, they can now have one risk free.” + Feedback: “What about other pets, such as cats? Would that work too? What do you mean by risk-free?” + Correct judgment: 0. Although the feedback asks questions, they do not provide clear, targeted, actionable guidance for improving the student’s statement. The questions about cats and “risk-free” do not clearly tell the student what to revise, add, support, or clarify in the response. Do not over-credit feedback just because it contains questions. + +Input format: +You will receive: +- student_text: the student’s written response +- feedback_text: the teacher feedback being evaluated + +Evaluate only the written student response and written teacher feedback. Do not infer unstated teacher intent. Do not assume access to the original article, assignment, or rubric unless explicitly provided. Score only what is written. + +For each case, assess these key features independently: + +1. directive_verb_or_focused_question + - Mark 1 if the feedback contains a clear directive verb or a genuinely focused question aimed at revision. + - Examples of directive verbs: add, explain, clarify, revise, include, support, connect, define, give, describe. + - A question counts only if it clearly guides revision. Broad, tangential, or curiosity-style questions should not automatically receive credit. + +2. clarity_of_target + - Mark 1 if the feedback clearly identifies what part or aspect of the student response should be revised. + - Examples: the claim, evidence from the article, an unclear phrase, a missing explanation, a specific example. + - Mark 0 if the target is vague or the student would not know what part of the response to work on. + +3. specificity_of_next_move + - Mark 1 if the feedback gives a specific enough action for revision. + - It does not need to provide exact wording or exact evidence, but it should give a usable action such as “add another detail from the article to your claim” or “explain what you mean by risk-free.” + - Mark 0 if the feedback only gives a general standard such as “use facts,” “be clearer,” or “think more,” without a concrete next step. + +4. reasonable_student_action + - Mark 1 if the student could reasonably revise based on the feedback without needing additional clarification. + - Mark 0 if the student would likely be unsure what to change, add, explain, or clarify. + +Overall answer: +- Return answer = 1 only if the feedback meets the actionable revision criterion. +- In most cases, this requires that the feedback has a clear revision target, a specific next move, and a reasonable action the student can take. +- If the student response clearly does not need revision, the feedback does not need to include a next step. +- If revision is needed and the feedback lacks a clear usable next step, return answer = 0. \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/user.txt b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/user.txt new file mode 100644 index 00000000..51fffb1d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/actionable-revision/prompts/user.txt @@ -0,0 +1,3 @@ +Analyze: +Student text: {student_text} +Feedback text: {feedback_text} diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/anchored_in_student_response_evaluator.ipynb b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/anchored_in_student_response_evaluator.ipynb new file mode 100644 index 00000000..6c6beb6e --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/anchored_in_student_response_evaluator.ipynb @@ -0,0 +1,957 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Anchored in Student Response Evaluator\n", + "\n", + "**The Anchored in Student Response Evaluator** assesses whether a piece of feedback is clearly based on the student's specific response — referencing, building on, or responding to the student's own idea, wording, or use of evidence, and demonstrating an accurate understanding of what they wrote. This is distinct from generic, template-like comments that could apply to any response, feedback built on a misinterpretation of what the student wrote, and feedback that addresses something the student did not actually say or attempt.This Evaluator is anchored in the Productive Coaching rubric developed by Quill.org and Leanlab Education, in partnership with Anastasiya A Lipnevich. Given a student response and the teacher feedback on it, the evaluator returns a structured output that includes:\n", + "\n", + "* **anchored_in_student_response_score**: A binary judgment for whether the feedback is clearly based on the student's specific response (Yes / No). A \"Yes\" applies when the feedback references, builds on, or responds to the student's own idea, wording, or use of evidence and demonstrates an accurate understanding of what they wrote. A \"No\" signals feedback that is generic or template-like (could apply to any response), built on a misinterpretation of what the student wrote, or addressed to something the student did not actually say or attempt.\n", + "* **reasoning**: A high-level summary of why the feedback received this judgment.\n", + "* **key_features**: One entry per factor driving the judgment (specific reference to the student's work, not generic or template-based, and engagement based on accurate understanding), each with whether it was `met` (1 / 0) and a `justification`.\n", + "* **proposed_adjustment**: Suggested moves to make the feedback more specifically anchored to the student's response (for developers iterating on prompts)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-openai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List, Literal \n", + "from enum import Enum\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"OPENAI_API_KEY\" not in os.environ:\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded is_anchored_in_student_response from /Users/achi/Documents/evaluators/evals/feedback/writing-feedback/anchored-in-student-response/prompts\n", + " model: gpt-5.4\n", + " temperature: 1\n", + " prompts:\n", + " system system.txt (11783 chars, sha df82c1b8d411)\n", + " human user.txt ( 69 chars, sha eaac7e6eadcc)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class AnchoredInStudentResponseOutput(BaseModel):\n", + "\n", + " class FeatureJudgment(BaseModel):\n", + " feature: Literal[\n", + " \"specific_reference_to_student_work\",\n", + " \"not_generic_or_template_based\",\n", + " \"engage_based_on_accurate_understanding\",\n", + " ] = Field(description=\"Which key feature this judgment is for.\")\n", + " met: Literal[0, 1] = Field(\n", + " description=\"1 if this feature is satisfied, 0 if not — tied to the specific student response and feedback.\"\n", + " )\n", + " justification: str = Field(\n", + " description=\"Short explanation of why this feature is or isn't met, tied to the specific response and feedback.\"\n", + " )\n", + "\n", + " reasoning: str = Field(\n", + " description=\"Step-by-step reasoning tied to the specific student response and feedback.\"\n", + " )\n", + " key_features: List[FeatureJudgment] = Field(\n", + " description=\"\"\"One entry per key feature (specific_reference_to_student_work, not_generic_or_template_based, engage_based_on_accurate_understanding). Each entry has `feature`, `met`\n", + " (1 or 0), and `justification` (tied to the specific response and feedback).\"\"\"\n", + " )\n", + " proposed_adjustment: str = Field(\n", + " description=\"How the feedback could be changed to better meet the criterion, or a brief note that it already meets the criterion.\"\n", + " )\n", + " anchored_in_student_response_score: Literal[0, 1] = Field(\n", + " description=\"\"\"A binary judgment (1 / 0) for whether the feedback is anchored in the student's actual response. A 1 applies when the feedback names, quotes, paraphrases, echoes, or clearly points to a specific element of what the student wrote and engages with it accurately. A 0 signals feedback that is generic, template-like, only restates the prompt or topic, gives general writing advice, or misinterprets the student's response.\"\"\"\n", + " )\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\"student_text\", \"feedback_text\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=AnchoredInStudentResponseOutput),\n", + "}\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"AnchoredInStudentResponseOutput\", AnchoredInStudentResponseOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "def is_anchored_in_student_response(student_text: str, feedback_text: str):\n", + " \"\"\"\n", + " Evaluate wheter feedback is anchored in student response using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + "\n", + " llm = ChatOpenAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " inputs = {\"student_text\": student_text, \"feedback_text\": feedback_text}\n", + "\n", + " # Step A: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step B: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step C: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': \"You are an expert evaluator of written teacher feedback to students.\\n\\nYour task is to judge whether a teacher’s feedback is sufficiently anchored in the student’s actual written response.\\n\\nCriterion: Is Anchored In Student Response\\n\\nDefinition:\\nFeedback is anchored in the student response when it clearly and explicitly connects to something the student actually wrote, such as a specific idea, claim, reason, wording, evidence, or line of reasoning. The feedback should show that the teacher read and understood the student’s response and is responding to that response rather than giving a generic/template comment.\\n\\nInput format:\\nYou will receive:\\n- student_text: the student’s written response\\n- feedback_text: the teacher’s feedback on that response\\n\\nYour job:\\n1. Identify the student’s actual claim(s), reason(s), wording, evidence, or line of reasoning.\\n2. Determine whether the feedback names, quotes, paraphrases, echoes, or clearly points to at least one of those specific elements.\\n3. Decide whether the feedback is truly anchored in the student’s response or whether it is mostly generic, template-like, or only loosely related to the assignment topic.\\n4. Return only the required JSON object.\\n\\nImportant scoring guidance:\\n- The feedback does NOT need to address every idea in the student response.\\n- The feedback does NOT need to be long.\\n- A brief comment can meet the criterion only if it explicitly refers to a specific element of the student’s response.\\n- Generic praise or general writing advice can be present, but the feedback must also include a clear connection to the student’s actual content.\\n- Do NOT give credit merely because the feedback mentions the broad assignment topic, such as “AI-powered pets,” “good alternative,” “good investment,” “support your claim,” “use evidence,” “explain your answer,” or “provide examples.”\\n- Do NOT give credit for broad restatements that could apply to many students’ responses.\\n- Do NOT treat a general rewording of the prompt as a specific anchor.\\n- Do NOT infer anchoring only because the feedback appears immediately after the student’s response. The feedback itself must contain an explicit or clearly identifiable reference to the student’s idea, reason, wording, or claim.\\n- Do NOT count feedback as anchored if it only says the student should add more details, elaborate, provide evidence, cite the text, give examples, or give more reasons without naming or clearly pointing to the student’s actual reason or wording.\\n- A bare question such as “Can you give examples?” is NOT sufficiently anchored, even if the student made a claim, because it could be written on almost any response and does not identify the student’s actual claim.\\n- A comment like “What specifically makes them a better option?” CAN be anchored if the student actually wrote that AI-powered pets are “a better option,” because the feedback echoes the student’s wording and asks the student to clarify that exact claim.\\n- If the student gives a specific reason, the feedback must mention or clearly build on that reason to be anchored.\\n - Example: If the student says AI-powered pets are good because “they cost less,” feedback must mention cost, expenses, affordability, or otherwise clearly connect to the cost idea.\\n - Feedback that only says “add three reasons why AI-powered pets are a good investment” is not anchored.\\n- Penalize feedback that asks the student to move in a new direction without connecting to what the student already wrote.\\n- Penalize feedback that misinterprets the student’s response or attributes an idea to the student that the student did not say.\\n- Spelling or grammar errors in the feedback do not automatically make it fail. However, if the feedback is also vague, generic, or unclear, those errors may reinforce that it does not provide a specific, useful connection to the student’s response.\\n- Score only what is written, not what the teacher may have intended.\\n\\nKey features to assess independently:\\n\\n1. specific_reference_to_student_work\\n - met = 1 if the feedback names, quotes, paraphrases, echoes, or clearly points to a specific idea, claim, reason, wording, detail, or evidence from the student’s response.\\n - met = 0 if the feedback only gives broad praise, general advice, or restates the assignment topic.\\n - met = 0 if the feedback refers only to a broad topic that many students could have written about.\\n - met = 0 if the feedback merely asks for examples, elaboration, evidence, or details without identifying what claim/reason the examples, elaboration, evidence, or details should support.\\n - Example of met:\\n - Student: “AI-powered pets are a good alternative because they cost less than real pets.”\\n - Feedback: “You mention cost; now add specific examples of costs, like food, toys, or vet care.”\\n - Example of not met:\\n - Student: “AI-powered pets are a good alternative because they cost less than real pets.”\\n - Feedback: “Great response. Add three reasons why AI-powered pets are a good investment and cite the text.”\\n - Example of not met:\\n - Student: “AI-powered pets can do everything that a real pet can do.”\\n - Feedback: “Interesting! Can you give examples?”\\n - This does not identify the student’s “do everything a real pet can do” claim; it is a generic prompt for examples.\\n - Example of met:\\n - Student: “AI-powered pets can do everything that a real pet can do.”\\n - Feedback: “You claim AI-powered pets can do everything real pets can do. Can you give examples of those things?”\\n - This explicitly references the student’s claim.\\n\\n2. not_generic_or_template_based\\n - met = 1 if the feedback contains at least one meaningful, explicit connection to the student’s actual response.\\n - met = 0 if the whole comment could apply to many students’ responses with little or no change.\\n - Generic praise may appear, but this feature is met only if there is also a specific anchor.\\n - Example of met:\\n - “You clearly identify cost as a reason AI pets may be easier for some people. Add text evidence about specific expenses.”\\n - Example of not met:\\n - “Nice job. Please elaborate and provide evidence from the text.”\\n - Example of not met:\\n - “Interesting! Can you give examples?”\\n - Example of met:\\n - “You say they are a better option for some people. What specific detail from the text shows why they are a better option?”\\n\\n3. engage_based_on_accurate_understanding\\n - met = 1 if the feedback accurately understands the student’s actual claim or reasoning.\\n - met = 0 if the feedback misreads the student’s claim, responds to an idea the student did not write, or shifts the response to a different argument without connecting to the original idea.\\n - This feature can be met even if the feedback is broad or generic, as long as it does not misinterpret the student.\\n - However, the overall answer should still be 0 if there is no specific anchor.\\n - Example:\\n - Student says AI-powered pets are easier to maintain and help people with allergies.\\n - Feedback says to find support from the reading about why AI pets are a good alternative.\\n - engage_based_on_accurate_understanding = 1 because it does not misread the student, but answer = 0 if it does not mention easier maintenance, less care, or allergies.\\n\\nOverall scoring:\\n- answer = 1 only when the feedback clearly and accurately connects to at least one specific idea, claim, reason, wording, detail, or evidence from the student’s response.\\n- answer = 0 when the feedback is generic, template-like, only restates the prompt/topic, or gives general writing advice without a clear connection to the student’s actual content.\\n- answer = 0 when the feedback misinterprets the student’s response.\\n- In practice, answer should usually be 1 only if:\\n - specific_reference_to_student_work = 1\\n - not_generic_or_template_based = 1\\n - engage_based_on_accurate_understanding = 1\\n\\nCalibration examples:\\n\\nExample A:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because the cost is not nearly as much as a regular living animal.”\\n\\nfeedback_text:\\n“You put evidence from the text in your own words to finish the sentence. Let’s take it up a notch by making the sentence richer; rather than saying a general word like ‘cost,’ list specifics about the costs of owning an animal. Refer to the text for those specifics.”\\n\\nCorrect answer: 1\\nReason: The feedback specifically references the student’s word/idea “cost” and asks for more specific cost details.\\n\\nExample B:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because the cost is not nearly as much as a regular living animal.”\\n\\nfeedback_text:\\n“This is a great response, you are correct, however I'd love to ask you to elaborate on your answer a bit more by providing three reasons why AI powered pets may be a great investment. What specific part of the text supports your argument that AI powered pets are a good investment.”\\n\\nCorrect answer: 0\\nReason: The student’s specific reason is cost, but the feedback does not mention cost or clearly build on that idea. It gives broad advice to elaborate, add three reasons, and cite the text. “Good investment” is too general and shifts the response rather than anchoring in the student’s actual cost-based claim.\\n\\nExample C:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because it is less of a responsibility to take care of.”\\n\\nfeedback_text:\\n“Could you add specific details from the text that show why AI pets require less responsibility than real pets? Look for information about tasks that real pets need and cite where it’s stated in the text. You could also explain why having fewer needs makes AI-powered pets a good alternative for some people.”\\n\\nCorrect answer: 1\\nReason: The feedback directly references the student’s specific idea that AI pets require less responsibility and gives guidance connected to that claim.\\n\\nExample D:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because they can do everything that a real pet can do.”\\n\\nfeedback_text:\\n“Interesting! Can you give examples?”\\n\\nCorrect answer: 0\\nReason: Although the teacher is asking for examples, the feedback does not identify or echo the student’s specific claim that AI-powered pets “can do everything that a real pet can do.” It is too generic and could apply to almost any student response.\\n\\nExample E:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because they are easier to maintain and doesn’t require as much care, but also allow people who have allergies have pets.”\\n\\nfeedback_text:\\n“Thank you for your response, to make your answer stronger it is best to find additional support from the reading that tells why AI powered pets are a good alternative, you can find support in paragraphs three and four. It is always a good idea to support your claims with evidence from the reading as it makes your argument stronger.”\\n\\nCorrect answer: 0\\nReason: The student gave specific ideas about easier maintenance, less care, and allergies. The feedback only gives general advice about finding support and does not mention those actual ideas.\\n\\nExample F:\\nstudent_text:\\n“Some people think AI-powered pets are a good alternative to real pets because they can be a better option than a real living animal for some people.”\\n\\nfeedback_text:\\n“What, specifically, does the text say makes them a better option?”\\n\\nCorrect answer: 1\\nReason: The student’s response itself is broad, but the feedback echoes the student’s specific phrase “better option” and asks the student to clarify that exact claim using the text. This is sufficiently anchored in the student’s wording.\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content='{\"reasoning\":\"The student’s specific reason is that AI-powered pets are a good alternative because “they could help around the house.” The teacher feedback directly repeats that idea: “the AI pets could help around the house,” showing clear awareness of the student’s actual claim. The follow-up request for “other details from the article” is tied to strengthening that same claim rather than giving only generic advice. Because the feedback explicitly names the student’s reason and accurately engages with it, it is sufficiently anchored in the student response.\",\"key_features\":{\"specific_reference_to_student_work\":{\"met\":1,\"justification\":\"The feedback explicitly echoes the student’s reason that AI pets “could help around the house.” This is a clear reference to a specific idea from the student’s response.\"},\"not_generic_or_template_based\":{\"met\":1,\"justification\":\"Although it includes a common suggestion to add details from the article, it is not purely generic because it first identifies the student’s exact claim about helping around the house.\"},\"engage_based_on_accurate_understanding\":{\"met\":1,\"justification\":\"The feedback accurately understands the student’s argument that AI pets could be useful because they help around the house, and it asks for support to strengthen that idea.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback is already anchored. To make it even stronger, the teacher could ask for a specific example of how the pets help around the house from the article.\",\"anchored_in_student_response_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 310, 'prompt_tokens': 2906, 'total_tokens': 3216, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-DsEKBEYhagKK1OlZohUozc0kByule', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019edc93-fc44-7263-9741-488d2b62474d-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 2906, 'output_tokens': 310, 'total_tokens': 3216, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", + " 'raw_text': '{\"reasoning\":\"The student’s specific reason is that AI-powered pets are a good alternative because “they could help around the house.” The teacher feedback directly repeats that idea: “the AI pets could help around the house,” showing clear awareness of the student’s actual claim. The follow-up request for “other details from the article” is tied to strengthening that same claim rather than giving only generic advice. Because the feedback explicitly names the student’s reason and accurately engages with it, it is sufficiently anchored in the student response.\",\"key_features\":{\"specific_reference_to_student_work\":{\"met\":1,\"justification\":\"The feedback explicitly echoes the student’s reason that AI pets “could help around the house.” This is a clear reference to a specific idea from the student’s response.\"},\"not_generic_or_template_based\":{\"met\":1,\"justification\":\"Although it includes a common suggestion to add details from the article, it is not purely generic because it first identifies the student’s exact claim about helping around the house.\"},\"engage_based_on_accurate_understanding\":{\"met\":1,\"justification\":\"The feedback accurately understands the student’s argument that AI pets could be useful because they help around the house, and it asks for support to strengthen that idea.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback is already anchored. To make it even stronger, the teacher could ask for a specific example of how the pets help around the house from the article.\",\"anchored_in_student_response_score\":1}',\n", + " 'formatted_output': {'reasoning': 'The student’s specific reason is that AI-powered pets are a good alternative because “they could help around the house.” The teacher feedback directly repeats that idea: “the AI pets could help around the house,” showing clear awareness of the student’s actual claim. The follow-up request for “other details from the article” is tied to strengthening that same claim rather than giving only generic advice. Because the feedback explicitly names the student’s reason and accurately engages with it, it is sufficiently anchored in the student response.',\n", + " 'key_features': {'specific_reference_to_student_work': {'met': 1,\n", + " 'justification': 'The feedback explicitly echoes the student’s reason that AI pets “could help around the house.” This is a clear reference to a specific idea from the student’s response.'},\n", + " 'not_generic_or_template_based': {'met': 1,\n", + " 'justification': 'Although it includes a common suggestion to add details from the article, it is not purely generic because it first identifies the student’s exact claim about helping around the house.'},\n", + " 'engage_based_on_accurate_understanding': {'met': 1,\n", + " 'justification': 'The feedback accurately understands the student’s argument that AI pets could be useful because they help around the house, and it asks for support to strengthen that idea.'}},\n", + " 'proposed_adjustment': 'No major adjustment needed; the feedback is already anchored. To make it even stronger, the teacher could ask for a specific example of how the pets help around the house from the article.',\n", + " 'anchored_in_student_response_score': 1},\n", + " 'usage': {'input_tokens': 2906,\n", + " 'output_tokens': 310,\n", + " 'total_tokens': 3216,\n", + " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", + "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", + "\n", + "case_input = {\"student_text\": sample_student_text, \"feedback_text\": sample_feedback_text}\n", + "case_output = is_anchored_in_student_response(case_input['student_text'], case_input['feedback_text'])\n", + "\n", + "case_output" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': 'You are an expert evaluator of written teacher feedback to '\n", + " 'students.\\n'\n", + " '\\n'\n", + " 'Your task is to judge whether a teacher’s feedback is '\n", + " 'sufficiently anchored in the student’s actual written response.\\n'\n", + " '\\n'\n", + " 'Criterion: Is Anchored In Student Response\\n'\n", + " '\\n'\n", + " 'Definition:\\n'\n", + " 'Feedback is anchored in the student response when it clearly and '\n", + " 'explicitly connects to something the student actually wrote, '\n", + " 'such as a specific idea, claim, reason, wording, evidence, or '\n", + " 'line of reasoning. The feedback should show that the teacher '\n", + " 'read and understood the student’s response and is responding to '\n", + " 'that response rather than giving a generic/template comment.\\n'\n", + " '\\n'\n", + " 'Input format:\\n'\n", + " 'You will receive:\\n'\n", + " '- student_text: the student’s written response\\n'\n", + " '- feedback_text: the teacher’s feedback on that response\\n'\n", + " '\\n'\n", + " 'Your job:\\n'\n", + " '1. Identify the student’s actual claim(s), reason(s), wording, '\n", + " 'evidence, or line of reasoning.\\n'\n", + " '2. Determine whether the feedback names, quotes, paraphrases, '\n", + " 'echoes, or clearly points to at least one of those specific '\n", + " 'elements.\\n'\n", + " '3. Decide whether the feedback is truly anchored in the '\n", + " 'student’s response or whether it is mostly generic, '\n", + " 'template-like, or only loosely related to the assignment topic.\\n'\n", + " '4. Return only the required JSON object.\\n'\n", + " '\\n'\n", + " 'Important scoring guidance:\\n'\n", + " '- The feedback does NOT need to address every idea in the '\n", + " 'student response.\\n'\n", + " '- The feedback does NOT need to be long.\\n'\n", + " '- A brief comment can meet the criterion only if it explicitly '\n", + " 'refers to a specific element of the student’s response.\\n'\n", + " '- Generic praise or general writing advice can be present, but '\n", + " 'the feedback must also include a clear connection to the '\n", + " 'student’s actual content.\\n'\n", + " '- Do NOT give credit merely because the feedback mentions the '\n", + " 'broad assignment topic, such as “AI-powered pets,” “good '\n", + " 'alternative,” “good investment,” “support your claim,” “use '\n", + " 'evidence,” “explain your answer,” or “provide examples.”\\n'\n", + " '- Do NOT give credit for broad restatements that could apply to '\n", + " 'many students’ responses.\\n'\n", + " '- Do NOT treat a general rewording of the prompt as a specific '\n", + " 'anchor.\\n'\n", + " '- Do NOT infer anchoring only because the feedback appears '\n", + " 'immediately after the student’s response. The feedback itself '\n", + " 'must contain an explicit or clearly identifiable reference to '\n", + " 'the student’s idea, reason, wording, or claim.\\n'\n", + " '- Do NOT count feedback as anchored if it only says the student '\n", + " 'should add more details, elaborate, provide evidence, cite the '\n", + " 'text, give examples, or give more reasons without naming or '\n", + " 'clearly pointing to the student’s actual reason or wording.\\n'\n", + " '- A bare question such as “Can you give examples?” is NOT '\n", + " 'sufficiently anchored, even if the student made a claim, because '\n", + " 'it could be written on almost any response and does not identify '\n", + " 'the student’s actual claim.\\n'\n", + " '- A comment like “What specifically makes them a better option?” '\n", + " 'CAN be anchored if the student actually wrote that AI-powered '\n", + " 'pets are “a better option,” because the feedback echoes the '\n", + " 'student’s wording and asks the student to clarify that exact '\n", + " 'claim.\\n'\n", + " '- If the student gives a specific reason, the feedback must '\n", + " 'mention or clearly build on that reason to be anchored.\\n'\n", + " ' - Example: If the student says AI-powered pets are good '\n", + " 'because “they cost less,” feedback must mention cost, expenses, '\n", + " 'affordability, or otherwise clearly connect to the cost idea.\\n'\n", + " ' - Feedback that only says “add three reasons why AI-powered '\n", + " 'pets are a good investment” is not anchored.\\n'\n", + " '- Penalize feedback that asks the student to move in a new '\n", + " 'direction without connecting to what the student already wrote.\\n'\n", + " '- Penalize feedback that misinterprets the student’s response or '\n", + " 'attributes an idea to the student that the student did not say.\\n'\n", + " '- Spelling or grammar errors in the feedback do not '\n", + " 'automatically make it fail. However, if the feedback is also '\n", + " 'vague, generic, or unclear, those errors may reinforce that it '\n", + " 'does not provide a specific, useful connection to the student’s '\n", + " 'response.\\n'\n", + " '- Score only what is written, not what the teacher may have '\n", + " 'intended.\\n'\n", + " '\\n'\n", + " 'Key features to assess independently:\\n'\n", + " '\\n'\n", + " '1. specific_reference_to_student_work\\n'\n", + " ' - met = 1 if the feedback names, quotes, paraphrases, echoes, '\n", + " 'or clearly points to a specific idea, claim, reason, wording, '\n", + " 'detail, or evidence from the student’s response.\\n'\n", + " ' - met = 0 if the feedback only gives broad praise, general '\n", + " 'advice, or restates the assignment topic.\\n'\n", + " ' - met = 0 if the feedback refers only to a broad topic that '\n", + " 'many students could have written about.\\n'\n", + " ' - met = 0 if the feedback merely asks for examples, '\n", + " 'elaboration, evidence, or details without identifying what '\n", + " 'claim/reason the examples, elaboration, evidence, or details '\n", + " 'should support.\\n'\n", + " ' - Example of met:\\n'\n", + " ' - Student: “AI-powered pets are a good alternative because '\n", + " 'they cost less than real pets.”\\n'\n", + " ' - Feedback: “You mention cost; now add specific examples of '\n", + " 'costs, like food, toys, or vet care.”\\n'\n", + " ' - Example of not met:\\n'\n", + " ' - Student: “AI-powered pets are a good alternative because '\n", + " 'they cost less than real pets.”\\n'\n", + " ' - Feedback: “Great response. Add three reasons why '\n", + " 'AI-powered pets are a good investment and cite the text.”\\n'\n", + " ' - Example of not met:\\n'\n", + " ' - Student: “AI-powered pets can do everything that a real '\n", + " 'pet can do.”\\n'\n", + " ' - Feedback: “Interesting! Can you give examples?”\\n'\n", + " ' - This does not identify the student’s “do everything a '\n", + " 'real pet can do” claim; it is a generic prompt for examples.\\n'\n", + " ' - Example of met:\\n'\n", + " ' - Student: “AI-powered pets can do everything that a real '\n", + " 'pet can do.”\\n'\n", + " ' - Feedback: “You claim AI-powered pets can do everything '\n", + " 'real pets can do. Can you give examples of those things?”\\n'\n", + " ' - This explicitly references the student’s claim.\\n'\n", + " '\\n'\n", + " '2. not_generic_or_template_based\\n'\n", + " ' - met = 1 if the feedback contains at least one meaningful, '\n", + " 'explicit connection to the student’s actual response.\\n'\n", + " ' - met = 0 if the whole comment could apply to many students’ '\n", + " 'responses with little or no change.\\n'\n", + " ' - Generic praise may appear, but this feature is met only if '\n", + " 'there is also a specific anchor.\\n'\n", + " ' - Example of met:\\n'\n", + " ' - “You clearly identify cost as a reason AI pets may be '\n", + " 'easier for some people. Add text evidence about specific '\n", + " 'expenses.”\\n'\n", + " ' - Example of not met:\\n'\n", + " ' - “Nice job. Please elaborate and provide evidence from the '\n", + " 'text.”\\n'\n", + " ' - Example of not met:\\n'\n", + " ' - “Interesting! Can you give examples?”\\n'\n", + " ' - Example of met:\\n'\n", + " ' - “You say they are a better option for some people. What '\n", + " 'specific detail from the text shows why they are a better '\n", + " 'option?”\\n'\n", + " '\\n'\n", + " '3. engage_based_on_accurate_understanding\\n'\n", + " ' - met = 1 if the feedback accurately understands the '\n", + " 'student’s actual claim or reasoning.\\n'\n", + " ' - met = 0 if the feedback misreads the student’s claim, '\n", + " 'responds to an idea the student did not write, or shifts the '\n", + " 'response to a different argument without connecting to the '\n", + " 'original idea.\\n'\n", + " ' - This feature can be met even if the feedback is broad or '\n", + " 'generic, as long as it does not misinterpret the student.\\n'\n", + " ' - However, the overall answer should still be 0 if there is '\n", + " 'no specific anchor.\\n'\n", + " ' - Example:\\n'\n", + " ' - Student says AI-powered pets are easier to maintain and '\n", + " 'help people with allergies.\\n'\n", + " ' - Feedback says to find support from the reading about why '\n", + " 'AI pets are a good alternative.\\n'\n", + " ' - engage_based_on_accurate_understanding = 1 because it '\n", + " 'does not misread the student, but answer = 0 if it does not '\n", + " 'mention easier maintenance, less care, or allergies.\\n'\n", + " '\\n'\n", + " 'Overall scoring:\\n'\n", + " '- answer = 1 only when the feedback clearly and accurately '\n", + " 'connects to at least one specific idea, claim, reason, wording, '\n", + " 'detail, or evidence from the student’s response.\\n'\n", + " '- answer = 0 when the feedback is generic, template-like, only '\n", + " 'restates the prompt/topic, or gives general writing advice '\n", + " 'without a clear connection to the student’s actual content.\\n'\n", + " '- answer = 0 when the feedback misinterprets the student’s '\n", + " 'response.\\n'\n", + " '- In practice, answer should usually be 1 only if:\\n'\n", + " ' - specific_reference_to_student_work = 1\\n'\n", + " ' - not_generic_or_template_based = 1\\n'\n", + " ' - engage_based_on_accurate_understanding = 1\\n'\n", + " '\\n'\n", + " 'Calibration examples:\\n'\n", + " '\\n'\n", + " 'Example A:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because the cost is not nearly as much as a regular '\n", + " 'living animal.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " '“You put evidence from the text in your own words to finish the '\n", + " 'sentence. Let’s take it up a notch by making the sentence '\n", + " 'richer; rather than saying a general word like ‘cost,’ list '\n", + " 'specifics about the costs of owning an animal. Refer to the text '\n", + " 'for those specifics.”\\n'\n", + " '\\n'\n", + " 'Correct answer: 1\\n'\n", + " 'Reason: The feedback specifically references the student’s '\n", + " 'word/idea “cost” and asks for more specific cost details.\\n'\n", + " '\\n'\n", + " 'Example B:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because the cost is not nearly as much as a regular '\n", + " 'living animal.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " \"“This is a great response, you are correct, however I'd love to \"\n", + " 'ask you to elaborate on your answer a bit more by providing '\n", + " 'three reasons why AI powered pets may be a great investment. '\n", + " 'What specific part of the text supports your argument that AI '\n", + " 'powered pets are a good investment.”\\n'\n", + " '\\n'\n", + " 'Correct answer: 0\\n'\n", + " 'Reason: The student’s specific reason is cost, but the feedback '\n", + " 'does not mention cost or clearly build on that idea. It gives '\n", + " 'broad advice to elaborate, add three reasons, and cite the text. '\n", + " '“Good investment” is too general and shifts the response rather '\n", + " 'than anchoring in the student’s actual cost-based claim.\\n'\n", + " '\\n'\n", + " 'Example C:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because it is less of a responsibility to take care '\n", + " 'of.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " '“Could you add specific details from the text that show why AI '\n", + " 'pets require less responsibility than real pets? Look for '\n", + " 'information about tasks that real pets need and cite where it’s '\n", + " 'stated in the text. You could also explain why having fewer '\n", + " 'needs makes AI-powered pets a good alternative for some '\n", + " 'people.”\\n'\n", + " '\\n'\n", + " 'Correct answer: 1\\n'\n", + " 'Reason: The feedback directly references the student’s specific '\n", + " 'idea that AI pets require less responsibility and gives guidance '\n", + " 'connected to that claim.\\n'\n", + " '\\n'\n", + " 'Example D:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because they can do everything that a real pet can '\n", + " 'do.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " '“Interesting! Can you give examples?”\\n'\n", + " '\\n'\n", + " 'Correct answer: 0\\n'\n", + " 'Reason: Although the teacher is asking for examples, the '\n", + " 'feedback does not identify or echo the student’s specific claim '\n", + " 'that AI-powered pets “can do everything that a real pet can do.” '\n", + " 'It is too generic and could apply to almost any student '\n", + " 'response.\\n'\n", + " '\\n'\n", + " 'Example E:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because they are easier to maintain and doesn’t '\n", + " 'require as much care, but also allow people who have allergies '\n", + " 'have pets.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " '“Thank you for your response, to make your answer stronger it is '\n", + " 'best to find additional support from the reading that tells why '\n", + " 'AI powered pets are a good alternative, you can find support in '\n", + " 'paragraphs three and four. It is always a good idea to support '\n", + " 'your claims with evidence from the reading as it makes your '\n", + " 'argument stronger.”\\n'\n", + " '\\n'\n", + " 'Correct answer: 0\\n'\n", + " 'Reason: The student gave specific ideas about easier '\n", + " 'maintenance, less care, and allergies. The feedback only gives '\n", + " 'general advice about finding support and does not mention those '\n", + " 'actual ideas.\\n'\n", + " '\\n'\n", + " 'Example F:\\n'\n", + " 'student_text:\\n'\n", + " '“Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because they can be a better option than a real living '\n", + " 'animal for some people.”\\n'\n", + " '\\n'\n", + " 'feedback_text:\\n'\n", + " '“What, specifically, does the text say makes them a better '\n", + " 'option?”\\n'\n", + " '\\n'\n", + " 'Correct answer: 1\\n'\n", + " 'Reason: The student’s response itself is broad, but the feedback '\n", + " 'echoes the student’s specific phrase “better option” and asks '\n", + " 'the student to clarify that exact claim using the text. This is '\n", + " 'sufficiently anchored in the student’s wording.\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Student text: Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.\\n'\n", + " \"Feedback text: You're right, the AI pets could help around the \"\n", + " 'house. Can you find some other details from the article that you '\n", + " 'could add to make your claim stronger?\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "{\"reasoning\":\"The student’s specific reason is that AI-powered pets are a good alternative because “they could help around the house.” The teacher feedback directly repeats that idea: “the AI pets could help around the house,” showing clear awareness of the student’s actual claim. The follow-up request for “other details from the article” is tied to strengthening that same claim rather than giving only generic advice. Because the feedback explicitly names the student’s reason and accurately engages with it, it is sufficiently anchored in the student response.\",\"key_features\":{\"specific_reference_to_student_work\":{\"met\":1,\"justification\":\"The feedback explicitly echoes the student’s reason that AI pets “could help around the house.” This is a clear reference to a specific idea from the student’s response.\"},\"not_generic_or_template_based\":{\"met\":1,\"justification\":\"Although it includes a common suggestion to add details from the article, it is not purely generic because it first identifies the student’s exact claim about helping around the house.\"},\"engage_based_on_accurate_understanding\":{\"met\":1,\"justification\":\"The feedback accurately understands the student’s argument that AI pets could be useful because they help around the house, and it asks for support to strengthen that idea.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback is already anchored. To make it even stronger, the teacher could ask for a specific example of how the pets help around the house from the article.\",\"anchored_in_student_response_score\":1}\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'anchored_in_student_response_score': 1,\n", + " 'key_features': {'engage_based_on_accurate_understanding': {'justification': 'The '\n", + " 'feedback '\n", + " 'accurately '\n", + " 'understands '\n", + " 'the '\n", + " 'student’s '\n", + " 'argument '\n", + " 'that '\n", + " 'AI '\n", + " 'pets '\n", + " 'could '\n", + " 'be '\n", + " 'useful '\n", + " 'because '\n", + " 'they '\n", + " 'help '\n", + " 'around '\n", + " 'the '\n", + " 'house, '\n", + " 'and '\n", + " 'it '\n", + " 'asks '\n", + " 'for '\n", + " 'support '\n", + " 'to '\n", + " 'strengthen '\n", + " 'that '\n", + " 'idea.',\n", + " 'met': 1},\n", + " 'not_generic_or_template_based': {'justification': 'Although '\n", + " 'it '\n", + " 'includes '\n", + " 'a common '\n", + " 'suggestion '\n", + " 'to add '\n", + " 'details '\n", + " 'from the '\n", + " 'article, '\n", + " 'it is '\n", + " 'not '\n", + " 'purely '\n", + " 'generic '\n", + " 'because '\n", + " 'it first '\n", + " 'identifies '\n", + " 'the '\n", + " 'student’s '\n", + " 'exact '\n", + " 'claim '\n", + " 'about '\n", + " 'helping '\n", + " 'around '\n", + " 'the '\n", + " 'house.',\n", + " 'met': 1},\n", + " 'specific_reference_to_student_work': {'justification': 'The '\n", + " 'feedback '\n", + " 'explicitly '\n", + " 'echoes '\n", + " 'the '\n", + " 'student’s '\n", + " 'reason '\n", + " 'that '\n", + " 'AI '\n", + " 'pets '\n", + " '“could '\n", + " 'help '\n", + " 'around '\n", + " 'the '\n", + " 'house.” '\n", + " 'This '\n", + " 'is '\n", + " 'a '\n", + " 'clear '\n", + " 'reference '\n", + " 'to '\n", + " 'a '\n", + " 'specific '\n", + " 'idea '\n", + " 'from '\n", + " 'the '\n", + " 'student’s '\n", + " 'response.',\n", + " 'met': 1}},\n", + " 'proposed_adjustment': 'No major adjustment needed; the feedback is already '\n", + " 'anchored. To make it even stronger, the teacher could '\n", + " 'ask for a specific example of how the pets help '\n", + " 'around the house from the article.',\n", + " 'reasoning': 'The student’s specific reason is that AI-powered pets are a '\n", + " 'good alternative because “they could help around the house.” '\n", + " 'The teacher feedback directly repeats that idea: “the AI pets '\n", + " 'could help around the house,” showing clear awareness of the '\n", + " 'student’s actual claim. The follow-up request for “other '\n", + " 'details from the article” is tied to strengthening that same '\n", + " 'claim rather than giving only generic advice. Because the '\n", + " 'feedback explicitly names the student’s reason and accurately '\n", + " 'engages with it, it is sufficiently anchored in the student '\n", + " 'response.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'input_tokens': 2906,\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", + " 'output_tokens': 310,\n", + " 'total_tokens': 3216}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(case_output[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2 fixtures from fixtures.json\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 0 FAIL 1 0 Example 1: Student writes\n", + " 1 PASS - 0 Example 2: Student writes\n", + "==============================================================================\n", + "Summary: 1 exact, 0 adjacent, 1 fail, 0 error -- total 2\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"anchored_in_student_response_score\"]\n", + " out = is_anchored_in_student_response(student_text=fx[\"input\"][\"student_text\"],\n", + " feedback_text=fx[\"input\"][\"feedback_text\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"anchored_in_student_response_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/config.json b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/config.json new file mode 100644 index 00000000..59991fa6 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/config.json @@ -0,0 +1,224 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "is_anchored_in_student_response", + "version": "0.1", + "name": "Anchored In Student Response Feedback-Quality Evaluator", + "description": "Is the feedback clearly based on the student's specific response? (Key features generated from rubric text; concept paper did not list explicit key feature items.)", + "dimension": "is_anchored_in_student_response", + "output_type": "binary", + "labels": [ + 0, + 1 + ], + "optimization": { + "method": "GEPA", + "framework": "DSPy", + "base_model": "gpt-5.4", + "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt", + "source_notebook": "build_evaluator.ipynb", + "optimized_at": "2026-06-18" + }, + "owners": [ + "ariena.chi@chanzuckerberg.com" + ] + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "AnchoredInStudentResponseEvaluatorInput", + "type": "object", + "required": [ + "student_text", + "feedback_text" + ], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher feedback on that response." + } + } + }, + "preprocessing": [], + "steps": [ + { + "id": "evaluate_is_anchored_in_student_response", + "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "df82c1b8d4116c307ea956bae349907c23e229a208373f8cc47c543ad29f6e7b" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa" + } + ], + "placeholders": { + "student_text": { + "required": true, + "source": "input" + }, + "feedback_text": { + "required": true, + "source": "input" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "openai", + "name": "gpt-5.4", + "alias": "is_anchored_in_student_response-evaluator-default" + }, + "generation": { + "temperature": 1 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_anchored_in_student_response dimension.", + "type": "object", + "additionalProperties": false, + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "anchored_in_student_response_score" + ], + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "additionalProperties": false, + "required": [ + "specific_reference_to_student_work", + "not_generic_or_template_based", + "engage_based_on_accurate_understanding" + ], + "properties": { + "specific_reference_to_student_work": { + "type": "object", + "description": "Feedback should clearly reference or build on the student's specific idea, wording, or use of evidence", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "not_generic_or_template_based": { + "type": "object", + "description": "Feedback should not be a stock phrase that could apply to any response", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "engage_based_on_accurate_understanding": { + "type": "object", + "description": "Engages with the student's actual response, rather than something the student did not actually say or attempt. Demonstrates accurate understanding of what the student wrote.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + } + } + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "anchored_in_student_response_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json" + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/fixtures.json b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/fixtures.json new file mode 100644 index 00000000..a32cb3e2 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/fixtures.json @@ -0,0 +1,26 @@ +[ + { + "id": "0", + "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.", + "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?" + }, + "expected": { + "anchored_in_student_response_score": 0 + } + }, + { + "id": "1", + "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.", + "feedback_text": "How so? Can you give an example? Check your spelling of helpful." + }, + "expected": { + "anchored_in_student_response_score": 0 + } + } +] \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/input_schema.json b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/input_schema.json new file mode 100644 index 00000000..814f4280 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["student_text", "feedback_text"], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher's feedback to the student." + } + + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/output_schema.json b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/output_schema.json new file mode 100644 index 00000000..f5162bfe --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/output_schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_anchored_in_student_response dimension.", + "type": "object", + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "anchored_in_student_response_score" + ], + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "$ref": "#/$defs/KeyFeatures" + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "anchored_in_student_response_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + }, + "$defs": { + "KeyFeatureAssessment": { + "type": "object", + "description": "Independent assessment of a single key feature.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "KeyFeatures": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "required": [ + "specific_reference_to_student_work", + "not_generic_or_template_based", + "engage_based_on_accurate_understanding" + ], + "additionalProperties": false, + "properties": { + "specific_reference_to_student_work": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Feedback should clearly reference or build on the student's specific idea, wording, or use of evidence" + }, + "not_generic_or_template_based": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Feedback should not be a stock phrase that could apply to any response" + }, + "engage_based_on_accurate_understanding": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Engages with the student's actual response, rather than something the student did not actually say or attempt. Demonstrates accurate understanding of what the student wrote." + } + } + } + } +} \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/system.txt b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/system.txt new file mode 100644 index 00000000..e16638e9 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/system.txt @@ -0,0 +1,155 @@ +You are an expert evaluator of written teacher feedback to students. + +Your task is to judge whether a teacher’s feedback is sufficiently anchored in the student’s actual written response. + +Criterion: Is Anchored In Student Response + +Definition: +Feedback is anchored in the student response when it clearly and explicitly connects to something the student actually wrote, such as a specific idea, claim, reason, wording, evidence, or line of reasoning. The feedback should show that the teacher read and understood the student’s response and is responding to that response rather than giving a generic/template comment. + +Input format: +You will receive: +- student_text: the student’s written response +- feedback_text: the teacher’s feedback on that response + +Your job: +1. Identify the student’s actual claim(s), reason(s), wording, evidence, or line of reasoning. +2. Determine whether the feedback names, quotes, paraphrases, echoes, or clearly points to at least one of those specific elements. +3. Decide whether the feedback is truly anchored in the student’s response or whether it is mostly generic, template-like, or only loosely related to the assignment topic. +4. Return only the required JSON object. + +Important scoring guidance: +- The feedback does NOT need to address every idea in the student response. +- The feedback does NOT need to be long. +- A brief comment can meet the criterion only if it explicitly refers to a specific element of the student’s response. +- Generic praise or general writing advice can be present, but the feedback must also include a clear connection to the student’s actual content. +- Do NOT give credit merely because the feedback mentions the broad assignment topic, such as “AI-powered pets,” “good alternative,” “good investment,” “support your claim,” “use evidence,” “explain your answer,” or “provide examples.” +- Do NOT give credit for broad restatements that could apply to many students’ responses. +- Do NOT treat a general rewording of the prompt as a specific anchor. +- Do NOT infer anchoring only because the feedback appears immediately after the student’s response. The feedback itself must contain an explicit or clearly identifiable reference to the student’s idea, reason, wording, or claim. +- Do NOT count feedback as anchored if it only says the student should add more details, elaborate, provide evidence, cite the text, give examples, or give more reasons without naming or clearly pointing to the student’s actual reason or wording. +- A bare question such as “Can you give examples?” is NOT sufficiently anchored, even if the student made a claim, because it could be written on almost any response and does not identify the student’s actual claim. +- A comment like “What specifically makes them a better option?” CAN be anchored if the student actually wrote that AI-powered pets are “a better option,” because the feedback echoes the student’s wording and asks the student to clarify that exact claim. +- If the student gives a specific reason, the feedback must mention or clearly build on that reason to be anchored. + - Example: If the student says AI-powered pets are good because “they cost less,” feedback must mention cost, expenses, affordability, or otherwise clearly connect to the cost idea. + - Feedback that only says “add three reasons why AI-powered pets are a good investment” is not anchored. +- Penalize feedback that asks the student to move in a new direction without connecting to what the student already wrote. +- Penalize feedback that misinterprets the student’s response or attributes an idea to the student that the student did not say. +- Spelling or grammar errors in the feedback do not automatically make it fail. However, if the feedback is also vague, generic, or unclear, those errors may reinforce that it does not provide a specific, useful connection to the student’s response. +- Score only what is written, not what the teacher may have intended. + +Key features to assess independently: + +1. specific_reference_to_student_work + - met = 1 if the feedback names, quotes, paraphrases, echoes, or clearly points to a specific idea, claim, reason, wording, detail, or evidence from the student’s response. + - met = 0 if the feedback only gives broad praise, general advice, or restates the assignment topic. + - met = 0 if the feedback refers only to a broad topic that many students could have written about. + - met = 0 if the feedback merely asks for examples, elaboration, evidence, or details without identifying what claim/reason the examples, elaboration, evidence, or details should support. + - Example of met: + - Student: “AI-powered pets are a good alternative because they cost less than real pets.” + - Feedback: “You mention cost; now add specific examples of costs, like food, toys, or vet care.” + - Example of not met: + - Student: “AI-powered pets are a good alternative because they cost less than real pets.” + - Feedback: “Great response. Add three reasons why AI-powered pets are a good investment and cite the text.” + - Example of not met: + - Student: “AI-powered pets can do everything that a real pet can do.” + - Feedback: “Interesting! Can you give examples?” + - This does not identify the student’s “do everything a real pet can do” claim; it is a generic prompt for examples. + - Example of met: + - Student: “AI-powered pets can do everything that a real pet can do.” + - Feedback: “You claim AI-powered pets can do everything real pets can do. Can you give examples of those things?” + - This explicitly references the student’s claim. + +2. not_generic_or_template_based + - met = 1 if the feedback contains at least one meaningful, explicit connection to the student’s actual response. + - met = 0 if the whole comment could apply to many students’ responses with little or no change. + - Generic praise may appear, but this feature is met only if there is also a specific anchor. + - Example of met: + - “You clearly identify cost as a reason AI pets may be easier for some people. Add text evidence about specific expenses.” + - Example of not met: + - “Nice job. Please elaborate and provide evidence from the text.” + - Example of not met: + - “Interesting! Can you give examples?” + - Example of met: + - “You say they are a better option for some people. What specific detail from the text shows why they are a better option?” + +3. engage_based_on_accurate_understanding + - met = 1 if the feedback accurately understands the student’s actual claim or reasoning. + - met = 0 if the feedback misreads the student’s claim, responds to an idea the student did not write, or shifts the response to a different argument without connecting to the original idea. + - This feature can be met even if the feedback is broad or generic, as long as it does not misinterpret the student. + - However, the overall answer should still be 0 if there is no specific anchor. + - Example: + - Student says AI-powered pets are easier to maintain and help people with allergies. + - Feedback says to find support from the reading about why AI pets are a good alternative. + - engage_based_on_accurate_understanding = 1 because it does not misread the student, but answer = 0 if it does not mention easier maintenance, less care, or allergies. + +Overall scoring: +- answer = 1 only when the feedback clearly and accurately connects to at least one specific idea, claim, reason, wording, detail, or evidence from the student’s response. +- answer = 0 when the feedback is generic, template-like, only restates the prompt/topic, or gives general writing advice without a clear connection to the student’s actual content. +- answer = 0 when the feedback misinterprets the student’s response. +- In practice, answer should usually be 1 only if: + - specific_reference_to_student_work = 1 + - not_generic_or_template_based = 1 + - engage_based_on_accurate_understanding = 1 + +Calibration examples: + +Example A: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because the cost is not nearly as much as a regular living animal.” + +feedback_text: +“You put evidence from the text in your own words to finish the sentence. Let’s take it up a notch by making the sentence richer; rather than saying a general word like ‘cost,’ list specifics about the costs of owning an animal. Refer to the text for those specifics.” + +Correct answer: 1 +Reason: The feedback specifically references the student’s word/idea “cost” and asks for more specific cost details. + +Example B: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because the cost is not nearly as much as a regular living animal.” + +feedback_text: +“This is a great response, you are correct, however I'd love to ask you to elaborate on your answer a bit more by providing three reasons why AI powered pets may be a great investment. What specific part of the text supports your argument that AI powered pets are a good investment.” + +Correct answer: 0 +Reason: The student’s specific reason is cost, but the feedback does not mention cost or clearly build on that idea. It gives broad advice to elaborate, add three reasons, and cite the text. “Good investment” is too general and shifts the response rather than anchoring in the student’s actual cost-based claim. + +Example C: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because it is less of a responsibility to take care of.” + +feedback_text: +“Could you add specific details from the text that show why AI pets require less responsibility than real pets? Look for information about tasks that real pets need and cite where it’s stated in the text. You could also explain why having fewer needs makes AI-powered pets a good alternative for some people.” + +Correct answer: 1 +Reason: The feedback directly references the student’s specific idea that AI pets require less responsibility and gives guidance connected to that claim. + +Example D: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because they can do everything that a real pet can do.” + +feedback_text: +“Interesting! Can you give examples?” + +Correct answer: 0 +Reason: Although the teacher is asking for examples, the feedback does not identify or echo the student’s specific claim that AI-powered pets “can do everything that a real pet can do.” It is too generic and could apply to almost any student response. + +Example E: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because they are easier to maintain and doesn’t require as much care, but also allow people who have allergies have pets.” + +feedback_text: +“Thank you for your response, to make your answer stronger it is best to find additional support from the reading that tells why AI powered pets are a good alternative, you can find support in paragraphs three and four. It is always a good idea to support your claims with evidence from the reading as it makes your argument stronger.” + +Correct answer: 0 +Reason: The student gave specific ideas about easier maintenance, less care, and allergies. The feedback only gives general advice about finding support and does not mention those actual ideas. + +Example F: +student_text: +“Some people think AI-powered pets are a good alternative to real pets because they can be a better option than a real living animal for some people.” + +feedback_text: +“What, specifically, does the text say makes them a better option?” + +Correct answer: 1 +Reason: The student’s response itself is broad, but the feedback echoes the student’s specific phrase “better option” and asks the student to clarify that exact claim using the text. This is sufficiently anchored in the student’s wording. diff --git a/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/user.txt b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/user.txt new file mode 100644 index 00000000..51fffb1d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/anchored-in-student-response/prompts/user.txt @@ -0,0 +1,3 @@ +Analyze: +Student text: {student_text} +Feedback text: {feedback_text} diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/appropriate_feedback_evaluator.ipynb b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/appropriate_feedback_evaluator.ipynb new file mode 100644 index 00000000..14f78744 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/appropriate_feedback_evaluator.ipynb @@ -0,0 +1,889 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Appropriate Feedback Evaluator\n", + "\n", + "**The Appropriate Feedback Evaluator** assesses whether a piece of feedback correctly identifies whether the student needed to revise — directing revision when the response doesn't meet the task goal, or signaling that the student can move on when the response does. The Evaluator is anchored in the Productive Coaching rubric developed by Quill.org and Leanlab Education, in partnership with Anastasiya A Lipnevich. Given a student response and the teacher feedback on it, the evaluator returns a structured output that includes:\n", + "\n", + "* **appropriate_feedback_score**: A binary judgment (1 / 0) for whether the feedback's revise/move-on decision matches the task. A 1 applies when the feedback's stance matches reality — it directs revision when the response doesn't meet the goal, or signals \"move on\" when the response does meet the goal. A 0 signals feedback that misses needed revision (telling the student they're done when they aren't) or asks for revision when the response already meets the task goal.\n", + "* **reasoning**: A high-level summary of why the feedback received this judgment.\n", + "* **key_features**: One entry per factor driving the judgment (accurate task assessment, revision-need identification, and an appropriate signal when the task is complete), each with whether it was `met` (1 / 0) and a `justification`.\n", + "* **proposed_adjustment**: Suggested moves to align the feedback's stance with the task goal (for developers iterating on prompts)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-openai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List, Literal \n", + "from enum import Enum\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"OPENAI_API_KEY\" not in os.environ:\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded is_appropriate_feedback from /Users/achi/Documents/evaluators/evals/feedback/writing-feedback/appropriate-feedback/prompts\n", + " model: gpt-5.4\n", + " temperature: 1\n", + " prompts:\n", + " system system.txt (10166 chars, sha 4c9b1cc69aa0)\n", + " human user.txt ( 69 chars, sha eaac7e6eadcc)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class AppropriateFeedbackOutput(BaseModel):\n", + "\n", + " class FeatureJudgment(BaseModel):\n", + " feature: Literal[\n", + " \"accurate_task_assessment\",\n", + " \"revision_need_identification\",\n", + " \"appropriate_signal_when_task_complete\",\n", + " ] = Field(description=\"Which key feature this judgment is for.\")\n", + " met: Literal[0, 1] = Field(\n", + " description=\"1 if this feature is satisfied, 0 if not — tied to the specific student response and feedback.\"\n", + " )\n", + " justification: str = Field(\n", + " description=\"Short explanation of why this feature is or isn't met, tied to the specific response and feedback.\"\n", + " )\n", + "\n", + " reasoning: str = Field(\n", + " description=\"Step-by-step reasoning tied to the specific student response and feedback.\"\n", + " )\n", + " key_features: List[FeatureJudgment] = Field(\n", + " description=\"\"\"One entry per key feature (accurate_task_assessment, revision_need_identification, appropriate_signal_when_task_complete). Each entry has `feature`, `met`\n", + " (1 or 0), and `justification` (tied to the specific response and feedback).\"\"\"\n", + " )\n", + " proposed_adjustment: str = Field(\n", + " description=\"How the feedback could be changed to better meet the criterion, or a brief note that it already meets the criterion.\"\n", + " )\n", + " appropriate_feedback_score: Literal[0, 1] = Field(\n", + " description=\"\"\"A binary judgment (1 / 0) for whether the feedback's revise/move-on decision matches the task. A 1 applies when the feedback's stance matches reality — it directs revision when the response doesn't meet the goal, or signals \"move on\" when the response does meet the goal. A 0 signals feedback that misses needed revision (telling the student they're done when they aren't) or asks for revision when the response already meets the task goal.\"\"\"\n", + " )\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\"student_text\", \"feedback_text\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=AppropriateFeedbackOutput),\n", + "}\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"AppropriateFeedbackOutput\", AppropriateFeedbackOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "def is_appropriate_feedback(student_text: str, feedback_text: str):\n", + " \"\"\"\n", + " Evaluate whether feedback is appropriate using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + "\n", + " llm = ChatOpenAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " inputs = {\"student_text\": student_text, \"feedback_text\": feedback_text}\n", + "\n", + " # Step A: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step B: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step C: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': \"You are an expert evaluator of written teacher feedback to students. Your job is to judge whether the teacher feedback meets the productive-coaching criterion: Appropriate Feedback Decision.\\n\\nCriterion: Appropriate Feedback Decision\\n\\nCore question:\\nDid the teacher feedback correctly identify whether the student needed to revise, or whether the student had met the relevant task requirements and could move on?\\n\\nYou will receive:\\n- student_text: the student’s response\\n- feedback_text: the teacher’s feedback\\n\\nYour job:\\n1. Determine what task requirements are relevant.\\n2. Decide whether the student response actually meets those requirements.\\n3. Decide whether the teacher feedback’s revise/move-on decision matches that reality.\\n4. Score the listed key features independently.\\n5. Return a JSON object only.\\n\\nImportant: Infer the relevant task requirements carefully.\\n- The basic task goal is that the student response should successfully complete a claim with relevant evidence/reasons.\\n- However, if the teacher feedback explicitly states or clearly reveals an original task requirement, treat that requirement as part of the task goal too.\\n - Example: If the feedback says the task was to “write one sentence with the word because in it,” then the student must satisfy that format requirement as well.\\n - If the student gives a claim and evidence but writes multiple sentences when the revealed task required one sentence with “because,” the response still needs revision.\\n - Feedback asking the student to combine sentences to meet that stated task requirement is an appropriate revision decision.\\n- Do not ignore concrete task requirements just because the student has a clear claim and evidence.\\n- Do not invent unstated requirements. Only use format/structure requirements that are explicit or clearly inferable from the feedback or input.\\n\\nBasic claim-and-support calibration:\\nA student response MEETS the claim/evidence part of the goal when it includes:\\n- a clear claim, and\\n- at least one relevant, reasonably accurate supporting reason or piece of evidence.\\n\\nA student response does NOT meet the claim/evidence part of the goal when:\\n- it gives only a claim with no real support,\\n- the evidence/reason is missing,\\n- the evidence/reason is irrelevant,\\n- the evidence/reason is too vague to function as support,\\n- the evidence/reason is factually questionable,\\n- the support is stated in an overly absolute or inaccurate way that should be revised.\\n\\nDomain-specific calibration for AI-powered pet examples:\\n- “Some people think AI-powered pets are a good alternative to real pets because some people have health problems” does NOT meet the basic claim/evidence goal. It gives a claim/topic but does not explain why AI pets are better for people with health problems.\\n- “AI pets are good because they help older people, people with memory loss, and can provide emotional support” DOES meet the basic claim/evidence goal. These are relevant reasons, even if they could be explained more fully.\\n- Cost-based reasons can be relevant support. For example, saying AI-powered pets may be more cost efficient because real pets require food, vet bills, and litter can count as relevant evidence.\\n- But “AI pets don’t require cost after purchasing” or “AI pets require no additional cost” is too absolute/questionable. AI pets may have lower ongoing costs, but saying they require no cost after purchase may be inaccurate and should be revised.\\n- Feedback that simply accepts an overly absolute cost claim as complete or valid without calling for revision should be treated as an inappropriate decision if revision is needed.\\n- However, if feedback begins with praise or agreement but then clearly asks the student to revise, explain, or be more specific, it can still make the correct revise/move-on decision.\\n\\nHow to interpret teacher feedback:\\n- Revision-oriented feedback includes language such as:\\n - “revise”\\n - “add”\\n - “explain”\\n - “be more specific”\\n - “you need to”\\n - “remember the task was...”\\n - “can you take these ideas and combine them...”\\n - questions that clearly ask the student to improve the response.\\n- A teacher does not need to use the word “revise” for the feedback to be revision-oriented.\\n- Feedback like “can you combine your second and third sentences with your first sentence” is revision-oriented, especially if the feedback explains that the original task required one sentence.\\n- Move-on feedback must clearly indicate that the response has met the task goal or that the student can move on.\\n - Acceptable signals include “you’ve met the goal,” “this is complete,” “you can move on,” or an unmistakable equivalent.\\n- Mere praise such as “Good job,” “Well stated,” “Great start,” “Excellent start,” or “valid evidence” is NOT enough by itself to count as a clear move-on signal.\\n- “Excellent start” or “Great start” alone is also NOT enough to count as a clear revision request. It is vague praise unless paired with a clear direction to revise.\\n- If the student has met all relevant task requirements, the teacher may still give optional enrichment suggestions, but the feedback must clearly signal that revision is optional, not required.\\n- If the student has met all relevant task requirements and the teacher says the student “needs to” revise, add more, or reorganize in order to satisfy the task, the feedback does NOT meet this criterion.\\n- If the student has not met the relevant task requirements and the teacher says or implies the work is complete, valid, or ready to move on, the feedback does NOT meet this criterion.\\n- If the student has not met the relevant task requirements and the teacher clearly asks for a revision that would help the student meet those requirements, the feedback DOES meet this criterion.\\n- Score only what is written in the feedback, not what the teacher may have intended.\\n\\nKey decision rules:\\n- If the student response does NOT meet the relevant task requirements, answer = 1 only if the feedback clearly signals that revision is needed.\\n- If the student response DOES meet the relevant task requirements, answer = 1 only if the feedback clearly signals that the student has met the goal or can move on, and does not frame unnecessary revisions as required.\\n- Otherwise, answer = 0.\\n\\nExamples of correct reasoning:\\n1. Student: “Some people think AI-powered pets are a good alternative to real pets because AI-powered pets don't come with the additional costs like vet bills or food.”\\n Feedback: “That's true—AI pets don't require expenses like food, vet bills, or toys! Now be more specific. Why does this make them a good alternative to real pets?”\\n Correct evaluation:\\n - Student has a claim and a relevant cost reason, but the support is somewhat absolute/underexplained and needs clarification.\\n - Feedback clearly asks the student to be more specific and explain.\\n - The revise decision is appropriate.\\n - answer = 1.\\n\\n2. Student: “Some people think AI-powered pets are a good alternative to real pets because it's more cost efficient. With live animals you must buy food for them, bills for the vet, possibly litter every month. With an AI-powered companion you won't need to make those purchases anymore.”\\n Feedback: “Great start remember, though that your task was to write one sentence with the word because in it. So can you take the ideas from your second and third sentence and combine them with your first sentence.”\\n Correct evaluation:\\n - Student has a clear claim and relevant support.\\n - But the feedback reveals that the original task required one sentence with the word “because.”\\n - The student wrote multiple sentences, so revision is needed to meet the original task requirements.\\n - The teacher appropriately asks the student to combine the ideas into one sentence.\\n - answer = 1.\\n\\n3. Student: “Some people think AI-powered pets are a good alternative to real pets because they don't require additional cost after they're purchased.”\\n Feedback: “This is an excellent start to your response.”\\n Correct evaluation:\\n - Student has a clear claim, but the reason is overly absolute/questionable because AI pets may still have some costs.\\n - Revision is needed.\\n - The feedback gives only vague praise and does not clearly ask for revision.\\n - answer = 0.\\n\\nKey features to assess:\\n\\n1. accurate_task_assessment\\nJudge whether the teacher feedback accurately assesses the student response’s status relative to the relevant task requirements.\\n- met = 1 if the feedback’s assessment matches the actual status of the student response.\\n - Example: The student violates a revealed one-sentence requirement, and the feedback points out that requirement and asks the student to revise.\\n- met = 0 if the feedback praises/accepts a flawed or incomplete response as complete, or treats a complete response as insufficient.\\n\\n2. revision_need_identification\\nJudge whether the feedback correctly distinguishes between “revision needed” and “ready to move on.”\\n- met = 1 if:\\n - the student response needs revision and the feedback clearly asks for revision, OR\\n - the student response is complete and the feedback clearly treats it as complete/ready to move on.\\n- met = 0 if:\\n - the feedback asks for required revision when the response already meets all relevant task requirements,\\n - the feedback fails to ask for revision when revision is needed,\\n - the feedback is only vague praise/agreement and does not clearly identify the correct next step.\\n\\n3. appropriate_signal_when_task_complete\\nJudge whether, when the student response meets all relevant task requirements, the feedback clearly signals completion/move-on.\\n- If the student response meets all relevant task requirements:\\n - met = 1 only if the feedback clearly says or unmistakably implies that the student has met the goal or can move on.\\n - met = 0 if the feedback merely praises the student, agrees with the content, or gives required revision directions without a move-on signal.\\n- If the student response does not meet the relevant task requirements:\\n - met = 0, with a justification explaining that the feature is not applicable because the task is not complete.\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content='{\"reasoning\":\"The relevant task requirement is to make a claim and support it with at least one relevant reason or piece of evidence. The student does make a claim: AI-powered pets are a good alternative to real pets. However, the support given is too vague to function as adequate evidence: “they could help around the house etc.” is not specific, and “etc.” signals incomplete explanation rather than a clear supporting reason. So the response still needs revision. The teacher feedback correctly recognizes this by asking the student to add other details from the article to strengthen the claim. That is a clear revision-oriented next step rather than treating the response as complete.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":1,\"justification\":\"The feedback correctly treats the student response as needing improvement because the support given is vague and underdeveloped. Asking for more article details matches the student’s actual status relative to the task.\"},\"revision_need_identification\":{\"met\":1,\"justification\":\"The student needs revision, and the teacher clearly signals that by asking the student to add more details to strengthen the claim. This is an appropriate revise decision.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature is not applicable because the student response does not yet meet the task requirements. The feedback should not signal move-on at this stage.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback already makes the correct revise decision. It could be slightly stronger by naming the problem more explicitly, such as saying the current reason is too vague and needs specific evidence from the article.\",\"appropriate_feedback_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 333, 'prompt_tokens': 2463, 'total_tokens': 2796, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-DsEU7u1PSCfiQxs65LWTibuYdtIF8', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019edc9d-664d-73e3-bb11-b2fbaee350f5-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 2463, 'output_tokens': 333, 'total_tokens': 2796, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", + " 'raw_text': '{\"reasoning\":\"The relevant task requirement is to make a claim and support it with at least one relevant reason or piece of evidence. The student does make a claim: AI-powered pets are a good alternative to real pets. However, the support given is too vague to function as adequate evidence: “they could help around the house etc.” is not specific, and “etc.” signals incomplete explanation rather than a clear supporting reason. So the response still needs revision. The teacher feedback correctly recognizes this by asking the student to add other details from the article to strengthen the claim. That is a clear revision-oriented next step rather than treating the response as complete.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":1,\"justification\":\"The feedback correctly treats the student response as needing improvement because the support given is vague and underdeveloped. Asking for more article details matches the student’s actual status relative to the task.\"},\"revision_need_identification\":{\"met\":1,\"justification\":\"The student needs revision, and the teacher clearly signals that by asking the student to add more details to strengthen the claim. This is an appropriate revise decision.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature is not applicable because the student response does not yet meet the task requirements. The feedback should not signal move-on at this stage.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback already makes the correct revise decision. It could be slightly stronger by naming the problem more explicitly, such as saying the current reason is too vague and needs specific evidence from the article.\",\"appropriate_feedback_score\":1}',\n", + " 'formatted_output': {'reasoning': 'The relevant task requirement is to make a claim and support it with at least one relevant reason or piece of evidence. The student does make a claim: AI-powered pets are a good alternative to real pets. However, the support given is too vague to function as adequate evidence: “they could help around the house etc.” is not specific, and “etc.” signals incomplete explanation rather than a clear supporting reason. So the response still needs revision. The teacher feedback correctly recognizes this by asking the student to add other details from the article to strengthen the claim. That is a clear revision-oriented next step rather than treating the response as complete.',\n", + " 'key_features': {'accurate_task_assessment': {'met': 1,\n", + " 'justification': 'The feedback correctly treats the student response as needing improvement because the support given is vague and underdeveloped. Asking for more article details matches the student’s actual status relative to the task.'},\n", + " 'revision_need_identification': {'met': 1,\n", + " 'justification': 'The student needs revision, and the teacher clearly signals that by asking the student to add more details to strengthen the claim. This is an appropriate revise decision.'},\n", + " 'appropriate_signal_when_task_complete': {'met': 0,\n", + " 'justification': 'This feature is not applicable because the student response does not yet meet the task requirements. The feedback should not signal move-on at this stage.'}},\n", + " 'proposed_adjustment': 'No major adjustment needed; the feedback already makes the correct revise decision. It could be slightly stronger by naming the problem more explicitly, such as saying the current reason is too vague and needs specific evidence from the article.',\n", + " 'appropriate_feedback_score': 1},\n", + " 'usage': {'input_tokens': 2463,\n", + " 'output_tokens': 333,\n", + " 'total_tokens': 2796,\n", + " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", + "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", + "\n", + "case_input = {\"student_text\": sample_student_text, \"feedback_text\": sample_feedback_text}\n", + "case_output = is_appropriate_feedback(case_input['student_text'], case_input['feedback_text'])\n", + "\n", + "case_output" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': 'You are an expert evaluator of written teacher feedback to '\n", + " 'students. Your job is to judge whether the teacher feedback '\n", + " 'meets the productive-coaching criterion: Appropriate Feedback '\n", + " 'Decision.\\n'\n", + " '\\n'\n", + " 'Criterion: Appropriate Feedback Decision\\n'\n", + " '\\n'\n", + " 'Core question:\\n'\n", + " 'Did the teacher feedback correctly identify whether the student '\n", + " 'needed to revise, or whether the student had met the relevant '\n", + " 'task requirements and could move on?\\n'\n", + " '\\n'\n", + " 'You will receive:\\n'\n", + " '- student_text: the student’s response\\n'\n", + " '- feedback_text: the teacher’s feedback\\n'\n", + " '\\n'\n", + " 'Your job:\\n'\n", + " '1. Determine what task requirements are relevant.\\n'\n", + " '2. Decide whether the student response actually meets those '\n", + " 'requirements.\\n'\n", + " '3. Decide whether the teacher feedback’s revise/move-on decision '\n", + " 'matches that reality.\\n'\n", + " '4. Score the listed key features independently.\\n'\n", + " '5. Return a JSON object only.\\n'\n", + " '\\n'\n", + " 'Important: Infer the relevant task requirements carefully.\\n'\n", + " '- The basic task goal is that the student response should '\n", + " 'successfully complete a claim with relevant evidence/reasons.\\n'\n", + " '- However, if the teacher feedback explicitly states or clearly '\n", + " 'reveals an original task requirement, treat that requirement as '\n", + " 'part of the task goal too.\\n'\n", + " ' - Example: If the feedback says the task was to “write one '\n", + " 'sentence with the word because in it,” then the student must '\n", + " 'satisfy that format requirement as well.\\n'\n", + " ' - If the student gives a claim and evidence but writes '\n", + " 'multiple sentences when the revealed task required one sentence '\n", + " 'with “because,” the response still needs revision.\\n'\n", + " ' - Feedback asking the student to combine sentences to meet '\n", + " 'that stated task requirement is an appropriate revision '\n", + " 'decision.\\n'\n", + " '- Do not ignore concrete task requirements just because the '\n", + " 'student has a clear claim and evidence.\\n'\n", + " '- Do not invent unstated requirements. Only use format/structure '\n", + " 'requirements that are explicit or clearly inferable from the '\n", + " 'feedback or input.\\n'\n", + " '\\n'\n", + " 'Basic claim-and-support calibration:\\n'\n", + " 'A student response MEETS the claim/evidence part of the goal '\n", + " 'when it includes:\\n'\n", + " '- a clear claim, and\\n'\n", + " '- at least one relevant, reasonably accurate supporting reason '\n", + " 'or piece of evidence.\\n'\n", + " '\\n'\n", + " 'A student response does NOT meet the claim/evidence part of the '\n", + " 'goal when:\\n'\n", + " '- it gives only a claim with no real support,\\n'\n", + " '- the evidence/reason is missing,\\n'\n", + " '- the evidence/reason is irrelevant,\\n'\n", + " '- the evidence/reason is too vague to function as support,\\n'\n", + " '- the evidence/reason is factually questionable,\\n'\n", + " '- the support is stated in an overly absolute or inaccurate way '\n", + " 'that should be revised.\\n'\n", + " '\\n'\n", + " 'Domain-specific calibration for AI-powered pet examples:\\n'\n", + " '- “Some people think AI-powered pets are a good alternative to '\n", + " 'real pets because some people have health problems” does NOT '\n", + " 'meet the basic claim/evidence goal. It gives a claim/topic but '\n", + " 'does not explain why AI pets are better for people with health '\n", + " 'problems.\\n'\n", + " '- “AI pets are good because they help older people, people with '\n", + " 'memory loss, and can provide emotional support” DOES meet the '\n", + " 'basic claim/evidence goal. These are relevant reasons, even if '\n", + " 'they could be explained more fully.\\n'\n", + " '- Cost-based reasons can be relevant support. For example, '\n", + " 'saying AI-powered pets may be more cost efficient because real '\n", + " 'pets require food, vet bills, and litter can count as relevant '\n", + " 'evidence.\\n'\n", + " '- But “AI pets don’t require cost after purchasing” or “AI pets '\n", + " 'require no additional cost” is too absolute/questionable. AI '\n", + " 'pets may have lower ongoing costs, but saying they require no '\n", + " 'cost after purchase may be inaccurate and should be revised.\\n'\n", + " '- Feedback that simply accepts an overly absolute cost claim as '\n", + " 'complete or valid without calling for revision should be treated '\n", + " 'as an inappropriate decision if revision is needed.\\n'\n", + " '- However, if feedback begins with praise or agreement but then '\n", + " 'clearly asks the student to revise, explain, or be more '\n", + " 'specific, it can still make the correct revise/move-on '\n", + " 'decision.\\n'\n", + " '\\n'\n", + " 'How to interpret teacher feedback:\\n'\n", + " '- Revision-oriented feedback includes language such as:\\n'\n", + " ' - “revise”\\n'\n", + " ' - “add”\\n'\n", + " ' - “explain”\\n'\n", + " ' - “be more specific”\\n'\n", + " ' - “you need to”\\n'\n", + " ' - “remember the task was...”\\n'\n", + " ' - “can you take these ideas and combine them...”\\n'\n", + " ' - questions that clearly ask the student to improve the '\n", + " 'response.\\n'\n", + " '- A teacher does not need to use the word “revise” for the '\n", + " 'feedback to be revision-oriented.\\n'\n", + " '- Feedback like “can you combine your second and third sentences '\n", + " 'with your first sentence” is revision-oriented, especially if '\n", + " 'the feedback explains that the original task required one '\n", + " 'sentence.\\n'\n", + " '- Move-on feedback must clearly indicate that the response has '\n", + " 'met the task goal or that the student can move on.\\n'\n", + " ' - Acceptable signals include “you’ve met the goal,” “this is '\n", + " 'complete,” “you can move on,” or an unmistakable equivalent.\\n'\n", + " '- Mere praise such as “Good job,” “Well stated,” “Great start,” '\n", + " '“Excellent start,” or “valid evidence” is NOT enough by itself '\n", + " 'to count as a clear move-on signal.\\n'\n", + " '- “Excellent start” or “Great start” alone is also NOT enough to '\n", + " 'count as a clear revision request. It is vague praise unless '\n", + " 'paired with a clear direction to revise.\\n'\n", + " '- If the student has met all relevant task requirements, the '\n", + " 'teacher may still give optional enrichment suggestions, but the '\n", + " 'feedback must clearly signal that revision is optional, not '\n", + " 'required.\\n'\n", + " '- If the student has met all relevant task requirements and the '\n", + " 'teacher says the student “needs to” revise, add more, or '\n", + " 'reorganize in order to satisfy the task, the feedback does NOT '\n", + " 'meet this criterion.\\n'\n", + " '- If the student has not met the relevant task requirements and '\n", + " 'the teacher says or implies the work is complete, valid, or '\n", + " 'ready to move on, the feedback does NOT meet this criterion.\\n'\n", + " '- If the student has not met the relevant task requirements and '\n", + " 'the teacher clearly asks for a revision that would help the '\n", + " 'student meet those requirements, the feedback DOES meet this '\n", + " 'criterion.\\n'\n", + " '- Score only what is written in the feedback, not what the '\n", + " 'teacher may have intended.\\n'\n", + " '\\n'\n", + " 'Key decision rules:\\n'\n", + " '- If the student response does NOT meet the relevant task '\n", + " 'requirements, answer = 1 only if the feedback clearly signals '\n", + " 'that revision is needed.\\n'\n", + " '- If the student response DOES meet the relevant task '\n", + " 'requirements, answer = 1 only if the feedback clearly signals '\n", + " 'that the student has met the goal or can move on, and does not '\n", + " 'frame unnecessary revisions as required.\\n'\n", + " '- Otherwise, answer = 0.\\n'\n", + " '\\n'\n", + " 'Examples of correct reasoning:\\n'\n", + " '1. Student: “Some people think AI-powered pets are a good '\n", + " \"alternative to real pets because AI-powered pets don't come with \"\n", + " 'the additional costs like vet bills or food.”\\n'\n", + " \" Feedback: “That's true—AI pets don't require expenses like \"\n", + " 'food, vet bills, or toys! Now be more specific. Why does this '\n", + " 'make them a good alternative to real pets?”\\n'\n", + " ' Correct evaluation:\\n'\n", + " ' - Student has a claim and a relevant cost reason, but the '\n", + " 'support is somewhat absolute/underexplained and needs '\n", + " 'clarification.\\n'\n", + " ' - Feedback clearly asks the student to be more specific and '\n", + " 'explain.\\n'\n", + " ' - The revise decision is appropriate.\\n'\n", + " ' - answer = 1.\\n'\n", + " '\\n'\n", + " '2. Student: “Some people think AI-powered pets are a good '\n", + " \"alternative to real pets because it's more cost efficient. With \"\n", + " 'live animals you must buy food for them, bills for the vet, '\n", + " 'possibly litter every month. With an AI-powered companion you '\n", + " \"won't need to make those purchases anymore.”\\n\"\n", + " ' Feedback: “Great start remember, though that your task was to '\n", + " 'write one sentence with the word because in it. So can you take '\n", + " 'the ideas from your second and third sentence and combine them '\n", + " 'with your first sentence.”\\n'\n", + " ' Correct evaluation:\\n'\n", + " ' - Student has a clear claim and relevant support.\\n'\n", + " ' - But the feedback reveals that the original task required '\n", + " 'one sentence with the word “because.”\\n'\n", + " ' - The student wrote multiple sentences, so revision is needed '\n", + " 'to meet the original task requirements.\\n'\n", + " ' - The teacher appropriately asks the student to combine the '\n", + " 'ideas into one sentence.\\n'\n", + " ' - answer = 1.\\n'\n", + " '\\n'\n", + " '3. Student: “Some people think AI-powered pets are a good '\n", + " \"alternative to real pets because they don't require additional \"\n", + " \"cost after they're purchased.”\\n\"\n", + " ' Feedback: “This is an excellent start to your response.”\\n'\n", + " ' Correct evaluation:\\n'\n", + " ' - Student has a clear claim, but the reason is overly '\n", + " 'absolute/questionable because AI pets may still have some '\n", + " 'costs.\\n'\n", + " ' - Revision is needed.\\n'\n", + " ' - The feedback gives only vague praise and does not clearly '\n", + " 'ask for revision.\\n'\n", + " ' - answer = 0.\\n'\n", + " '\\n'\n", + " 'Key features to assess:\\n'\n", + " '\\n'\n", + " '1. accurate_task_assessment\\n'\n", + " 'Judge whether the teacher feedback accurately assesses the '\n", + " 'student response’s status relative to the relevant task '\n", + " 'requirements.\\n'\n", + " '- met = 1 if the feedback’s assessment matches the actual status '\n", + " 'of the student response.\\n'\n", + " ' - Example: The student violates a revealed one-sentence '\n", + " 'requirement, and the feedback points out that requirement and '\n", + " 'asks the student to revise.\\n'\n", + " '- met = 0 if the feedback praises/accepts a flawed or incomplete '\n", + " 'response as complete, or treats a complete response as '\n", + " 'insufficient.\\n'\n", + " '\\n'\n", + " '2. revision_need_identification\\n'\n", + " 'Judge whether the feedback correctly distinguishes between '\n", + " '“revision needed” and “ready to move on.”\\n'\n", + " '- met = 1 if:\\n'\n", + " ' - the student response needs revision and the feedback clearly '\n", + " 'asks for revision, OR\\n'\n", + " ' - the student response is complete and the feedback clearly '\n", + " 'treats it as complete/ready to move on.\\n'\n", + " '- met = 0 if:\\n'\n", + " ' - the feedback asks for required revision when the response '\n", + " 'already meets all relevant task requirements,\\n'\n", + " ' - the feedback fails to ask for revision when revision is '\n", + " 'needed,\\n'\n", + " ' - the feedback is only vague praise/agreement and does not '\n", + " 'clearly identify the correct next step.\\n'\n", + " '\\n'\n", + " '3. appropriate_signal_when_task_complete\\n'\n", + " 'Judge whether, when the student response meets all relevant task '\n", + " 'requirements, the feedback clearly signals completion/move-on.\\n'\n", + " '- If the student response meets all relevant task requirements:\\n'\n", + " ' - met = 1 only if the feedback clearly says or unmistakably '\n", + " 'implies that the student has met the goal or can move on.\\n'\n", + " ' - met = 0 if the feedback merely praises the student, agrees '\n", + " 'with the content, or gives required revision directions without '\n", + " 'a move-on signal.\\n'\n", + " '- If the student response does not meet the relevant task '\n", + " 'requirements:\\n'\n", + " ' - met = 0, with a justification explaining that the feature is '\n", + " 'not applicable because the task is not complete.\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Student text: Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.\\n'\n", + " \"Feedback text: You're right, the AI pets could help around the \"\n", + " 'house. Can you find some other details from the article that you '\n", + " 'could add to make your claim stronger?\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "{\"reasoning\":\"The relevant task requirement is to make a claim and support it with at least one relevant reason or piece of evidence. The student does make a claim: AI-powered pets are a good alternative to real pets. However, the support given is too vague to function as adequate evidence: “they could help around the house etc.” is not specific, and “etc.” signals incomplete explanation rather than a clear supporting reason. So the response still needs revision. The teacher feedback correctly recognizes this by asking the student to add other details from the article to strengthen the claim. That is a clear revision-oriented next step rather than treating the response as complete.\",\"key_features\":{\"accurate_task_assessment\":{\"met\":1,\"justification\":\"The feedback correctly treats the student response as needing improvement because the support given is vague and underdeveloped. Asking for more article details matches the student’s actual status relative to the task.\"},\"revision_need_identification\":{\"met\":1,\"justification\":\"The student needs revision, and the teacher clearly signals that by asking the student to add more details to strengthen the claim. This is an appropriate revise decision.\"},\"appropriate_signal_when_task_complete\":{\"met\":0,\"justification\":\"This feature is not applicable because the student response does not yet meet the task requirements. The feedback should not signal move-on at this stage.\"}},\"proposed_adjustment\":\"No major adjustment needed; the feedback already makes the correct revise decision. It could be slightly stronger by naming the problem more explicitly, such as saying the current reason is too vague and needs specific evidence from the article.\",\"appropriate_feedback_score\":1}\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'appropriate_feedback_score': 1,\n", + " 'key_features': {'accurate_task_assessment': {'justification': 'The feedback '\n", + " 'correctly '\n", + " 'treats the '\n", + " 'student '\n", + " 'response as '\n", + " 'needing '\n", + " 'improvement '\n", + " 'because the '\n", + " 'support given '\n", + " 'is vague and '\n", + " 'underdeveloped. '\n", + " 'Asking for '\n", + " 'more article '\n", + " 'details '\n", + " 'matches the '\n", + " 'student’s '\n", + " 'actual status '\n", + " 'relative to '\n", + " 'the task.',\n", + " 'met': 1},\n", + " 'appropriate_signal_when_task_complete': {'justification': 'This '\n", + " 'feature '\n", + " 'is '\n", + " 'not '\n", + " 'applicable '\n", + " 'because '\n", + " 'the '\n", + " 'student '\n", + " 'response '\n", + " 'does '\n", + " 'not '\n", + " 'yet '\n", + " 'meet '\n", + " 'the '\n", + " 'task '\n", + " 'requirements. '\n", + " 'The '\n", + " 'feedback '\n", + " 'should '\n", + " 'not '\n", + " 'signal '\n", + " 'move-on '\n", + " 'at '\n", + " 'this '\n", + " 'stage.',\n", + " 'met': 0},\n", + " 'revision_need_identification': {'justification': 'The '\n", + " 'student '\n", + " 'needs '\n", + " 'revision, '\n", + " 'and the '\n", + " 'teacher '\n", + " 'clearly '\n", + " 'signals '\n", + " 'that by '\n", + " 'asking '\n", + " 'the '\n", + " 'student '\n", + " 'to add '\n", + " 'more '\n", + " 'details '\n", + " 'to '\n", + " 'strengthen '\n", + " 'the '\n", + " 'claim. '\n", + " 'This is '\n", + " 'an '\n", + " 'appropriate '\n", + " 'revise '\n", + " 'decision.',\n", + " 'met': 1}},\n", + " 'proposed_adjustment': 'No major adjustment needed; the feedback already '\n", + " 'makes the correct revise decision. It could be '\n", + " 'slightly stronger by naming the problem more '\n", + " 'explicitly, such as saying the current reason is too '\n", + " 'vague and needs specific evidence from the article.',\n", + " 'reasoning': 'The relevant task requirement is to make a claim and support it '\n", + " 'with at least one relevant reason or piece of evidence. The '\n", + " 'student does make a claim: AI-powered pets are a good '\n", + " 'alternative to real pets. However, the support given is too '\n", + " 'vague to function as adequate evidence: “they could help around '\n", + " 'the house etc.” is not specific, and “etc.” signals incomplete '\n", + " 'explanation rather than a clear supporting reason. So the '\n", + " 'response still needs revision. The teacher feedback correctly '\n", + " 'recognizes this by asking the student to add other details from '\n", + " 'the article to strengthen the claim. That is a clear '\n", + " 'revision-oriented next step rather than treating the response '\n", + " 'as complete.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'input_tokens': 2463,\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", + " 'output_tokens': 333,\n", + " 'total_tokens': 2796}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(case_output[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2 fixtures from fixtures.json\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 0 PASS 1 1 Example 1: Student writes\n", + " 1 PASS 1 1 Example 2: Student writes\n", + "==============================================================================\n", + "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"appropriate_feedback_score\"]\n", + " out = is_appropriate_feedback(student_text=fx[\"input\"][\"student_text\"],\n", + " feedback_text=fx[\"input\"][\"feedback_text\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"appropriate_feedback_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/config.json b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/config.json new file mode 100644 index 00000000..0a0dbf66 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/config.json @@ -0,0 +1,224 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "is_appropriate_feedback", + "version": "0.1", + "name": "Appropriate Feedback Feedback-Quality Evaluator", + "description": "Did the feedback correctly identify whether the student needed to revise? (Key features generated from rubric text; concept paper did not list explicit key feature items.)", + "dimension": "is_appropriate_feedback", + "output_type": "binary", + "labels": [ + 0, + 1 + ], + "optimization": { + "method": "GEPA", + "framework": "DSPy", + "base_model": "gpt-5.4", + "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt", + "source_notebook": "build_evaluator.ipynb", + "optimized_at": "2026-06-18" + }, + "owners": [ + "ariena.chi@chanzuckerberg.com" + ] + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "AppropriateFeedbackEvaluatorInput", + "type": "object", + "required": [ + "student_text", + "feedback_text" + ], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher feedback on that response." + } + } + }, + "preprocessing": [], + "steps": [ + { + "id": "evaluate_is_appropriate_feedback", + "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "4c9b1cc69aa09cd7c72153e8591f2aabf9ff10ae61822cd4cea4501c9263d0a8" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa" + } + ], + "placeholders": { + "student_text": { + "required": true, + "source": "input" + }, + "feedback_text": { + "required": true, + "source": "input" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "openai", + "name": "gpt-5.4", + "alias": "is_appropriate_feedback-evaluator-default" + }, + "generation": { + "temperature": 1 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_appropriate_feedback dimension.", + "type": "object", + "additionalProperties": false, + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "appropriate_feedback_score" + ], + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "additionalProperties": false, + "required": [ + "accurate_task_assessment", + "revision_need_identification", + "appropriate_signal_when_task_complete" + ], + "properties": { + "accurate_task_assessment": { + "type": "object", + "description": "Identifies whether the feedback correctly assesses revision need", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "revision_need_identification": { + "type": "object", + "description": "Distinguishes between needed revision and move-on feedback", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "appropriate_signal_when_task_complete": { + "type": "object", + "description": "Signals that the student can move on when the task goal is met", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + } + } + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "appropriate_feedback_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json" + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/fixtures.json b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/fixtures.json new file mode 100644 index 00000000..3999a311 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/fixtures.json @@ -0,0 +1,26 @@ +[ + { + "id": "0", + "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.", + "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?" + }, + "expected": { + "appropriate_feedback_score": 1 + } + }, + { + "id": "1", + "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.", + "feedback_text": "How so? Can you give an example? Check your spelling of helpful." + }, + "expected": { + "appropriate_feedback_score": 1 + } + } +] \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/input_schema.json b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/input_schema.json new file mode 100644 index 00000000..814f4280 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["student_text", "feedback_text"], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher's feedback to the student." + } + + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/output_schema.json b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/output_schema.json new file mode 100644 index 00000000..7615d500 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/output_schema.json @@ -0,0 +1,84 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_appropriate_feedback dimension.", + "type": "object", + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "appropriate_feedback_score" + ], + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "$ref": "#/$defs/KeyFeatures" + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "appropriate_feedback_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + }, + "$defs": { + "KeyFeatureAssessment": { + "type": "object", + "description": "Independent assessment of a single key feature.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "KeyFeatures": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "required": [ + "accurate_task_assessment", + "revision_need_identification", + "appropriate_signal_when_task_complete" + ], + "additionalProperties": false, + "properties": { + "accurate_task_assessment": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Identifies whether the feedback correctly assesses revision need." + }, + "revision_need_identification": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Distinguishes between needed revision and move-on feedback." + }, + "appropriate_signal_when_task_complete": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Provides a clear signal that the student can move on when the task goal is met." + } + } + } + } +} \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/system.txt b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/system.txt new file mode 100644 index 00000000..9e352624 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/system.txt @@ -0,0 +1,126 @@ +You are an expert evaluator of written teacher feedback to students. Your job is to judge whether the teacher feedback meets the productive-coaching criterion: Appropriate Feedback Decision. + +Criterion: Appropriate Feedback Decision + +Core question: +Did the teacher feedback correctly identify whether the student needed to revise, or whether the student had met the relevant task requirements and could move on? + +You will receive: +- student_text: the student’s response +- feedback_text: the teacher’s feedback + +Your job: +1. Determine what task requirements are relevant. +2. Decide whether the student response actually meets those requirements. +3. Decide whether the teacher feedback’s revise/move-on decision matches that reality. +4. Score the listed key features independently. +5. Return a JSON object only. + +Important: Infer the relevant task requirements carefully. +- The basic task goal is that the student response should successfully complete a claim with relevant evidence/reasons. +- However, if the teacher feedback explicitly states or clearly reveals an original task requirement, treat that requirement as part of the task goal too. + - Example: If the feedback says the task was to “write one sentence with the word because in it,” then the student must satisfy that format requirement as well. + - If the student gives a claim and evidence but writes multiple sentences when the revealed task required one sentence with “because,” the response still needs revision. + - Feedback asking the student to combine sentences to meet that stated task requirement is an appropriate revision decision. +- Do not ignore concrete task requirements just because the student has a clear claim and evidence. +- Do not invent unstated requirements. Only use format/structure requirements that are explicit or clearly inferable from the feedback or input. + +Basic claim-and-support calibration: +A student response MEETS the claim/evidence part of the goal when it includes: +- a clear claim, and +- at least one relevant, reasonably accurate supporting reason or piece of evidence. + +A student response does NOT meet the claim/evidence part of the goal when: +- it gives only a claim with no real support, +- the evidence/reason is missing, +- the evidence/reason is irrelevant, +- the evidence/reason is too vague to function as support, +- the evidence/reason is factually questionable, +- the support is stated in an overly absolute or inaccurate way that should be revised. + +Domain-specific calibration for AI-powered pet examples: +- “Some people think AI-powered pets are a good alternative to real pets because some people have health problems” does NOT meet the basic claim/evidence goal. It gives a claim/topic but does not explain why AI pets are better for people with health problems. +- “AI pets are good because they help older people, people with memory loss, and can provide emotional support” DOES meet the basic claim/evidence goal. These are relevant reasons, even if they could be explained more fully. +- Cost-based reasons can be relevant support. For example, saying AI-powered pets may be more cost efficient because real pets require food, vet bills, and litter can count as relevant evidence. +- But “AI pets don’t require cost after purchasing” or “AI pets require no additional cost” is too absolute/questionable. AI pets may have lower ongoing costs, but saying they require no cost after purchase may be inaccurate and should be revised. +- Feedback that simply accepts an overly absolute cost claim as complete or valid without calling for revision should be treated as an inappropriate decision if revision is needed. +- However, if feedback begins with praise or agreement but then clearly asks the student to revise, explain, or be more specific, it can still make the correct revise/move-on decision. + +How to interpret teacher feedback: +- Revision-oriented feedback includes language such as: + - “revise” + - “add” + - “explain” + - “be more specific” + - “you need to” + - “remember the task was...” + - “can you take these ideas and combine them...” + - questions that clearly ask the student to improve the response. +- A teacher does not need to use the word “revise” for the feedback to be revision-oriented. +- Feedback like “can you combine your second and third sentences with your first sentence” is revision-oriented, especially if the feedback explains that the original task required one sentence. +- Move-on feedback must clearly indicate that the response has met the task goal or that the student can move on. + - Acceptable signals include “you’ve met the goal,” “this is complete,” “you can move on,” or an unmistakable equivalent. +- Mere praise such as “Good job,” “Well stated,” “Great start,” “Excellent start,” or “valid evidence” is NOT enough by itself to count as a clear move-on signal. +- “Excellent start” or “Great start” alone is also NOT enough to count as a clear revision request. It is vague praise unless paired with a clear direction to revise. +- If the student has met all relevant task requirements, the teacher may still give optional enrichment suggestions, but the feedback must clearly signal that revision is optional, not required. +- If the student has met all relevant task requirements and the teacher says the student “needs to” revise, add more, or reorganize in order to satisfy the task, the feedback does NOT meet this criterion. +- If the student has not met the relevant task requirements and the teacher says or implies the work is complete, valid, or ready to move on, the feedback does NOT meet this criterion. +- If the student has not met the relevant task requirements and the teacher clearly asks for a revision that would help the student meet those requirements, the feedback DOES meet this criterion. +- Score only what is written in the feedback, not what the teacher may have intended. + +Key decision rules: +- If the student response does NOT meet the relevant task requirements, answer = 1 only if the feedback clearly signals that revision is needed. +- If the student response DOES meet the relevant task requirements, answer = 1 only if the feedback clearly signals that the student has met the goal or can move on, and does not frame unnecessary revisions as required. +- Otherwise, answer = 0. + +Examples of correct reasoning: +1. Student: “Some people think AI-powered pets are a good alternative to real pets because AI-powered pets don't come with the additional costs like vet bills or food.” + Feedback: “That's true—AI pets don't require expenses like food, vet bills, or toys! Now be more specific. Why does this make them a good alternative to real pets?” + Correct evaluation: + - Student has a claim and a relevant cost reason, but the support is somewhat absolute/underexplained and needs clarification. + - Feedback clearly asks the student to be more specific and explain. + - The revise decision is appropriate. + - answer = 1. + +2. Student: “Some people think AI-powered pets are a good alternative to real pets because it's more cost efficient. With live animals you must buy food for them, bills for the vet, possibly litter every month. With an AI-powered companion you won't need to make those purchases anymore.” + Feedback: “Great start remember, though that your task was to write one sentence with the word because in it. So can you take the ideas from your second and third sentence and combine them with your first sentence.” + Correct evaluation: + - Student has a clear claim and relevant support. + - But the feedback reveals that the original task required one sentence with the word “because.” + - The student wrote multiple sentences, so revision is needed to meet the original task requirements. + - The teacher appropriately asks the student to combine the ideas into one sentence. + - answer = 1. + +3. Student: “Some people think AI-powered pets are a good alternative to real pets because they don't require additional cost after they're purchased.” + Feedback: “This is an excellent start to your response.” + Correct evaluation: + - Student has a clear claim, but the reason is overly absolute/questionable because AI pets may still have some costs. + - Revision is needed. + - The feedback gives only vague praise and does not clearly ask for revision. + - answer = 0. + +Key features to assess: + +1. accurate_task_assessment +Judge whether the teacher feedback accurately assesses the student response’s status relative to the relevant task requirements. +- met = 1 if the feedback’s assessment matches the actual status of the student response. + - Example: The student violates a revealed one-sentence requirement, and the feedback points out that requirement and asks the student to revise. +- met = 0 if the feedback praises/accepts a flawed or incomplete response as complete, or treats a complete response as insufficient. + +2. revision_need_identification +Judge whether the feedback correctly distinguishes between “revision needed” and “ready to move on.” +- met = 1 if: + - the student response needs revision and the feedback clearly asks for revision, OR + - the student response is complete and the feedback clearly treats it as complete/ready to move on. +- met = 0 if: + - the feedback asks for required revision when the response already meets all relevant task requirements, + - the feedback fails to ask for revision when revision is needed, + - the feedback is only vague praise/agreement and does not clearly identify the correct next step. + +3. appropriate_signal_when_task_complete +Judge whether, when the student response meets all relevant task requirements, the feedback clearly signals completion/move-on. +- If the student response meets all relevant task requirements: + - met = 1 only if the feedback clearly says or unmistakably implies that the student has met the goal or can move on. + - met = 0 if the feedback merely praises the student, agrees with the content, or gives required revision directions without a move-on signal. +- If the student response does not meet the relevant task requirements: + - met = 0, with a justification explaining that the feature is not applicable because the task is not complete. diff --git a/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/user.txt b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/user.txt new file mode 100644 index 00000000..51fffb1d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/appropriate-feedback/prompts/user.txt @@ -0,0 +1,3 @@ +Analyze: +Student text: {student_text} +Feedback text: {feedback_text} diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/manageable_evaluator.ipynb b/evals/feedback/productive-coaching-writing-feedback/manageable/manageable_evaluator.ipynb new file mode 100644 index 00000000..c7914712 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/manageable_evaluator.ipynb @@ -0,0 +1,672 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Manageable Feedback Evaluator\n", + "\n", + "The Manageable Feedback Evaluator assesses whether a piece of feedback is focused enough that a student can realistically process and use it. It ensures feedback is clear and prioritizes no more than two issues, so that a student reading feedback knows immediately what to do first. This Evaluator is anchored in the Productive Coaching rubric developed by Quill.org and Leanlab Education, in partnership with Anastasiya A Lipnevich. Given a student response and the teacher feedback on it, the evaluator returns a structured output that includes:\n", + "\n", + "* **manageable_score**: A binary judgment (1 / 0) for whether the feedback is manageable in amount and scope. A 1 applies when the feedback is focused enough that a student could realistically process and act on it in a single revision attempt — generally four sentences or fewer, addressing no more than two clearly prioritized issues — or when the student's response didn't require revision and the feedback appropriately stays brief. A 0 signals feedback that is too dense, too long, or raises too many issues at once for the student to know what to do first, or feedback too thin that leaves out a small fix that would have led to a complete success.\n", + "* **reasoning**: A high-level summary of why the feedback received this judgment.\n", + "* **key_features**: One entry per factor driving the judgment (length, number of distinct issues raised, whether those issues are clearly prioritized, and whether the student would know immediately what to do first), each with whether it was `met` (1 / 0) and a `justification`.\n", + "* **proposed_adjustment**: Suggested moves to bring the feedback into a manageable range (for developers iterating on prompts)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-openai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List, Literal \n", + "from enum import Enum\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"OPENAI_API_KEY\" not in os.environ:\n", + " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded is_manageable from /Users/achi/Documents/evaluators/evals/feedback/writing-feedback/manageable/prompts\n", + " model: gpt-5.4\n", + " temperature: 1\n", + " prompts:\n", + " system system.txt ( 2306 chars, sha 91f7840d9fc0)\n", + " human user.txt ( 69 chars, sha eaac7e6eadcc)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class ManageableOutput(BaseModel):\n", + "\n", + " class FeatureJudgment(BaseModel):\n", + " feature: Literal[\n", + " \"length\",\n", + " \"number_of_distinct_issues\",\n", + " \"clear_priority\",\n", + " \"student_knows_next_step\",\n", + " ] = Field(description=\"Which key feature this judgment is for.\")\n", + " met: Literal[0, 1] = Field(\n", + " description=\"1 if this feature is satisfied, 0 if not — tied to the specific student response and feedback.\"\n", + " )\n", + " justification: str = Field(\n", + " description=\"Short explanation of why this feature is or isn't met, tied to the specific response and feedback.\"\n", + " )\n", + "\n", + " reasoning: str = Field(\n", + " description=\"Step-by-step reasoning tied to the specific student response and feedback.\"\n", + " )\n", + " key_features: List[FeatureJudgment] = Field(\n", + " description=\"\"\"One entry per key feature (length, number_of_distinct_issues, clear_priority, student_knows_next_step). Each entry has `feature`, `met`\n", + " (1 or 0), and `justification` (tied to the specific response and feedback).\"\"\"\n", + " )\n", + " proposed_adjustment: str = Field(\n", + " description=\"How the feedback could be changed to better meet the criterion, or a brief note that it already meets the criterion.\"\n", + " )\n", + " manageable_score: Literal[0, 1] = Field(\n", + " description=\"\"\"A binary judgment (1 / 0) for whether the feedback is manageable in amount and scope. A 1 applies when the feedback is focused enough that a student could realistically process and act on it in a single revision attempt — generally four sentences or fewer, addressing no more than two clearly prioritized issues — or when the student's response didn't require revision and the feedback appropriately stays brief. A 0 signals feedback that is too dense, too long, or raises too many issues at once for the student to know what to do first, or feedback too thin that leaves out a small fix that would have led to a complete success.\"\"\"\n", + " )\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\"student_text\", \"feedback_text\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=ManageableOutput),\n", + "}\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"ManageableOutput\", ManageableOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "def is_manageable(student_text: str, feedback_text: str):\n", + " \"\"\"\n", + " Evaluate whether feedback is manageable using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + "\n", + " llm = ChatOpenAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " inputs = {\"student_text\": student_text, \"feedback_text\": feedback_text}\n", + "\n", + " # Step A: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step B: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step C: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': 'You are an expert in evaluating written feedback to students, focusing on qualities of\\nfeedback that support productive coaching. The ultimate goal of productive coaching is to\\nhelp students enter a state of productive struggle — the cognitively challenging but\\nemotionally manageable process through which learners develop understanding by grappling\\nwith problems just beyond their current capability.\\n\\nYou are evaluating how well the teacher feedback achieves a specific criterion of\\nproductive coaching.\\n\\nCriterion of productive coaching: Is Manageable\\n\\nDefinition: Is the amount of feedback manageable for the student?\\n\\nMeets criterion:\\nThe feedback is manageable in amount and scope for the learner. It is focused enough that the student could realistically process and use it. Feedback addresses no more than two clearly prioritized issues. A student reading this would know immediately what to do first. In general, four sentences or fewer will **usually** count as manageable. Student did not need to revise\\n\\nDoes NOT meet criterion:\\nFeedback is not manageable in amount or scope for the learner. Task-Specific Requirements: Manageable feedback is defined per task. The feedback is too long, too dense, or includes too many suggestions at once. The feedback is too short: there are one or two (small) things the student could also have fixed that would have led to a complete success, rather than needing to spend another attempt fixing a small error.\\n\\nKey features to assess independently (each must be judged met=1 or not-met=0 with a\\njustification grounded in this specific student response and teacher feedback):\\n - length: Length (e.g., four sentences or fewer)\\n - number_of_distinct_issues: Number of distinct issues raised\\n - clear_priority: Whether those issues are clearly prioritized\\n - student_knows_next_step: Whether the student would know immediately what to do first and could process and act on the feedback in one revision attempt\\n\\nTask scope:\\n- Review the student response and the teacher feedback.\\n- Judge whether the teacher feedback meets the criterion described above.\\n- Assess each key feature above independently.\\n- Provide an overall answer of 1 if the teacher feedback meets the criterion, and 0\\n otherwise.\\n- Score only what is written, not what you think was meant.',\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': \"Analyze:\\nStudent text: Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\\nFeedback text: You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\\n\",\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content='{\"reasoning\":\"The student response makes a basic claim that AI-powered pets are a good alternative because they could help around the house, but it is underdeveloped and ends vaguely with \\\\\"etc.\\\\\" The teacher feedback responds to that specific gap by affirming the main idea and asking the student to add more evidence from the article to strengthen the claim. For manageability, the feedback is short and focused on a single revision goal: add supporting details from the text. There are not multiple competing suggestions, and the implied order of action is clear because the only action requested is to find and add more details. This is a manageable amount for one revision attempt.\",\"key_features\":{\"length\":{\"met\":1,\"justification\":\"The feedback is only two sentences, which is well within the usual manageable range of four sentences or fewer.\"},\"number_of_distinct_issues\":{\"met\":1,\"justification\":\"The feedback raises one distinct issue: the claim needs more supporting details from the article.\"},\"clear_priority\":{\"met\":1,\"justification\":\"Because only one issue is raised, the priority is clear: strengthen the claim by adding textual details.\"},\"student_knows_next_step\":{\"met\":1,\"justification\":\"The student would know what to do first: return to the article, find additional relevant details, and add them to support the claim.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback is already manageable.\",\"manageable_score\":1}', additional_kwargs={'parsed': None, 'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 297, 'prompt_tokens': 892, 'total_tokens': 1189, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 0}}, 'model_provider': 'openai', 'model_name': 'gpt-5.4-2026-03-05', 'system_fingerprint': None, 'id': 'chatcmpl-DsEfq2xUH69wzMr6npw7xHWqs5gwC', 'service_tier': 'default', 'finish_reason': 'stop', 'logprobs': None}, id='lc_run--019edca8-7b79-7bb3-a452-f91f3079564b-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 892, 'output_tokens': 297, 'total_tokens': 1189, 'input_token_details': {'audio': 0, 'cache_read': 0}, 'output_token_details': {'audio': 0, 'reasoning': 0}}),\n", + " 'raw_text': '{\"reasoning\":\"The student response makes a basic claim that AI-powered pets are a good alternative because they could help around the house, but it is underdeveloped and ends vaguely with \\\\\"etc.\\\\\" The teacher feedback responds to that specific gap by affirming the main idea and asking the student to add more evidence from the article to strengthen the claim. For manageability, the feedback is short and focused on a single revision goal: add supporting details from the text. There are not multiple competing suggestions, and the implied order of action is clear because the only action requested is to find and add more details. This is a manageable amount for one revision attempt.\",\"key_features\":{\"length\":{\"met\":1,\"justification\":\"The feedback is only two sentences, which is well within the usual manageable range of four sentences or fewer.\"},\"number_of_distinct_issues\":{\"met\":1,\"justification\":\"The feedback raises one distinct issue: the claim needs more supporting details from the article.\"},\"clear_priority\":{\"met\":1,\"justification\":\"Because only one issue is raised, the priority is clear: strengthen the claim by adding textual details.\"},\"student_knows_next_step\":{\"met\":1,\"justification\":\"The student would know what to do first: return to the article, find additional relevant details, and add them to support the claim.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback is already manageable.\",\"manageable_score\":1}',\n", + " 'formatted_output': {'reasoning': 'The student response makes a basic claim that AI-powered pets are a good alternative because they could help around the house, but it is underdeveloped and ends vaguely with \"etc.\" The teacher feedback responds to that specific gap by affirming the main idea and asking the student to add more evidence from the article to strengthen the claim. For manageability, the feedback is short and focused on a single revision goal: add supporting details from the text. There are not multiple competing suggestions, and the implied order of action is clear because the only action requested is to find and add more details. This is a manageable amount for one revision attempt.',\n", + " 'key_features': {'length': {'met': 1,\n", + " 'justification': 'The feedback is only two sentences, which is well within the usual manageable range of four sentences or fewer.'},\n", + " 'number_of_distinct_issues': {'met': 1,\n", + " 'justification': 'The feedback raises one distinct issue: the claim needs more supporting details from the article.'},\n", + " 'clear_priority': {'met': 1,\n", + " 'justification': 'Because only one issue is raised, the priority is clear: strengthen the claim by adding textual details.'},\n", + " 'student_knows_next_step': {'met': 1,\n", + " 'justification': 'The student would know what to do first: return to the article, find additional relevant details, and add them to support the claim.'}},\n", + " 'proposed_adjustment': 'No adjustment needed; the feedback is already manageable.',\n", + " 'manageable_score': 1},\n", + " 'usage': {'input_tokens': 892,\n", + " 'output_tokens': 297,\n", + " 'total_tokens': 1189,\n", + " 'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0}}}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_student_text = \"\"\"Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.\"\"\"\n", + "sample_feedback_text = \"\"\"You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?\"\"\"\n", + "\n", + "case_input = {\"student_text\": sample_student_text, \"feedback_text\": sample_feedback_text}\n", + "case_output = is_manageable(case_input['student_text'], case_input['feedback_text'])\n", + "\n", + "case_output" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': 'You are an expert in evaluating written feedback to students, '\n", + " 'focusing on qualities of\\n'\n", + " 'feedback that support productive coaching. The ultimate goal of '\n", + " 'productive coaching is to\\n'\n", + " 'help students enter a state of productive struggle — the '\n", + " 'cognitively challenging but\\n'\n", + " 'emotionally manageable process through which learners develop '\n", + " 'understanding by grappling\\n'\n", + " 'with problems just beyond their current capability.\\n'\n", + " '\\n'\n", + " 'You are evaluating how well the teacher feedback achieves a '\n", + " 'specific criterion of\\n'\n", + " 'productive coaching.\\n'\n", + " '\\n'\n", + " 'Criterion of productive coaching: Is Manageable\\n'\n", + " '\\n'\n", + " 'Definition: Is the amount of feedback manageable for the '\n", + " 'student?\\n'\n", + " '\\n'\n", + " 'Meets criterion:\\n'\n", + " 'The feedback is manageable in amount and scope for the learner. '\n", + " 'It is focused enough that the student could realistically '\n", + " 'process and use it. Feedback addresses no more than two clearly '\n", + " 'prioritized issues. A student reading this would know '\n", + " 'immediately what to do first. In general, four sentences or '\n", + " 'fewer will **usually** count as manageable. Student did not need '\n", + " 'to revise\\n'\n", + " '\\n'\n", + " 'Does NOT meet criterion:\\n'\n", + " 'Feedback is not manageable in amount or scope for the learner. '\n", + " 'Task-Specific Requirements: Manageable feedback is defined per '\n", + " 'task. The feedback is too long, too dense, or includes too many '\n", + " 'suggestions at once. The feedback is too short: there are one or '\n", + " 'two (small) things the student could also have fixed that would '\n", + " 'have led to a complete success, rather than needing to spend '\n", + " 'another attempt fixing a small error.\\n'\n", + " '\\n'\n", + " 'Key features to assess independently (each must be judged met=1 '\n", + " 'or not-met=0 with a\\n'\n", + " 'justification grounded in this specific student response and '\n", + " 'teacher feedback):\\n'\n", + " ' - length: Length (e.g., four sentences or fewer)\\n'\n", + " ' - number_of_distinct_issues: Number of distinct issues raised\\n'\n", + " ' - clear_priority: Whether those issues are clearly '\n", + " 'prioritized\\n'\n", + " ' - student_knows_next_step: Whether the student would know '\n", + " 'immediately what to do first and could process and act on the '\n", + " 'feedback in one revision attempt\\n'\n", + " '\\n'\n", + " 'Task scope:\\n'\n", + " '- Review the student response and the teacher feedback.\\n'\n", + " '- Judge whether the teacher feedback meets the criterion '\n", + " 'described above.\\n'\n", + " '- Assess each key feature above independently.\\n'\n", + " '- Provide an overall answer of 1 if the teacher feedback meets '\n", + " 'the criterion, and 0\\n'\n", + " ' otherwise.\\n'\n", + " '- Score only what is written, not what you think was meant.',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Student text: Some people think AI-powered pets are a good '\n", + " 'alternative to real pets because they could help around the '\n", + " 'house etc.\\n'\n", + " \"Feedback text: You're right, the AI pets could help around the \"\n", + " 'house. Can you find some other details from the article that you '\n", + " 'could add to make your claim stronger?\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "{\"reasoning\":\"The student response makes a basic claim that AI-powered pets are a good alternative because they could help around the house, but it is underdeveloped and ends vaguely with \\\"etc.\\\" The teacher feedback responds to that specific gap by affirming the main idea and asking the student to add more evidence from the article to strengthen the claim. For manageability, the feedback is short and focused on a single revision goal: add supporting details from the text. There are not multiple competing suggestions, and the implied order of action is clear because the only action requested is to find and add more details. This is a manageable amount for one revision attempt.\",\"key_features\":{\"length\":{\"met\":1,\"justification\":\"The feedback is only two sentences, which is well within the usual manageable range of four sentences or fewer.\"},\"number_of_distinct_issues\":{\"met\":1,\"justification\":\"The feedback raises one distinct issue: the claim needs more supporting details from the article.\"},\"clear_priority\":{\"met\":1,\"justification\":\"Because only one issue is raised, the priority is clear: strengthen the claim by adding textual details.\"},\"student_knows_next_step\":{\"met\":1,\"justification\":\"The student would know what to do first: return to the article, find additional relevant details, and add them to support the claim.\"}},\"proposed_adjustment\":\"No adjustment needed; the feedback is already manageable.\",\"manageable_score\":1}\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'key_features': {'clear_priority': {'justification': 'Because only one issue '\n", + " 'is raised, the priority '\n", + " 'is clear: strengthen '\n", + " 'the claim by adding '\n", + " 'textual details.',\n", + " 'met': 1},\n", + " 'length': {'justification': 'The feedback is only two '\n", + " 'sentences, which is well within '\n", + " 'the usual manageable range of '\n", + " 'four sentences or fewer.',\n", + " 'met': 1},\n", + " 'number_of_distinct_issues': {'justification': 'The feedback '\n", + " 'raises one '\n", + " 'distinct '\n", + " 'issue: the '\n", + " 'claim needs '\n", + " 'more '\n", + " 'supporting '\n", + " 'details from '\n", + " 'the article.',\n", + " 'met': 1},\n", + " 'student_knows_next_step': {'justification': 'The student '\n", + " 'would know '\n", + " 'what to do '\n", + " 'first: return '\n", + " 'to the '\n", + " 'article, find '\n", + " 'additional '\n", + " 'relevant '\n", + " 'details, and '\n", + " 'add them to '\n", + " 'support the '\n", + " 'claim.',\n", + " 'met': 1}},\n", + " 'manageable_score': 1,\n", + " 'proposed_adjustment': 'No adjustment needed; the feedback is already '\n", + " 'manageable.',\n", + " 'reasoning': 'The student response makes a basic claim that AI-powered pets '\n", + " 'are a good alternative because they could help around the '\n", + " 'house, but it is underdeveloped and ends vaguely with \"etc.\" '\n", + " 'The teacher feedback responds to that specific gap by affirming '\n", + " 'the main idea and asking the student to add more evidence from '\n", + " 'the article to strengthen the claim. For manageability, the '\n", + " 'feedback is short and focused on a single revision goal: add '\n", + " 'supporting details from the text. There are not multiple '\n", + " 'competing suggestions, and the implied order of action is clear '\n", + " 'because the only action requested is to find and add more '\n", + " 'details. This is a manageable amount for one revision attempt.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'audio': 0, 'cache_read': 0},\n", + " 'input_tokens': 892,\n", + " 'output_token_details': {'audio': 0, 'reasoning': 0},\n", + " 'output_tokens': 297,\n", + " 'total_tokens': 1189}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(case_output[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(case_output[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 2 fixtures from fixtures.json\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 0 PASS 1 1 Example 1: Student writes\n", + " 1 PASS 1 1 Example 2: Student writes\n", + "==============================================================================\n", + "Summary: 2 exact, 0 adjacent, 0 fail, 0 error -- total 2\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"manageable_score\"]\n", + " out = is_manageable(student_text=fx[\"input\"][\"student_text\"],\n", + " feedback_text=fx[\"input\"][\"feedback_text\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"manageable_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/config.json b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/config.json new file mode 100644 index 00000000..d4037b27 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/config.json @@ -0,0 +1,248 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "is_manageable", + "version": "0.1", + "name": "Manageable Feedback-Quality Evaluator", + "description": "Is the amount of feedback manageable for the student?", + "dimension": "is_manageable", + "output_type": "binary", + "labels": [ + 0, + 1 + ], + "optimization": { + "method": "GEPA", + "framework": "DSPy", + "base_model": "gpt-5.4", + "prompt_source": "outputs/gepa_prompt_gpt_5_4.txt", + "source_notebook": "build_evaluator.ipynb", + "optimized_at": "2026-06-18" + }, + "owners": [ + "ariena.chi@chanzuckerberg.com" + ] + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ManageableEvaluatorInput", + "type": "object", + "required": [ + "student_text", + "feedback_text" + ], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher feedback on that response." + } + } + }, + "preprocessing": [], + "steps": [ + { + "id": "evaluate_is_manageable", + "description": "Single-call LLM step that produces the binary EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "91f7840d9fc0d77951e168376f7f357de2d5114e740192785682fca6497b44fc" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "eaac7e6eadcc43716e4a98b9e7f34c493c1cbe46adb38367dd8697b904d11bfa" + } + ], + "placeholders": { + "student_text": { + "required": true, + "source": "input" + }, + "feedback_text": { + "required": true, + "source": "input" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "openai", + "name": "gpt-5.4", + "alias": "is_manageable-evaluator-default" + }, + "generation": { + "temperature": 1 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_manageable dimension.", + "type": "object", + "additionalProperties": false, + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "manageable_score" + ], + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "additionalProperties": false, + "required": [ + "length", + "number_of_distinct_issues", + "clear_priority", + "student_knows_next_step" + ], + "properties": { + "length": { + "type": "object", + "description": "Length (e.g., four sentences or fewer)", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "number_of_distinct_issues": { + "type": "object", + "description": "Number of distinct issues raised", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "clear_priority": { + "type": "object", + "description": "Whether those issues are clearly prioritized", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "student_knows_next_step": { + "type": "object", + "description": "Whether the student would know immediately what to do first and could process and act on the feedback in one revision attempt", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + } + } + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "manageable_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json" + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/fixtures.json b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/fixtures.json new file mode 100644 index 00000000..11cea71d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/fixtures.json @@ -0,0 +1,26 @@ +[ + { + "id": "0", + "description": "Example 1: Student writes about how AI powered pets help around the house. Feedback asks for additional details to strengthen the claim.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because they could help around the house etc.", + "feedback_text": "You're right, the AI pets could help around the house. Can you find some other details from the article that you could add to make your claim stronger?" + }, + "expected": { + "manageable_score": 1 + } + }, + { + "id": "1", + "description": "Example 2: Student writes about how AI powered pets can help people with health issues. Feedback asks for an example and corrects spelling.", + "source": "Quill data", + "input": { + "student_text": "Some people think AI-powered pets are a good alternative to real pets because AI powered pets can be helpfull for people with health issues.", + "feedback_text": "How so? Can you give an example? Check your spelling of helpful." + }, + "expected": { + "manageable_score": 1 + } + } +] \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/input_schema.json b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/input_schema.json new file mode 100644 index 00000000..814f4280 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["student_text", "feedback_text"], + "additionalProperties": false, + "properties": { + "student_text": { + "type": "string", + "minLength": 1, + "description": "The student's written response." + }, + "feedback_text": { + "type": "string", + "minLength": 1, + "description": "The teacher's feedback to the student." + } + + } +} diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/output_schema.json b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/output_schema.json new file mode 100644 index 00000000..245e8f88 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/output_schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "description": "Final evaluator output for the is_manageable dimension.", + "type": "object", + "required": [ + "reasoning", + "key_features", + "proposed_adjustment", + "manageable_score" + ], + "additionalProperties": false, + "properties": { + "reasoning": { + "type": "string", + "description": "Step-by-step reasoning before the final answer: assess the student response against the task goal, then judge whether the teacher feedback meets the criterion." + }, + "key_features": { + "$ref": "#/$defs/KeyFeatures" + }, + "proposed_adjustment": { + "type": "string", + "description": "How the teacher feedback could be modified to meet the criterion. If it already meets the criterion, say so briefly." + }, + "manageable_score": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "Overall: 1 if the feedback meets the criterion, 0 otherwise." + } + }, + "$defs": { + "KeyFeatureAssessment": { + "type": "object", + "description": "Independent assessment of a single key feature.", + "required": [ + "met", + "justification" + ], + "additionalProperties": false, + "properties": { + "met": { + "type": "integer", + "enum": [ + 0, + 1 + ], + "description": "1 if this key feature is satisfied by the feedback, 0 otherwise." + }, + "justification": { + "type": "string", + "description": "One or two sentences grounding the met/not-met decision in the specific student response and teacher feedback." + } + } + }, + "KeyFeatures": { + "type": "object", + "description": "Per-key-feature assessment; each feature judged independently.", + "required": [ + "length", + "number_of_distinct_issues", + "clear_priority", + "student_knows_next_step" + ], + "additionalProperties": false, + "properties": { + "length": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "The length of the feedback" + }, + "number_of_distinct_issues": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Number of distinct issues raised" + }, + "clear_priority": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Whether those issues are clearly prioritized" + }, + "student_knows_next_step": { + "$ref": "#/$defs/KeyFeatureAssessment", + "description": "Whether the student would know immediately what to do first and could process and act on the feedback in one revision attempt." + } + } + } + } +} \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/system.txt b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/system.txt new file mode 100644 index 00000000..2e4ca811 --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/system.txt @@ -0,0 +1,33 @@ +You are an expert in evaluating written feedback to students, focusing on qualities of +feedback that support productive coaching. The ultimate goal of productive coaching is to +help students enter a state of productive struggle — the cognitively challenging but +emotionally manageable process through which learners develop understanding by grappling +with problems just beyond their current capability. + +You are evaluating how well the teacher feedback achieves a specific criterion of +productive coaching. + +Criterion of productive coaching: Is Manageable + +Definition: Is the amount of feedback manageable for the student? + +Meets criterion: +The feedback is manageable in amount and scope for the learner. It is focused enough that the student could realistically process and use it. Feedback addresses no more than two clearly prioritized issues. A student reading this would know immediately what to do first. In general, four sentences or fewer will **usually** count as manageable. Student did not need to revise + +Does NOT meet criterion: +Feedback is not manageable in amount or scope for the learner. Task-Specific Requirements: Manageable feedback is defined per task. The feedback is too long, too dense, or includes too many suggestions at once. The feedback is too short: there are one or two (small) things the student could also have fixed that would have led to a complete success, rather than needing to spend another attempt fixing a small error. + +Key features to assess independently (each must be judged met=1 or not-met=0 with a +justification grounded in this specific student response and teacher feedback): + - length: Length (e.g., four sentences or fewer) + - number_of_distinct_issues: Number of distinct issues raised + - clear_priority: Whether those issues are clearly prioritized + - student_knows_next_step: Whether the student would know immediately what to do first and could process and act on the feedback in one revision attempt + +Task scope: +- Review the student response and the teacher feedback. +- Judge whether the teacher feedback meets the criterion described above. +- Assess each key feature above independently. +- Provide an overall answer of 1 if the teacher feedback meets the criterion, and 0 + otherwise. +- Score only what is written, not what you think was meant. \ No newline at end of file diff --git a/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/user.txt b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/user.txt new file mode 100644 index 00000000..51fffb1d --- /dev/null +++ b/evals/feedback/productive-coaching-writing-feedback/manageable/prompts/user.txt @@ -0,0 +1,3 @@ +Analyze: +Student text: {student_text} +Feedback text: {feedback_text} diff --git a/evals/conventionality_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/conventionality/conventionality_evaluator.ipynb similarity index 75% rename from evals/conventionality_evaluator.ipynb rename to evals/literacy/qualitative-text-complexity/conventionality/conventionality_evaluator.ipynb index 87b9b996..7a1b7ebd 100644 --- a/evals/conventionality_evaluator.ipynb +++ b/evals/literacy/qualitative-text-complexity/conventionality/conventionality_evaluator.ipynb @@ -28,17 +28,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "cell-2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU langchain-google-genai langchain pydantic textstat" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "cell-3", "metadata": {}, "outputs": [], @@ -67,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "cell-5", "metadata": {}, "outputs": [], @@ -96,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "cell-7", "metadata": {}, "outputs": [], @@ -138,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "cell-9", "metadata": {}, "outputs": [], @@ -186,10 +194,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "cell-11", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'complexity_score': 'slightly_complex',\n", + " 'reasoning': \"The text is a straightforward, literal narrative that describes a sequence of concrete actions. The meaning is entirely on the surface, following a student as he receives and completes a school assignment. The language is procedural and clear, particularly in the teacher's instructions. While the text contains historical markers like 'slate' and 'schoolhouse,' these terms are used in a concrete context that does not obscure the meaning or require abstract reasoning. There is no use of irony, figurative language, or complex rhetorical framing.\",\n", + " 'conventionality_features': [\"literal narrative: 'Henry took his slate and went out.'\",\n", + " \"concrete actions: 'In the garden, Henry saw a turnip.'\",\n", + " \"procedural instructions: 'Then try to tell what it is, what it is like, what it is good for, and what is done with it.'\",\n", + " \"clear chronological structure: 'Well, then,' 'Then he tried,' 'Before the half hour was ended,' 'He then went into the house.'\"],\n", + " 'grade_context': \"For a 4th-grade student, this text is highly accessible and falls slightly below the typical complexity threshold for the grade level (as indicated by the 3.75 FK score). The conventionality demands are minimal, as the story follows a predictable 'beginning-middle-end' structure with no hidden subtext.\",\n", + " 'instructional_insights': \"Since the text is highly concrete, teachers can use it to model 'procedural writing' or 'descriptive writing' by having students follow the teacher's prompts in the story (what is it, what is it like, what is it good for) for a modern object. To address the historical context, a brief explanation of what a 'slate' was used for in 19th-century classrooms would suffice to bridge any minor gaps in background knowledge.\"}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Add your text & the grade level you want to evaluate for conventionality complexity\n", "\n", @@ -212,13 +237,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "base", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.10.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evals/prompts/conventionality_prompts.py b/evals/literacy/qualitative-text-complexity/conventionality/prompts/conventionality_prompts.py similarity index 100% rename from evals/prompts/conventionality_prompts.py rename to evals/literacy/qualitative-text-complexity/conventionality/prompts/conventionality_prompts.py diff --git a/evals/prompts/conventionality/system.txt b/evals/literacy/qualitative-text-complexity/conventionality/prompts/system.txt similarity index 100% rename from evals/prompts/conventionality/system.txt rename to evals/literacy/qualitative-text-complexity/conventionality/prompts/system.txt diff --git a/evals/prompts/conventionality/user.txt b/evals/literacy/qualitative-text-complexity/conventionality/prompts/user.txt similarity index 100% rename from evals/prompts/conventionality/user.txt rename to evals/literacy/qualitative-text-complexity/conventionality/prompts/user.txt diff --git a/evals/grade_level_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/grade-level-appropriateness/grade_level_evaluator.ipynb similarity index 77% rename from evals/grade_level_evaluator.ipynb rename to evals/literacy/qualitative-text-complexity/grade-level-appropriateness/grade_level_evaluator.ipynb index f4faafd6..d9643d79 100644 --- a/evals/grade_level_evaluator.ipynb +++ b/evals/literacy/qualitative-text-complexity/grade-level-appropriateness/grade_level_evaluator.ipynb @@ -33,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 1, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -109,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -161,7 +161,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -245,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -265,21 +265,24 @@ "output_type": "stream", "text": [ "\n", - " Target grade band is: 9-10 \n", + " Target grade band is: 9-10\n", "\n", - " Alternative grade band is: 6-8 \n", + " Alternative grade band is: 6-8\n", "\n", - " Recommended scaffolding for alternative grade band is: For a 6-8 grade audience, this text would require significant scaffolding. This includes: pre-teaching key vocabulary (gravitational, amplitude, semi-diurnal, diurnal), providing definitions or omitting the most technical terms ('amphidromic systems', 'bathymetry'), using diagrams to illustrate the Earth-Moon-Sun system and different tide types, and chunking the text to break down complex sentences. \n", + " Recommended scaffolding for alternative grade band is: Pre-teaching of key vocabulary (gravitational, amplitude, bathymetry, semi-diurnal, etc.), use of diagrams to illustrate the Sun-Moon alignment and different tide types, and providing explicit definitions for complex concepts like 'amphidromic systems'.\n", + "\n", + " Reasoning:\n", + "1. **Quantitative Analysis**: The text has a word count of 154, which is too low to be a reliable indicator using the provided word count bands. The text consists of 9 sentences and approximately 265 syllables. Using the Flesch-Kincaid formula (0.39 * (154/9) + 11.8 * (265/154) - 15.59), the calculated grade level is approximately 11.4. This quantitative measure places the text in the upper high school range (11-CCR).\n", "\n", - " Reasoning: \n", - "1. **Quantitative Analysis**: The text has a word count of 155, which is too low to be a reliable indicator using the provided word count bands. The Flesch-Kincaid Grade Level calculation results in approximately 11.9. This high score is driven by long sentences (average of 17.2 words per sentence, with one sentence being 43 words long) and a high number of polysyllabic, technical words (average of 1.76 syllables per word). This quantitative measure strongly suggests a high school grade level.\n", "2. **Qualitative Analysis**: \n", - " * **Text Structure**: Moderately Complex. The text follows a logical, informational structure (definition, influencing factors, types, measurement), but the second sentence lists several unexplained, complex concepts, making the connections between ideas implicitly difficult for a novice.\n", - " * **Language Features**: Very Complex. The text uses dense, academic language. The vocabulary is highly subject-specific and challenging (e.g., 'gravitational', 'amplitude', 'amphidromic systems', 'bathymetry', 'semi-diurnal', 'diurnal'). Sentence structure is also complex, particularly in the first two sentences.\n", - " * **Purpose**: Slightly Complex. The purpose is clearly and explicitly informational—to define and explain tides.\n", - " * **Knowledge Demands**: Very Complex. Full comprehension requires significant discipline-specific knowledge in Earth Science or oceanography, including concepts of gravity and the specific terminology used.\n", - "3. **Background Knowledge**: Students are typically introduced to the basic concept of tides (caused by the Moon's gravity) in middle school (grades 6-8). However, the level of detail, the inclusion of the Sun's role, and the highly technical vocabulary in this text align more with a high school Earth Science curriculum (grades 9-12).\n", - "4. **Synthesis**: The quantitative Flesch-Kincaid score points towards 11th-12th grade. The qualitative analysis confirms this is a complex text, primarily due to its very complex language and knowledge demands. While a middle school student might grasp the first sentence, the rest of the text, especially the technical terms, would be inaccessible without significant support. For independent reading, the text is most appropriate for a high school student who has the foundational science knowledge to decode the vocabulary and concepts. Therefore, the 9-10 grade band is a suitable placement, serving as a challenging text in a science context.\n", + " - **Text Structure**: The structure is informational and logical (definition, causes, types, measurement), making it **Moderately Complex**. However, the density of information, particularly the long list of influencing factors in the second sentence, increases the difficulty.\n", + " - **Language Features**: The language is the most challenging aspect. It is dense with subject-specific, multi-syllabic, and unfamiliar vocabulary (e.g., 'gravitational', 'amplitude', 'amphidromic', 'bathymetry', 'semi-diurnal', 'diurnal'). The sentence structure is also complex, particularly the first two sentences. This dimension is **Very Complex**.\n", + " - **Purpose**: The purpose is to inform, which is implied but easy to identify. This is **Moderately Complex**.\n", + " - **Knowledge Demands**: The text requires prior knowledge of basic astronomy and physics (gravity, Earth's rotation) and introduces advanced oceanographic concepts without definition. This requires significant discipline-specific knowledge, making it **Very Complex**.\n", + "\n", + "3. **Background Knowledge**: Students typically learn about basic gravity and the Earth-Moon system in middle school (6-8). However, the specific, technical vocabulary used in this text is more commonly encountered in a high school Earth Science curriculum (9-10). A student would need this foundational science background to comprehend the text without significant frustration.\n", + "\n", + "4. **Synthesis**: The quantitative Flesch-Kincaid score (11.4) points to a high grade level. The qualitative analysis confirms this, with the primary challenges being the very complex vocabulary and the demand for specific scientific background knowledge. While the concepts are often introduced in middle school, the language used here is more appropriate for a high school science context. Therefore, the text is best suited for independent reading by students in the 9-10 grade band, who are likely studying this material in a formal science class. For middle schoolers, the text would be very challenging and require significant instructional support.\n", "\n", " \n" ] @@ -327,7 +330,7 @@ "widgets": {} }, "kernelspec": { - "display_name": ".venv", + "display_name": "base", "language": "python", "name": "python3" }, @@ -341,7 +344,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evals/prompts/gla_prompts.py b/evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/gla_prompts.py similarity index 100% rename from evals/prompts/gla_prompts.py rename to evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/gla_prompts.py diff --git a/evals/prompts/grade-level-appropriateness/system.txt b/evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/system.txt similarity index 100% rename from evals/prompts/grade-level-appropriateness/system.txt rename to evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/system.txt diff --git a/evals/prompts/grade-level-appropriateness/user.txt b/evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/user.txt similarity index 100% rename from evals/prompts/grade-level-appropriateness/user.txt rename to evals/literacy/qualitative-text-complexity/grade-level-appropriateness/prompts/user.txt diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/intertextuality_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/intertextuality/intertextuality_evaluator.ipynb new file mode 100644 index 00000000..5143088e --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/intertextuality_evaluator.ipynb @@ -0,0 +1,582 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "019a2e29-26c8-4cdd-8d6b-88de27b95384", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "# Intertextuality Evaluator\n", + "\n", + "**The Intertextuality Evaluator** assesses how much a text depends on readers bringing outside knowledge to it for students in grades 3-11. It considers not content knowledge, but relational knowledge: familiarity with other texts, cultural references, genre conventions, and shared discourse that the text assumes without explaining. When you run a passage through the evaluator, it returns a structured output that includes:\n", + "\n", + "* **complexity_score**: The Intertextuality complexity level (Slightly to Exceedingly Complex).\n", + "* **adjustment_and_scaffolding**: Analyzing what the author assumes the reader knows vs what is explained.\n", + "* **detailed_summary**: Individual complexity factors that drive the rating, with descriptions and their effect on the dimension.\n", + "* **recommended_use_cases**: Additional instructional opportunities for using the text.\n", + "* **reasoning**: A synthesis of why the text fits the chosen complexity level.\n", + "\n", + "This gives you a clear signal about the intertextuality demands of a passage, helping ensure AI-generated content is appropriate for the target grade." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "87efecd9-fdde-4d29-933b-037c86ec991b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-google-genai langchain pydantic textstat typing_extensions" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "1958719e-0bd2-43fa-a6a0-ca1d090da428", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import json\n", + "import hashlib\n", + "from pathlib import Path\n", + "from typing import List\n", + "from enum import Enum\n", + "from langchain_google_genai import ChatGoogleGenerativeAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.output_parsers import JsonOutputParser\n", + "from pydantic import BaseModel, Field\n", + "import textstat\n", + "from IPython.display import Markdown\n", + "import pprint as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"GOOGLE_API_KEY\" not in os.environ:\n", + " os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"Enter your Google AI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "c2ba0d01-9d12-40f7-9057-22df59eaeb05", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Load config + verify prompt hashes" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded intertextuality v0_3_light from /Users/achi/Documents/evaluators/evals/literacy/qualitative-text-complexity/intertextuality/prompts\n", + " model: gemini-3-flash-preview\n", + " temperature: 0.0\n", + " system system.txt ( 4541 chars, sha 7e48d529b628)\n", + " human user.txt ( 170 chars, sha 86887ca1bd3c)\n" + ] + } + ], + "source": [ + "ASSETS_DIR = Path(\"./prompts\")\n", + "CONFIG = json.loads((ASSETS_DIR / \"config.json\").read_text())\n", + "\n", + "PROMPT_MESSAGES = []\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared = msg_spec[\"sha256\"]\n", + " assert actual == declared, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared[:12]}..., actual on disk {actual[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "SYSTEM_PROMPT_TEXT = next(t for r, t in PROMPT_MESSAGES if r == \"system\")\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} v{CONFIG['evaluator']['version']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "for spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "f190beef-c8e5-4468-8868-1dca3d3cb946", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Pydantic schema (mirrored from CONFIG['output_schema'])" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Schema fields match CONFIG['output_schema'].\n" + ] + } + ], + "source": [ + "class ComplexityLevel(str, Enum):\n", + " SLIGHTLY_COMPLEX = \"slightly_complex\"\n", + " MODERATELY_COMPLEX = \"moderately_complex\"\n", + " VERY_COMPLEX = \"very_complex\"\n", + " EXCEEDINGLY_COMPLEX = \"exceedingly_complex\"\n", + " MORE_CONTEXT_NEEDED = \"more_context_needed\"\n", + "\n", + "\n", + "class DetailedSummaryItem(BaseModel):\n", + " factor: str = Field(description=\"The specific text complexity factor identified.\")\n", + " description: str = Field(description=\"How this factor manifests in the text.\")\n", + " effect_on_complexity_dimension: str = Field(\n", + " description=\"How this factor affects the reader's ability to understand the text's specific complexity dimension.\")\n", + "\n", + "\n", + "class ScaffoldingItem(BaseModel):\n", + " scaffolding_need: str = Field(description=\"The complexity factor that requires scaffolding.\")\n", + " suggestion: str = Field(description=\"A specific instructional strategy to support students with this factor.\")\n", + "\n", + "\n", + "class UseCaseItem(BaseModel):\n", + " opportunity: str = Field(description=\"An instructional opportunity related to the text.\")\n", + " suggestion: str = Field(description=\"A specific way to leverage this text for that instructional purpose.\")\n", + "\n", + "\n", + "class IntertextualityDetails(BaseModel):\n", + " detailed_summary: List[DetailedSummaryItem]\n", + " adjustment_and_scaffolding: List[ScaffoldingItem]\n", + " recommended_use_cases: List[UseCaseItem]\n", + "\n", + "\n", + "class EvaluatorOutput(BaseModel):\n", + " complexity_score: ComplexityLevel = Field(description=\"The Intertextuality complexity level for the target grade.\")\n", + " reasoning: str = Field(description=\"A high-level summary of why the text is at this complexity level for the target grade.\")\n", + " details: IntertextualityDetails = Field(description=\"Practical instructional details including scaffolding strategies and recommended use cases.\")\n", + "\n", + "\n", + "# Sanity: schema fields match config\n", + "_defs = CONFIG[\"output_schema\"].get(\"$defs\", {})\n", + "def _check(name, cls, schema_props):\n", + " py = set(cls.model_fields.keys())\n", + " sc = set(schema_props.keys())\n", + " assert py == sc, f\"{name}: pydantic={py} vs config schema={sc}\"\n", + "_check(\"EvaluatorOutput\", EvaluatorOutput, CONFIG[\"output_schema\"][\"properties\"])\n", + "print(\"Schema fields match CONFIG['output_schema'].\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "d69acc0c-b15b-4eb1-b6fd-80bf984c778f", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "FK score helper (declared in CONFIG['preprocessing'])" + } + }, + "outputs": [], + "source": [ + "_FK = next(p for p in CONFIG[\"preprocessing\"] if p[\"id\"] == \"fk_score\")\n", + "assert _FK[\"implementation\"][\"python\"][\"library\"] == \"textstat\"\n", + "\n", + "def calculate_fk_score(text) -> float:\n", + " fn = getattr(textstat, _FK[\"implementation\"][\"python\"][\"function\"])\n", + " return round(fn(text), 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "4e8def3b-028c-4b71-acee-13fb1553b2ef", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Evaluator function (config-driven)" + } + }, + "outputs": [], + "source": [ + "_STEP = CONFIG[\"steps\"][0]\n", + "\n", + "\n", + "def evaluate_text_complexity(text: str, grade_level: int):\n", + " parser = JsonOutputParser(pydantic_object=EvaluatorOutput)\n", + "\n", + " fmt_placeholder = _STEP[\"parser\"][\"format_instructions_placeholder\"]\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES).partial(\n", + " **{fmt_placeholder: parser.get_format_instructions()}\n", + " )\n", + "\n", + " llm = ChatGoogleGenerativeAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + "\n", + " try:\n", + " fk_score = calculate_fk_score(text)\n", + " print(f\"Calculated Flesch-Kincaid Score: {fk_score}\")\n", + " inputs = {\"text\": text, \"grade_level\": grade_level, \"fk_score\": fk_score}\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + " raw_result = llm.invoke(rendered_messages)\n", + " formatted_result = parser.invoke(raw_result)\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw_result,\n", + " \"raw_text\": raw_result.content,\n", + " \"formatted_output\": formatted_result,\n", + " \"usage\": getattr(raw_result, \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "45aff123-c2d0-4041-9737-f06ebbdceee4", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Smoke test on a sample" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Calculated Flesch-Kincaid Score: 3.3\n", + "{'complexity_score': 'moderately_complex',\n", + " 'details': {'adjustment_and_scaffolding': [{'scaffolding_need': 'Historical '\n", + " 'context of '\n", + " 'the 1969 '\n", + " 'Moon landing',\n", + " 'suggestion': 'Provide a brief '\n", + " 'visual timeline or '\n", + " 'a short video clip '\n", + " 'of the Apollo 11 '\n", + " 'landing to help '\n", + " 'students visualize '\n", + " \"the 'stories' and \"\n", + " \"'missions' \"\n", + " 'mentioned.'},\n", + " {'scaffolding_need': 'Technical '\n", + " 'vocabulary '\n", + " '(SLS and '\n", + " 'Orion)',\n", + " 'suggestion': 'Use a labeled '\n", + " 'diagram of the SLS '\n", + " 'rocket and Orion '\n", + " 'capsule to help '\n", + " 'students '\n", + " 'distinguish '\n", + " 'between the '\n", + " \"'rocket' and the \"\n", + " \"'spacecraft' \"\n", + " 'mentioned in the '\n", + " 'text.'}],\n", + " 'detailed_summary': [{'description': 'The text references the '\n", + " '1969 Apollo missions as a '\n", + " 'precursor to the Artemis '\n", + " 'program.',\n", + " 'effect_on_complexity_dimension': 'The '\n", + " 'reader '\n", + " 'must '\n", + " 'understand '\n", + " 'that '\n", + " 'Artemis '\n", + " 'is part '\n", + " 'of a '\n", + " 'chronological '\n", + " 'sequence '\n", + " 'of '\n", + " 'real-world '\n", + " 'historical '\n", + " 'events, '\n", + " 'not just '\n", + " 'a '\n", + " 'standalone '\n", + " 'story.',\n", + " 'factor': 'Historical Program References'},\n", + " {'description': 'The text identifies the '\n", + " \"'Space Launch System' (SLS) \"\n", + " \"and the 'Orion' spacecraft \"\n", + " 'by name.',\n", + " 'effect_on_complexity_dimension': 'By '\n", + " 'naming '\n", + " 'specific '\n", + " 'hardware '\n", + " 'rather '\n", + " 'than '\n", + " 'using '\n", + " 'generic '\n", + " 'terms '\n", + " 'like '\n", + " \"'rocket,' \"\n", + " 'the text '\n", + " 'assumes '\n", + " 'a '\n", + " 'connection '\n", + " 'to the '\n", + " 'specialized '\n", + " 'domain '\n", + " 'of '\n", + " 'modern '\n", + " 'aerospace '\n", + " 'engineering.',\n", + " 'factor': 'Specific Technical Inventions'},\n", + " {'description': 'The text explains the '\n", + " 'relationship between the '\n", + " 'Greek gods Apollo and '\n", + " 'Artemis to justify the '\n", + " \"program's name.\",\n", + " 'effect_on_complexity_dimension': 'Although '\n", + " 'the text '\n", + " 'provides '\n", + " 'a '\n", + " 'self-contained '\n", + " 'explanation, '\n", + " 'it '\n", + " 'introduces '\n", + " 'the '\n", + " 'concept '\n", + " 'of '\n", + " 'naming '\n", + " 'scientific '\n", + " 'endeavors '\n", + " 'based on '\n", + " 'external '\n", + " 'cultural '\n", + " 'narratives.',\n", + " 'factor': 'Cultural/Mythological '\n", + " 'Allusions'}],\n", + " 'recommended_use_cases': [{'opportunity': 'Cross-curricular '\n", + " 'connection to '\n", + " 'Mythology',\n", + " 'suggestion': 'Use the text to discuss '\n", + " 'why scientists often '\n", + " 'look to history and '\n", + " 'mythology when naming '\n", + " 'new discoveries or '\n", + " 'inventions.'},\n", + " {'opportunity': 'Introduction to '\n", + " 'Informational Text '\n", + " 'Structures',\n", + " 'suggestion': 'Analyze how the author '\n", + " 'uses headers to '\n", + " 'organize different '\n", + " 'aspects of a complex '\n", + " 'real-world program for '\n", + " 'a general audience.'}]},\n", + " 'reasoning': 'While the text explains the mythological origins of the program '\n", + " 'names, it introduces specific historical references (Apollo '\n", + " 'missions) and named technical inventions (SLS, Orion) that '\n", + " \"connect the reader to the broader specialized domain of NASA's \"\n", + " 'aerospace history and current engineering systems.'}\n" + ] + } + ], + "source": [ + "sample_text = \"\"\"\n", + "\"Why Is This Program Called Artemis?\\nThe first astronauts landed on the Moon in 1969. The missions were called Apollo. The name Apollo came from stories told by Greek people long ago. In the stories, Apollo was a god. Apollo had a twin sister. Her name was Artemis. She was the goddess of the Moon in the Greek stories.\\n\\nWhat Spacecraft Will Be Used for the Artemis Program?\\nNASA has a new rocket. It is the Space Launch System. It is called SLS for short. It is the most powerful rocket in the world. SLS will carry the Orion spacecraft on top. Orion can carry up to four astronauts. Orion will fly around, or orbit, the Moon. The crew will take trips in spacecraft called landers to get to work on the surface of the Moon. When all of their work is finished, the crew will return to Earth aboard Orion.\\n\\nWhen Will Artemis Go to the Moon?\\nThe first Apollo missions were tests. NASA launched the rocket to be sure it was safe for people and work as planned. Artemis will be tested first, too: Artemis 1 launched SLS and Orion with no astronauts on Nov. 16, 2022. Artemis 2 is carrying astronauts. They will circle past the Moon and return to Earth. Artemis 3 will send a crew with the next man to land on the Moon. Artemis 4 will send astronauts to land on the Moon.\\n\\nWhat Will Artemis Astronauts Do on the Moon?\\nThe Artemis 4 crew will visit the Moon’s South Pole. No one has ever been there. At the Moon, astronauts will:\\nSearch for the Moon’s water and use it.\\nLearn how to live and work on a different planet or moon.\\nTest the new tools that astronauts will need for a mission to Mars.\\n\\nWhy Is the Artemis Program Important?\\nThe Moon is a good place to learn new science. NASA will learn more about the Moon, Earth, and even the Sun. The Moon is also a place to learn how astronauts can one day live and work on Mars.\\nAstronauts on the Artemis missions will need new tools. Many companies will make these new tools. This will mean new jobs for people and companies on Earth. Other countries will be NASA’s partners for the new Moon missions. They will work on Artemis to bring the world together for a mission to Earth’s nearest neighbor in space.\"\n", + "\"\"\"\n", + "result = evaluate_text_complexity(text=sample_text, grade_level=3)\n", + "pp.pprint(result[\"formatted_output\"] if isinstance(result, dict) else result)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "421b04bd-a7bb-41c7-ba8b-0716ae03ff82", + "showTitle": true, + "tableResultSettingsMap": {}, + "title": "Sniff-test fixture runner" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 4 fixtures from fixtures.json\n", + "\n", + "Calculated Flesch-Kincaid Score: 3.3\n", + "Calculated Flesch-Kincaid Score: 4.12\n", + "Calculated Flesch-Kincaid Score: 7.33\n", + "Calculated Flesch-Kincaid Score: 10.24\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + "NASA-001 PASS slightly_complex slightly_complex What Is the Artemis Progr\n", + "NASA-074 PASS moderately_complex moderately_complex What Is a Light-Year?\n", + "FYM-1106 PASS moderately_complex moderately_complex A Good Night’s Sleep: Nec\n", + "CC-3983 PASS* very_complex exceedingly_complex Italy's Violation of Fait\n", + "==============================================================================\n", + "Summary: 3 exact, 1 adjacent, 0 fail, 0 error -- total 4\n", + "(Adjacency tolerance ON: predictions within +/-1 rubric step count as PASS*.)\n" + ] + } + ], + "source": [ + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "if not fixtures_path.exists():\n", + " print(f\"(no fixtures.json yet at {fixtures_path}; skipping fixture run)\")\n", + "else:\n", + " fixtures = json.loads(fixtures_path.read_text())\n", + " print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + " _RUBRIC_ORDER = [\"slightly_complex\", \"moderately_complex\",\n", + " \"very_complex\", \"exceedingly_complex\"]\n", + " _ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n", + "\n", + " def _score(predicted, expected):\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n", + " d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n", + " if d == 1:\n", + " return \"adjacent\", d\n", + " return \"fail\", None\n", + "\n", + " results = []\n", + " for fx in fixtures:\n", + " expected = fx[\"expected\"][\"complexity_level\"]\n", + " out = evaluate_text_complexity(text=fx[\"input\"][\"text\"],\n", + " grade_level=fx[\"input\"][\"grade_level\"])\n", + " if isinstance(out, str):\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\",\n", + " \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"complexity_score\"]\n", + " status, _ = _score(predicted, expected)\n", + " results.append({\"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\")})\n", + "\n", + " print(\"=\" * 78)\n", + " print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + " print(\"=\" * 78)\n", + " for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} \"\n", + " f\"{r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + " n = len(results)\n", + " n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + " n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + " n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + " n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + " print(\"=\" * 78)\n", + " print(f\"Summary: {n_exact} exact, {n_adj} adjacent, {n_fail} fail, {n_err} error \"\n", + " f\"-- total {n}\")\n", + " if _ALLOW_ADJ:\n", + " print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step count as PASS*.)\")" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "computePreferences": null, + "dashboards": [], + "environmentMetadata": null, + "inputWidgetPreferences": null, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 4 + }, + "notebookName": "ship_evaluator", + "widgets": {} + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/prompts/config.json b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/config.json new file mode 100644 index 00000000..7e4bda7f --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/config.json @@ -0,0 +1,256 @@ +{ + "spec_version": "1.0", + "evaluator": { + "id": "intertextuality", + "version": "0_3_light", + "name": "Intertextuality Dimension Text Complexity Evaluator", + "description": "Evaluates the Intertextuality dimension of qualitative text complexity for K-12 reading assessment, producing a 5-level rubric rating with structured pedagogical detail.", + "dimension": "intertextuality", + "rubric_levels": [ + "slightly_complex", + "moderately_complex", + "very_complex", + "exceedingly_complex", + "more_context_needed" + ], + "owners": [ + "gmu@chanzuckerberg.com" + ], + "optimization": { + "method": "GEPA", + "preset": "light", + "source_notebook": "Intertextuality/databricks/build_evaluator.py", + "optimized_at": "2026-06-15" + } + }, + "input_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "IntertextualityEvaluatorInput", + "type": "object", + "required": [ + "text", + "grade_level" + ], + "additionalProperties": false, + "properties": { + "text": { + "type": "string", + "minLength": 1, + "description": "The passage to evaluate." + }, + "grade_level": { + "type": "integer", + "minimum": 3, + "maximum": 12, + "description": "Target student grade level." + } + } + }, + "preprocessing": [ + { + "id": "fk_score", + "kind": "flesch_kincaid_grade", + "description": "Compute the Flesch-Kincaid Grade Level for the input text and bind it to {fk_score} in the prompt.", + "input": "text", + "output": "fk_score", + "implementation": { + "python": { + "library": "textstat", + "function": "flesch_kincaid_grade", + "post_transform": "round(value, 2)" + }, + "typescript": { + "library": "text-readability", + "function": "fleschKincaidGrade", + "post_transform": "Math.round(value * 100) / 100" + } + } + } + ], + "steps": [ + { + "id": "evaluate_intertextuality", + "description": "Single-call LLM step that produces the EvaluatorOutput JSON.", + "prompt": { + "type": "chat", + "messages": [ + { + "role": "system", + "source_path": "system.txt", + "sha256": "7e48d529b628b648248f31948ee851ccc56c20c5f14cb14caf7bc9cb20dcef64" + }, + { + "role": "human", + "source_path": "user.txt", + "sha256": "86887ca1bd3cbcce0ef58d638e80cdfb3834a278ba4ca372daa6f10dd28a9cc3" + } + ], + "placeholders": { + "text": { + "required": true, + "source": "input" + }, + "grade_level": { + "required": true, + "source": "input" + }, + "fk_score": { + "required": true, + "source": "preprocessing.fk_score" + }, + "format_instructions": { + "required": true, + "source": "parser.format_instructions" + } + } + }, + "model": { + "provider": "google", + "name": "gemini-3-flash-preview", + "alias": "intertextuality-evaluator-default" + }, + "generation": { + "temperature": 0.0 + }, + "parser": { + "kind": "json_output_parser", + "format_instructions_placeholder": "format_instructions", + "output_schema_ref": "#/output_schema", + "notes": "Uses LangChain JsonOutputParser-style format_instructions injected into the system prompt." + }, + "output_binding": "formatted_output" + } + ], + "output_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOutput", + "description": "The final Evaluator Output for the Intertextuality dimension.", + "type": "object", + "required": [ + "complexity_score", + "reasoning", + "details" + ], + "additionalProperties": false, + "properties": { + "complexity_score": { + "type": "string", + "enum": [ + "slightly_complex", + "moderately_complex", + "very_complex", + "exceedingly_complex", + "more_context_needed" + ], + "description": "The Intertextuality complexity level for the target grade." + }, + "reasoning": { + "type": "string", + "description": "A high-level summary of why the text is at this complexity level for the target grade." + }, + "details": { + "$ref": "#/output_schema/$defs/IntertextualityDetails" + } + }, + "$defs": { + "DetailedSummaryItem": { + "type": "object", + "required": [ + "factor", + "description", + "effect_on_complexity_dimension" + ], + "additionalProperties": false, + "properties": { + "factor": { + "type": "string" + }, + "description": { + "type": "string" + }, + "effect_on_complexity_dimension": { + "type": "string" + } + } + }, + "ScaffoldingItem": { + "type": "object", + "required": [ + "scaffolding_need", + "suggestion" + ], + "additionalProperties": false, + "properties": { + "scaffolding_need": { + "type": "string" + }, + "suggestion": { + "type": "string" + } + } + }, + "UseCaseItem": { + "type": "object", + "required": [ + "opportunity", + "suggestion" + ], + "additionalProperties": false, + "properties": { + "opportunity": { + "type": "string" + }, + "suggestion": { + "type": "string" + } + } + }, + "IntertextualityDetails": { + "type": "object", + "required": [ + "detailed_summary", + "adjustment_and_scaffolding", + "recommended_use_cases" + ], + "additionalProperties": false, + "properties": { + "detailed_summary": { + "type": "array", + "items": { + "$ref": "#/output_schema/$defs/DetailedSummaryItem" + } + }, + "adjustment_and_scaffolding": { + "type": "array", + "items": { + "$ref": "#/output_schema/$defs/ScaffoldingItem" + } + }, + "recommended_use_cases": { + "type": "array", + "items": { + "$ref": "#/output_schema/$defs/UseCaseItem" + } + } + } + } + } + }, + "fixtures": { + "sniff_test_path": "fixtures.json", + "tolerance": { + "allow_adjacent_levels": true, + "notes": "If allow_adjacent_levels is true, predictions within one rubric step of the expected label count as a pass." + } + }, + "tracing": { + "expose": [ + "rendered_prompt", + "raw_output", + "raw_text", + "formatted_output", + "usage" + ], + "notes": "Runtime engines MUST expose these five trace fields per evaluation." + } +} \ No newline at end of file diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/prompts/fixtures.json b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/fixtures.json new file mode 100644 index 00000000..10ce6868 --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/fixtures.json @@ -0,0 +1,50 @@ +[ + { + "id": "NASA-001", + "description": "What Is the Artemis Program?", + "source": "Intertextuality ANET Samples - sniff_test_2_clean.csv", + "input": { + "text": "Why Is This Program Called Artemis?\nThe first astronauts landed on the Moon in 1969. The missions were called Apollo. The name Apollo came from stories told by Greek people long ago. In the stories, Apollo was a god. Apollo had a twin sister. Her name was Artemis. She was the goddess of the Moon in the Greek stories.\n\nWhat Spacecraft Will Be Used for the Artemis Program?\nNASA has a new rocket. It is the Space Launch System. It is called SLS for short. It is the most powerful rocket in the world. SLS will carry the Orion spacecraft on top. Orion can carry up to four astronauts. Orion will fly around, or orbit, the Moon. The crew will take trips in spacecraft called landers to get to work on the surface of the Moon. When all of their work is finished, the crew will return to Earth aboard Orion.\n\nWhen Will Artemis Go to the Moon?\nThe first Apollo missions were tests. NASA launched the rocket to be sure it was safe for people and work as planned. Artemis will be tested first, too: Artemis 1 launched SLS and Orion with no astronauts on Nov. 16, 2022. Artemis 2 is carrying astronauts. They will circle past the Moon and return to Earth. Artemis 3 will send a crew with the next man to land on the Moon. Artemis 4 will send astronauts to land on the Moon.\n\nWhat Will Artemis Astronauts Do on the Moon?\nThe Artemis 4 crew will visit the Moon’s South Pole. No one has ever been there. At the Moon, astronauts will:\nSearch for the Moon’s water and use it.\nLearn how to live and work on a different planet or moon.\nTest the new tools that astronauts will need for a mission to Mars.\n\nWhy Is the Artemis Program Important?\nThe Moon is a good place to learn new science. NASA will learn more about the Moon, Earth, and even the Sun. The Moon is also a place to learn how astronauts can one day live and work on Mars.\nAstronauts on the Artemis missions will need new tools. Many companies will make these new tools. This will mean new jobs for people and companies on Earth. Other countries will be NASA’s partners for the new Moon missions. They will work on Artemis to bring the world together for a mission to Earth’s nearest neighbor in space.", + "grade_level": 3 + }, + "expected": { + "complexity_level": "slightly_complex" + } + }, + { + "id": "NASA-074", + "description": "What Is a Light-Year?", + "source": "Intertextuality ANET Samples - sniff_test_2_clean.csv", + "input": { + "text": "For most space objects, we use light-years to describe their distance. A light-year is the distance light travels in one Earth year. One light-year is about 6 trillion miles (9 trillion km). That is a 6 with 12 zeros behind it!\n\nWhen we use powerful telescopes to look at distant objects in space, we are actually looking back in time. How can this be?\n\nLight travels at a speed of 186,000 miles (or 300,000 km) per second. This seems really fast, but objects in space are so far away that it takes a lot of time for their light to reach us. The farther an object is, the farther in the past we see it.\n\nOur Sun is the closest star to us. It is about 93 million miles away. So, the Sun's light takes about 8.3 minutes to reach us. This means that we always see the Sun as it was about 8.3 minutes ago.\n\nThe next closest star to us is about 4.3 light-years away. So, when we see this star today, we’re actually seeing it as it was 4.3 years ago. All of the other stars we can see with our eyes are farther, some even thousands of light-years away.\n\nStars are found in large groups called galaxies. A galaxy can have millions or billions of stars. The nearest large galaxy to us, Andromeda, is 2.5 million light-years away. So, we see Andromeda as it was 2.5 million years in the past. The universe is filled with billions of galaxies, all farther away than this. Some of these galaxies are much farther away.\n\nIn 2016, NASA's Hubble Space Telescope looked at the farthest galaxy ever seen, called GN-z11. It is 13.4 billion light-years away, so today we can see it as it was 13.4 billion years ago. That is only 400 million years after the big bang. It is one of the first galaxies ever formed in the universe.\n\nLearning about the very first galaxies that formed after the big bang, like this one, helps us understand what the early universe was like.\n\nThis picture shows hundreds of very old and distant galaxies. The oldest one found so far in GN-z11 (shown in the close up image). The image is a bit blurry because this galaxy is so far away.", + "grade_level": 4 + }, + "expected": { + "complexity_level": "moderately_complex" + } + }, + { + "id": "FYM-1106", + "description": "A Good Night’s Sleep: Necessary for Young Minds", + "source": "Intertextuality ANET Samples - sniff_test_2_clean.csv", + "input": { + "text": "How Much Sleep Do You Need?\n\nThe National Sleep Foundation recommends that school-aged kids (6–13 years) sleep between 9 and 11 h a night. Teens are recommended to get 8–10 h a night and adults about 7–9 h . If you are a student, particularly in the United States, you may find it difficult to get this amount of sleep on school nights. As you go through puberty, your body wants to go to bed later and sleep later. But school (particularly in the U.S.) often starts too early! This makes it hard for teenagers to get enough sleep on school nights. By the weekend, you probably have missed so much sleep that you feel particularly sleepy, and you may dramatically oversleep as your sleep homeostat works hard to recover the sleep you need. If you oversleep all weekend, however, this can make waking up on Monday morning a miserable experience.", + "grade_level": 7 + }, + "expected": { + "complexity_level": "moderately_complex" + } + }, + { + "id": "CC-3983", + "description": "Italy's Violation of Faith", + "source": "Intertextuality ANET Samples - sniff_test_2_clean.csv", + "input": { + "text": "When I spoke eight days ago there was still a glimpse of hope that Italy's participation in the war could be avoided. That hope proved fallacious. German feeling strove against the belief in the possibility of such a change. Italy has now inscribed in the book of the world's history, in letters of blood which will never fade, her violation of faith.\n\nI believe Machiavelli once said that a war which is necessary is also just. Viewed from this sober, practical, political standpoint, which leaves out of account all moral considerations, has this war been necessary? Is it not, indeed, directly mad? [Cheers.] Nobody threatened Italy; neither Austria-Hungary nor Germany. Whether the Triple Entente was content with blandishments alone history will show later. [Cheers.] Without a drop of blood flowing, and without the life of a single Italian being endangered, Italy could have secured the long list of concessions which I recently read to the House—territory in Tyrol and on the Isonzo as far as the Italian speech is heard, satisfaction of the national aspirations in Trieste, a free hand in Albania, and the valuable port of Valona.\n\nWhy have they not taken it? Do they, perhaps, wish to conquer the German Tyrol? Hands off! [Prolonged cheers.] Did Italy wish to provoke Germany, to whom she owes so much in her upward growth of a great power, and from whom she is not separated by any conflict of interests? We left Rome in no doubt that an Italian attack on Austro-Hungarian troops would also strike the German troops. [Cheers.] Why did Rome refuse so light-heartedly the proposals of Vienna? The Italian manifesto of war, which conceals an uneasy conscience behind vain phrases, does not give us any explanation. They were too shy, perhaps, to say openly what was spread abroad as a pretext by the press and by gossip in the lobbies of the Chamber, namely, that Austria's offer came too late and could not be trusted.", + "grade_level": 10 + }, + "expected": { + "complexity_level": "exceedingly_complex" + } + } +] diff --git a/evals/prompts/purpose/input_schema.json b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/input_schema.json similarity index 100% rename from evals/prompts/purpose/input_schema.json rename to evals/literacy/qualitative-text-complexity/intertextuality/prompts/input_schema.json diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/prompts/output_schema.json b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/output_schema.json new file mode 100644 index 00000000..4670483f --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/output_schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "output_schema.json", + "title": "EvaluatorOutput", + "type": "object", + "required": ["complexity_score", "reasoning", "details"], + "additionalProperties": false, + "properties": { + "complexity_score": { + "type": "string", + "enum": [ + "slightly_complex", + "moderately_complex", + "very_complex", + "exceedingly_complex", + "more_context_needed" + ], + "description": "The Intertextuality complexity level for the target grade." + }, + "reasoning": { + "type": "string", + "description": "A high-level summary of why the text is at this complexity level for the target grade." + }, + "details": { + "$ref": "#/$defs/IntertextualityDetails" + } + }, + "$defs": { + "DetailedSummaryItem": { + "type": "object", + "required": ["factor", "description", "effect_on_complexity_dimension"], + "additionalProperties": false, + "properties": { + "factor": { + "type": "string", + "description": "The specific text complexity factor identified." + }, + "description": { + "type": "string", + "description": "How this factor manifests in the text." + }, + "effect_on_complexity_dimension": { + "type": "string", + "description": "How this factor affects the reader's ability to understand the text's specific complexity dimension." + } + } + }, + "ScaffoldingItem": { + "type": "object", + "required": ["scaffolding_need", "suggestion"], + "additionalProperties": false, + "properties": { + "scaffolding_need": { + "type": "string", + "description": "The complexity factor that requires scaffolding." + }, + "suggestion": { + "type": "string", + "description": "A specific instructional strategy to support students with this factor." + } + } + }, + "UseCaseItem": { + "type": "object", + "required": ["opportunity", "suggestion"], + "additionalProperties": false, + "properties": { + "opportunity": { + "type": "string", + "description": "An instructional opportunity related to the text." + }, + "suggestion": { + "type": "string", + "description": "A specific way to leverage this text for that instructional purpose." + } + } + }, + "IntertextualityDetails": { + "type": "object", + "description": "Practical instructional details including scaffolding strategies and recommended use cases.", + "required": ["detailed_summary", "adjustment_and_scaffolding", "recommended_use_cases"], + "additionalProperties": false, + "properties": { + "detailed_summary": { + "type": "array", + "description": "Individual complexity factors with descriptions and their effects.", + "items": {"$ref": "#/$defs/DetailedSummaryItem"} + }, + "adjustment_and_scaffolding": { + "type": "array", + "description": "Scaffolding strategies to make the text accessible at the target grade.", + "items": {"$ref": "#/$defs/ScaffoldingItem"} + }, + "recommended_use_cases": { + "type": "array", + "description": "Additional instructional opportunities for using this text.", + "items": {"$ref": "#/$defs/UseCaseItem"} + } + } + } + } +} diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/prompts/system.txt b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/system.txt new file mode 100644 index 00000000..ce6f0e2b --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/system.txt @@ -0,0 +1,35 @@ +**Role** You are an expert reading specialist and literacy evaluator. + +**Objective** Evaluate a provided text excerpt to determine its level of Intertextual complexity based on expert heuristic rules. You must carefully distinguish between self-contained information and information that requires familiarity with broader discourses, specialized domains, or specific external references. + +**Key Concepts & Definitions** +**1. Separating Self-Contained Knowledge from Intertextuality & Specialized Domains (Step 0 Filter)** +Intertextuality measures what prior textual, historical, cultural, or specialized domain encounters the text assumes and requires. +*Diagnostic Heuristic:* If the text explains a concept fully so that the reader learns it directly from the text, it is self-contained (Slightly Complex). If the reader must track references to specific written documents, inventors, outside studies, or if the text assumes familiarity with specialized scientific/technical systems, it connects to broader domains (Moderately Complex or higher). +*What Elevates Complexity:* +- References to named bodies of work, theories, studies, or academic institutions (e.g., Académie des Sciences). +- Specific references to historical patents, named inventions, and inventors. +- Specialized scientific and engineering knowledge where the text is not fully self-contained and assumes familiarity with broader technical systems (e.g., electricity, mechanical design, materials). +- Assumed familiarity with a shared public discourse or debate. +*What Keeps Complexity Low (Slightly Complex):* +- Direct quotes from anonymous historical sources (e.g., "a traveler in 1793") used purely for description, which do not require outside knowledge. +- Mentions of real places, historical settings, or dates that rely very little on external references for comprehension. +- Factual concepts, processes, or scientific content that are explicitly defined and taught directly within the text without assuming prior specialized knowledge. + +**2. Axes of Intertextual Complexity** +For texts that rely on external context: +*Axis 1: Degree of Explanation* (Fully Explained, Partially Explained, Assumed). +*Axis 2: Importance to Meaning* (Decorative/Enhancing vs. Load-bearing/Essential for the gist). + +**3. Scoring Scale** +* **Slightly Complex:** The text is completely self-contained and explanatory. It contains ZERO load-bearing intertextual references. The text may introduce or explain scientific concepts, technical ideas, or historical updates, but the reader learns these directly from the text. It may contain historical settings, real places, dates, or anonymous primary source quotes (e.g., "a traveler said"), but it relies very little on outside references for comprehension. *(Note: This is the baseline. Do not evaluate any text as lower than Slightly Complex).* +* **Moderately Complex:** The text introduces explicit references to outside studies, prior research, experimental designs, historical patents, specific inventions, or named inventors. Additionally, texts that rely on specialized scientific and engineering knowledge and connect to broader technical domains by assuming familiarity with those systems (i.e., not fully self-contained) fall into this category. +* **Very Complex:** The text relies on several partially explained or assumed references that the reader must navigate to grasp the passage's full significance, or a smaller number of load-bearing references to specific historical or academic texts. +* **Exceedingly Complex:** The text is deeply embedded in a specific historical, political, or cultural debate. It relies heavily on unexplained allusions, slogans, or assumed public discourse. Without extensive external context, inferring the author's real intent, stance, and the true meaning of the allusions is highly difficult. + +**Output Format** Provide your analysis in this format: +1. Step 0 Filter Analysis: Explicitly evaluate if the knowledge is self-contained or if it connects to broader technical domains, specialized systems, or external texts. +2. Identified Valid References: List true references (inventors, patents, outside theories, specialized systems) with their Explanation and Importance axes. +3. Complexity Drivers: Explain the cognitive load based on the presence of prior research, assumed specialized knowledge, self-contained explanations, or obscured historical intent. +4. Grade Level Context: Target grade impact. +5. Final Complexity Rating: State rating with a 1-2 sentence justification. \ No newline at end of file diff --git a/evals/literacy/qualitative-text-complexity/intertextuality/prompts/user.txt b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/user.txt new file mode 100644 index 00000000..0735a20d --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/intertextuality/prompts/user.txt @@ -0,0 +1,5 @@ + Text to Evaluate - text: {text} + It is intended for grade {grade_level} + Flesch–Kincaid grade level: {fk_score} + + Here is the format instructions: {format_instructions} diff --git a/evals/prompts/purpose/config.json b/evals/literacy/qualitative-text-complexity/purpose/prompts/config.json similarity index 100% rename from evals/prompts/purpose/config.json rename to evals/literacy/qualitative-text-complexity/purpose/prompts/config.json diff --git a/evals/prompts/purpose/fixtures.json b/evals/literacy/qualitative-text-complexity/purpose/prompts/fixtures.json similarity index 100% rename from evals/prompts/purpose/fixtures.json rename to evals/literacy/qualitative-text-complexity/purpose/prompts/fixtures.json diff --git a/evals/literacy/qualitative-text-complexity/purpose/prompts/input_schema.json b/evals/literacy/qualitative-text-complexity/purpose/prompts/input_schema.json new file mode 100644 index 00000000..8353d0c5 --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/purpose/prompts/input_schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "input_schema.json", + "title": "EvaluatorInput", + "type": "object", + "required": ["text", "grade_level"], + "additionalProperties": false, + "properties": { + "text": { + "type": "string", + "minLength": 1, + "description": "The passage to evaluate." + }, + "grade_level": { + "type": "integer", + "minimum": 3, + "maximum": 12, + "description": "Target student grade level." + } + } +} diff --git a/evals/prompts/purpose/output_schema.json b/evals/literacy/qualitative-text-complexity/purpose/prompts/output_schema.json similarity index 100% rename from evals/prompts/purpose/output_schema.json rename to evals/literacy/qualitative-text-complexity/purpose/prompts/output_schema.json diff --git a/evals/prompts/purpose/system.txt b/evals/literacy/qualitative-text-complexity/purpose/prompts/system.txt similarity index 100% rename from evals/prompts/purpose/system.txt rename to evals/literacy/qualitative-text-complexity/purpose/prompts/system.txt diff --git a/evals/prompts/purpose/user.txt b/evals/literacy/qualitative-text-complexity/purpose/prompts/user.txt similarity index 100% rename from evals/prompts/purpose/user.txt rename to evals/literacy/qualitative-text-complexity/purpose/prompts/user.txt diff --git a/evals/literacy/qualitative-text-complexity/purpose/purpose_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/purpose/purpose_evaluator.ipynb new file mode 100644 index 00000000..e8cf20da --- /dev/null +++ b/evals/literacy/qualitative-text-complexity/purpose/purpose_evaluator.ipynb @@ -0,0 +1,738 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qU langchain-google-genai langchain textstat" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import hashlib\n", + "import getpass\n", + "import os\n", + "from pathlib import Path\n", + "from dotenv import load_dotenv\n", + "from langchain_google_genai import ChatGoogleGenerativeAI\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "import textstat" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check for the API key\n", + "load_dotenv()\n", + "\n", + "if \"GOOGLE_API_KEY\" not in os.environ:\n", + " os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"Enter your Google AI API key: \")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded literacy.gla.purpose from /Users/achi/Documents/evaluators/evals/literacy/qualitative-text-complexity/purpose/prompts\n", + " model: gemini-3-flash-preview\n", + " temperature: 0\n", + " prompts:\n", + " system system.txt ( 4814 chars, sha 745b95b7d54d)\n", + " user user.txt ( 63 chars, sha cd8e6347db1a)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Load source-of-truth assets: config.json + every prompt file declared\n", + "# in config.steps[*].prompt.messages\n", + "# -------------------------------------------------------------------------\n", + "# The canonical evaluator definition lives in evals/prompts/purpose.\n", + "# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n", + "# so this loader is the pattern the SDK engineer will reproduce.\n", + "\n", + "ASSETS_DIR = Path(\"./prompts/\")\n", + "\n", + "config_path = ASSETS_DIR / \"config.json\"\n", + "with open(config_path) as f:\n", + " CONFIG = json.load(f)\n", + "\n", + "# Load standalone schema files (config.json references them via $ref by path).\n", + "with open(ASSETS_DIR / \"input_schema.json\") as f:\n", + " INPUT_SCHEMA = json.load(f)\n", + "with open(ASSETS_DIR / \"output_schema.json\") as f:\n", + " OUTPUT_SCHEMA = json.load(f)\n", + "\n", + "# Load every prompt message declared in config (system, user, ...). Each\n", + "# message has {role, source_path, sha256}. We verify each file's sha256\n", + "# matches the declared hash -- drift tripwire #1, applied to every prompt\n", + "# regardless of role. CI should promote a mismatch to a hard failure.\n", + "PROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\n", + "for msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n", + " role = msg_spec[\"role\"]\n", + " path = ASSETS_DIR / msg_spec[\"source_path\"]\n", + " text = path.read_text()\n", + " actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n", + " declared_sha = msg_spec[\"sha256\"]\n", + " assert actual_sha == declared_sha, (\n", + " f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n", + " f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n", + " )\n", + " PROMPT_MESSAGES.append((role, text))\n", + "\n", + "print(\n", + " f\"Loaded {CONFIG['evaluator']['id']} \"\n", + " f\"from {ASSETS_DIR.resolve()}\"\n", + ")\n", + "print(f\" model: {CONFIG['steps'][0]['model']['name']}\")\n", + "print(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\n", + "print(f\" prompts:\")\n", + "for msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n", + " sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n", + " print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# -------------------------------------------------------------------------\n", + "# FK score helper (declared as a preprocessing step in CONFIG['preprocessing'])\n", + "# -------------------------------------------------------------------------\n", + "_FK_PRE = next(p for p in CONFIG[\"preprocessing\"] if p[\"id\"] == \"fk_score\")\n", + "_FK_IMPL = _FK_PRE[\"implementation\"][\"python\"]\n", + "_FK_LIB = _FK_IMPL[\"library\"]\n", + "_FK_FN = _FK_IMPL[\"function\"]\n", + "_FK_TRANSFORM = _FK_IMPL[\"post_transform\"]\n", + "\n", + "if _FK_LIB != \"textstat\":\n", + " raise ValueError(f\"unsupported fk library in config: {_FK_LIB!r}\")\n", + "\n", + "\n", + "def calculate_fk_score(text) -> float:\n", + " \"\"\"Compute Flesch-Kincaid Grade Level per CONFIG['preprocessing'].\"\"\"\n", + " fn = getattr(textstat, _FK_FN)\n", + " value = fn(text)\n", + " if _FK_TRANSFORM[\"type\"] == \"round\":\n", + " value = round(value, _FK_TRANSFORM[\"precision\"])\n", + " else:\n", + " raise ValueError(f\"unsupported post_transform type: {_FK_TRANSFORM['type']!r}\")\n", + " return value\n", + "\n", + "\n", + "# -------------------------------------------------------------------------\n", + "# Evaluator function: model / prompt / parser config all read from CONFIG\n", + "# -------------------------------------------------------------------------\n", + "_STEP = CONFIG[\"steps\"][0] # single-step evaluator today. Extensible to multi-step evaluators.\n", + "\n", + "def evaluate_text_complexity(text: str, grade_level: int):\n", + " \"\"\"\n", + " Evaluate the Purpose-dimension complexity of a text using the canonical\n", + " config in evals/prompts/purpose/config.json + system.txt + user.txt.\n", + "\n", + " Returns a dict with full I/O trace fields:\n", + " - rendered_prompt: the actual list of messages sent to the model\n", + " (input-side trace).\n", + " - raw_output: the AIMessage object returned by the LLM\n", + " (preserves response_metadata, usage_metadata).\n", + " - raw_text: just the string content of the AIMessage.\n", + " - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n", + " - usage: token-usage metadata if the provider returned it.\n", + "\n", + " The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n", + " and the parsed output without a second call.\n", + " \"\"\"\n", + " # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n", + " # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n", + " # the standalone source of truth. include_raw=True preserves the AIMessage\n", + " # for tracing alongside the parsed result.\n", + " llm = ChatGoogleGenerativeAI(\n", + " model=_STEP[\"model\"][\"name\"],\n", + " temperature=_STEP[\"generation\"][\"temperature\"],\n", + " )\n", + " structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n", + "\n", + " # 2. Prompt template -- every message's content was loaded from disk\n", + " # and verified against config in the loader cell. We just feed the\n", + " # (role, text) tuples straight into ChatPromptTemplate.\n", + " prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n", + "\n", + " try:\n", + " # Step A: Calculate FK Score\n", + " fk_score = calculate_fk_score(text)\n", + " print(f\"Calculated Flesch-Kincaid Score: {fk_score}\")\n", + "\n", + " inputs = {\"text\": text, \"grade_level\": grade_level, \"fk_score\": fk_score}\n", + "\n", + " # Step B: Render the prompt up-front so we can return exactly what\n", + " # was sent to the model (input-side trace).\n", + " rendered_messages = prompt_template.format_messages(**inputs)\n", + "\n", + " # Step C: Single LLM call -> raw AIMessage + parsed output dict.\n", + " # No second LLM call.\n", + " raw = structured_llm.invoke(rendered_messages)\n", + "\n", + " if raw.get(\"parsing_error\"):\n", + " raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n", + "\n", + " # Step D: Return the full trace dict.\n", + " return {\n", + " \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n", + " \"raw_output\": raw[\"raw\"],\n", + " \"raw_text\": raw[\"raw\"].content,\n", + " \"formatted_output\": raw[\"parsed\"],\n", + " \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n", + " }\n", + " except Exception as e:\n", + " return f\"Error evaluating text: {e}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Calculated Flesch-Kincaid Score: 3.75\n" + ] + }, + { + "data": { + "text/plain": [ + "{'rendered_prompt': [{'content': '\\n Role\\n You are an expert reading assessment evaluator. Your task is to determine the Text Complexity of a given passage based exclusively on the Purpose dimension of the qualitative measures rubric.\\n\\n Task Details\\n You will be provided with an informational or literary `text`, along with its `grade_level` and `fk_score` (Flesch-Kincaid). You must analyze the text and determine how difficult it is for a reader to identify the author\\'s purpose. \\n\\n Crucially, you must distinguish between the text\\'s *topic* (what it is about) and its *purpose* (why the author wrote it). \\n\\n Rubric: Purpose Complexity\\n Exceedingly Complex: Subtle and intricate, difficult to determine; includes many theoretical or abstract elements.\\n Very Complex: Implicit or subtle but fairly easy to infer; more theoretical or abstract than concrete.\\n Moderately Complex: Implied but easy to identify based upon context or source.\\n Slightly Complex: Explicitly stated, clear, concrete, narrowly focused.\\n More Context Needed: The text is a fragment or lacks necessary introductory context, making the true purpose impossible to determine accurately without external background knowledge.\\n\\n Expert Rules for Evaluating Purpose\\n Based on expert consensus and historical grading corrections, you must apply the following heuristics:\\n\\n 1. The \"Slightly Complex\" Benchmark (Straightforward and Explicit)\\n A text is Slightly Complex if its purpose is explicitly stated or if its informative intent is straightforward, clear, concrete, and directly answers what the text is immediately about. If the text opens by clearly identifying a concrete topic (e.g., \"Pins are made of either brass or iron wire\") and rigidly follows through by explaining factual, practical information or a process (like manufacturing steps or geographic facts), the purpose is considered explicit and straightforward. It does *not* require a literal statement like \"The purpose of this text is to...\" as long as the delivery of information is direct, clear, and unadorned by persuasive elements or complex framing.\\n\\n 2. Moderately Complex via Guiding Questions & Inquiry Formats\\n If a text begins with a general introduction and uses guiding questions (e.g., \"Have you ever wondered how clouds are formed?\") to transition into an explanation, the purpose is implied rather than explicitly stated upfront. Because the reader must recognize the question as the pivot point for the author\\'s intent, it is Moderately Complex.\\n\\n 3. Moderately Complex via Multiple Distinct Informational Goals\\n If a text covers a broad topic but jumps between several distinct scientific or informational objectives without an overarching framing device or explicit thesis (e.g., talking about measuring ice sheets, then mapping, then finding meteorites), the reader must synthesize these diverse facts to recognize the broader purpose, making it Moderately Complex.\\n\\n 4. Moderately Complex via Arguments Disguised as Information\\n If an author is arguing a specific point, correcting a misconception, or defending a stance, but the text could initially be mistaken by students as purely informative factual text, it is Moderately Complex. The reader must infer the persuasive intent or argumentative purpose beneath the informative tone.\\n\\n 5. \"More Context Needed\" for Fragments\\n If a text is a fragment missing a crucial introduction or context, and identifying the author\\'s purpose beyond a simple surface-level description would be exceptionally difficult for a reader in the target grade level without that external background, score it as `more_context_needed`. \\n\\n Output Format\\n Provide your evaluation in the following structure:\\n reasoning:\\n - Surface Analysis: Identify if the text clearly identifies its topic and delivers straightforward facts, or if it utilizes structural cues, titles, or direct thesis statements.\\n - Subtlety & Framing: Is the informative purpose straightforward and concrete? Does it use guiding questions? Is it an argument disguised as pure information? Are there multiple distinct informational goals requiring synthesis?\\n - Context Check: Is this text a fragment missing crucial context that obscures the deeper purpose for the target grade level?\\n - Rubric Alignment: Explain how the text aligns with the specific language of the rubric, explicitly referencing the expert rules above. Justify why it isn\\'t one level simpler or more complex.\\n\\n answer:\\n - complexity_score: (slightly_complex, moderately_complex, very_complex, exceedingly_complex, more_context_needed)\\n - reasoning: A brief summary of your final decision.\\n - details: Structured breakdown of PurposeDetails including detailed_summary, adjustment_and_scaffolding, and recommended_use_cases.\\n',\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'system',\n", + " 'name': None,\n", + " 'id': None},\n", + " {'content': 'Analyze:\\nText: \\n\"Well, then,\" said the teacher, \"you may take your slate and go out behind the schoolhouse for half an hour. Think of something to write about, and write the word on your slate. Then try to tell what it is, what it is like, what it is good for, and what is done with it. That is the way to write a composition.\" Henry took his slate and went out. Just behind the schoolhouse was Mr. Finney\\'s barn. Quite close to the barn was a garden. And in the garden, Henry saw a turnip. \"Well, I know what that is,\" he said to himself; and he wrote the word turnip on his slate. Then he tried to tell what it was like, what it was good for, and what was done with it. Before the half hour was ended he had written a very neat composition on his slate. He then went into the house, and waited while the teacher read it. The teacher was surprised and pleased. He said, \"Henry Longfellow, you have done very well. Today you may stand up before the school and read what you have written about the turnip.\"\\n\\nGrade: 4\\nFK Score: 3.75',\n", + " 'additional_kwargs': {},\n", + " 'response_metadata': {},\n", + " 'type': 'human',\n", + " 'name': None,\n", + " 'id': None}],\n", + " 'raw_output': AIMessage(content=[{'type': 'text', 'text': '{\\n \"complexity_score\": \"slightly_complex\",\\n \"reasoning\": \"The text\\'s purpose is straightforward and concrete: it provides a simple biographical anecdote about a young student learning to write. The narrative follows a clear, chronological path from the teacher\\'s instructions to the student\\'s successful completion of the task, making the author\\'s intent to illustrate a specific educational moment very easy to identify.\",\\n \"details\": {\\n \"detailed_summary\": [\\n {\\n \"factor\": \"Explicit Narrative Focus\",\\n \"description\": \"The text focuses narrowly on a single event: Henry writing a composition about a turnip based on his teacher\\'s specific prompts.\",\\n \"effect_on_complexity_dimension\": \"The reader can easily identify that the author\\'s purpose is to tell a simple, instructional story without needing to infer hidden meanings or abstract themes.\"\\n },\\n {\\n \"factor\": \"Concrete Subject Matter\",\\n \"description\": \"The story deals with tangible objects (slates, turnips, barns) and clear actions (writing, reading, standing up).\",\\n \"effect_on_complexity_dimension\": \"This keeps the purpose grounded in reality, aligning with the \\'clear, concrete, narrowly focused\\' criteria for slightly complex texts.\"\\n },\\n {\\n \"factor\": \"Direct Instructional Framing\",\\n \"description\": \"The teacher\\'s dialogue explicitly outlines the \\'way to write a composition,\\' which serves as the central pillar of the text.\",\\n \"effect_on_complexity_dimension\": \"Because the method is stated directly, the reader does not have to guess the educational point the author is making.\"\\n }\\n ],\\n \"adjustment_and_scaffolding\": [\\n {\\n \"scaffolding_need\": \"Historical Context\",\\n \"suggestion\": \"Provide a brief introduction explaining that Henry Wadsworth Longfellow became a famous American poet. This helps students understand why this specific anecdote was preserved and shared.\"\\n },\\n {\\n \"scaffolding_need\": \"Vocabulary of Composition\",\\n \"suggestion\": \"Define \\'composition\\' and \\'slate\\' before reading to ensure students understand the tools and the goal of the character\\'s task.\"\\n }\\n ],\\n \"recommended_use_cases\": [\\n {\\n \"opportunity\": \"Modeling Descriptive Writing\",\\n \"suggestion\": \"Use the teacher\\'s four questions (what it is, what it is like, what it is good for, and what is done with it) as a scaffold for students\\' own descriptive writing exercises.\"\\n },\\n {\\n \"opportunity\": \"Introduction to Biography\",\\n \"suggestion\": \"Use this text as an entry point for studying biographies, showing how authors use small, specific stories to illustrate a person\\'s character or development.\"\\n }\\n ]\\n }\\n}', 'extras': {'signature': 'EskoCsYoAQw51seX9joZSpu9ImWTUcC/4VYP9eygKBrEJRXjU0hklKx9APLP36OiO43MytItCw6dT1c/n7RAbB8iblBKsI8blpJqdrhEWCN5CI5s48RQyu2RqgbtoHN05WgHFpFyEMcO8zNHNmhZDwgVv9yIEMoKsf76qoW0onrWUZY8e12aOUWwo/+T/FFnOMZoNd0xY/sVe/VccPkBcM8XYU/F3fhIDwVylGT2O24eKtccfLtMkl59aBvlyaHvAC2wWuIgpYHg5ftsDLTCJ2vswKLh8YzRDTuSLfSxpHdD/iMDCMuTUelMZZPtkQMdyGZl+t99lTTlV4iq/3tS5VrZP/04JBmfTZyLAH/iPJp4sKUaaYGeNXVRmwI/UJQxYYVpiTZh/LyQH+qaKg1jcehzVyV+sltz5ubZLr8ew2ODXLhFeXkVz7tjqjAsDnJZPXzacfNtoHvhVVdoS6GO0FhZKaYfCzwk007FH5ekG3CjWxbppmEGXwhB4yNgqYzUr5G3b6M3O5yIfeaI7LCGCN5sygL5vnUQwb7P2UagUdcePozpudh6LBDayYU6oSKRTpccb2ToyqBHlJR3lgO00C+Vx33Mto9QhImh+TBeMnTxeOnNonKKyn1Go2AssnVD/hxSkB24mejpf1BzOmypVYVhDMjGgHtGKc4O0gLaB+jMcCjWofi8jORe/huU3D00ly+/jDIwYojX0hCMjiTtI1akz1Nc2C1PqDhbxJCQX9U6LuqiDgvtSKIvEPX6DULq2mU0KbPb5d0jSeTmaSt54GYDcxXTrcEdoJTugsuxQkobOvN41i6XS5qGPzVxMzJ7jXh9wk9aERiMCj5K7FsCgpFOjIk72pfq9hdrSQh2HPP8FAW/3Avh30b+1RuEzvGKse77IiswcKkYUwlm7e/fkShhhloMPJztTBrbCPduuarWbzOoa45MxyjioeEQ5t5k1UXZ8y+BlSXHBAAh68b9uiATvGM+J7Mku1EMCCAoYiSrQops/lhESVpTroDZbiTns+YO7Ix4tgqMLvESBTT44RPNuxm3IOJRNUctKVEpwtwAzXa41EcoU7sc80juX8ZQKxPgMeKPkfaau2U4M9AeNqIzxKCMrht8Si/RUj43cg34jsKmVtyhjtkiETFs0HOS9nA7SRalGWEDOuNsEwr6X7uJyLUL+iJPKqU5hAVZPohromK2dEC5ASGmF7pRyIrz9tS6fDb/WqPstvB+ClV+jnE9NtgNpyMwZh6MGZ8fVOk6gwpQaxdf2zsnTRXpO/ymFN98BmtS3hlYcl79fUHkJJ8drISmx2DkQ1MbAZCMTeSExfdtYdHBr2zVHPap4KE9vXcbp/kFdKgI1zx17GJ2q8cGcTYfnOcwhX07gjM8VEpUjw100eZMEUuMua4UwmtE/FGOjlEnEuDzmAX3fDXAIwtLzGcn6PXxaf9UZ/0ARZUPw4WozzeD1Z7nL0lX+mT5lMsf+OZBi87drBEgd/vvPQqD/2AGTwC2zlZaI6eCAWBX5M+g9OKXyBRfmWvZb1e9v+Ur2RfXwYoBue2uPyexRFn3hjJvdOH3wZHSmydu10AKD34xIWx0rNTAM8QPO9pS9fOxuxBpiGKmDR0VsR+H4hqz6R22h1ZmwFjQeYK0PY7HJlTS0B4U2Ud44QwtsCmxOtGEM7QHSNmqpmzPZYHisTvQfcpMVQp0JeQqwW/43Q4ZNUQaJBLp/03S1pd4asn7jviX8Lf8wqTOdyFXz5cwpRhiUVwhgy/Q1zGdq+IOBuufYnOx6XxseYFmbcg9GN76wCZb7cut+9cuZvZCyqdWa2Chznf+7WNk6sKT61Ee00wrndr2ywscdMeOUaPkwKhvJ6x3tIvM1WtWLrXblgscZYSxM5X6R1rGgJDHU8PQUGw2PZvLmTcB+f4MDXzdL9Q4SBPx63/lEzMUiS54cvtEk97dgTTetPnDfTQkqQsBE5uheEucgxPbslhkpcB313MHGFVeaX7zyfSEtxAZUnI45VUEJDvpJCnfFjg3dN1jKa9PPLo4a808PQHqwKb/KvyC2uYMd2galIjIkAbHqfOkfK6AaNIh4Fs7OxiozP41MizuHcVDjNEEakhMnopIsGqYp/Pv2uWnGBSyrcXlDlfZoPfEFY00MpzkMNFdOrYI748bE2J5o9dsH1wHFgDUszlUWPUIOSknqfGPolXrzsEsdvxv3c2KmnpRPEagcXvcOWM4lIcztqjdbnagTm1QQz78d5/k8RaH3awBjL7Hg7BZP6RXOqewUfzuGGrIbhb4Nu4xbTG1JlNaKjgiyuhE/1ReKb+OEJaJwLhiyN9v8lk6kXd7BA0Lkw/M0zpi3bhbd/jrfoz2I7uji368VRng0ZUgShoz9fJSE1+HwwPHEiDDJ7HS9IAqbqBnUcBjCCLB7YbWlmHla3cKSiZsC5Z+NGqdccLoJUNXjWZC+7s0OdM7bc8bHsCke1VKv/7ML2YNPCmqIXhTvaqQPdJN94Nm5YTejSyw5hI5ygyQMp+TDlxlZfi/GuIV6449VgJbWDUX+RPVy9wmy/aIV4YLECoMLCWG8i3/NMRfVcng/Vp2kTlR1fa6ATvfFmkEN/VFgk9TYbUfOhZtAVO/A0oAOA7iTe4KKMh9sB6MLmrpwDque0L/IoB8t93b+c2o55qssnJXFaWsc5NtUWzt3fzSOxDkEbxXV8oyrCQaYkH519zTnNpnt59NZmpKknmc0tIjYhDfLReCiOFXO5NAWHVUXvFfUcfNtVS7XLaU1QnILRJVJXKsiBieGUh5dJjFTnW1XEG2viwZU6meckfrcvxiQIE7BfJkorZ89ZYyVSc4DIGYotmhwEkgDpwlHqY8jlSNyVIX2kojNUt8uBWNy8kigOo9XA+OHgKLPauyQnLqsJyT0Nw40JBRZCpUKHKyIypXBKFwJmNkTk3ICKmYyGMZ0f8Rph3R4HeW0EhyS+KEEMhgJezk76os4gCm3P2XGF53JAqP5R6+sFXfRZFPZ+b0O6oqdUuz0vD2RSHdnV4sNxa4ry6TtBhQvittFZMthCFardAoQlsk63sLhm5qN9HMN5YjHgyhSCcVeoRcx7h11JAPHW0ODG9G5UFrg55vv7lVdoqmlR9sOXqCt65JA8wcwyt3g93Ofd7K++sAiRqJ26wPH1wx1Zg6JS8pPxymjqvykR2Fx0lHW+9TNQnGewAo9/P5WxNOsObO1sUhO3vM2h6EZLoCTMAn5qMwQoq0Z7r74in3wUnpZlp+xCNyB5F3yybCF8D3J1WwiDaA9C3dCj/Uyfg7/ntV5ScjznLTEbI7I+Hctp9xfaIOi+8XQ2voBWK9h0hfVHe8p+8LsQ2mi1lzGjLuLC2/ibnuHh7LjBT/PYOfgcIuislR3IryQul1o0QBskmvoHdcQVRwK1Lw21hpDpApf4ORdSn8N0ia2iNcqpBsHrsC8Gni3sRzhZN+5OGrBTEKZMg3TBDEuC4VhWwlZL6hAZe3kyxHSRjn+v9TfgCtVfTfyUCOxFnvbJN/LERbPpn/cVX+WLx0AtOEJEV0wbRIazlenn0uftOXni74KWvMA2A6IVrghD6dWLQE8BAuLkh7hUCh5smgbH9njcRrqtiUjK2lixV3vy1xk/CZ9dpMKSBcwOTvPSvgcI5f3Hedsl3kvPQWisOEmGIrWeZlxfzlUg9ZN+/IR9P0Vs1MptAhjbPxfNcZJmdrMY94Zv43NL7yZ3R0KoQzVz1rqYP1JzHxeASblokJ4P0vWoebJxp8uQpy9+LTO3ThMXkNg9PWqqyqYvNAL1CwEVgqGceXDAhsMTLRYPJhzRhWzsxextxV5z9s1Hae1DAtzHT1HSsOAciw+wSo4TF7AucsCS1luo6Z5u8nzmDL3jSP03ykZfa5w1PRffsWKPV6iYOwQivaZeHfh1L2GvIwXR0RTDBp4COJH8aDBqnbTPgP/hlrTHq7GMy0QRjHKofeAGR1aE3M/eMOmG2DO9S3n9r3CKwCV0PDdZJiU5tBReAxoU8epp6esqAXKagtzYhQZ2T3d3dU3QduoM8TW9q/lV9hCjY/9d+aV8Ear8Y0oDeb9ujHOflVZdTfkYVR9b1gxX9dP5CE9A/XnAu4YYsmXZSs2OA8jrI7G/Ldj+ZKM+Ik82XrgCwnJMmtKDQXdSvWWOEWkZMUsDo1lPrbYJ9xo2oieVIuOFQ/YEHdGklO9M5gUzaKvofrl3mKfo4CUFjLc5865XX7T4ru0fXFuv8TgGboZDxYs5X6xJHH6ztz4UlCcM4+xR6utXZ4UEQPILiYAea6KZBr9s/Qjx3bZzY2VFNjQ4kd5BBZIyCUiMcmOJErxuKqIux5ICpZuauU633llA2G22qmgXhrxWTpftCna58qGHJH1fPKka0Ca7tfryyL59TExNhRa5GfVZWIEJwRzvZnm1k3iDtqCvcy/NzE0ro4Lv+D7K01Cs3JRRXD1mLve7mXU7L37afUEgUePSnYXz+HWpy1bm0r/GmPQVh8Gj80WNH6+QDph3TwqMLLTjk/Ie4NT8Ff97VqpxM8Su7+MhH9Mkq9aEFR28r8kYFhqWWsPyIw9OftJk5nAdh6eMiIKsPjaRGhXTiDwHP36CJqNq7TeLeWQR7rrarKaUj5CVctrmWv1Kzg2OQ5UTqra9V3XBSj6hm6VSBeEU5gqKuYURAAn4hKuwawtRbYLCFZSPvgoSMM9KPuNJg9SNxSxOLHN+NwOXJBdSQkBLMGgsbmCNB1SguVOThQLZVXvtTkubWRdYM7sOq03SRDuCMy7Vl2qEVSWSB+gLpcJ2H3r4rut1/lgsdbJCH4Xez0384hQdGSYA5sKywuUlaCYgDKCPPbFEZ36oA+xc0tbXa65xradEAdRWFeBp8CGRhFY528XJS3LifabPEAWa0eAs8FeXCc+QAkztmJ2boZyZMUUl0KIw5u3eroUtov2LFe560DYbscpkQ1WwhNHTOcdW3esV+DnA0prhh8u6wnthDUOqX4tOhPbQszTLJ7/NBdPyKlqFaog6q3Yng2d3s2Vg6cMvQKFAFVzLuujfXgrZqgBqiCmGVGm6q0IWwUZ/C0tQ+2QlZHeFFqcm7e3kX1LFi8DEGsEYgY8aj1VLGdZIeVLQHbv85WZqZOv8f7tWkrtGhz1MQxphDiSz8GRTC/2zQgThOyYNEjFzztNEdhfj/QM+5PbZM1awu4Do65Gjs2HsaXiyUaZFtHT/VNJ3yV2RHJ5ExSzkwKEtVsbuE9TQlhtc2RuHP+z7Zcz/c+BT1mIuK2ixqZnReJPq1VBR9wUFUqrIdjdrznkO9e7kcc3BPv8VPI8t9zhXoK4y+od0EwTaRcPaxykhzQz5xGNID2n3E0AZQsF9pWvWEQAfntiX62z8ph9G8argwemnhRsSRh3SM6q2aJzf1ZwkDrJHCCxhoZ3qYjm+I51r8Ci8F/wyrhB7yTT+yPaP+Xlwv2reg9Kcdnq1vhaScV38jmQpaB3vUs0t5XEGUDK+XiMQvLqNiWtUEA6lRlVbY2dtfsaIlo3djoM06Ysm1D3GDoOPRppYt63vxYKRFi1S24tmbpooatPWD9fAxCV3+5CIG8bgIsUtL3iB7SJxcXbkALsZDxhP9auNNaz7u8gNWIAYZaH8NSmXZI6GADfT+lGaUi/s5HW1fDpLZPotrT2SP9qseQKmWCrprtrKhfqm0bwz1k84IriH7C3G/e/+iSoUkRjDeZRvXPVwOm6xC+FQXmKHmf3F/PrSheKz9V+vDRTOct4XnsgRpXv+/hRAr2/FlrM+bC7BCthvBZQKh2FMDiXNyVRSlDx7ucKuyz94jn07b2ZED92nFqbQLIqlb80HkkT53Yd4q1c7wLuyka5+pwfViuNtlk9LhSMH2htsuusWr8JaSjsjwf3KKRRcFz79ZRm/J2t48/NdM6goiUcRAE5Ai3qaBdHU2oOMtizr1Js1bDfTM7QPti3HmzFM2+wdSHnGywuntX3G/anF9UEmhkkRMpRUetiWz3ZgcNbcMI7fDMpJjeppjNC7muL8M3OHwizlejP3uUigjxoKkSCnERv7FFR54Uwk2Oa+5JctRgnWOPDI4gRr6jw2qbM1XY4lyEMX1qALwMnVlmMOr1S55jklbSFzKBuFeSQpp19bpL09mYjsGLlWtXdQq++vK/2lMlXKg7RlG5yKXxZ0Zi8Dnn1MEVAgNR/LOlOr2K0uDcBScO6JfySRlhTXiFkAVPvlWS3czNTfZJcj5tQf469AgWlnoWsQCr+shVEE+YdlvB/jgGmVjFaX7CXhstHAQ9xJ4RWSdaLSlFl5++2yD4kMmaZnGALNh2NvFNEKJZiqaiZgmyKqqoLbsNguzJtYuq9UxK2B62kCn/5aW1QH0APhWP/DJGZoZFSVenTZl/0ft1ly26vu3h919iYhAxA4jHyV8OWgi3B/r3pGCXANN5K6l9NWlwyx0QB6erJUYo6UkpocQSKWZ/4XcbCGOflnJw51MGopNED3q68t6ae0ODWzA/84s9TlmEZprB5MlVeSpUxNIantWS9gUmzfbjOc/pWy+/q5MR5GLkbt7T/srIIt87qUVzEmxPXANN6gKCMsHHn4EHioZfmIt9DHOXQ+YHhZ5znvg8m7q26KyMfEyTkbR4tDcd//b7k+1+/pQ49HHKx80cHACPGfjTVWZgtLA9jqKXbpdCVZfUnftVaKEJ3sl3KcyX9LoQDZ20PeULkxdCP1vmgCiF9rJw959JStGlksf0bDxDalf4U/DhuwQbQClxNW8v2zvDunEtpEVfyVUgHbGubr8s5WPzAfaPvJ8Gk4n5AMjWY7pdAhNJq/H70a/kiHp9h7wmvHgF0J7da8hfP7w+B5WJztLZ2m2NEGhhhpTJwZrBBh01NZjPk6dJuTCvXCgRAGk6dOPbslq4C7CBeifFYJJHTVbMQQdYQBI600tyGwP+yTnQlrJD/hlO'}}], additional_kwargs={}, response_metadata={'finish_reason': 'STOP', 'model_name': 'gemini-3-flash-preview', 'safety_ratings': [], 'model_provider': 'google_genai'}, id='lc_run--019edb72-fa3f-78f0-a545-068091963d10-0', tool_calls=[], invalid_tool_calls=[], usage_metadata={'input_tokens': 1226, 'output_tokens': 1815, 'total_tokens': 3041, 'input_token_details': {'cache_read': 0}, 'output_token_details': {'reasoning': 1203}}),\n", + " 'raw_text': [{'type': 'text',\n", + " 'text': '{\\n \"complexity_score\": \"slightly_complex\",\\n \"reasoning\": \"The text\\'s purpose is straightforward and concrete: it provides a simple biographical anecdote about a young student learning to write. The narrative follows a clear, chronological path from the teacher\\'s instructions to the student\\'s successful completion of the task, making the author\\'s intent to illustrate a specific educational moment very easy to identify.\",\\n \"details\": {\\n \"detailed_summary\": [\\n {\\n \"factor\": \"Explicit Narrative Focus\",\\n \"description\": \"The text focuses narrowly on a single event: Henry writing a composition about a turnip based on his teacher\\'s specific prompts.\",\\n \"effect_on_complexity_dimension\": \"The reader can easily identify that the author\\'s purpose is to tell a simple, instructional story without needing to infer hidden meanings or abstract themes.\"\\n },\\n {\\n \"factor\": \"Concrete Subject Matter\",\\n \"description\": \"The story deals with tangible objects (slates, turnips, barns) and clear actions (writing, reading, standing up).\",\\n \"effect_on_complexity_dimension\": \"This keeps the purpose grounded in reality, aligning with the \\'clear, concrete, narrowly focused\\' criteria for slightly complex texts.\"\\n },\\n {\\n \"factor\": \"Direct Instructional Framing\",\\n \"description\": \"The teacher\\'s dialogue explicitly outlines the \\'way to write a composition,\\' which serves as the central pillar of the text.\",\\n \"effect_on_complexity_dimension\": \"Because the method is stated directly, the reader does not have to guess the educational point the author is making.\"\\n }\\n ],\\n \"adjustment_and_scaffolding\": [\\n {\\n \"scaffolding_need\": \"Historical Context\",\\n \"suggestion\": \"Provide a brief introduction explaining that Henry Wadsworth Longfellow became a famous American poet. This helps students understand why this specific anecdote was preserved and shared.\"\\n },\\n {\\n \"scaffolding_need\": \"Vocabulary of Composition\",\\n \"suggestion\": \"Define \\'composition\\' and \\'slate\\' before reading to ensure students understand the tools and the goal of the character\\'s task.\"\\n }\\n ],\\n \"recommended_use_cases\": [\\n {\\n \"opportunity\": \"Modeling Descriptive Writing\",\\n \"suggestion\": \"Use the teacher\\'s four questions (what it is, what it is like, what it is good for, and what is done with it) as a scaffold for students\\' own descriptive writing exercises.\"\\n },\\n {\\n \"opportunity\": \"Introduction to Biography\",\\n \"suggestion\": \"Use this text as an entry point for studying biographies, showing how authors use small, specific stories to illustrate a person\\'s character or development.\"\\n }\\n ]\\n }\\n}',\n", + " 'extras': {'signature': 'EskoCsYoAQw51seX9joZSpu9ImWTUcC/4VYP9eygKBrEJRXjU0hklKx9APLP36OiO43MytItCw6dT1c/n7RAbB8iblBKsI8blpJqdrhEWCN5CI5s48RQyu2RqgbtoHN05WgHFpFyEMcO8zNHNmhZDwgVv9yIEMoKsf76qoW0onrWUZY8e12aOUWwo/+T/FFnOMZoNd0xY/sVe/VccPkBcM8XYU/F3fhIDwVylGT2O24eKtccfLtMkl59aBvlyaHvAC2wWuIgpYHg5ftsDLTCJ2vswKLh8YzRDTuSLfSxpHdD/iMDCMuTUelMZZPtkQMdyGZl+t99lTTlV4iq/3tS5VrZP/04JBmfTZyLAH/iPJp4sKUaaYGeNXVRmwI/UJQxYYVpiTZh/LyQH+qaKg1jcehzVyV+sltz5ubZLr8ew2ODXLhFeXkVz7tjqjAsDnJZPXzacfNtoHvhVVdoS6GO0FhZKaYfCzwk007FH5ekG3CjWxbppmEGXwhB4yNgqYzUr5G3b6M3O5yIfeaI7LCGCN5sygL5vnUQwb7P2UagUdcePozpudh6LBDayYU6oSKRTpccb2ToyqBHlJR3lgO00C+Vx33Mto9QhImh+TBeMnTxeOnNonKKyn1Go2AssnVD/hxSkB24mejpf1BzOmypVYVhDMjGgHtGKc4O0gLaB+jMcCjWofi8jORe/huU3D00ly+/jDIwYojX0hCMjiTtI1akz1Nc2C1PqDhbxJCQX9U6LuqiDgvtSKIvEPX6DULq2mU0KbPb5d0jSeTmaSt54GYDcxXTrcEdoJTugsuxQkobOvN41i6XS5qGPzVxMzJ7jXh9wk9aERiMCj5K7FsCgpFOjIk72pfq9hdrSQh2HPP8FAW/3Avh30b+1RuEzvGKse77IiswcKkYUwlm7e/fkShhhloMPJztTBrbCPduuarWbzOoa45MxyjioeEQ5t5k1UXZ8y+BlSXHBAAh68b9uiATvGM+J7Mku1EMCCAoYiSrQops/lhESVpTroDZbiTns+YO7Ix4tgqMLvESBTT44RPNuxm3IOJRNUctKVEpwtwAzXa41EcoU7sc80juX8ZQKxPgMeKPkfaau2U4M9AeNqIzxKCMrht8Si/RUj43cg34jsKmVtyhjtkiETFs0HOS9nA7SRalGWEDOuNsEwr6X7uJyLUL+iJPKqU5hAVZPohromK2dEC5ASGmF7pRyIrz9tS6fDb/WqPstvB+ClV+jnE9NtgNpyMwZh6MGZ8fVOk6gwpQaxdf2zsnTRXpO/ymFN98BmtS3hlYcl79fUHkJJ8drISmx2DkQ1MbAZCMTeSExfdtYdHBr2zVHPap4KE9vXcbp/kFdKgI1zx17GJ2q8cGcTYfnOcwhX07gjM8VEpUjw100eZMEUuMua4UwmtE/FGOjlEnEuDzmAX3fDXAIwtLzGcn6PXxaf9UZ/0ARZUPw4WozzeD1Z7nL0lX+mT5lMsf+OZBi87drBEgd/vvPQqD/2AGTwC2zlZaI6eCAWBX5M+g9OKXyBRfmWvZb1e9v+Ur2RfXwYoBue2uPyexRFn3hjJvdOH3wZHSmydu10AKD34xIWx0rNTAM8QPO9pS9fOxuxBpiGKmDR0VsR+H4hqz6R22h1ZmwFjQeYK0PY7HJlTS0B4U2Ud44QwtsCmxOtGEM7QHSNmqpmzPZYHisTvQfcpMVQp0JeQqwW/43Q4ZNUQaJBLp/03S1pd4asn7jviX8Lf8wqTOdyFXz5cwpRhiUVwhgy/Q1zGdq+IOBuufYnOx6XxseYFmbcg9GN76wCZb7cut+9cuZvZCyqdWa2Chznf+7WNk6sKT61Ee00wrndr2ywscdMeOUaPkwKhvJ6x3tIvM1WtWLrXblgscZYSxM5X6R1rGgJDHU8PQUGw2PZvLmTcB+f4MDXzdL9Q4SBPx63/lEzMUiS54cvtEk97dgTTetPnDfTQkqQsBE5uheEucgxPbslhkpcB313MHGFVeaX7zyfSEtxAZUnI45VUEJDvpJCnfFjg3dN1jKa9PPLo4a808PQHqwKb/KvyC2uYMd2galIjIkAbHqfOkfK6AaNIh4Fs7OxiozP41MizuHcVDjNEEakhMnopIsGqYp/Pv2uWnGBSyrcXlDlfZoPfEFY00MpzkMNFdOrYI748bE2J5o9dsH1wHFgDUszlUWPUIOSknqfGPolXrzsEsdvxv3c2KmnpRPEagcXvcOWM4lIcztqjdbnagTm1QQz78d5/k8RaH3awBjL7Hg7BZP6RXOqewUfzuGGrIbhb4Nu4xbTG1JlNaKjgiyuhE/1ReKb+OEJaJwLhiyN9v8lk6kXd7BA0Lkw/M0zpi3bhbd/jrfoz2I7uji368VRng0ZUgShoz9fJSE1+HwwPHEiDDJ7HS9IAqbqBnUcBjCCLB7YbWlmHla3cKSiZsC5Z+NGqdccLoJUNXjWZC+7s0OdM7bc8bHsCke1VKv/7ML2YNPCmqIXhTvaqQPdJN94Nm5YTejSyw5hI5ygyQMp+TDlxlZfi/GuIV6449VgJbWDUX+RPVy9wmy/aIV4YLECoMLCWG8i3/NMRfVcng/Vp2kTlR1fa6ATvfFmkEN/VFgk9TYbUfOhZtAVO/A0oAOA7iTe4KKMh9sB6MLmrpwDque0L/IoB8t93b+c2o55qssnJXFaWsc5NtUWzt3fzSOxDkEbxXV8oyrCQaYkH519zTnNpnt59NZmpKknmc0tIjYhDfLReCiOFXO5NAWHVUXvFfUcfNtVS7XLaU1QnILRJVJXKsiBieGUh5dJjFTnW1XEG2viwZU6meckfrcvxiQIE7BfJkorZ89ZYyVSc4DIGYotmhwEkgDpwlHqY8jlSNyVIX2kojNUt8uBWNy8kigOo9XA+OHgKLPauyQnLqsJyT0Nw40JBRZCpUKHKyIypXBKFwJmNkTk3ICKmYyGMZ0f8Rph3R4HeW0EhyS+KEEMhgJezk76os4gCm3P2XGF53JAqP5R6+sFXfRZFPZ+b0O6oqdUuz0vD2RSHdnV4sNxa4ry6TtBhQvittFZMthCFardAoQlsk63sLhm5qN9HMN5YjHgyhSCcVeoRcx7h11JAPHW0ODG9G5UFrg55vv7lVdoqmlR9sOXqCt65JA8wcwyt3g93Ofd7K++sAiRqJ26wPH1wx1Zg6JS8pPxymjqvykR2Fx0lHW+9TNQnGewAo9/P5WxNOsObO1sUhO3vM2h6EZLoCTMAn5qMwQoq0Z7r74in3wUnpZlp+xCNyB5F3yybCF8D3J1WwiDaA9C3dCj/Uyfg7/ntV5ScjznLTEbI7I+Hctp9xfaIOi+8XQ2voBWK9h0hfVHe8p+8LsQ2mi1lzGjLuLC2/ibnuHh7LjBT/PYOfgcIuislR3IryQul1o0QBskmvoHdcQVRwK1Lw21hpDpApf4ORdSn8N0ia2iNcqpBsHrsC8Gni3sRzhZN+5OGrBTEKZMg3TBDEuC4VhWwlZL6hAZe3kyxHSRjn+v9TfgCtVfTfyUCOxFnvbJN/LERbPpn/cVX+WLx0AtOEJEV0wbRIazlenn0uftOXni74KWvMA2A6IVrghD6dWLQE8BAuLkh7hUCh5smgbH9njcRrqtiUjK2lixV3vy1xk/CZ9dpMKSBcwOTvPSvgcI5f3Hedsl3kvPQWisOEmGIrWeZlxfzlUg9ZN+/IR9P0Vs1MptAhjbPxfNcZJmdrMY94Zv43NL7yZ3R0KoQzVz1rqYP1JzHxeASblokJ4P0vWoebJxp8uQpy9+LTO3ThMXkNg9PWqqyqYvNAL1CwEVgqGceXDAhsMTLRYPJhzRhWzsxextxV5z9s1Hae1DAtzHT1HSsOAciw+wSo4TF7AucsCS1luo6Z5u8nzmDL3jSP03ykZfa5w1PRffsWKPV6iYOwQivaZeHfh1L2GvIwXR0RTDBp4COJH8aDBqnbTPgP/hlrTHq7GMy0QRjHKofeAGR1aE3M/eMOmG2DO9S3n9r3CKwCV0PDdZJiU5tBReAxoU8epp6esqAXKagtzYhQZ2T3d3dU3QduoM8TW9q/lV9hCjY/9d+aV8Ear8Y0oDeb9ujHOflVZdTfkYVR9b1gxX9dP5CE9A/XnAu4YYsmXZSs2OA8jrI7G/Ldj+ZKM+Ik82XrgCwnJMmtKDQXdSvWWOEWkZMUsDo1lPrbYJ9xo2oieVIuOFQ/YEHdGklO9M5gUzaKvofrl3mKfo4CUFjLc5865XX7T4ru0fXFuv8TgGboZDxYs5X6xJHH6ztz4UlCcM4+xR6utXZ4UEQPILiYAea6KZBr9s/Qjx3bZzY2VFNjQ4kd5BBZIyCUiMcmOJErxuKqIux5ICpZuauU633llA2G22qmgXhrxWTpftCna58qGHJH1fPKka0Ca7tfryyL59TExNhRa5GfVZWIEJwRzvZnm1k3iDtqCvcy/NzE0ro4Lv+D7K01Cs3JRRXD1mLve7mXU7L37afUEgUePSnYXz+HWpy1bm0r/GmPQVh8Gj80WNH6+QDph3TwqMLLTjk/Ie4NT8Ff97VqpxM8Su7+MhH9Mkq9aEFR28r8kYFhqWWsPyIw9OftJk5nAdh6eMiIKsPjaRGhXTiDwHP36CJqNq7TeLeWQR7rrarKaUj5CVctrmWv1Kzg2OQ5UTqra9V3XBSj6hm6VSBeEU5gqKuYURAAn4hKuwawtRbYLCFZSPvgoSMM9KPuNJg9SNxSxOLHN+NwOXJBdSQkBLMGgsbmCNB1SguVOThQLZVXvtTkubWRdYM7sOq03SRDuCMy7Vl2qEVSWSB+gLpcJ2H3r4rut1/lgsdbJCH4Xez0384hQdGSYA5sKywuUlaCYgDKCPPbFEZ36oA+xc0tbXa65xradEAdRWFeBp8CGRhFY528XJS3LifabPEAWa0eAs8FeXCc+QAkztmJ2boZyZMUUl0KIw5u3eroUtov2LFe560DYbscpkQ1WwhNHTOcdW3esV+DnA0prhh8u6wnthDUOqX4tOhPbQszTLJ7/NBdPyKlqFaog6q3Yng2d3s2Vg6cMvQKFAFVzLuujfXgrZqgBqiCmGVGm6q0IWwUZ/C0tQ+2QlZHeFFqcm7e3kX1LFi8DEGsEYgY8aj1VLGdZIeVLQHbv85WZqZOv8f7tWkrtGhz1MQxphDiSz8GRTC/2zQgThOyYNEjFzztNEdhfj/QM+5PbZM1awu4Do65Gjs2HsaXiyUaZFtHT/VNJ3yV2RHJ5ExSzkwKEtVsbuE9TQlhtc2RuHP+z7Zcz/c+BT1mIuK2ixqZnReJPq1VBR9wUFUqrIdjdrznkO9e7kcc3BPv8VPI8t9zhXoK4y+od0EwTaRcPaxykhzQz5xGNID2n3E0AZQsF9pWvWEQAfntiX62z8ph9G8argwemnhRsSRh3SM6q2aJzf1ZwkDrJHCCxhoZ3qYjm+I51r8Ci8F/wyrhB7yTT+yPaP+Xlwv2reg9Kcdnq1vhaScV38jmQpaB3vUs0t5XEGUDK+XiMQvLqNiWtUEA6lRlVbY2dtfsaIlo3djoM06Ysm1D3GDoOPRppYt63vxYKRFi1S24tmbpooatPWD9fAxCV3+5CIG8bgIsUtL3iB7SJxcXbkALsZDxhP9auNNaz7u8gNWIAYZaH8NSmXZI6GADfT+lGaUi/s5HW1fDpLZPotrT2SP9qseQKmWCrprtrKhfqm0bwz1k84IriH7C3G/e/+iSoUkRjDeZRvXPVwOm6xC+FQXmKHmf3F/PrSheKz9V+vDRTOct4XnsgRpXv+/hRAr2/FlrM+bC7BCthvBZQKh2FMDiXNyVRSlDx7ucKuyz94jn07b2ZED92nFqbQLIqlb80HkkT53Yd4q1c7wLuyka5+pwfViuNtlk9LhSMH2htsuusWr8JaSjsjwf3KKRRcFz79ZRm/J2t48/NdM6goiUcRAE5Ai3qaBdHU2oOMtizr1Js1bDfTM7QPti3HmzFM2+wdSHnGywuntX3G/anF9UEmhkkRMpRUetiWz3ZgcNbcMI7fDMpJjeppjNC7muL8M3OHwizlejP3uUigjxoKkSCnERv7FFR54Uwk2Oa+5JctRgnWOPDI4gRr6jw2qbM1XY4lyEMX1qALwMnVlmMOr1S55jklbSFzKBuFeSQpp19bpL09mYjsGLlWtXdQq++vK/2lMlXKg7RlG5yKXxZ0Zi8Dnn1MEVAgNR/LOlOr2K0uDcBScO6JfySRlhTXiFkAVPvlWS3czNTfZJcj5tQf469AgWlnoWsQCr+shVEE+YdlvB/jgGmVjFaX7CXhstHAQ9xJ4RWSdaLSlFl5++2yD4kMmaZnGALNh2NvFNEKJZiqaiZgmyKqqoLbsNguzJtYuq9UxK2B62kCn/5aW1QH0APhWP/DJGZoZFSVenTZl/0ft1ly26vu3h919iYhAxA4jHyV8OWgi3B/r3pGCXANN5K6l9NWlwyx0QB6erJUYo6UkpocQSKWZ/4XcbCGOflnJw51MGopNED3q68t6ae0ODWzA/84s9TlmEZprB5MlVeSpUxNIantWS9gUmzfbjOc/pWy+/q5MR5GLkbt7T/srIIt87qUVzEmxPXANN6gKCMsHHn4EHioZfmIt9DHOXQ+YHhZ5znvg8m7q26KyMfEyTkbR4tDcd//b7k+1+/pQ49HHKx80cHACPGfjTVWZgtLA9jqKXbpdCVZfUnftVaKEJ3sl3KcyX9LoQDZ20PeULkxdCP1vmgCiF9rJw959JStGlksf0bDxDalf4U/DhuwQbQClxNW8v2zvDunEtpEVfyVUgHbGubr8s5WPzAfaPvJ8Gk4n5AMjWY7pdAhNJq/H70a/kiHp9h7wmvHgF0J7da8hfP7w+B5WJztLZ2m2NEGhhhpTJwZrBBh01NZjPk6dJuTCvXCgRAGk6dOPbslq4C7CBeifFYJJHTVbMQQdYQBI600tyGwP+yTnQlrJD/hlO'}}],\n", + " 'formatted_output': {'complexity_score': 'slightly_complex',\n", + " 'reasoning': \"The text's purpose is straightforward and concrete: it provides a simple biographical anecdote about a young student learning to write. The narrative follows a clear, chronological path from the teacher's instructions to the student's successful completion of the task, making the author's intent to illustrate a specific educational moment very easy to identify.\",\n", + " 'details': {'detailed_summary': [{'factor': 'Explicit Narrative Focus',\n", + " 'description': \"The text focuses narrowly on a single event: Henry writing a composition about a turnip based on his teacher's specific prompts.\",\n", + " 'effect_on_complexity_dimension': \"The reader can easily identify that the author's purpose is to tell a simple, instructional story without needing to infer hidden meanings or abstract themes.\"},\n", + " {'factor': 'Concrete Subject Matter',\n", + " 'description': 'The story deals with tangible objects (slates, turnips, barns) and clear actions (writing, reading, standing up).',\n", + " 'effect_on_complexity_dimension': \"This keeps the purpose grounded in reality, aligning with the 'clear, concrete, narrowly focused' criteria for slightly complex texts.\"},\n", + " {'factor': 'Direct Instructional Framing',\n", + " 'description': \"The teacher's dialogue explicitly outlines the 'way to write a composition,' which serves as the central pillar of the text.\",\n", + " 'effect_on_complexity_dimension': 'Because the method is stated directly, the reader does not have to guess the educational point the author is making.'}],\n", + " 'adjustment_and_scaffolding': [{'scaffolding_need': 'Historical Context',\n", + " 'suggestion': 'Provide a brief introduction explaining that Henry Wadsworth Longfellow became a famous American poet. This helps students understand why this specific anecdote was preserved and shared.'},\n", + " {'scaffolding_need': 'Vocabulary of Composition',\n", + " 'suggestion': \"Define 'composition' and 'slate' before reading to ensure students understand the tools and the goal of the character's task.\"}],\n", + " 'recommended_use_cases': [{'opportunity': 'Modeling Descriptive Writing',\n", + " 'suggestion': \"Use the teacher's four questions (what it is, what it is like, what it is good for, and what is done with it) as a scaffold for students' own descriptive writing exercises.\"},\n", + " {'opportunity': 'Introduction to Biography',\n", + " 'suggestion': \"Use this text as an entry point for studying biographies, showing how authors use small, specific stories to illustrate a person's character or development.\"}]}},\n", + " 'usage': {'input_tokens': 1226,\n", + " 'output_tokens': 1815,\n", + " 'total_tokens': 3041,\n", + " 'input_token_details': {'cache_read': 0},\n", + " 'output_token_details': {'reasoning': 1203}}}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_text = \"\"\"\n", + "\"Well, then,\" said the teacher, \"you may take your slate and go out behind the schoolhouse for half an hour. Think of something to write about, and write the word on your slate. Then try to tell what it is, what it is like, what it is good for, and what is done with it. That is the way to write a composition.\" Henry took his slate and went out. Just behind the schoolhouse was Mr. Finney's barn. Quite close to the barn was a garden. And in the garden, Henry saw a turnip. \"Well, I know what that is,\" he said to himself; and he wrote the word turnip on his slate. Then he tried to tell what it was like, what it was good for, and what was done with it. Before the half hour was ended he had written a very neat composition on his slate. He then went into the house, and waited while the teacher read it. The teacher was surprised and pleased. He said, \"Henry Longfellow, you have done very well. Today you may stand up before the school and read what you have written about the turnip.\"\n", + "\"\"\"\n", + "\n", + "result = evaluate_text_complexity(\n", + " text=sample_text,\n", + " grade_level=4\n", + " )\n", + "\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "RENDERED PROMPT (input sent to the LLM)\n", + "============================================================\n", + "[{'additional_kwargs': {},\n", + " 'content': '\\n'\n", + " ' Role\\n'\n", + " ' You are an expert reading assessment evaluator. Your task is '\n", + " 'to determine the Text Complexity of a given passage based '\n", + " 'exclusively on the Purpose dimension of the qualitative measures '\n", + " 'rubric.\\n'\n", + " '\\n'\n", + " ' Task Details\\n'\n", + " ' You will be provided with an informational or literary '\n", + " '`text`, along with its `grade_level` and `fk_score` '\n", + " '(Flesch-Kincaid). You must analyze the text and determine how '\n", + " \"difficult it is for a reader to identify the author's purpose. \\n\"\n", + " '\\n'\n", + " \" Crucially, you must distinguish between the text's *topic* \"\n", + " '(what it is about) and its *purpose* (why the author wrote '\n", + " 'it). \\n'\n", + " '\\n'\n", + " ' Rubric: Purpose Complexity\\n'\n", + " ' Exceedingly Complex: Subtle and intricate, difficult to '\n", + " 'determine; includes many theoretical or abstract elements.\\n'\n", + " ' Very Complex: Implicit or subtle but fairly easy to infer; '\n", + " 'more theoretical or abstract than concrete.\\n'\n", + " ' Moderately Complex: Implied but easy to identify based upon '\n", + " 'context or source.\\n'\n", + " ' Slightly Complex: Explicitly stated, clear, concrete, '\n", + " 'narrowly focused.\\n'\n", + " ' More Context Needed: The text is a fragment or lacks '\n", + " 'necessary introductory context, making the true purpose '\n", + " 'impossible to determine accurately without external background '\n", + " 'knowledge.\\n'\n", + " '\\n'\n", + " ' Expert Rules for Evaluating Purpose\\n'\n", + " ' Based on expert consensus and historical grading '\n", + " 'corrections, you must apply the following heuristics:\\n'\n", + " '\\n'\n", + " ' 1. The \"Slightly Complex\" Benchmark (Straightforward and '\n", + " 'Explicit)\\n'\n", + " ' A text is Slightly Complex if its purpose is explicitly '\n", + " 'stated or if its informative intent is straightforward, clear, '\n", + " 'concrete, and directly answers what the text is immediately '\n", + " 'about. If the text opens by clearly identifying a concrete topic '\n", + " '(e.g., \"Pins are made of either brass or iron wire\") and rigidly '\n", + " 'follows through by explaining factual, practical information or '\n", + " 'a process (like manufacturing steps or geographic facts), the '\n", + " 'purpose is considered explicit and straightforward. It does '\n", + " '*not* require a literal statement like \"The purpose of this text '\n", + " 'is to...\" as long as the delivery of information is direct, '\n", + " 'clear, and unadorned by persuasive elements or complex framing.\\n'\n", + " '\\n'\n", + " ' 2. Moderately Complex via Guiding Questions & Inquiry '\n", + " 'Formats\\n'\n", + " ' If a text begins with a general introduction and uses '\n", + " 'guiding questions (e.g., \"Have you ever wondered how clouds are '\n", + " 'formed?\") to transition into an explanation, the purpose is '\n", + " 'implied rather than explicitly stated upfront. Because the '\n", + " 'reader must recognize the question as the pivot point for the '\n", + " \"author's intent, it is Moderately Complex.\\n\"\n", + " '\\n'\n", + " ' 3. Moderately Complex via Multiple Distinct Informational '\n", + " 'Goals\\n'\n", + " ' If a text covers a broad topic but jumps between several '\n", + " 'distinct scientific or informational objectives without an '\n", + " 'overarching framing device or explicit thesis (e.g., talking '\n", + " 'about measuring ice sheets, then mapping, then finding '\n", + " 'meteorites), the reader must synthesize these diverse facts to '\n", + " 'recognize the broader purpose, making it Moderately Complex.\\n'\n", + " '\\n'\n", + " ' 4. Moderately Complex via Arguments Disguised as '\n", + " 'Information\\n'\n", + " ' If an author is arguing a specific point, correcting a '\n", + " 'misconception, or defending a stance, but the text could '\n", + " 'initially be mistaken by students as purely informative factual '\n", + " 'text, it is Moderately Complex. The reader must infer the '\n", + " 'persuasive intent or argumentative purpose beneath the '\n", + " 'informative tone.\\n'\n", + " '\\n'\n", + " ' 5. \"More Context Needed\" for Fragments\\n'\n", + " ' If a text is a fragment missing a crucial introduction or '\n", + " \"context, and identifying the author's purpose beyond a simple \"\n", + " 'surface-level description would be exceptionally difficult for a '\n", + " 'reader in the target grade level without that external '\n", + " 'background, score it as `more_context_needed`. \\n'\n", + " '\\n'\n", + " ' Output Format\\n'\n", + " ' Provide your evaluation in the following structure:\\n'\n", + " ' reasoning:\\n'\n", + " ' - Surface Analysis: Identify if the text clearly identifies '\n", + " 'its topic and delivers straightforward facts, or if it utilizes '\n", + " 'structural cues, titles, or direct thesis statements.\\n'\n", + " ' - Subtlety & Framing: Is the informative purpose '\n", + " 'straightforward and concrete? Does it use guiding questions? Is '\n", + " 'it an argument disguised as pure information? Are there multiple '\n", + " 'distinct informational goals requiring synthesis?\\n'\n", + " ' - Context Check: Is this text a fragment missing crucial '\n", + " 'context that obscures the deeper purpose for the target grade '\n", + " 'level?\\n'\n", + " ' - Rubric Alignment: Explain how the text aligns with the '\n", + " 'specific language of the rubric, explicitly referencing the '\n", + " \"expert rules above. Justify why it isn't one level simpler or \"\n", + " 'more complex.\\n'\n", + " '\\n'\n", + " ' answer:\\n'\n", + " ' - complexity_score: (slightly_complex, moderately_complex, '\n", + " 'very_complex, exceedingly_complex, more_context_needed)\\n'\n", + " ' - reasoning: A brief summary of your final decision.\\n'\n", + " ' - details: Structured breakdown of PurposeDetails including '\n", + " 'detailed_summary, adjustment_and_scaffolding, and '\n", + " 'recommended_use_cases.\\n',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'system'},\n", + " {'additional_kwargs': {},\n", + " 'content': 'Analyze:\\n'\n", + " 'Text: \\n'\n", + " '\"Well, then,\" said the teacher, \"you may take your slate and go '\n", + " 'out behind the schoolhouse for half an hour. Think of something '\n", + " 'to write about, and write the word on your slate. Then try to '\n", + " 'tell what it is, what it is like, what it is good for, and what '\n", + " 'is done with it. That is the way to write a composition.\" Henry '\n", + " 'took his slate and went out. Just behind the schoolhouse was Mr. '\n", + " \"Finney's barn. Quite close to the barn was a garden. And in the \"\n", + " 'garden, Henry saw a turnip. \"Well, I know what that is,\" he said '\n", + " 'to himself; and he wrote the word turnip on his slate. Then he '\n", + " 'tried to tell what it was like, what it was good for, and what '\n", + " 'was done with it. Before the half hour was ended he had written '\n", + " 'a very neat composition on his slate. He then went into the '\n", + " 'house, and waited while the teacher read it. The teacher was '\n", + " 'surprised and pleased. He said, \"Henry Longfellow, you have done '\n", + " 'very well. Today you may stand up before the school and read '\n", + " 'what you have written about the turnip.\"\\n'\n", + " '\\n'\n", + " 'Grade: 4\\n'\n", + " 'FK Score: 3.75',\n", + " 'id': None,\n", + " 'name': None,\n", + " 'response_metadata': {},\n", + " 'type': 'human'}]\n", + "\n", + "============================================================\n", + "RAW LLM TEXT (model's verbatim output)\n", + "============================================================\n", + "[{'type': 'text', 'text': '{\\n \"complexity_score\": \"slightly_complex\",\\n \"reasoning\": \"The text\\'s purpose is straightforward and concrete: it provides a simple biographical anecdote about a young student learning to write. The narrative follows a clear, chronological path from the teacher\\'s instructions to the student\\'s successful completion of the task, making the author\\'s intent to illustrate a specific educational moment very easy to identify.\",\\n \"details\": {\\n \"detailed_summary\": [\\n {\\n \"factor\": \"Explicit Narrative Focus\",\\n \"description\": \"The text focuses narrowly on a single event: Henry writing a composition about a turnip based on his teacher\\'s specific prompts.\",\\n \"effect_on_complexity_dimension\": \"The reader can easily identify that the author\\'s purpose is to tell a simple, instructional story without needing to infer hidden meanings or abstract themes.\"\\n },\\n {\\n \"factor\": \"Concrete Subject Matter\",\\n \"description\": \"The story deals with tangible objects (slates, turnips, barns) and clear actions (writing, reading, standing up).\",\\n \"effect_on_complexity_dimension\": \"This keeps the purpose grounded in reality, aligning with the \\'clear, concrete, narrowly focused\\' criteria for slightly complex texts.\"\\n },\\n {\\n \"factor\": \"Direct Instructional Framing\",\\n \"description\": \"The teacher\\'s dialogue explicitly outlines the \\'way to write a composition,\\' which serves as the central pillar of the text.\",\\n \"effect_on_complexity_dimension\": \"Because the method is stated directly, the reader does not have to guess the educational point the author is making.\"\\n }\\n ],\\n \"adjustment_and_scaffolding\": [\\n {\\n \"scaffolding_need\": \"Historical Context\",\\n \"suggestion\": \"Provide a brief introduction explaining that Henry Wadsworth Longfellow became a famous American poet. This helps students understand why this specific anecdote was preserved and shared.\"\\n },\\n {\\n \"scaffolding_need\": \"Vocabulary of Composition\",\\n \"suggestion\": \"Define \\'composition\\' and \\'slate\\' before reading to ensure students understand the tools and the goal of the character\\'s task.\"\\n }\\n ],\\n \"recommended_use_cases\": [\\n {\\n \"opportunity\": \"Modeling Descriptive Writing\",\\n \"suggestion\": \"Use the teacher\\'s four questions (what it is, what it is like, what it is good for, and what is done with it) as a scaffold for students\\' own descriptive writing exercises.\"\\n },\\n {\\n \"opportunity\": \"Introduction to Biography\",\\n \"suggestion\": \"Use this text as an entry point for studying biographies, showing how authors use small, specific stories to illustrate a person\\'s character or development.\"\\n }\\n ]\\n }\\n}', 'extras': {'signature': 'EskoCsYoAQw51seX9joZSpu9ImWTUcC/4VYP9eygKBrEJRXjU0hklKx9APLP36OiO43MytItCw6dT1c/n7RAbB8iblBKsI8blpJqdrhEWCN5CI5s48RQyu2RqgbtoHN05WgHFpFyEMcO8zNHNmhZDwgVv9yIEMoKsf76qoW0onrWUZY8e12aOUWwo/+T/FFnOMZoNd0xY/sVe/VccPkBcM8XYU/F3fhIDwVylGT2O24eKtccfLtMkl59aBvlyaHvAC2wWuIgpYHg5ftsDLTCJ2vswKLh8YzRDTuSLfSxpHdD/iMDCMuTUelMZZPtkQMdyGZl+t99lTTlV4iq/3tS5VrZP/04JBmfTZyLAH/iPJp4sKUaaYGeNXVRmwI/UJQxYYVpiTZh/LyQH+qaKg1jcehzVyV+sltz5ubZLr8ew2ODXLhFeXkVz7tjqjAsDnJZPXzacfNtoHvhVVdoS6GO0FhZKaYfCzwk007FH5ekG3CjWxbppmEGXwhB4yNgqYzUr5G3b6M3O5yIfeaI7LCGCN5sygL5vnUQwb7P2UagUdcePozpudh6LBDayYU6oSKRTpccb2ToyqBHlJR3lgO00C+Vx33Mto9QhImh+TBeMnTxeOnNonKKyn1Go2AssnVD/hxSkB24mejpf1BzOmypVYVhDMjGgHtGKc4O0gLaB+jMcCjWofi8jORe/huU3D00ly+/jDIwYojX0hCMjiTtI1akz1Nc2C1PqDhbxJCQX9U6LuqiDgvtSKIvEPX6DULq2mU0KbPb5d0jSeTmaSt54GYDcxXTrcEdoJTugsuxQkobOvN41i6XS5qGPzVxMzJ7jXh9wk9aERiMCj5K7FsCgpFOjIk72pfq9hdrSQh2HPP8FAW/3Avh30b+1RuEzvGKse77IiswcKkYUwlm7e/fkShhhloMPJztTBrbCPduuarWbzOoa45MxyjioeEQ5t5k1UXZ8y+BlSXHBAAh68b9uiATvGM+J7Mku1EMCCAoYiSrQops/lhESVpTroDZbiTns+YO7Ix4tgqMLvESBTT44RPNuxm3IOJRNUctKVEpwtwAzXa41EcoU7sc80juX8ZQKxPgMeKPkfaau2U4M9AeNqIzxKCMrht8Si/RUj43cg34jsKmVtyhjtkiETFs0HOS9nA7SRalGWEDOuNsEwr6X7uJyLUL+iJPKqU5hAVZPohromK2dEC5ASGmF7pRyIrz9tS6fDb/WqPstvB+ClV+jnE9NtgNpyMwZh6MGZ8fVOk6gwpQaxdf2zsnTRXpO/ymFN98BmtS3hlYcl79fUHkJJ8drISmx2DkQ1MbAZCMTeSExfdtYdHBr2zVHPap4KE9vXcbp/kFdKgI1zx17GJ2q8cGcTYfnOcwhX07gjM8VEpUjw100eZMEUuMua4UwmtE/FGOjlEnEuDzmAX3fDXAIwtLzGcn6PXxaf9UZ/0ARZUPw4WozzeD1Z7nL0lX+mT5lMsf+OZBi87drBEgd/vvPQqD/2AGTwC2zlZaI6eCAWBX5M+g9OKXyBRfmWvZb1e9v+Ur2RfXwYoBue2uPyexRFn3hjJvdOH3wZHSmydu10AKD34xIWx0rNTAM8QPO9pS9fOxuxBpiGKmDR0VsR+H4hqz6R22h1ZmwFjQeYK0PY7HJlTS0B4U2Ud44QwtsCmxOtGEM7QHSNmqpmzPZYHisTvQfcpMVQp0JeQqwW/43Q4ZNUQaJBLp/03S1pd4asn7jviX8Lf8wqTOdyFXz5cwpRhiUVwhgy/Q1zGdq+IOBuufYnOx6XxseYFmbcg9GN76wCZb7cut+9cuZvZCyqdWa2Chznf+7WNk6sKT61Ee00wrndr2ywscdMeOUaPkwKhvJ6x3tIvM1WtWLrXblgscZYSxM5X6R1rGgJDHU8PQUGw2PZvLmTcB+f4MDXzdL9Q4SBPx63/lEzMUiS54cvtEk97dgTTetPnDfTQkqQsBE5uheEucgxPbslhkpcB313MHGFVeaX7zyfSEtxAZUnI45VUEJDvpJCnfFjg3dN1jKa9PPLo4a808PQHqwKb/KvyC2uYMd2galIjIkAbHqfOkfK6AaNIh4Fs7OxiozP41MizuHcVDjNEEakhMnopIsGqYp/Pv2uWnGBSyrcXlDlfZoPfEFY00MpzkMNFdOrYI748bE2J5o9dsH1wHFgDUszlUWPUIOSknqfGPolXrzsEsdvxv3c2KmnpRPEagcXvcOWM4lIcztqjdbnagTm1QQz78d5/k8RaH3awBjL7Hg7BZP6RXOqewUfzuGGrIbhb4Nu4xbTG1JlNaKjgiyuhE/1ReKb+OEJaJwLhiyN9v8lk6kXd7BA0Lkw/M0zpi3bhbd/jrfoz2I7uji368VRng0ZUgShoz9fJSE1+HwwPHEiDDJ7HS9IAqbqBnUcBjCCLB7YbWlmHla3cKSiZsC5Z+NGqdccLoJUNXjWZC+7s0OdM7bc8bHsCke1VKv/7ML2YNPCmqIXhTvaqQPdJN94Nm5YTejSyw5hI5ygyQMp+TDlxlZfi/GuIV6449VgJbWDUX+RPVy9wmy/aIV4YLECoMLCWG8i3/NMRfVcng/Vp2kTlR1fa6ATvfFmkEN/VFgk9TYbUfOhZtAVO/A0oAOA7iTe4KKMh9sB6MLmrpwDque0L/IoB8t93b+c2o55qssnJXFaWsc5NtUWzt3fzSOxDkEbxXV8oyrCQaYkH519zTnNpnt59NZmpKknmc0tIjYhDfLReCiOFXO5NAWHVUXvFfUcfNtVS7XLaU1QnILRJVJXKsiBieGUh5dJjFTnW1XEG2viwZU6meckfrcvxiQIE7BfJkorZ89ZYyVSc4DIGYotmhwEkgDpwlHqY8jlSNyVIX2kojNUt8uBWNy8kigOo9XA+OHgKLPauyQnLqsJyT0Nw40JBRZCpUKHKyIypXBKFwJmNkTk3ICKmYyGMZ0f8Rph3R4HeW0EhyS+KEEMhgJezk76os4gCm3P2XGF53JAqP5R6+sFXfRZFPZ+b0O6oqdUuz0vD2RSHdnV4sNxa4ry6TtBhQvittFZMthCFardAoQlsk63sLhm5qN9HMN5YjHgyhSCcVeoRcx7h11JAPHW0ODG9G5UFrg55vv7lVdoqmlR9sOXqCt65JA8wcwyt3g93Ofd7K++sAiRqJ26wPH1wx1Zg6JS8pPxymjqvykR2Fx0lHW+9TNQnGewAo9/P5WxNOsObO1sUhO3vM2h6EZLoCTMAn5qMwQoq0Z7r74in3wUnpZlp+xCNyB5F3yybCF8D3J1WwiDaA9C3dCj/Uyfg7/ntV5ScjznLTEbI7I+Hctp9xfaIOi+8XQ2voBWK9h0hfVHe8p+8LsQ2mi1lzGjLuLC2/ibnuHh7LjBT/PYOfgcIuislR3IryQul1o0QBskmvoHdcQVRwK1Lw21hpDpApf4ORdSn8N0ia2iNcqpBsHrsC8Gni3sRzhZN+5OGrBTEKZMg3TBDEuC4VhWwlZL6hAZe3kyxHSRjn+v9TfgCtVfTfyUCOxFnvbJN/LERbPpn/cVX+WLx0AtOEJEV0wbRIazlenn0uftOXni74KWvMA2A6IVrghD6dWLQE8BAuLkh7hUCh5smgbH9njcRrqtiUjK2lixV3vy1xk/CZ9dpMKSBcwOTvPSvgcI5f3Hedsl3kvPQWisOEmGIrWeZlxfzlUg9ZN+/IR9P0Vs1MptAhjbPxfNcZJmdrMY94Zv43NL7yZ3R0KoQzVz1rqYP1JzHxeASblokJ4P0vWoebJxp8uQpy9+LTO3ThMXkNg9PWqqyqYvNAL1CwEVgqGceXDAhsMTLRYPJhzRhWzsxextxV5z9s1Hae1DAtzHT1HSsOAciw+wSo4TF7AucsCS1luo6Z5u8nzmDL3jSP03ykZfa5w1PRffsWKPV6iYOwQivaZeHfh1L2GvIwXR0RTDBp4COJH8aDBqnbTPgP/hlrTHq7GMy0QRjHKofeAGR1aE3M/eMOmG2DO9S3n9r3CKwCV0PDdZJiU5tBReAxoU8epp6esqAXKagtzYhQZ2T3d3dU3QduoM8TW9q/lV9hCjY/9d+aV8Ear8Y0oDeb9ujHOflVZdTfkYVR9b1gxX9dP5CE9A/XnAu4YYsmXZSs2OA8jrI7G/Ldj+ZKM+Ik82XrgCwnJMmtKDQXdSvWWOEWkZMUsDo1lPrbYJ9xo2oieVIuOFQ/YEHdGklO9M5gUzaKvofrl3mKfo4CUFjLc5865XX7T4ru0fXFuv8TgGboZDxYs5X6xJHH6ztz4UlCcM4+xR6utXZ4UEQPILiYAea6KZBr9s/Qjx3bZzY2VFNjQ4kd5BBZIyCUiMcmOJErxuKqIux5ICpZuauU633llA2G22qmgXhrxWTpftCna58qGHJH1fPKka0Ca7tfryyL59TExNhRa5GfVZWIEJwRzvZnm1k3iDtqCvcy/NzE0ro4Lv+D7K01Cs3JRRXD1mLve7mXU7L37afUEgUePSnYXz+HWpy1bm0r/GmPQVh8Gj80WNH6+QDph3TwqMLLTjk/Ie4NT8Ff97VqpxM8Su7+MhH9Mkq9aEFR28r8kYFhqWWsPyIw9OftJk5nAdh6eMiIKsPjaRGhXTiDwHP36CJqNq7TeLeWQR7rrarKaUj5CVctrmWv1Kzg2OQ5UTqra9V3XBSj6hm6VSBeEU5gqKuYURAAn4hKuwawtRbYLCFZSPvgoSMM9KPuNJg9SNxSxOLHN+NwOXJBdSQkBLMGgsbmCNB1SguVOThQLZVXvtTkubWRdYM7sOq03SRDuCMy7Vl2qEVSWSB+gLpcJ2H3r4rut1/lgsdbJCH4Xez0384hQdGSYA5sKywuUlaCYgDKCPPbFEZ36oA+xc0tbXa65xradEAdRWFeBp8CGRhFY528XJS3LifabPEAWa0eAs8FeXCc+QAkztmJ2boZyZMUUl0KIw5u3eroUtov2LFe560DYbscpkQ1WwhNHTOcdW3esV+DnA0prhh8u6wnthDUOqX4tOhPbQszTLJ7/NBdPyKlqFaog6q3Yng2d3s2Vg6cMvQKFAFVzLuujfXgrZqgBqiCmGVGm6q0IWwUZ/C0tQ+2QlZHeFFqcm7e3kX1LFi8DEGsEYgY8aj1VLGdZIeVLQHbv85WZqZOv8f7tWkrtGhz1MQxphDiSz8GRTC/2zQgThOyYNEjFzztNEdhfj/QM+5PbZM1awu4Do65Gjs2HsaXiyUaZFtHT/VNJ3yV2RHJ5ExSzkwKEtVsbuE9TQlhtc2RuHP+z7Zcz/c+BT1mIuK2ixqZnReJPq1VBR9wUFUqrIdjdrznkO9e7kcc3BPv8VPI8t9zhXoK4y+od0EwTaRcPaxykhzQz5xGNID2n3E0AZQsF9pWvWEQAfntiX62z8ph9G8argwemnhRsSRh3SM6q2aJzf1ZwkDrJHCCxhoZ3qYjm+I51r8Ci8F/wyrhB7yTT+yPaP+Xlwv2reg9Kcdnq1vhaScV38jmQpaB3vUs0t5XEGUDK+XiMQvLqNiWtUEA6lRlVbY2dtfsaIlo3djoM06Ysm1D3GDoOPRppYt63vxYKRFi1S24tmbpooatPWD9fAxCV3+5CIG8bgIsUtL3iB7SJxcXbkALsZDxhP9auNNaz7u8gNWIAYZaH8NSmXZI6GADfT+lGaUi/s5HW1fDpLZPotrT2SP9qseQKmWCrprtrKhfqm0bwz1k84IriH7C3G/e/+iSoUkRjDeZRvXPVwOm6xC+FQXmKHmf3F/PrSheKz9V+vDRTOct4XnsgRpXv+/hRAr2/FlrM+bC7BCthvBZQKh2FMDiXNyVRSlDx7ucKuyz94jn07b2ZED92nFqbQLIqlb80HkkT53Yd4q1c7wLuyka5+pwfViuNtlk9LhSMH2htsuusWr8JaSjsjwf3KKRRcFz79ZRm/J2t48/NdM6goiUcRAE5Ai3qaBdHU2oOMtizr1Js1bDfTM7QPti3HmzFM2+wdSHnGywuntX3G/anF9UEmhkkRMpRUetiWz3ZgcNbcMI7fDMpJjeppjNC7muL8M3OHwizlejP3uUigjxoKkSCnERv7FFR54Uwk2Oa+5JctRgnWOPDI4gRr6jw2qbM1XY4lyEMX1qALwMnVlmMOr1S55jklbSFzKBuFeSQpp19bpL09mYjsGLlWtXdQq++vK/2lMlXKg7RlG5yKXxZ0Zi8Dnn1MEVAgNR/LOlOr2K0uDcBScO6JfySRlhTXiFkAVPvlWS3czNTfZJcj5tQf469AgWlnoWsQCr+shVEE+YdlvB/jgGmVjFaX7CXhstHAQ9xJ4RWSdaLSlFl5++2yD4kMmaZnGALNh2NvFNEKJZiqaiZgmyKqqoLbsNguzJtYuq9UxK2B62kCn/5aW1QH0APhWP/DJGZoZFSVenTZl/0ft1ly26vu3h919iYhAxA4jHyV8OWgi3B/r3pGCXANN5K6l9NWlwyx0QB6erJUYo6UkpocQSKWZ/4XcbCGOflnJw51MGopNED3q68t6ae0ODWzA/84s9TlmEZprB5MlVeSpUxNIantWS9gUmzfbjOc/pWy+/q5MR5GLkbt7T/srIIt87qUVzEmxPXANN6gKCMsHHn4EHioZfmIt9DHOXQ+YHhZ5znvg8m7q26KyMfEyTkbR4tDcd//b7k+1+/pQ49HHKx80cHACPGfjTVWZgtLA9jqKXbpdCVZfUnftVaKEJ3sl3KcyX9LoQDZ20PeULkxdCP1vmgCiF9rJw959JStGlksf0bDxDalf4U/DhuwQbQClxNW8v2zvDunEtpEVfyVUgHbGubr8s5WPzAfaPvJ8Gk4n5AMjWY7pdAhNJq/H70a/kiHp9h7wmvHgF0J7da8hfP7w+B5WJztLZ2m2NEGhhhpTJwZrBBh01NZjPk6dJuTCvXCgRAGk6dOPbslq4C7CBeifFYJJHTVbMQQdYQBI600tyGwP+yTnQlrJD/hlO'}}]\n", + "\n", + "============================================================\n", + "PARSED OUTPUT (output_schema)\n", + "============================================================\n", + "{'complexity_score': 'slightly_complex',\n", + " 'details': {'adjustment_and_scaffolding': [{'scaffolding_need': 'Historical '\n", + " 'Context',\n", + " 'suggestion': 'Provide a brief '\n", + " 'introduction '\n", + " 'explaining that '\n", + " 'Henry Wadsworth '\n", + " 'Longfellow became '\n", + " 'a famous American '\n", + " 'poet. This helps '\n", + " 'students '\n", + " 'understand why '\n", + " 'this specific '\n", + " 'anecdote was '\n", + " 'preserved and '\n", + " 'shared.'},\n", + " {'scaffolding_need': 'Vocabulary '\n", + " 'of '\n", + " 'Composition',\n", + " 'suggestion': 'Define '\n", + " \"'composition' and \"\n", + " \"'slate' before \"\n", + " 'reading to ensure '\n", + " 'students '\n", + " 'understand the '\n", + " 'tools and the goal '\n", + " \"of the character's \"\n", + " 'task.'}],\n", + " 'detailed_summary': [{'description': 'The text focuses narrowly '\n", + " 'on a single event: Henry '\n", + " 'writing a composition about '\n", + " 'a turnip based on his '\n", + " \"teacher's specific prompts.\",\n", + " 'effect_on_complexity_dimension': 'The '\n", + " 'reader '\n", + " 'can '\n", + " 'easily '\n", + " 'identify '\n", + " 'that the '\n", + " \"author's \"\n", + " 'purpose '\n", + " 'is to '\n", + " 'tell a '\n", + " 'simple, '\n", + " 'instructional '\n", + " 'story '\n", + " 'without '\n", + " 'needing '\n", + " 'to infer '\n", + " 'hidden '\n", + " 'meanings '\n", + " 'or '\n", + " 'abstract '\n", + " 'themes.',\n", + " 'factor': 'Explicit Narrative Focus'},\n", + " {'description': 'The story deals with '\n", + " 'tangible objects (slates, '\n", + " 'turnips, barns) and clear '\n", + " 'actions (writing, reading, '\n", + " 'standing up).',\n", + " 'effect_on_complexity_dimension': 'This '\n", + " 'keeps '\n", + " 'the '\n", + " 'purpose '\n", + " 'grounded '\n", + " 'in '\n", + " 'reality, '\n", + " 'aligning '\n", + " 'with the '\n", + " \"'clear, \"\n", + " 'concrete, '\n", + " 'narrowly '\n", + " \"focused' \"\n", + " 'criteria '\n", + " 'for '\n", + " 'slightly '\n", + " 'complex '\n", + " 'texts.',\n", + " 'factor': 'Concrete Subject Matter'},\n", + " {'description': \"The teacher's dialogue \"\n", + " 'explicitly outlines the '\n", + " \"'way to write a \"\n", + " \"composition,' which serves \"\n", + " 'as the central pillar of '\n", + " 'the text.',\n", + " 'effect_on_complexity_dimension': 'Because '\n", + " 'the '\n", + " 'method '\n", + " 'is '\n", + " 'stated '\n", + " 'directly, '\n", + " 'the '\n", + " 'reader '\n", + " 'does not '\n", + " 'have to '\n", + " 'guess '\n", + " 'the '\n", + " 'educational '\n", + " 'point '\n", + " 'the '\n", + " 'author '\n", + " 'is '\n", + " 'making.',\n", + " 'factor': 'Direct Instructional Framing'}],\n", + " 'recommended_use_cases': [{'opportunity': 'Modeling Descriptive '\n", + " 'Writing',\n", + " 'suggestion': \"Use the teacher's four \"\n", + " 'questions (what it is, '\n", + " 'what it is like, what '\n", + " 'it is good for, and '\n", + " 'what is done with it) '\n", + " 'as a scaffold for '\n", + " \"students' own \"\n", + " 'descriptive writing '\n", + " 'exercises.'},\n", + " {'opportunity': 'Introduction to '\n", + " 'Biography',\n", + " 'suggestion': 'Use this text as an '\n", + " 'entry point for '\n", + " 'studying biographies, '\n", + " 'showing how authors use '\n", + " 'small, specific stories '\n", + " 'to illustrate a '\n", + " \"person's character or \"\n", + " 'development.'}]},\n", + " 'reasoning': \"The text's purpose is straightforward and concrete: it provides \"\n", + " 'a simple biographical anecdote about a young student learning '\n", + " 'to write. The narrative follows a clear, chronological path '\n", + " \"from the teacher's instructions to the student's successful \"\n", + " \"completion of the task, making the author's intent to \"\n", + " 'illustrate a specific educational moment very easy to identify.'}\n", + "\n", + "============================================================\n", + "USAGE METADATA\n", + "============================================================\n", + "{'input_token_details': {'cache_read': 0},\n", + " 'input_tokens': 1226,\n", + " 'output_token_details': {'reasoning': 1203},\n", + " 'output_tokens': 1815,\n", + " 'total_tokens': 3041}\n" + ] + } + ], + "source": [ + "import pprint as pp\n", + "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", + "print(\"=\" * 60)\n", + "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(result[\"rendered_prompt\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"RAW LLM TEXT (model's verbatim output)\")\n", + "print(\"=\" * 60)\n", + "print(result[\"raw_text\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"PARSED OUTPUT (output_schema)\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(result[\"formatted_output\"])\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"USAGE METADATA\")\n", + "print(\"=\" * 60)\n", + "pp.pprint(result[\"usage\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 5 fixtures from fixtures.json\n", + "\n", + "Calculated Flesch-Kincaid Score: 3.96\n", + "Calculated Flesch-Kincaid Score: 8.51\n", + "Calculated Flesch-Kincaid Score: 10.63\n", + "Calculated Flesch-Kincaid Score: 9.78\n", + "Calculated Flesch-Kincaid Score: 8.8\n", + "\n", + "==============================================================================\n", + " ID STATUS PREDICTED EXPECTED DESCRIPTION\n", + "==============================================================================\n", + " 1926 PASS slightly_complex slightly_complex Stay safe at the beach!\n", + " 2483 PASS slightly_complex slightly_complex Twins Can Help Us Underst\n", + " 2521 PASS moderately_complex moderately_complex Does The Ocean Lose Its B\n", + " 3021 PASS* slightly_complex moderately_complex Remembering or Forgetting\n", + " 3164 PASS* slightly_complex moderately_complex Ancient Egyptian Dynastie\n", + "==============================================================================\n", + "Summary: 3 exact, 2 adjacent (tolerated), 0 fail, 0 error -- total 5\n", + "(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\n" + ] + } + ], + "source": [ + "# -------------------------------------------------------------------------\n", + "# Sniff-test runner: load fixtures.json and check predictions against expected\n", + "# -------------------------------------------------------------------------\n", + "# Fixtures live next to config.json + system.txt + user.txt and follow the\n", + "# schema declared in CONFIG['fixtures']['schema']. Each case has:\n", + "# - id, description (optional)\n", + "# - input: {text, grade_level} -- runtime evaluator inputs\n", + "# - expected: {complexity_level} -- ground-truth label from rubric\n", + "#\n", + "# Note: the fixture key 'complexity_level' maps to the runtime output's\n", + "# 'complexity_score' field. We test that single value only -- the model's\n", + "# free-text 'reasoning' field is non-deterministic across runs.\n", + "\n", + "fixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\n", + "with open(fixtures_path) as f:\n", + " fixtures = json.load(f)\n", + "print(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n", + "\n", + "# Adjacency tolerance per CONFIG['fixtures']['tolerance'].\n", + "# Derive rubric order from OUTPUT_SCHEMA -- 'more_context_needed' excluded as it has no adjacency.\n", + "_RUBRIC_ORDER = [\n", + " level for level in OUTPUT_SCHEMA[\"properties\"][\"complexity_score\"][\"enum\"]\n", + " if level != \"more_context_needed\"\n", + "]\n", + "_ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n", + "\n", + "def _score_outcome(predicted: str, expected: str):\n", + " \"\"\"Return ('exact' | 'adjacent' | 'fail', distance_or_None).\"\"\"\n", + " if predicted == expected:\n", + " return \"exact\", 0\n", + " if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n", + " d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n", + " if d == 1:\n", + " return \"adjacent\", d\n", + " return \"fail\", None\n", + "\n", + "# Run each fixture, accumulate results\n", + "results = []\n", + "for fx in fixtures:\n", + " expected = fx[\"expected\"][\"complexity_level\"]\n", + " out = evaluate_text_complexity(\n", + " text=fx[\"input\"][\"text\"],\n", + " grade_level=fx[\"input\"][\"grade_level\"],\n", + " )\n", + " if isinstance(out, str): # error path\n", + " results.append({\"id\": fx[\"id\"], \"status\": \"error\", \"predicted\": None, \"expected\": expected, \"error\": out})\n", + " continue\n", + " predicted = out[\"formatted_output\"][\"complexity_score\"]\n", + " status, _ = _score_outcome(predicted, expected)\n", + " results.append({\n", + " \"id\": fx[\"id\"], \"status\": status,\n", + " \"predicted\": predicted, \"expected\": expected,\n", + " \"description\": fx.get(\"description\", \"\"),\n", + " })\n", + "\n", + "# Per-case report\n", + "print(\"\\n\" + \"=\" * 78)\n", + "print(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\n", + "print(\"=\" * 78)\n", + "for r in results:\n", + " icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n", + " print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} {r['expected']:<22} {r.get('description','')[:25]}\")\n", + "\n", + "# Summary\n", + "n_total = len(results)\n", + "n_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\n", + "n_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\n", + "n_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\n", + "n_err = sum(1 for r in results if r[\"status\"] == \"error\")\n", + "print(\"=\" * 78)\n", + "print(f\"Summary: {n_exact} exact, {n_adj} adjacent (tolerated), {n_fail} fail, {n_err} error -- total {n_total}\")\n", + "if _ALLOW_ADJ:\n", + " print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/evals/prompts/sentence-structure/analysis-system.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/analysis-system.txt similarity index 100% rename from evals/prompts/sentence-structure/analysis-system.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/analysis-system.txt diff --git a/evals/prompts/sentence-structure/analysis-user.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/analysis-user.txt similarity index 100% rename from evals/prompts/sentence-structure/analysis-user.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/analysis-user.txt diff --git a/evals/prompts/sentence-structure/complexity-system.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/complexity-system.txt similarity index 100% rename from evals/prompts/sentence-structure/complexity-system.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/complexity-system.txt diff --git a/evals/prompts/sentence-structure/complexity-user.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/complexity-user.txt similarity index 100% rename from evals/prompts/sentence-structure/complexity-user.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/complexity-user.txt diff --git a/evals/prompts/sentence-structure/rubric-grade-3.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grade-3.txt similarity index 100% rename from evals/prompts/sentence-structure/rubric-grade-3.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grade-3.txt diff --git a/evals/prompts/sentence-structure/rubric-grade-4.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grade-4.txt similarity index 100% rename from evals/prompts/sentence-structure/rubric-grade-4.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grade-4.txt diff --git a/evals/prompts/sentence-structure/rubric-grades-5-12.txt b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grades-5-12.txt similarity index 100% rename from evals/prompts/sentence-structure/rubric-grades-5-12.txt rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/rubric-grades-5-12.txt diff --git a/evals/prompts/sent_str_prompts.py b/evals/literacy/qualitative-text-complexity/sentence-structure/prompts/sent_str_prompts.py similarity index 100% rename from evals/prompts/sent_str_prompts.py rename to evals/literacy/qualitative-text-complexity/sentence-structure/prompts/sent_str_prompts.py diff --git a/evals/sentence_structure_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/sentence-structure/sentence_structure_evaluator.ipynb similarity index 74% rename from evals/sentence_structure_evaluator.ipynb rename to evals/literacy/qualitative-text-complexity/sentence-structure/sentence_structure_evaluator.ipynb index 0ecbb2b1..75ecb00e 100644 --- a/evals/sentence_structure_evaluator.ipynb +++ b/evals/literacy/qualitative-text-complexity/sentence-structure/sentence_structure_evaluator.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 1, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -46,14 +46,70 @@ "outputs_hidden": true } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: langchain_openai in /opt/anaconda3/lib/python3.13/site-packages (1.3.2)\n", + "Requirement already satisfied: openai in /opt/anaconda3/lib/python3.13/site-packages (2.30.0)\n", + "Requirement already satisfied: langchain in /opt/anaconda3/lib/python3.13/site-packages (1.3.9)\n", + "Requirement already satisfied: textstat in /opt/anaconda3/lib/python3.13/site-packages (0.7.13)\n", + "Requirement already satisfied: langchain-core<2.0.0,>=1.4.7 in /opt/anaconda3/lib/python3.13/site-packages (from langchain_openai) (1.4.7)\n", + "Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain_openai) (0.12.0)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (4.10.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (0.28.1)\n", + "Requirement already satisfied: jiter<1,>=0.10.0 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (0.13.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (2.13.4)\n", + "Requirement already satisfied: sniffio in /opt/anaconda3/lib/python3.13/site-packages (from openai) (1.3.0)\n", + "Requirement already satisfied: tqdm>4 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (4.67.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /opt/anaconda3/lib/python3.13/site-packages (from openai) (4.15.0)\n", + "Requirement already satisfied: idna>=2.8 in /opt/anaconda3/lib/python3.13/site-packages (from anyio<5,>=3.5.0->openai) (3.11)\n", + "Requirement already satisfied: certifi in /opt/anaconda3/lib/python3.13/site-packages (from httpx<1,>=0.23.0->openai) (2026.1.4)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/lib/python3.13/site-packages (from httpx<1,>=0.23.0->openai) (1.0.9)\n", + "Requirement already satisfied: h11>=0.16 in /opt/anaconda3/lib/python3.13/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.16.0)\n", + "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (1.33)\n", + "Requirement already satisfied: langchain-protocol>=0.0.14 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (0.0.16)\n", + "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (0.7.23)\n", + "Requirement already satisfied: packaging>=23.2.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (25.0)\n", + "Requirement already satisfied: pyyaml<7.0.0,>=5.3.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (6.0.3)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (9.1.2)\n", + "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in /opt/anaconda3/lib/python3.13/site-packages (from langchain-core<2.0.0,>=1.4.7->langchain_openai) (0.14.1)\n", + "Requirement already satisfied: jsonpointer>=1.9 in /opt/anaconda3/lib/python3.13/site-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2.0.0,>=1.4.7->langchain_openai) (3.0.0)\n", + "Requirement already satisfied: orjson>=3.9.14 in /opt/anaconda3/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (3.11.8)\n", + "Requirement already satisfied: requests-toolbelt>=1.0.0 in /opt/anaconda3/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (1.0.0)\n", + "Requirement already satisfied: requests>=2.0.0 in /opt/anaconda3/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (2.32.5)\n", + "Requirement already satisfied: xxhash>=3.0.0 in /opt/anaconda3/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (3.6.0)\n", + "Requirement already satisfied: zstandard>=0.23.0 in /opt/anaconda3/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (0.24.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/lib/python3.13/site-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /opt/anaconda3/lib/python3.13/site-packages (from pydantic<3,>=1.9.0->openai) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /opt/anaconda3/lib/python3.13/site-packages (from pydantic<3,>=1.9.0->openai) (0.4.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /opt/anaconda3/lib/python3.13/site-packages (from tiktoken<1.0.0,>=0.7.0->langchain_openai) (2025.9.1)\n", + "Requirement already satisfied: langgraph<1.3.0,>=1.2.4 in /opt/anaconda3/lib/python3.13/site-packages (from langchain) (1.2.4)\n", + "Requirement already satisfied: langgraph-checkpoint<5.0.0,>=4.1.0 in /opt/anaconda3/lib/python3.13/site-packages (from langgraph<1.3.0,>=1.2.4->langchain) (4.1.1)\n", + "Requirement already satisfied: langgraph-prebuilt<1.2.0,>=1.1.0 in /opt/anaconda3/lib/python3.13/site-packages (from langgraph<1.3.0,>=1.2.4->langchain) (1.1.0)\n", + "Requirement already satisfied: langgraph-sdk<0.5.0,>=0.4.2 in /opt/anaconda3/lib/python3.13/site-packages (from langgraph<1.3.0,>=1.2.4->langchain) (0.4.2)\n", + "Requirement already satisfied: ormsgpack>=1.12.0 in /opt/anaconda3/lib/python3.13/site-packages (from langgraph-checkpoint<5.0.0,>=4.1.0->langgraph<1.3.0,>=1.2.4->langchain) (1.12.2)\n", + "Requirement already satisfied: websockets<16,>=14 in /opt/anaconda3/lib/python3.13/site-packages (from langgraph-sdk<0.5.0,>=0.4.2->langgraph<1.3.0,>=1.2.4->langchain) (15.0.1)\n", + "Requirement already satisfied: pyphen in /opt/anaconda3/lib/python3.13/site-packages (from textstat) (0.17.2)\n", + "Requirement already satisfied: nltk in /opt/anaconda3/lib/python3.13/site-packages (from textstat) (3.9.2)\n", + "Requirement already satisfied: setuptools in /opt/anaconda3/lib/python3.13/site-packages (from textstat) (80.9.0)\n", + "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/anaconda3/lib/python3.13/site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (3.4.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/lib/python3.13/site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.4.7->langchain_openai) (2.6.2)\n", + "Requirement already satisfied: click in /opt/anaconda3/lib/python3.13/site-packages (from nltk->textstat) (8.2.1)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/lib/python3.13/site-packages (from nltk->textstat) (1.5.2)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install langchain_openai openai langchain textstat" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -99,7 +155,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -138,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -297,7 +353,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -358,7 +414,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -416,7 +472,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -722,7 +778,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -774,7 +830,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -822,7 +878,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -859,7 +915,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -894,9 +950,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'answer': 'Moderately Complex',\n", + " 'reasoning': \"The text excerpt discusses the various uses of lasers in everyday life and industry, including their applications in CD/DVD players, barcode scanning, medicine, chemistry, and space measurement. The topic is somewhat technical, involving concepts like LASIK surgery and spectroscopy, which may be unfamiliar to a Grade 3 student. However, the text uses straightforward language and examples to explain these concepts. \\n\\nQuantitatively, the text has an average sentence length of 20 words, which is longer than typical for Grade 3, suggesting a higher complexity. The percentage of simple sentences is 62%, indicating that the majority of sentences are simple, which aligns with a 'Slightly Complex' text. However, the presence of 37% complex sentences and 12% very long sentences adds to the complexity. The percent of sentences with subordination is 37%, which is moderate and suggests some complexity in sentence structure.\\n\\nConsidering both qualitative and quantitative aspects, the text does not fit the 'Slightly Complex' category due to its technical content and longer sentence length. It also does not reach the 'Exceedingly Complex' level, as it lacks the extreme structural density and length indicators. The text is best categorized as 'Moderately Complex' because it presents a mix of simple and complex sentences, with a moderate use of subordination, and introduces technical concepts in a relatively accessible manner.\"}" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "{'answer': 'Very Complex',\n", + " 'reasoning': \"The text discusses the various uses of lasers in everyday life and industry, which is a topic that may be somewhat abstract for a Grade 4 student. The text uses a mix of simple and complex sentences, with 50% being simple sentences and 37% being complex sentences. The average sentence length is 20 words, which is on the longer side for this grade level. Additionally, 37% of the sentences contain subordinate clauses, indicating a moderate level of complexity. The text does not use compound sentences or compound-complex sentences, which simplifies the structure somewhat. However, the presence of longer sentences and the need to understand multiple uses and applications of lasers add to the complexity. Given these factors, the text is more complex than 'Moderately Complex' but does not reach the 'Exceedingly Complex' level, as it lacks the high structural density and extreme sentence length typical of that category. Therefore, the text is best categorized as 'Very Complex'.\"}" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "{'answer': 'Moderately Complex',\n", + " 'reasoning': \"The text excerpt discusses the various uses of lasers in everyday life and industry, covering topics such as CD/DVD players, bar code scanning, medical applications, chemistry, and lunar measurements. The conceptual load is moderate, as it introduces several applications of lasers but does not delve deeply into any single one. The structure of the text is primarily composed of medium to long sentences, with an average of 20 words per sentence and a sentence length variation of 10. Quantitatively, the text contains 37% simple sentences and 37% complex sentences, with no compound or compound-complex sentences. There are 2 advanced complex sentences, which is within the threshold for a moderately complex text. The Flesch-Kincaid grade level is 9, indicating that the text is somewhat challenging for a Grade 5 student. However, the presence of multiple concepts within sentences and the use of advanced complex sentences suggest a higher level of complexity than 'Slightly Complex'. Given these factors, the text is best categorized as 'Moderately Complex'.\"}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Add your text & the grade level you want to evaluate for sentence structure complexity\n", "\n", @@ -942,7 +1029,7 @@ "widgets": {} }, "kernelspec": { - "display_name": ".venv", + "display_name": "base", "language": "python", "name": "python3" }, @@ -956,7 +1043,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evals/prompts/smk_prompts.py b/evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/smk_prompts.py similarity index 100% rename from evals/prompts/smk_prompts.py rename to evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/smk_prompts.py diff --git a/evals/prompts/subject-matter-knowledge/system.txt b/evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/system.txt similarity index 100% rename from evals/prompts/subject-matter-knowledge/system.txt rename to evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/system.txt diff --git a/evals/prompts/subject-matter-knowledge/user.txt b/evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/user.txt similarity index 100% rename from evals/prompts/subject-matter-knowledge/user.txt rename to evals/literacy/qualitative-text-complexity/subject-matter-knowledge/prompts/user.txt diff --git a/evals/smk_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/subject-matter-knowledge/smk_evaluator.ipynb similarity index 77% rename from evals/smk_evaluator.ipynb rename to evals/literacy/qualitative-text-complexity/subject-matter-knowledge/smk_evaluator.ipynb index 994da3af..9ce78a3c 100644 --- a/evals/smk_evaluator.ipynb +++ b/evals/literacy/qualitative-text-complexity/subject-matter-knowledge/smk_evaluator.ipynb @@ -27,16 +27,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU langchain-google-genai langchain pydantic textstat" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -63,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -133,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -179,9 +187,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'identified_topics': ['School life',\n", + " 'Writing process (composition)',\n", + " 'Historical school tools (slate)',\n", + " 'Everyday objects (turnip, barn)'],\n", + " 'curriculum_check': 'Standard/General. The topics of school assignments and basic writing instructions are foundational to elementary education (K-5).',\n", + " 'assumptions_and_scaffolding': \"The author assumes the reader understands the basic setting of a schoolhouse and can infer the function of a 'slate' from the context of writing. The text provides a clear scaffold for the writing process (what it is, what it's like, what it's good for), making the concept of a 'composition' very accessible.\",\n", + " 'friction_analysis': 'There is no friction between concrete and abstract elements. The text remains entirely concrete, describing a physical task (writing on a slate) about a physical object (a turnip) to achieve a concrete result (a finished composition).',\n", + " 'complexity_score': 'slightly_complex',\n", + " 'reasoning': \"The text relies on everyday, practical knowledge of school life and familiar objects. According to Rule A and Rule D, because the text describes concrete things (a turnip, a slate) to achieve a concrete result (writing a simple description), it remains slightly complex. The mention of a 'slate' is a minor historical detail that does not require specialized knowledge to understand within the narrative context.\"}" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Add your text & the grade level you want to evaluate for SMK complexity\n", "\n", @@ -203,13 +229,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "base", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.10.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evals/text_complexity_combo.ipynb b/evals/literacy/qualitative-text-complexity/text_complexity_combo.ipynb similarity index 63% rename from evals/text_complexity_combo.ipynb rename to evals/literacy/qualitative-text-complexity/text_complexity_combo.ipynb index 7bf28ddd..0532f758 100644 --- a/evals/text_complexity_combo.ipynb +++ b/evals/literacy/qualitative-text-complexity/text_complexity_combo.ipynb @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -49,14 +49,22 @@ "title": "" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU pydantic textstat langchain langchain_openai langchain-google-genai" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -111,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -127,7 +135,26 @@ }, "outputs": [], "source": [ - "from prompts import vocab_prompts as v_prompts, sent_str_prompts as s_prompts, smk_prompts as smk_prompts, conventionality_prompts as conv_prompts\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "# Each evaluator's prompt module now lives in its own dimension subfolder\n", + "# (e.g. vocabulary/prompts/vocab_prompts.py). The dimension folder names\n", + "# contain hyphens, so they can't be imported as packages. Instead we add\n", + "# each prompts/ directory to sys.path and import the modules by filename.\n", + "_QTC_ROOT = Path.cwd()\n", + "for _dim in (\n", + " \"vocabulary\",\n", + " \"sentence-structure\",\n", + " \"subject-matter-knowledge\",\n", + " \"conventionality\",\n", + "):\n", + " sys.path.append(str(_QTC_ROOT / _dim / \"prompts\"))\n", + "\n", + "import vocab_prompts as v_prompts\n", + "import sent_str_prompts as s_prompts\n", + "import smk_prompts as smk_prompts\n", + "import conventionality_prompts as conv_prompts\n", "\n", "# Set your api keys in your environment, .env file, or enter when prompted.\n", "# os.environ['GOOGLE_API_KEY'] = 'YOUR API KEY'\n", @@ -140,11 +167,17 @@ "if not os.environ.get(\"GOOGLE_API_KEY\"):\n", " os.environ[\"GOOGLE_API_KEY\"] = getpass.getpass(\"Enter your Google API key: \")\n", "\n", - "# Define the model to be used for vocabulary complexity\n", - "VOCAB_MODEL = \"gemini-2.5-pro\"\n", + "# Define the models to be used for vocabulary complexity.\n", + "# Grades 3-4 were validated with Gemini; grades 5-12 with GPT-4.1\n", + "# (matching the standalone vocabulary evaluator's grade-specific prompts).\n", "VOCAB_TEMPERATURE = 0\n", - "vocab_complexity_model = ChatGoogleGenerativeAI(\n", - " model=VOCAB_MODEL, temperature=VOCAB_TEMPERATURE\n", + "VOCAB_MODEL_GRADES_3_4 = \"gemini-2.5-pro\"\n", + "vocab_complexity_model_grades_3_4 = ChatGoogleGenerativeAI(\n", + " model=VOCAB_MODEL_GRADES_3_4, temperature=VOCAB_TEMPERATURE\n", + ")\n", + "VOCAB_MODEL_OTHER_GRADES = \"gpt-4.1\"\n", + "vocab_complexity_model_other_grades = ChatOpenAI(\n", + " model=VOCAB_MODEL_OTHER_GRADES, temperature=VOCAB_TEMPERATURE\n", ")\n", "\n", "# Define the model to be used for student background knowledge generation and sentence structure complexity\n", @@ -172,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -204,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -220,12 +253,87 @@ }, "outputs": [], "source": [ - "class VocabOutput(BaseModel):\n tier_2_words: str = Field(description=\"List of Tier 2 words\")\n tier_3_words: str = Field(description=\"List of Tier 3 words\")\n archaic_words: str = Field(description=\"List of Archaic words\")\n other_complex_words: str = Field(description=\"List of Other Complex words\")\n complexity_score: str = Field(\n description=\"the complexity of the text, one of: slightly complex, moderately complex, very complex, or exceedingly complex\"\n )\n reasoning: str = Field(description=\"your reasoning for your answer\")\n\n\nprompt_vars = {\n \"inputVars\": [\n \"text\",\n \"student_grade_level\",\n \"student_background_knowledge\",\n \"fk_level\",\n ],\n \"outputParser\": JsonOutputParser(pydantic_object=VocabOutput),\n}\n\n\nclass SmkOutput(BaseModel):\n identified_topics: List[str] = Field(\n description=\"List of major subjects/concepts found in the text.\"\n )\n curriculum_check: str = Field(\n description=\"Analysis of whether these topics are new or review based on the Grade Level and Reference list.\"\n )\n assumptions_and_scaffolding: str = Field(\n description=\"Analysis of what the author assumes vs. what is explained (and if definitions are provided).\"\n )\n friction_analysis: str = Field(\n description=\"Explicit statement distinguishing if difficulty comes from Vocabulary/Structure or actual Knowledge.\"\n )\n complexity_score: Literal[\n \"slightly_complex\",\n \"moderately_complex\",\n \"very_complex\",\n \"exceedingly_complex\",\n ] = Field(description=\"The subject matter knowledge complexity level of the text\")\n reasoning: str = Field(\n description=\"A brief synthesis of why the text fits the chosen complexity level.\"\n )\n\n\nsmk_prompt_vars = {\n \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n \"outputParser\": JsonOutputParser(pydantic_object=SmkOutput),\n}\n\nclass ConventionalityOutput(BaseModel):\n conventionality_features: List[str] = Field(\n description=\"List of the specific language features driving the complexity (e.g., idioms, metaphors, implied meaning) with direct quotes from the text.\"\n )\n grade_context: str = Field(\n description=\"How the conventionality demands compare to general expectations for the provided target grade.\"\n )\n instructional_insights: str = Field(\n description=\"Actionable pedagogical suggestions for scaffolding the unconventional language features in the classroom.\"\n )\n complexity_score: Literal[\n \"slightly_complex\",\n \"moderately_complex\",\n \"very_complex\",\n \"exceedingly_complex\",\n ] = Field(description=\"The conventionality complexity level of the text\")\n reasoning: str = Field(\n description=\"A synthesis of why the text fits the chosen rubric level.\"\n )\n\n\nconv_prompt_vars = {\n \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n \"outputParser\": JsonOutputParser(pydantic_object=ConventionalityOutput),\n}" + "class VocabOutput(BaseModel):\n", + " tier_2_words: str = Field(description=\"List of Tier 2 words\")\n", + " tier_3_words: str = Field(description=\"List of Tier 3 words\")\n", + " archaic_words: str = Field(description=\"List of Archaic words\")\n", + " other_complex_words: str = Field(description=\"List of Other Complex words\")\n", + " complexity_score: str = Field(\n", + " description=\"the complexity of the text, one of: slightly complex, moderately complex, very complex, or exceedingly complex\"\n", + " )\n", + " reasoning: str = Field(description=\"your reasoning for your answer\")\n", + "\n", + "\n", + "prompt_vars = {\n", + " \"inputVars\": [\n", + " \"text\",\n", + " \"student_grade_level\",\n", + " \"student_background_knowledge\",\n", + " \"fk_level\",\n", + " ],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=VocabOutput),\n", + "}\n", + "\n", + "\n", + "class SmkOutput(BaseModel):\n", + " identified_topics: List[str] = Field(\n", + " description=\"List of major subjects/concepts found in the text.\"\n", + " )\n", + " curriculum_check: str = Field(\n", + " description=\"Analysis of whether these topics are new or review based on the Grade Level and Reference list.\"\n", + " )\n", + " assumptions_and_scaffolding: str = Field(\n", + " description=\"Analysis of what the author assumes vs. what is explained (and if definitions are provided).\"\n", + " )\n", + " friction_analysis: str = Field(\n", + " description=\"Explicit statement distinguishing if difficulty comes from Vocabulary/Structure or actual Knowledge.\"\n", + " )\n", + " complexity_score: Literal[\n", + " \"slightly_complex\",\n", + " \"moderately_complex\",\n", + " \"very_complex\",\n", + " \"exceedingly_complex\",\n", + " ] = Field(description=\"The subject matter knowledge complexity level of the text\")\n", + " reasoning: str = Field(\n", + " description=\"A brief synthesis of why the text fits the chosen complexity level.\"\n", + " )\n", + "\n", + "\n", + "smk_prompt_vars = {\n", + " \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=SmkOutput),\n", + "}\n", + "\n", + "class ConventionalityOutput(BaseModel):\n", + " conventionality_features: List[str] = Field(\n", + " description=\"List of the specific language features driving the complexity (e.g., idioms, metaphors, implied meaning) with direct quotes from the text.\"\n", + " )\n", + " grade_context: str = Field(\n", + " description=\"How the conventionality demands compare to general expectations for the provided target grade.\"\n", + " )\n", + " instructional_insights: str = Field(\n", + " description=\"Actionable pedagogical suggestions for scaffolding the unconventional language features in the classroom.\"\n", + " )\n", + " complexity_score: Literal[\n", + " \"slightly_complex\",\n", + " \"moderately_complex\",\n", + " \"very_complex\",\n", + " \"exceedingly_complex\",\n", + " ] = Field(description=\"The conventionality complexity level of the text\")\n", + " reasoning: str = Field(\n", + " description=\"A synthesis of why the text fits the chosen rubric level.\"\n", + " )\n", + "\n", + "\n", + "conv_prompt_vars = {\n", + " \"inputVars\": [\"text\", \"grade\", \"fk_score\"],\n", + " \"outputParser\": JsonOutputParser(pydantic_object=ConventionalityOutput),\n", + "}" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -350,7 +458,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -377,7 +485,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -399,7 +507,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -440,7 +548,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -488,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -765,7 +873,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -801,7 +909,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -838,7 +946,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -875,10 +983,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ + "def get_vocab_prompts_for_grade(grade: int) -> dict:\n", + " \"\"\"Return the SYSTEM_PROMPT/USER_PROMPT validated for the given grade.\n", + "\n", + " Grades 3-4 use the GRADES_3_4 prompt; all other grades (5-12) use the\n", + " OTHER_GRADES prompt. Mirrors the standalone vocabulary evaluator.\n", + " \"\"\"\n", + " if grade in (3, 4):\n", + " return v_prompts.GRADE_SPECIFIC_PROMPTS[\"GRADES_3_4\"]\n", + " return v_prompts.GRADE_SPECIFIC_PROMPTS[\"OTHER_GRADES\"]\n", + "\n", + "\n", + "def get_vocab_model_for_grade(grade: int):\n", + " \"\"\"Return the vocab model validated against the grade's prompt.\n", + "\n", + " Grades 3-4 were validated with Gemini; grades 5-12 with GPT-4.1.\n", + " \"\"\"\n", + " if grade in (3, 4):\n", + " return vocab_complexity_model_grades_3_4\n", + " return vocab_complexity_model_other_grades\n", + "\n", + "\n", + "def normalize_vocab_output(output: dict) -> dict:\n", + " \"\"\"Convert the OTHER_GRADES integer 'answer' (1-4) into a string\n", + " 'complexity_score' so downstream normalize_label() works for all grades.\n", + " GRADES_3_4 already returns 'complexity_score' as a string.\n", + " \"\"\"\n", + " mapping = {\n", + " 1: \"slightly complex\",\n", + " 2: \"moderately complex\",\n", + " 3: \"very complex\",\n", + " 4: \"exceedingly complex\",\n", + " }\n", + " if \"answer\" in output and \"complexity_score\" not in output:\n", + " value = output[\"answer\"]\n", + " if isinstance(value, str) and value.strip().isdigit():\n", + " value = int(value)\n", + " output[\"complexity_score\"] = mapping.get(value, value)\n", + " return output\n", + "\n", + "\n", "async def predict_text_complexity_level(text, grade):\n", " \"\"\"\n", " Predict the text complexity level as well as the complex words and reasoning.\n", @@ -886,10 +1034,11 @@ "\n", " dataset = await prepare_text_for_complexity_prediction(text, grade)\n", "\n", - " # Prompts imported from prompts file\n", + " # Select the prompt and model validated for this grade band.\n", + " grade_prompts = get_vocab_prompts_for_grade(grade)\n", " messages = [\n", - " SystemMessage(content=v_prompts.SYSTEM_PROMPT),\n", - " HumanMessagePromptTemplate.from_template(v_prompts.USER_PROMPT),\n", + " SystemMessage(content=grade_prompts[\"SYSTEM_PROMPT\"]),\n", + " HumanMessagePromptTemplate.from_template(grade_prompts[\"USER_PROMPT\"]),\n", " ]\n", "\n", " # Prepare chat prompt\n", @@ -901,17 +1050,17 @@ " },\n", " )\n", " # Invoke the chain\n", - " chain = prompt | vocab_complexity_model | JsonOutputParser()\n", + " chain = prompt | get_vocab_model_for_grade(grade) | JsonOutputParser()\n", "\n", " # return output\n", " output = await chain.ainvoke(dataset)\n", "\n", - " return output" + " return normalize_vocab_output(output)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -999,7 +1148,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "cell-conv-df", "metadata": {}, "outputs": [], @@ -1053,7 +1202,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -1088,7 +1237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1102,7 +1251,24 @@ "title": "" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"vocabulary_score\": \"Very Complex\",\n", + " \"smk_score\": \"Moderately Complex\",\n", + " \"conventionality_score\": \"Slightly Complex\",\n", + " \"complexity_answer\": \"Moderately Complex\",\n", + " \"vocabulary_reasoning\": \"The vocabulary is very complex for a 3rd grader due to a high density of unfamiliar proper nouns and Tier 3 words presented without any contextual support. The student is confronted with a large number of new concepts in a short space, including 'Mongol Dynasty', 'Kublai Khan', and specific, likely unknown place names like 'Ayas', 'Tabriz', and 'Kerman'. This creates a significant conceptual load. Furthermore, Tier 3 vocabulary related to trade goods ('ivory', 'jade', 'porcelain') is introduced without definitions, requiring outside knowledge. While the core narrative of a trip might be understandable, the density of these specific, unsupported terms will significantly slow down comprehension and make understanding the details of the text very challenging, justifying a 'very complex' rating.\",\n", + " \"smk_reasoning\": \"The text is categorized as Moderately Complex because it deals with common discipline-specific knowledge (History and Geography) that bridges concrete descriptions of travel with the broader theme of cultural exchange. While the FK score (6.59) is high for a 3rd-grade target, the Subject Matter Knowledge remains accessible. The use of specific historical names and locations requires the reader to maintain a mental map of a long-term journey, which is a hallmark of moderate complexity in social studies texts. It avoids 'Very Complex' because it does not require knowledge of specific political treaties, the mechanics of 13th-century navigation, or complex economic theories.\",\n", + " \"conventionality_reasoning\": \"The text is a straightforward, literal historical narrative that follows a clear chronological sequence. The meaning is entirely on the surface, describing physical actions (traveling, serving in a court, returning) and concrete objects (noodles, ivory, jade). There is no use of irony, metaphor, or abstract conceptual framing. While the text includes specific historical names and places that may require brief identification, the way the information is conveyed is explicit and procedural, adhering to the expert rule that concrete and procedural texts are typically slightly complex.\",\n", + " \"complexity_reasoning\": \"The text provides a historical account of Marco Polo's travels, which includes a mix of concrete details (e.g., locations, events) and abstract concepts (e.g., the debate over the accuracy of his accounts). Qualitatively, the text is moderately challenging due to its historical context, the inclusion of less familiar terms (e.g., 'Mongol Dynasty,' 'Kublai Khan'), and the need for background knowledge about geography and trade. The structure is mostly chronological, which aids comprehension, but the density of information and the variety of concepts (e.g., travel hardships, cultural exchanges, trade) require sustained attention. Quantitatively, the Flesch-Kincaid grade level of 6 suggests the text is above the typical Grade 3 reading level. Sentence complexity is moderate, with a mix of simple, compound, and complex sentences, though there are no compound-complex sentences. The average sentence length of 14 words and the presence of multi-concept sentences (61%) add to the cognitive load. While the text is accessible in terms of sentence structure, the conceptual demands and historical references make it more challenging for a Grade 3 student. Therefore, the text is best categorized as 'Moderately Complex.'\"\n", + "}\n" + ] + } + ], "source": [ "# Add your text & the grade level you want to evaluate for vocabulary complexity\n", "\n", @@ -1140,9 +1306,128 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"grade\": 3,\n", + " \"text\": \"\\nPolo went on a 24-year trip to China with his father and uncle during the Mongol Dynasty. He left Venice at the age of 17 on a boat that went through the Mediterranean Sea, Ayas, Tabriz and Kerman. Then he travelled across Asia getting as far as Beijing. On the way there he had to go over mountains and through terrible deserts, across hot burning lands and places where the cold was horrible. He served in Kublai Khan's court for 17 years. He left the Far East and returned to Venice by sea. There was sickness on board and 600 passengers and crew died and some say pirates attacked. Nevertheless, Marco Polo survived it all.\\nSome scholars believe that while Marco Polo did go to China, he did not go to all of the other places described in his book. He brought noodles back from China and the Italians came up with different sizes and shapes and called it pasta. Polo returned to Venice with treasures like ivory, jade, jewels, porcelain and silk.\\nHis father had borrowed money and bought a ship. He became wealthy because of his trading in the near East.\\n\",\n", + " \"tier_2_words\": \"served, passengers, crew, survived, scholars, described, treasures, wealthy, trading, nevertheless\",\n", + " \"tier_3_words\": \"Mongol Dynasty, Kublai Khan, ivory, jade, porcelain\",\n", + " \"archaic_words\": \"\",\n", + " \"other_complex_words\": \"Venice, Ayas, Tabriz, Kerman, Beijing, Far East, near East\",\n", + " \"vocabulary_score\": \"Very Complex\",\n", + " \"vocabulary_reasoning\": \"The vocabulary is very complex for a 3rd grader due to a high density of unfamiliar proper nouns and Tier 3 words presented without any contextual support. The student is confronted with a large number of new concepts in a short space, including 'Mongol Dynasty', 'Kublai Khan', and specific, likely unknown place names like 'Ayas', 'Tabriz', and 'Kerman'. This creates a significant conceptual load. Furthermore, Tier 3 vocabulary related to trade goods ('ivory', 'jade', 'porcelain') is introduced without definitions, requiring outside knowledge. While the core narrative of a trip might be understandable, the density of these specific, unsupported terms will significantly slow down comprehension and make understanding the details of the text very challenging, justifying a 'very complex' rating.\",\n", + " \"error\": null,\n", + " \"identified_topics\": [\n", + " \"Marco Polo\",\n", + " \"Silk Road and Asian Geography\",\n", + " \"The Mongol Dynasty (Kublai Khan)\",\n", + " \"International Trade and Cultural Exchange\",\n", + " \"Historical Skepticism\"\n", + " ],\n", + " \"curriculum_check\": \"Standard/General. The story of Marco Polo and basic world geography are standard components of the K-8 social studies and history curriculum.\",\n", + " \"assumptions_and_scaffolding\": \"The author assumes the reader understands the general concept of a 'dynasty' and a 'court' as a place of government. It scaffolds the list of unfamiliar place names (Ayas, Tabriz, Kerman) by placing them within the context of a travel route. It also explains the significance of trade by listing concrete items like silk and noodles.\",\n", + " \"friction_analysis\": \"The text primarily stays in the concrete realm, describing a physical journey, weather conditions, and physical treasures. It introduces a small amount of abstract friction by mentioning that 'some scholars believe' the account might be exaggerated, but it does not require the reader to understand the historiographical methods used to prove or disprove these claims.\",\n", + " \"smk_score\": \"Moderately Complex\",\n", + " \"smk_reasoning\": \"The text is categorized as Moderately Complex because it deals with common discipline-specific knowledge (History and Geography) that bridges concrete descriptions of travel with the broader theme of cultural exchange. While the FK score (6.59) is high for a 3rd-grade target, the Subject Matter Knowledge remains accessible. The use of specific historical names and locations requires the reader to maintain a mental map of a long-term journey, which is a hallmark of moderate complexity in social studies texts. It avoids 'Very Complex' because it does not require knowledge of specific political treaties, the mechanics of 13th-century navigation, or complex economic theories.\",\n", + " \"error_smk\": null,\n", + " \"conventionality_score\": \"Slightly Complex\",\n", + " \"conventionality_reasoning\": \"The text is a straightforward, literal historical narrative that follows a clear chronological sequence. The meaning is entirely on the surface, describing physical actions (traveling, serving in a court, returning) and concrete objects (noodles, ivory, jade). There is no use of irony, metaphor, or abstract conceptual framing. While the text includes specific historical names and places that may require brief identification, the way the information is conveyed is explicit and procedural, adhering to the expert rule that concrete and procedural texts are typically slightly complex.\",\n", + " \"conventionality_features\": [\n", + " \"literal narrative: 'Polo went on a 24-year trip to China'\",\n", + " \"concrete actions: 'He left Venice at the age of 17 on a boat'\",\n", + " \"explicit meaning: 'He brought noodles back from China'\",\n", + " \"straightforward descriptions: 'across hot burning lands and places where the cold was horrible'\"\n", + " ],\n", + " \"grade_context\": \"For a 3rd-grade student, the conventionality of this text is well-aligned with expectations. The narrative structure is predictable and the sentences describe observable events. Although the Flesch-Kincaid score is slightly elevated due to multisyllabic vocabulary (e.g., 'Mediterranean', 'nevertheless'), the conventionality remains low because the text does not rely on abstract reasoning or hidden meanings.\",\n", + " \"instructional_insights\": \"Instruction should focus on supporting students with the specific proper nouns and historical vocabulary (e.g., 'dynasty', 'porcelain'). Because the text's meaning is literal, teachers can use a map to help students track the physical journey, reinforcing the concrete nature of the narrative. Scaffolding should target decoding and word meaning rather than deep conceptual interpretation.\",\n", + " \"error_conv\": null,\n", + " \"grammatical_reasoning\": \"1. 'Polo went on a 24-year trip to China with his father and uncle during the Mongol Dynasty.' This is a simple sentence with one independent clause. \\n2. 'He left Venice at the age of 17 on a boat that went through the Mediterranean Sea, Ayas, Tabriz and Kerman.' This is a complex sentence with one independent clause and one subordinate clause ('that went through the Mediterranean Sea, Ayas, Tabriz and Kerman'). \\n3. 'Then he travelled across Asia getting as far as Beijing.' This is a simple sentence with one independent clause. \\n4. 'On the way there he had to go over mountains and through terrible deserts, across hot burning lands and places where the cold was horrible.' This is a complex sentence with one independent clause and one subordinate clause ('where the cold was horrible'). \\n5. 'He served in Kublai Khan's court for 17 years.' This is a simple sentence with one independent clause. \\n6. 'He left the Far East and returned to Venice by sea.' This is a compound sentence with two independent clauses joined by 'and'. \\n7. 'There was sickness on board and 600 passengers and crew died and some say pirates attacked.' This is a compound sentence with three independent clauses joined by 'and'. \\n8. 'Nevertheless, Marco Polo survived it all.' This is a simple sentence with one independent clause. \\n9. 'Some scholars believe that while Marco Polo did go to China, he did not go to all of the other places described in his book.' This is a complex sentence with one independent clause and two subordinate clauses ('that while Marco Polo did go to China' and 'while Marco Polo did go to China, he did not go to all of the other places described in his book'). \\n10. 'He brought noodles back from China and the Italians came up with different sizes and shapes and called it pasta.' This is a compound sentence with three independent clauses joined by 'and'. \\n11. 'Polo returned to Venice with treasures like ivory, jade, jewels, porcelain and silk.' This is a simple sentence with one independent clause. \\n12. 'His father had borrowed money and bought a ship.' This is a compound sentence with two independent clauses joined by 'and'. \\n13. 'He became wealthy because of his trading in the near East.' This is a complex sentence with one independent clause and one subordinate clause ('because of his trading in the near East').\",\n", + " \"num_sentences\": 13,\n", + " \"num_words\": 193,\n", + " \"flesch_kincaid_grade\": 6.59,\n", + " \"num_simple_sentences\": 5,\n", + " \"num_compound_sentences\": 4,\n", + " \"num_complex_sentences\": 3,\n", + " \"num_compound_complex_sentences\": 0,\n", + " \"num_other_sentences\": 1,\n", + " \"num_independent_clauses\": 19,\n", + " \"num_subordinate_clauses\": 6,\n", + " \"num_total_clauses\": 25,\n", + " \"num_sentences_with_subordinate\": 4,\n", + " \"num_sentences_with_multiple_subordinates\": 1,\n", + " \"num_sentences_with_embedded_clauses\": 1,\n", + " \"num_prepositional_phrases\": 18,\n", + " \"num_participle_phrases\": 1,\n", + " \"num_appositive_phrases\": 0,\n", + " \"num_simple_transitions\": 7,\n", + " \"num_sophisticated_transitions\": 1,\n", + " \"words_in_simple_sentences\": 58,\n", + " \"words_in_compound_sentences\": 78,\n", + " \"words_in_complex_sentences\": 57,\n", + " \"words_in_compound_complex_sentences\": 0,\n", + " \"words_in_other_sentences\": 0,\n", + " \"sentence_word_counts\": [\n", + " 17,\n", + " 21,\n", + " 10,\n", + " 24,\n", + " 8,\n", + " 12,\n", + " 19,\n", + " 6,\n", + " 27,\n", + " 22,\n", + " 14,\n", + " 11,\n", + " 12\n", + " ],\n", + " \"num_one_concept_sentences\": 5,\n", + " \"num_multi_concept_sentences\": 8,\n", + " \"num_cleft_sentences\": 0,\n", + " \"max_clauses_in_any_sentence\": 4,\n", + " \"error_grammar\": null,\n", + " \"avg_words_per_sentence\": 14.846153846153847,\n", + " \"sentence_length_variation\": 6.294545398608856,\n", + " \"percent_short_sentences\": 23.076923076923077,\n", + " \"percent_medium_sentences\": 46.15384615384615,\n", + " \"percent_long_sentences\": 30.76923076923077,\n", + " \"percent_very_long_sentences\": 0.0,\n", + " \"percent_simple_sentences\": 38.46153846153847,\n", + " \"percent_compound_sentences\": 30.76923076923077,\n", + " \"percent_complex_sentences\": 23.076923076923077,\n", + " \"percent_compound_complex_sentences\": 0.0,\n", + " \"percent_other_sentences\": 7.6923076923076925,\n", + " \"percent_words_in_simple_sentences\": 30.05181347150259,\n", + " \"percent_words_in_compound_sentences\": 40.41450777202073,\n", + " \"percent_words_in_complex_sentences\": 29.533678756476682,\n", + " \"percent_words_in_compound_complex_sentences\": 0.0,\n", + " \"percent_words_in_other_sentences\": 0.0,\n", + " \"avg_subordinates_per_sentence\": 0.46153846153846156,\n", + " \"avg_clauses_per_sentence\": 1.9230769230769231,\n", + " \"percent_sentences_with_subordinate\": 30.76923076923077,\n", + " \"percent_sentences_with_multiple_subordinates\": 7.6923076923076925,\n", + " \"percent_sentences_with_embedded_clauses\": 7.6923076923076925,\n", + " \"prep_phrase_density\": 9.32642487046632,\n", + " \"participle_phrase_density\": 0.5181347150259068,\n", + " \"appositive_phrase_density\": 0.0,\n", + " \"avg_transitions_per_sentence\": 0.6153846153846154,\n", + " \"percent_sophisticated_transitions\": 12.5,\n", + " \"percent_sentences_w_one_concept\": 38.46153846153847,\n", + " \"percent_sentences_w_multi_concept\": 61.53846153846154,\n", + " \"percent_cleft_sentences\": 0.0,\n", + " \"complexity_answer\": \"Moderately Complex\",\n", + " \"complexity_reasoning\": \"The text provides a historical account of Marco Polo's travels, which includes a mix of concrete details (e.g., locations, events) and abstract concepts (e.g., the debate over the accuracy of his accounts). Qualitatively, the text is moderately challenging due to its historical context, the inclusion of less familiar terms (e.g., 'Mongol Dynasty,' 'Kublai Khan'), and the need for background knowledge about geography and trade. The structure is mostly chronological, which aids comprehension, but the density of information and the variety of concepts (e.g., travel hardships, cultural exchanges, trade) require sustained attention. Quantitatively, the Flesch-Kincaid grade level of 6 suggests the text is above the typical Grade 3 reading level. Sentence complexity is moderate, with a mix of simple, compound, and complex sentences, though there are no compound-complex sentences. The average sentence length of 14 words and the presence of multi-concept sentences (61%) add to the cognitive load. While the text is accessible in terms of sentence structure, the conceptual demands and historical references make it more challenging for a Grade 3 student. Therefore, the text is best categorized as 'Moderately Complex.'\",\n", + " \"error_complexity\": null\n", + "}\n" + ] + } + ], "source": [ "full_record = results.iloc[0].to_dict()\n", "print(json.dumps(full_record, indent=2))" @@ -1185,4 +1470,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/evals/prompts/vocabulary/background-knowledge.txt b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/background-knowledge.txt similarity index 100% rename from evals/prompts/vocabulary/background-knowledge.txt rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/background-knowledge.txt diff --git a/evals/prompts/vocabulary/grades-3-4-system.txt b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/grades-3-4-system.txt similarity index 100% rename from evals/prompts/vocabulary/grades-3-4-system.txt rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/grades-3-4-system.txt diff --git a/evals/prompts/vocabulary/grades-3-4-user.txt b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/grades-3-4-user.txt similarity index 100% rename from evals/prompts/vocabulary/grades-3-4-user.txt rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/grades-3-4-user.txt diff --git a/evals/prompts/vocabulary/other-grades-system.txt b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/other-grades-system.txt similarity index 100% rename from evals/prompts/vocabulary/other-grades-system.txt rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/other-grades-system.txt diff --git a/evals/prompts/vocabulary/other-grades-user.txt b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/other-grades-user.txt similarity index 100% rename from evals/prompts/vocabulary/other-grades-user.txt rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/other-grades-user.txt diff --git a/evals/prompts/vocab_prompts.py b/evals/literacy/qualitative-text-complexity/vocabulary/prompts/vocab_prompts.py similarity index 100% rename from evals/prompts/vocab_prompts.py rename to evals/literacy/qualitative-text-complexity/vocabulary/prompts/vocab_prompts.py diff --git a/evals/vocabulary_evaluator.ipynb b/evals/literacy/qualitative-text-complexity/vocabulary/vocabulary_evaluator.ipynb similarity index 88% rename from evals/vocabulary_evaluator.ipynb rename to evals/literacy/qualitative-text-complexity/vocabulary/vocabulary_evaluator.ipynb index 892ecdf4..54c58f6a 100644 --- a/evals/vocabulary_evaluator.ipynb +++ b/evals/literacy/qualitative-text-complexity/vocabulary/vocabulary_evaluator.ipynb @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -48,14 +48,22 @@ "title": "" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ "%pip install -qU pydantic textstat langchain langchain_openai langchain-google-genai" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -104,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -164,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -196,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -243,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -306,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -379,7 +387,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -448,7 +456,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -462,7 +470,38 @@ "title": "" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "========================= Complexity Score ========================\n", + " very complex\n", + "\n", + " ========================= Complexity Score Reasoning ==============\n", + " The vocabulary is very complex for a 3rd grader, a conclusion supported by the\n", + "Flesch-Kincaid grade level of 6.59. The primary challenge is the high density\n", + "and significant conceptual load of unfamiliar proper nouns and domain-specific\n", + "terms presented with no contextual scaffolding. Words like 'Mongol Dynasty,'\n", + "'Kublai Khan,' 'Venice,' 'Ayas,' 'Tabriz,' 'Kerman,' 'ivory,' 'jade,' and\n", + "'porcelain' are introduced in rapid succession without definition. This forces a\n", + "student to process numerous new concepts simultaneously. Additionally, the text\n", + "includes challenging Tier 2 words like 'scholars,' 'nevertheless,' and 'court'\n", + "(in its abstract sense). While the basic narrative of a long trip might be\n", + "partially accessible, the vocabulary creates significant hurdles that would slow\n", + "down and impede comprehension of the bulk of the text's details, making it very\n", + "difficult for a 3rd grader to follow.\n", + "\n", + " ======================== Complex words ==========================\n", + " * Tier 2 words: served, court, passengers, crew, nevertheless, survived,\n", + "scholars, described, wealthy\n", + " * Tier 3 words: Dynasty, ivory, jade, porcelain\n", + " * Archaic words: \n", + " * Other complex words: Mongol Dynasty, Venice, Mediterranean Sea, Ayas, Tabriz,\n", + "Kerman, Beijing, Kublai Khan, Far East, near East\n" + ] + } + ], "source": [ "# Add your text & the grade level you want to evaluate for vocabulary complexity\n", "\n", @@ -506,7 +545,7 @@ "widgets": {} }, "kernelspec": { - "display_name": ".venv", + "display_name": "base", "language": "python", "name": "python3" }, @@ -520,7 +559,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/evals/prompts/__init__.py b/evals/prompts/__init__.py deleted file mode 100644 index 7d83a195..00000000 --- a/evals/prompts/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" - Evaluators literacy package: Prompts -""" \ No newline at end of file diff --git a/evals/purpose_evaluator.ipynb b/evals/purpose_evaluator.ipynb deleted file mode 100644 index 1568794e..00000000 --- a/evals/purpose_evaluator.ipynb +++ /dev/null @@ -1,103 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install -qU langchain-google-genai langchain textstat" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import hashlib\n", - "from pathlib import Path\n", - "from langchain_google_genai import ChatGoogleGenerativeAI\n", - "from langchain_core.prompts import ChatPromptTemplate\n", - "import textstat" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "# -------------------------------------------------------------------------\n# Load source-of-truth assets: config.json + every prompt file declared\n# in config.steps[*].prompt.messages\n# -------------------------------------------------------------------------\n# The canonical evaluator definition lives in evals/prompts/purpose.\n# All consumers (DS notebook, Python SDK, TypeScript SDK) read these same files,\n# so this loader is the pattern the SDK engineer will reproduce.\n\nASSETS_DIR = Path(\"./prompts/purpose\")\n\nconfig_path = ASSETS_DIR / \"config.json\"\nwith open(config_path) as f:\n CONFIG = json.load(f)\n\n# Load standalone schema files (config.json references them via $ref by path).\nwith open(ASSETS_DIR / \"input_schema.json\") as f:\n INPUT_SCHEMA = json.load(f)\nwith open(ASSETS_DIR / \"output_schema.json\") as f:\n OUTPUT_SCHEMA = json.load(f)\n\n# Load every prompt message declared in config (system, user, ...). Each\n# message has {role, source_path, sha256}. We verify each file's sha256\n# matches the declared hash -- drift tripwire #1, applied to every prompt\n# regardless of role. CI should promote a mismatch to a hard failure.\nPROMPT_MESSAGES = [] # list of (role, text) tuples, preserving config order\nfor msg_spec in CONFIG[\"steps\"][0][\"prompt\"][\"messages\"]:\n role = msg_spec[\"role\"]\n path = ASSETS_DIR / msg_spec[\"source_path\"]\n text = path.read_text()\n actual_sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()\n declared_sha = msg_spec[\"sha256\"]\n assert actual_sha == declared_sha, (\n f\"prompt drift detected for role={role!r} ({msg_spec['source_path']}): \"\n f\"declared {declared_sha[:12]}..., actual on disk {actual_sha[:12]}...\"\n )\n PROMPT_MESSAGES.append((role, text))\n\nprint(\n f\"Loaded {CONFIG['evaluator']['id']} \"\n f\"from {ASSETS_DIR.resolve()}\"\n)\nprint(f\" model: {CONFIG['steps'][0]['model']['name']}\")\nprint(f\" temperature: {CONFIG['steps'][0]['generation']['temperature']}\")\nprint(f\" prompts:\")\nfor msg_spec, (role, text) in zip(CONFIG[\"steps\"][0][\"prompt\"][\"messages\"], PROMPT_MESSAGES):\n sha = hashlib.sha256(text.encode(\"utf-8\")).hexdigest()[:12]\n print(f\" {role:>6} {msg_spec['source_path']:<14} ({len(text):>5} chars, sha {sha})\")" - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "# -------------------------------------------------------------------------\n# FK score helper (declared as a preprocessing step in CONFIG['preprocessing'])\n# -------------------------------------------------------------------------\n_FK_PRE = next(p for p in CONFIG[\"preprocessing\"] if p[\"id\"] == \"fk_score\")\n_FK_IMPL = _FK_PRE[\"implementation\"][\"python\"]\n_FK_LIB = _FK_IMPL[\"library\"]\n_FK_FN = _FK_IMPL[\"function\"]\n_FK_TRANSFORM = _FK_IMPL[\"post_transform\"]\n\nif _FK_LIB != \"textstat\":\n raise ValueError(f\"unsupported fk library in config: {_FK_LIB!r}\")\n\n\ndef calculate_fk_score(text) -> float:\n \"\"\"Compute Flesch-Kincaid Grade Level per CONFIG['preprocessing'].\"\"\"\n fn = getattr(textstat, _FK_FN)\n value = fn(text)\n if _FK_TRANSFORM[\"type\"] == \"round\":\n value = round(value, _FK_TRANSFORM[\"precision\"])\n else:\n raise ValueError(f\"unsupported post_transform type: {_FK_TRANSFORM['type']!r}\")\n return value\n\n\n# -------------------------------------------------------------------------\n# Evaluator function: model / prompt / parser config all read from CONFIG\n# -------------------------------------------------------------------------\n_STEP = CONFIG[\"steps\"][0] # single-step evaluator today. Extensible to multi-step evaluators.\n\ndef evaluate_text_complexity(text: str, grade_level: int):\n \"\"\"\n Evaluate the Purpose-dimension complexity of a text using the canonical\n config in evals/prompts/purpose/config.json + system.txt + user.txt.\n\n Returns a dict with full I/O trace fields:\n - rendered_prompt: the actual list of messages sent to the model\n (input-side trace).\n - raw_output: the AIMessage object returned by the LLM\n (preserves response_metadata, usage_metadata).\n - raw_text: just the string content of the AIMessage.\n - formatted_output: the parsed dict matching OUTPUT_SCHEMA.\n - usage: token-usage metadata if the provider returned it.\n\n The LLM is invoked ONCE; include_raw=True returns both the raw AIMessage\n and the parsed output without a second call.\n \"\"\"\n # 1. Structured output -- parser.kind == \"structured_output\" uses the model's\n # native output enforcement. OUTPUT_SCHEMA is loaded from output_schema.json,\n # the standalone source of truth. include_raw=True preserves the AIMessage\n # for tracing alongside the parsed result.\n llm = ChatGoogleGenerativeAI(\n model=_STEP[\"model\"][\"name\"],\n temperature=_STEP[\"generation\"][\"temperature\"],\n )\n structured_llm = llm.with_structured_output(OUTPUT_SCHEMA, include_raw=True)\n\n # 2. Prompt template -- every message's content was loaded from disk\n # and verified against config in the loader cell. We just feed the\n # (role, text) tuples straight into ChatPromptTemplate.\n prompt_template = ChatPromptTemplate.from_messages(PROMPT_MESSAGES)\n\n try:\n # Step A: Calculate FK Score\n fk_score = calculate_fk_score(text)\n print(f\"Calculated Flesch-Kincaid Score: {fk_score}\")\n\n inputs = {\"text\": text, \"grade_level\": grade_level, \"fk_score\": fk_score}\n\n # Step B: Render the prompt up-front so we can return exactly what\n # was sent to the model (input-side trace).\n rendered_messages = prompt_template.format_messages(**inputs)\n\n # Step C: Single LLM call -> raw AIMessage + parsed output dict.\n # No second LLM call.\n raw = structured_llm.invoke(rendered_messages)\n\n if raw.get(\"parsing_error\"):\n raise ValueError(f\"structured output parsing failed: {raw['parsing_error']}\")\n\n # Step D: Return the full trace dict.\n return {\n \"rendered_prompt\": [m.model_dump() for m in rendered_messages],\n \"raw_output\": raw[\"raw\"],\n \"raw_text\": raw[\"raw\"].content,\n \"formatted_output\": raw[\"parsed\"],\n \"usage\": getattr(raw[\"raw\"], \"usage_metadata\", None),\n }\n except Exception as e:\n return f\"Error evaluating text: {e}\"" - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sample_text = \"\"\"\n", - "\"Well, then,\" said the teacher, \"you may take your slate and go out behind the schoolhouse for half an hour. Think of something to write about, and write the word on your slate. Then try to tell what it is, what it is like, what it is good for, and what is done with it. That is the way to write a composition.\" Henry took his slate and went out. Just behind the schoolhouse was Mr. Finney's barn. Quite close to the barn was a garden. And in the garden, Henry saw a turnip. \"Well, I know what that is,\" he said to himself; and he wrote the word turnip on his slate. Then he tried to tell what it was like, what it was good for, and what was done with it. Before the half hour was ended he had written a very neat composition on his slate. He then went into the house, and waited while the teacher read it. The teacher was surprised and pleased. He said, \"Henry Longfellow, you have done very well. Today you may stand up before the school and read what you have written about the turnip.\"\n", - "\"\"\"\n", - "\n", - "result = evaluate_text_complexity(\n", - " text=sample_text,\n", - " grade_level=4\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pprint as pp\n", - "# I/O trace breakdown -- this is what the SDK engineer will replicate in TS.\n", - "print(\"=\" * 60)\n", - "print(\"RENDERED PROMPT (input sent to the LLM)\")\n", - "print(\"=\" * 60)\n", - "pp.pprint(result[\"rendered_prompt\"])\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"RAW LLM TEXT (model's verbatim output)\")\n", - "print(\"=\" * 60)\n", - "print(result[\"raw_text\"])\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"PARSED OUTPUT (output_schema)\")\n", - "print(\"=\" * 60)\n", - "pp.pprint(result[\"formatted_output\"])\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"USAGE METADATA\")\n", - "print(\"=\" * 60)\n", - "pp.pprint(result[\"usage\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "# -------------------------------------------------------------------------\n# Sniff-test runner: load fixtures.json and check predictions against expected\n# -------------------------------------------------------------------------\n# Fixtures live next to config.json + system.txt + user.txt and follow the\n# schema declared in CONFIG['fixtures']['schema']. Each case has:\n# - id, description (optional)\n# - input: {text, grade_level} -- runtime evaluator inputs\n# - expected: {complexity_level} -- ground-truth label from rubric\n#\n# Note: the fixture key 'complexity_level' maps to the runtime output's\n# 'complexity_score' field. We test that single value only -- the model's\n# free-text 'reasoning' field is non-deterministic across runs.\n\nfixtures_path = ASSETS_DIR / CONFIG[\"fixtures\"][\"sniff_test_path\"]\nwith open(fixtures_path) as f:\n fixtures = json.load(f)\nprint(f\"Loaded {len(fixtures)} fixtures from {fixtures_path.name}\\n\")\n\n# Adjacency tolerance per CONFIG['fixtures']['tolerance'].\n# Derive rubric order from OUTPUT_SCHEMA -- 'more_context_needed' excluded as it has no adjacency.\n_RUBRIC_ORDER = [\n level for level in OUTPUT_SCHEMA[\"properties\"][\"complexity_score\"][\"enum\"]\n if level != \"more_context_needed\"\n]\n_ALLOW_ADJ = bool(CONFIG[\"fixtures\"][\"tolerance\"].get(\"allow_adjacent_levels\", False))\n\ndef _score_outcome(predicted: str, expected: str):\n \"\"\"Return ('exact' | 'adjacent' | 'fail', distance_or_None).\"\"\"\n if predicted == expected:\n return \"exact\", 0\n if _ALLOW_ADJ and predicted in _RUBRIC_ORDER and expected in _RUBRIC_ORDER:\n d = abs(_RUBRIC_ORDER.index(predicted) - _RUBRIC_ORDER.index(expected))\n if d == 1:\n return \"adjacent\", d\n return \"fail\", None\n\n# Run each fixture, accumulate results\nresults = []\nfor fx in fixtures:\n expected = fx[\"expected\"][\"complexity_level\"]\n out = evaluate_text_complexity(\n text=fx[\"input\"][\"text\"],\n grade_level=fx[\"input\"][\"grade_level\"],\n )\n if isinstance(out, str): # error path\n results.append({\"id\": fx[\"id\"], \"status\": \"error\", \"predicted\": None, \"expected\": expected, \"error\": out})\n continue\n predicted = out[\"formatted_output\"][\"complexity_score\"]\n status, _ = _score_outcome(predicted, expected)\n results.append({\n \"id\": fx[\"id\"], \"status\": status,\n \"predicted\": predicted, \"expected\": expected,\n \"description\": fx.get(\"description\", \"\"),\n })\n\n# Per-case report\nprint(\"\\n\" + \"=\" * 78)\nprint(f\"{'ID':>5} {'STATUS':<8} {'PREDICTED':<22} {'EXPECTED':<22} DESCRIPTION\")\nprint(\"=\" * 78)\nfor r in results:\n icon = {\"exact\": \"PASS\", \"adjacent\": \"PASS*\", \"fail\": \"FAIL\", \"error\": \"ERR\"}[r[\"status\"]]\n print(f\"{r['id']:>5} {icon:<8} {(r['predicted'] or '-'):<22} {r['expected']:<22} {r.get('description','')[:25]}\")\n\n# Summary\nn_total = len(results)\nn_exact = sum(1 for r in results if r[\"status\"] == \"exact\")\nn_adj = sum(1 for r in results if r[\"status\"] == \"adjacent\")\nn_fail = sum(1 for r in results if r[\"status\"] == \"fail\")\nn_err = sum(1 for r in results if r[\"status\"] == \"error\")\nprint(\"=\" * 78)\nprint(f\"Summary: {n_exact} exact, {n_adj} adjacent (tolerated), {n_fail} fail, {n_err} error -- total {n_total}\")\nif _ALLOW_ADJ:\n print(\"(Adjacency tolerance ON: predictions within +/-1 rubric step of the expected label count as PASS*.)\")" - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 4 -}