Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/src/agent/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def reflection_router(state: OverallState) -> list[Send] | str:
)
builder.add_edge("denoising_refiner", END)

# TODO(priority=High, complexity=Medium): [SOTA Deep Research] Graph Wiring
# TODO(priority=High, complexity=Medium, owner=agent): [SOTA Deep Research] Graph Wiring
# Add conditional edges to route from 'reflection' or 'update_plan' to 'research_subgraph'.
# research_subgraph results should then flow back into 'update_plan' or merge into the state.

Expand Down
12 changes: 6 additions & 6 deletions backend/src/agent/mcp_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,26 +47,26 @@ def validate(settings: MCPSettings) -> None:

# Fine-grained implementation guide for MCP Integration:
#
# TODO(priority=High, complexity=Low): [MCP:1] Define SSE client interface
# TODO(priority=High, complexity=Low, owner=agent): [MCP:1] Define SSE client interface
# - Create abstract base class for MCP transport
# - Define methods: connect(), disconnect(), send_message(), receive_stream()
#
# TODO(priority=High, complexity=Medium): [MCP:2] Implement SSE transport
# TODO(priority=High, complexity=Medium, owner=agent): [MCP:2] Implement SSE transport
# - Use httpx or aiohttp for Server-Sent Events
# - Handle reconnection with exponential backoff
# - Parse SSE event format (event:, data:, id:)
#
# TODO(priority=Medium, complexity=Medium): [MCP:3] Connection pooling
# TODO(priority=Medium, complexity=Medium, owner=agent): [MCP:3] Connection pooling
# - Maintain pool of persistent connections
# - Implement health checks and automatic reconnection
# - Thread-safe connection acquisition/release
#
# TODO(priority=Medium, complexity=Low): [MCP:4] Error recovery
# TODO(priority=Medium, complexity=Low, owner=agent): [MCP:4] Error recovery
# - Catch and log transport errors
# - Retry failed tool calls with backoff
# - Return graceful fallback on persistent failure
#
# TODO(priority=Low, complexity=Low): [MCP:5] Metrics and observability
# TODO(priority=Low, complexity=Low, owner=agent): [MCP:5] Metrics and observability
# - Track connection latency, success/failure rates
# - Integrate with Langfuse spans
class McpConnectionManager:
Expand Down Expand Up @@ -94,7 +94,7 @@ def get_persistence_tools(self) -> List:
]

async def get_tools(self):
# TODO(priority=High, complexity=Medium): [MCP:6] Implement actual SSE tool discovery
# TODO(priority=High, complexity=Medium, owner=agent): [MCP:6] Implement actual SSE tool discovery
# - Connect to MCP endpoint from settings
# - Fetch tool list via SSE stream
# - Convert to LangChain StructuredTool format
Expand Down
20 changes: 10 additions & 10 deletions backend/src/agent/nodes.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# TODO(priority=Low, complexity=Low): See docs/tasks/upstream_compatibility.md for future splitting of this file into _nodes.py (upstream) and nodes.py (evolved).
# TODO(priority=Low, complexity=Low, owner=agent): See docs/tasks/upstream_compatibility.md for future splitting of this file into _nodes.py (upstream) and nodes.py (evolved).
#
# TODO(priority=Medium, complexity=Medium): [SOTA Deep Research] Benchmarking
# TODO(priority=Medium, complexity=Medium, owner=agent): [SOTA Deep Research] Benchmarking
# See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md
# Subtask: MLE-bench Integration (Evaluate on Kaggle engineering tasks).
# Subtask: DeepResearch-Bench Setup (Load tasks from muset-ai space).

# TODO(priority=Medium, complexity=High): Investigate and integrate 'deepagents' patterns if applicable.
# TODO(priority=Medium, complexity=High, owner=agent): Investigate and integrate 'deepagents' patterns if applicable.
# See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md
# Subtask: Review 'deepagents' repo for relevant nodes (e.g. hierarchical planning).
# Subtask: Adapt useful patterns to `backend/src/agent/nodes.py`.
Expand Down Expand Up @@ -151,7 +151,7 @@ def scoping_node(state: OverallState, config: RunnableConfig) -> OverallState:
If yes -> Generates questions and sets status to 'active' (interrupt).
If no -> Sets status to 'complete' (proceed).

TODO(priority=High, complexity=High): [SOTA Deep Research] Verify full alignment with Open Deep Research (Clarification Loop).
TODO(priority=High, complexity=High, owner=agent): [SOTA Deep Research] Verify full alignment with Open Deep Research (Clarification Loop).
See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md
Subtask: Implement `scoping_node` logic: Analyze input query. If ambiguous, generate clarifying questions and interrupt graph.
"""
Expand Down Expand Up @@ -1035,25 +1035,25 @@ def flow_update(state: OverallState, config: RunnableConfig) -> OverallState:

Fine-grained implementation guide:

TODO(priority=High, complexity=Low): [flow_update:1] Extract current task from state
TODO(priority=High, complexity=Low, owner=agent): [flow_update:1] Extract current task from state
- Read `current_task_idx` and `plan` from state
- Get the task object being evaluated

TODO(priority=High, complexity=Medium): [flow_update:2] Analyze task completion
TODO(priority=High, complexity=Medium, owner=agent): [flow_update:2] Analyze task completion
- Compare task query against `web_research_result`
- Use fuzzy matching or LLM to determine if task is adequately answered
- Return completion_score (0.0-1.0)

TODO(priority=High, complexity=Medium): [flow_update:3] Identify knowledge gaps
TODO(priority=High, complexity=Medium, owner=agent): [flow_update:3] Identify knowledge gaps
- Parse research results for "unclear", "contradictory", or "insufficient" signals
- Generate list of follow-up questions if gaps detected

TODO(priority=Medium, complexity=High): [flow_update:4] DAG expansion logic
TODO(priority=Medium, complexity=High, owner=agent): [flow_update:4] DAG expansion logic
- If gaps detected: Create new tasks and insert into plan
- If task complete: Mark status='done' and increment current_task_idx
- If no more tasks: Set research_complete=True

TODO(priority=Low, complexity=Low): [flow_update:5] Return updated state
TODO(priority=Low, complexity=Low, owner=agent): [flow_update:5] Return updated state
- Return dict with updated `plan`, `current_task_idx`, `research_complete`

See docs/tasks/04_SOTA_DEEP_RESEARCH_TASKS.md
Expand Down Expand Up @@ -1157,7 +1157,7 @@ def content_reader(state: OverallState, config: RunnableConfig) -> OverallState:
return {"evidence_bank": extracted_evidence}


# TODO(priority=High, complexity=Medium): [SOTA Deep Research] Recursive Trigger
# TODO(priority=High, complexity=Medium, owner=agent): [SOTA Deep Research] Recursive Trigger
# Implement logic in reflection or a new 'router' node to decide when to call 'research_subgraph'.
# This should happen when a complex sub-topic is identified that requires its own full research loop.
def research_subgraph(state: OverallState, config: RunnableConfig) -> OverallState:
Expand Down
2 changes: 1 addition & 1 deletion backend/src/agent/rag.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,7 +537,7 @@ class Resource:
def create_rag_tool(resources):
"""Legacy compatibility stub - returns None.

TODO(priority=Low, complexity=Medium): [rag:legacy] Replace stub with real implementation
TODO(priority=Low, complexity=Medium, owner=agent): [rag:legacy] Replace stub with real implementation
- Migrate callers to use DeepSearchRAG directly
- Remove this function once all callers are updated
- Update tests that mock this function
Expand Down
24 changes: 12 additions & 12 deletions backend/src/evaluation/deep_research_bench.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
# Fine-grained implementation guide for DeepResearch-Bench Evaluation:
#
# TODO(priority=High, complexity=Low): [deep_bench:1] Dataset loader
# TODO(priority=High, complexity=Low, owner=agent): [deep_bench:1] Dataset loader
# - Connect to muset-ai/DeepResearch-Bench on HuggingFace
# - Implement load_deep_research_dataset() -> List[Task]
# - Each Task: {id, query, gold_report, evaluation_criteria}
#
# TODO(priority=High, complexity=Medium): [deep_bench:2] Agent runner
# TODO(priority=High, complexity=Medium, owner=agent): [deep_bench:2] Agent runner
# - Import graph from agent.graph
# - Configure for full research mode (scoping -> planning -> research -> synthesis)
# - Capture final report and all intermediate artifacts
#
# TODO(priority=Medium, complexity=High): [deep_bench:3] Report scorer
# TODO(priority=Medium, complexity=High, owner=agent): [deep_bench:3] Report scorer
# - Compare generated report against gold_report
# - Use metrics: ROUGE-L, BERTScore, factual accuracy (via NLI)
# - Return composite score (0.0-1.0)
#
# TODO(priority=Medium, complexity=Medium): [deep_bench:4] Citation verifier
# TODO(priority=Medium, complexity=Medium, owner=agent): [deep_bench:4] Citation verifier
# - Check that all claims are backed by sources
# - Verify source URLs are valid and content matches claims
# - Return citation_coverage score
#
# TODO(priority=Medium, complexity=Low): [deep_bench:5] Metrics aggregator
# TODO(priority=Medium, complexity=Low, owner=agent): [deep_bench:5] Metrics aggregator
# - Aggregate scores across all tasks
# - Compute mean, std, percentiles
# - Track token usage and latency
#
# TODO(priority=Low, complexity=Low): [deep_bench:6] Report generator
# TODO(priority=Low, complexity=Low, owner=agent): [deep_bench:6] Report generator
# - Output results to JSON and Markdown
# - Generate comparison charts (if multiple runs)
#
Expand All @@ -34,30 +34,30 @@

def evaluate_deep_research():
"""Evaluates the agent on DeepResearch-Bench (muset-ai)."""
# TODO(priority=High, complexity=Low): [deep_bench:1] Load dataset
# TODO(priority=High, complexity=Low, owner=agent): [deep_bench:1] Load dataset
dataset = [] # load_deep_research_dataset()

# TODO(priority=High, complexity=Medium): [deep_bench:2] Run agent
# TODO(priority=High, complexity=Medium, owner=agent): [deep_bench:2] Run agent
results = []
for task in dataset:
# report = run_full_research(task.query)
# results.append({"task_id": task.id, "report": report})
_ = task # placeholder until implementation is complete

# TODO(priority=Medium, complexity=High): [deep_bench:3] Score reports
# TODO(priority=Medium, complexity=High, owner=agent): [deep_bench:3] Score reports
scores = []
# for result in results:
# score = score_report(result["report"], gold_report)
# scores.append(score)

# TODO(priority=Medium, complexity=Medium): [deep_bench:4] Verify citations
# TODO(priority=Medium, complexity=Medium, owner=agent): [deep_bench:4] Verify citations
# for result in results:
# citation_score = verify_citations(result["report"])

# TODO(priority=Medium, complexity=Low): [deep_bench:5] Aggregate
# TODO(priority=Medium, complexity=Low, owner=agent): [deep_bench:5] Aggregate
# mean_score = sum(scores) / len(scores) if scores else 0

# TODO(priority=Low, complexity=Low): [deep_bench:6] Report
# TODO(priority=Low, complexity=Low, owner=agent): [deep_bench:6] Report
print("DeepResearch-Bench evaluation not yet implemented")


Expand Down
20 changes: 10 additions & 10 deletions backend/src/evaluation/mle_bench.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
# Fine-grained implementation guide for MLE-bench Evaluation:
#
# TODO(priority=High, complexity=Low): [mle_bench:1] Dataset loader
# TODO(priority=High, complexity=Low, owner=agent): [mle_bench:1] Dataset loader
# - Define path to MLE-bench dataset (HuggingFace or local)
# - Implement load_mle_dataset() -> List[Task]
# - Each Task: {id, prompt, expected_output, metadata}
#
# TODO(priority=High, complexity=Medium): [mle_bench:2] Agent runner
# TODO(priority=High, complexity=Medium, owner=agent): [mle_bench:2] Agent runner
# - Import graph from agent.graph
# - Run graph.invoke({"messages": [task.prompt]})
# - Capture final output and execution time
#
# TODO(priority=Medium, complexity=Medium): [mle_bench:3] Output evaluator
# TODO(priority=Medium, complexity=Medium, owner=agent): [mle_bench:3] Output evaluator
# - Compare agent output against expected_output
# - Implement exact_match, fuzzy_match, and llm_judge scoring
# - Return score (0.0-1.0) per task
#
# TODO(priority=Medium, complexity=Low): [mle_bench:4] Metrics aggregator
# TODO(priority=Medium, complexity=Low, owner=agent): [mle_bench:4] Metrics aggregator
# - Compute Pass@1 (% tasks with score >= threshold)
# - Compute average score across all tasks
# - Track latency percentiles (p50, p95, p99)
#
# TODO(priority=Low, complexity=Low): [mle_bench:5] Report generator
# TODO(priority=Low, complexity=Low, owner=agent): [mle_bench:5] Report generator
# - Output results to JSON and Markdown
# - Include per-task breakdown and aggregate stats
#
Expand All @@ -29,27 +29,27 @@

def evaluate_mle_bench():
"""Evaluates the agent on MLE-bench tasks."""
# TODO(priority=High, complexity=Low): [mle_bench:1] Load dataset
# TODO(priority=High, complexity=Low, owner=agent): [mle_bench:1] Load dataset
dataset = [] # load_mle_dataset()

# TODO(priority=High, complexity=Medium): [mle_bench:2] Run agent
# TODO(priority=High, complexity=Medium, owner=agent): [mle_bench:2] Run agent
results = []
for task in dataset:
# output = run_agent(task.prompt)
# results.append({"task_id": task.id, "output": output})
_ = task # placeholder until implementation is complete

# TODO(priority=Medium, complexity=Medium): [mle_bench:3] Evaluate
# TODO(priority=Medium, complexity=Medium, owner=agent): [mle_bench:3] Evaluate
scores = []
# for result in results:
# score = evaluate_output(result["output"], ...)
# scores.append(score)

# TODO(priority=Medium, complexity=Low): [mle_bench:4] Aggregate
# TODO(priority=Medium, complexity=Low, owner=agent): [mle_bench:4] Aggregate
# pass_at_1 = sum(1 for s in scores if s >= 0.5) / len(scores)
# avg_score = sum(scores) / len(scores)

# TODO(priority=Low, complexity=Low): [mle_bench:5] Report
# TODO(priority=Low, complexity=Low, owner=agent): [mle_bench:5] Report
print("MLE-bench evaluation not yet implemented")


Expand Down
8 changes: 4 additions & 4 deletions backend/tests/test_mcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@

# Fine-grained implementation guide for MCP Tests:
#
# TODO(priority=Medium, complexity=Low): [test_mcp:1] Test disabled MCP returns empty list
# TODO(priority=Medium, complexity=Low, owner=agent): [test_mcp:1] Test disabled MCP returns empty list
# - Create MCPSettings with enabled=False
# - Verify get_tools_from_mcp returns []
#
# TODO(priority=Medium, complexity=Medium): [test_mcp:2] Test connection error handling
# TODO(priority=Medium, complexity=Medium, owner=agent): [test_mcp:2] Test connection error handling
# - Mock SSEConnection to raise ConnectionError
# - Verify graceful fallback (empty list, logged warning)
#
# TODO(priority=Medium, complexity=Medium): [test_mcp:3] Test tool whitelist filtering
# TODO(priority=Medium, complexity=Medium, owner=agent): [test_mcp:3] Test tool whitelist filtering
# - Load multiple tools from mock MCP
# - Set tool_whitelist to subset
# - Verify only whitelisted tools returned
#
# TODO(priority=Low, complexity=Medium): [test_mcp:4] Test tool execution with real MCP server
# TODO(priority=Low, complexity=Medium, owner=agent): [test_mcp:4] Test tool execution with real MCP server
# - Skip if MCP_ENDPOINT not set (integration test)
# - Connect to real server, call a tool, verify response format
#
Expand Down
2 changes: 1 addition & 1 deletion docs/benchmarks/PLAN.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# TODO(priority=High, complexity=Large): Benchmarking & Evaluation Framework
# TODO(priority=High, complexity=Large, owner=agent): Benchmarking & Evaluation Framework

We need to implement a systematic evaluation framework to measure improvements in report quality, relevance, and accuracy.

Expand Down
4 changes: 2 additions & 2 deletions scripts/extract_todos_structured.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def extract_todos(root_dir):
dirs[:] = [d for d in dirs if d not in exclude_dirs]

for file in files:
if file.endswith(('.py', '.tsx', '.ts', '.js', '.jsx', '.md')):
if file.endswith(('.py', '.tsx', '.ts', '.js', '.jsx', '.md')) and file != 'extract_todos_structured.py':
filepath = os.path.join(root, file)
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
Expand All @@ -22,7 +22,7 @@ def extract_todos(root_dir):
# Simple parser
content = line.strip()
# Try to parse structured TODOs if they exist
# Format: TODO(priority=<Level>, complexity=<Level>):
# Format: TODO(priority=<Level>, complexity=<Level>, owner=<Owner>):
priority = "Unknown"
complexity = "Unknown"

Expand Down
Loading
Loading