Spaces:
Running
Running
File size: 5,231 Bytes
f9e2361 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | """
BaseResearchAgent - The standard interface for DR-Bench agents.
All agents submitted to the benchmark must implement this interface.
The benchmark framework provides standardized LLM and WebSearch clients
to ensure fair comparison (same model, same search provider for all agents).
The agent's job is purely ORCHESTRATION:
- Deciding what to search for
- How to decompose the research topic
- How to synthesize search results into a final report
- How many iterations/turns to use
Example minimal agent:
class MyAgent(BaseResearchAgent):
@property
def name(self) -> str:
return "my-simple-agent"
@property
def description(self) -> str:
return "Simple single-pass research agent"
@property
def author(self) -> str:
return "Jane Doe"
async def research(self, topic, llm, websearch, **kwargs):
# 1. Generate search queries
queries_response = await llm.chat.completions.create(
model=self.model_name,
messages=[{"role": "user", "content": f"Generate 3 search queries for: {topic}"}],
)
queries = queries_response.choices[0].message.content.split("\\n")
# 2. Search
results = await websearch.search(queries[:3], **kwargs)
# 3. Synthesize
synthesis = await llm.chat.completions.create(
model=self.model_name,
messages=[
{"role": "system", "content": "Synthesize search results into a report."},
{"role": "user", "content": f"Topic: {topic}\\n\\nSearch Results:\\n{results}"},
],
)
return ResearchOutput(
report=synthesis.choices[0].message.content,
searches_made=[{"queries": queries[:3]}],
)
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Optional, TYPE_CHECKING
if TYPE_CHECKING:
from openai import AsyncOpenAI
from benchmark.websearch import BenchmarkWebSearchClient
@dataclass
class ResearchOutput:
"""Output from a research agent."""
report: str
searches_made: list[dict[str, Any]] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
class BaseResearchAgent(ABC):
"""
Abstract base class for all DR-Bench research agents.
Agents receive:
- An AsyncOpenAI client (connected to the benchmark's standardized LLM)
- A BenchmarkWebSearchClient (connected to the benchmark's Brightdata-based search)
- The model name to use with the LLM client
Agents must implement:
- name: Agent identifier (alphanumeric + hyphens)
- description: Short description of the methodology
- author: Author name / team
- research(): The core research method
Agents may optionally import any Python libraries they need for their
orchestration logic (LangGraph, DSPy, custom code, etc.), as long as
all LLM calls go through the provided client and all web searches go
through the provided websearch client.
"""
def __init__(self, model_name: str = "gpt-4o"):
"""
Initialize the agent.
Args:
model_name: The LLM model name to use (provided by the benchmark).
All agents use the same model for fairness.
"""
self.model_name = model_name
@property
@abstractmethod
def name(self) -> str:
"""Unique identifier for this agent (alphanumeric + hyphens, e.g. 'my-agent-v1')."""
...
@property
@abstractmethod
def description(self) -> str:
"""Short description of the agent's methodology."""
...
@property
@abstractmethod
def author(self) -> str:
"""Author name or team."""
...
@abstractmethod
async def research(
self,
topic: str,
llm: AsyncOpenAI,
websearch: BenchmarkWebSearchClient,
*,
start_date: Optional[str] = None,
end_date: Optional[str] = None,
**kwargs: Any,
) -> ResearchOutput:
"""
Run research on a given topic.
Args:
topic: The research prompt / topic to investigate.
llm: AsyncOpenAI client - use for all LLM calls.
Call with: await llm.chat.completions.create(model=self.model_name, ...)
websearch: WebSearch client - use for all web searches.
Call with: await websearch.search(queries, start_date=..., end_date=...)
start_date: Optional search date filter start (YYYY-MM-DD).
end_date: Optional search date filter end (YYYY-MM-DD).
**kwargs: Additional benchmark-provided parameters.
Returns:
ResearchOutput with the research report and metadata.
"""
...
def get_info(self) -> dict[str, str]:
"""Return agent metadata."""
return {
"name": self.name,
"description": self.description,
"author": self.author,
"model_name": self.model_name,
}
|