from __future__ import annotations
import os
import re
from smolagents import (
ToolCallingAgent,
OpenAIModel,
PythonInterpreterTool,
FinalAnswerTool,
)
from gaia_tools import build_tools
import gemini_limits as limits
import providers
import vertex_auth
AGENT_INSTRUCTIONS = r"""
You are a high-reliability solver for the Hugging Face Agents Course Unit 4
GAIA benchmark (Level 1 questions). Answers are scored by EXACT MATCH, so the
final answer must be short, precise and in the format the question asks for.
GENERAL RULES
1. Read the entire question first. Note exactly what is asked (a number, a
name, a list, a single word, a move), any range (inclusive?), order, units
and formatting requirements.
2. Pick the right tool for the job (playbook below). Never repeat a call that
already failed or returned nothing: change the query, the tool or the source.
3. Do not stop at the first plausible answer: sanity-check it against the
question (right entity, right year, right range, right unit).
4. Do arithmetic, counting, sorting and filtering in python_interpreter, not in
your head. List the items explicitly before counting them.
5. You have a limited number of steps. Be efficient: one good tool call beats
three vague ones.
PLAYBOOK BY QUESTION TYPE
- English Wikipedia facts, lists, discographies, tables, statistics: call
wikipedia_page(title, year?, section?). Use year=2022 when the question says
"2022 version"/"as of 2022"; use section="Discography" etc. to jump to a table.
Wikipedia project pages also work (e.g. "Wikipedia:Featured articles promoted
in 2016", "Wikipedia:Featured article candidates/
/archive1"). Use
wikipedia_search to find the exact title.
- Hard web lookups (papers, awards, grant numbers, rosters, obscure facts), or
when web_search returns irrelevant pages: use google_search with a specific
question, then open a cited source with visit_webpage to confirm.
web_search: plain keywords (no site: operators); visit_webpage: any URL.
- YouTube: youtube_transcript for spoken words; analyze_youtube_video for
anything visual (counting things on screen) or when no transcript exists, or
when you must hear the audio. Never web-search a video ID.
- Attachments (the task lists a file): ALWAYS call inspect_attachment first.
* image: read the extracted text. CHESS: take the FEN it prints, call
chess_best_move, and answer with the move in algebraic notation for the
correct side to move (double-check the FEN piece count and orientation).
* audio (.mp3): inspect_attachment returns a transcript; extract exactly what
is asked (e.g. only ingredients, sorted, without quantities).
* .py: call run_attached_python to get the real output; if it is random or
fails, reason from the source.
* .xlsx/.csv: the dump shows the data and its local path; compute with
python_interpreter (pandas is available: pd.read_excel(path)); respect
exclusions in the question (e.g. "not including drinks").
- Text that looks reversed, scrambled or encoded: decode it with Python first
(e.g. text[::-1]), then answer the decoded question literally (if it asks for
the opposite of a word, answer just that word).
- Tables or rules given in the question (e.g. an operation table): copy them
into Python and check every case by brute force.
- Classification questions (e.g. botany): apply the strict definition the
question implies (botanical fruit vs. culinary vegetable), go item by item,
then format as asked.
- python_interpreter has NO internet; use web tools for lookups.
ANSWER FORMAT (exact match!)
- Call final_answer with ONLY the answer: no "Final Answer:", no explanation,
no citations, no Markdown, no quotation marks, no trailing period.
- A number: digits only (no thousands separators, no units) unless the question
asks for units or a specific format.
- A list: comma-separated, in the order/sorting the question asks (alphabetical
if asked), no brackets, no "and".
- Names: give exactly the part asked (e.g. only the last name, or the full
name). Use the spelling used by the source. Words in lowercase unless they
are proper names.
- If after real effort you cannot verify the answer, still give your single
best answer (never say "unknown" or explain).
"""
def _clean_answer(value: object) -> str:
if value is None:
return ""
text = str(value).strip()
# Remove accidental Markdown fences.
if text.startswith("```") and text.endswith("```"):
lines = text.splitlines()
if len(lines) >= 2:
text = "\n".join(lines[1:-1]).strip()
# Remove common wrapper labels.
text = re.sub(
r"^\s*(FINAL ANSWER|Final Answer|Answer|Final)\s*:\s*",
"",
text,
flags=re.IGNORECASE,
)
text = text.strip().strip('"').strip("'")
# Step-limit / tool-call leftovers are not answers.
if text.lower().startswith(("calling tools:", "error:", "agent_error:")):
return ""
# A single trailing period on a short answer ("Right.") breaks exact match.
if text.endswith(".") and text.count(".") == 1 and len(text.split()) <= 8:
text = text[:-1].rstrip()
return text
class RouterModel(OpenAIModel):
"""One model object that fails over across providers AND models.
Before every request it picks the first endpoint (provider + model) with
spare per-minute budget. On 429 / daily-quota / rejected key / provider
quirks the endpoint is parked or dropped and the SAME conversation simply
continues on the next one, instead of restarting the question.
"""
def __init__(self, endpoints, **kwargs):
self.endpoints = {e.key: e for e in endpoints}
self._clients = {}
self._last_key = None
first = endpoints[0]
# retry=False: smolagents' built-in retry just sleeps and hammers the
# same rate-limited model; failover is handled here.
super().__init__(
model_id=first.model, api_base=first.base_url,
api_key=first.api_key, retry=False, **kwargs,
)
def _client_for(self, endpoint):
if endpoint.provider == "gemini":
# Vertex tokens expire; obtain a fresh ADC token for every request.
return vertex_auth.openai_client()
import openai
if endpoint.provider not in self._clients:
self._clients[endpoint.provider] = openai.OpenAI(
api_key=endpoint.api_key, base_url=endpoint.base_url,
max_retries=0, # the SDK's hidden sleep-and-retry is unwanted
timeout=120,
)
return self._clients[endpoint.provider]
def generate(self, messages, *args, **kwargs):
# Providers with tiny per-minute token budgets (e.g. Groq free) are
# skipped for big prompts instead of failing with 413/429.
est_tokens = len(str(messages)) / 3.5
keys = [
k for k, e in self.endpoints.items()
if not (e.max_prompt_tokens and est_tokens > e.max_prompt_tokens)
] or list(self.endpoints)
last_exc = None
for _ in range(len(keys) * 6):
key = limits.acquire(keys) # may raise limits.QuotaExhausted
ep = self.endpoints[key]
self.client = self._client_for(ep)
self.model_id = ep.model
# Non-Gemini providers are happiest with plain-string content.
self.flatten_messages_as_text = ep.provider != "gemini"
if key != self._last_key:
print(f"[LLM] using {key}")
self._last_key = key
try:
return super().generate(messages, *args, **kwargs)
except Exception as exc:
last_exc = exc
if not limits.report_error(key, exc):
raise
raise last_exc
def _log_step(step, agent=None):
"""Print what each step did so failures are visible (verbosity is off)."""
try:
calls = ", ".join(
f"{tc.name}({str(tc.arguments)[:120]})"
for tc in (getattr(step, "tool_calls", None) or [])
)
obs = str(getattr(step, "observations", "") or "").replace("\n", " ")
err = getattr(step, "error", None)
print(f"[STEP {step.step_number}] {calls or '(no tool call)'}")
if err:
print(f" ERROR: {str(err)[:200]}")
elif obs:
print(f" -> {obs[:200]}")
except Exception:
pass
def _find_quota_error(exc):
seen = set()
while exc is not None and id(exc) not in seen:
if isinstance(exc, limits.QuotaExhausted):
return exc
seen.add(id(exc))
exc = exc.__cause__ or exc.__context__
return None
class GaiaAgent:
def __init__(self, task_map):
self.task_map = task_map
# Raises a clear error if no provider key is configured.
self.endpoints = providers.build_endpoints()
names = ", ".join(dict.fromkeys(e.provider for e in self.endpoints))
print(f"[LLM] providers in order: {names} "
f"({len(self.endpoints)} model endpoints)")
def _build_agent(self):
model = RouterModel(
self.endpoints,
temperature=0.0,
max_tokens=2500,
)
tools = build_tools(self.task_map)
tools.append(
PythonInterpreterTool(
authorized_imports=[
"math", "statistics", "datetime", "json", "re", "csv",
"collections", "itertools", "fractions", "decimal",
"string", "unicodedata", "random", "time",
"pandas", "pandas.*", "numpy", "numpy.*", "openpyxl",
"openpyxl.*",
]
)
)
tools.append(FinalAnswerTool())
return ToolCallingAgent(
tools=tools,
model=model,
instructions=AGENT_INSTRUCTIONS,
max_steps=int(os.getenv("MAX_STEPS", "12")),
planning_interval=None,
verbosity_level=0,
step_callbacks=[_log_step],
)
def solve(self, question, task_id, file_name=""):
prompt = f"""
GAIA TASK ID:
{task_id}
ATTACHMENT:
{file_name or "None"}
USER QUESTION:
{question}
Solve this task accurately.
Use tools when needed.
Verify important facts.
Follow the requested output format exactly.
When finished, return ONLY the answer.
Do not include explanations, citations, Markdown, or "Final Answer:".
"""
agent = self._build_agent()
try:
result = agent.run(prompt)
except Exception as exc:
# smolagents wraps model errors (AgentGenerationError), so look
# through the exception chain for our own quota error.
quota = _find_quota_error(exc)
if quota is not None:
print(f"[AGENT] out of quota: {quota}")
return f"AGENT_ERROR: QuotaExhausted: {quota}"
print(f"[AGENT] failed: {type(exc).__name__}: {exc}")
return f"AGENT_ERROR: {type(exc).__name__}: {exc}"
answer = _clean_answer(result)
if answer:
return answer
# Ran out of steps: ask once, without tools, for a best answer based
# on everything gathered so far.
try:
messages = agent.write_memory_to_messages()
messages.append({
"role": "user",
"content": [{"type": "text", "text": (
"You are out of steps. Using only what you found so far "
"(and your own knowledge if needed), reply with your best "
"final answer ONLY: no explanation, no tool calls."
)}],
})
reply = agent.model.generate(messages)
answer = _clean_answer(reply.content)
if answer:
print("[AGENT] step limit reached; used best-effort answer.")
return answer
except Exception as exc:
print(f"[AGENT] rescue failed: {type(exc).__name__}: {exc}")
return "AGENT_ERROR: no answer produced (step limit reached or tool failures)"