Upload folder using huggingface_hub

agent.py

@@ -1,6 +1,12 @@
-"""GAIA agent — the part you own. Edit the tools and the prompt to raise your score."""
+"""GAIA agent for the Hugging Face Agents Course, Unit 4.
+
+Answers questions from a 20-question subset of the GAIA benchmark. The provider is
+selected at runtime from whichever API key is present, because the free tiers differ
+sharply in what they allow — see make_model() for the measured limits of each.
+"""
 
 import os
+import time
 
 from smolagents import (
     CodeAgent,
@@ -10,6 +16,7 @@ from smolagents import (
     PythonInterpreterTool,
     InferenceClientModel,
     LiteLLMModel,
+    OpenAIServerModel,
     tool,
 )
 
@@ -17,19 +24,69 @@ from smolagents import (
 # Groq is preferred: roughly 10x faster than Gemini's thinking model and it follows
 # the CodeAgent format cleanly. GEMINI_API_KEY is still worth setting alongside it —
 # the vision and audio tools in gaia_tools.py call Gemini directly regardless.
-GROQ_MODEL = "groq/llama-3.3-70b-versatile"
-GEMINI_MODEL = "gemini/gemini-flash-latest"
+NVIDIA_MODEL_ID = "nvidia/nemotron-3-ultra-550b-a55b"
+GROQ_MODEL_ID = "llama-3.3-70b-versatile"
+# Model choice is quota-driven as much as quality-driven — check what actually has
+# headroom before assuming. On this key: flash-latest and 3.6-flash return 429,
+# 2.5-flash is 404, 3-flash-preview works but is 4x slower. gemini-3.5-flash answered
+# a multi-hop research question correctly in 6.3s and is the strongest available.
+# flash-lite is the fallback: fast, but too weak to chain several lookups.
+GEMINI_MODEL_ID = "gemini-3.5-flash"
 HF_MODEL_ID = "Qwen/Qwen2.5-Coder-32B-Instruct"
 
 
 def make_model():
     """Returns a model backed by whichever provider has a key configured."""
+    if os.getenv("NVIDIA_API_KEY"):
+        # Preferred. NVIDIA removed its credit cap, so this is 40 requests/minute with
+        # no daily token ceiling — the only provider tried that can run the full set
+        # repeatedly. Groq is faster per call but dies after ~100K tokens/day; Gemini's
+        # stronger models cap at 5 RPM. Slower per call, but it actually finishes.
+        print(f"[model] {NVIDIA_MODEL_ID} (nim)")
+        return OpenAIServerModel(
+            model_id=NVIDIA_MODEL_ID,
+            api_base="https://integrate.api.nvidia.com/v1",
+            api_key=os.environ["NVIDIA_API_KEY"],
+            temperature=0.2,
+            requests_per_minute=35,
+            retry=False,
+            client_kwargs={"max_retries": 0, "timeout": 120},
+        )
     if os.getenv("GROQ_API_KEY"):
-        print(f"[model] {GROQ_MODEL}")
-        return LiteLLMModel(model_id=GROQ_MODEL, temperature=0.2)
+        # Groq through LiteLLM stalls for minutes per call; the same request through
+        # the OpenAI SDK returns in under a second. Groq is OpenAI-compatible, so go
+        # direct. Measured 0.7s vs several minutes on identical prompts.
+        print(f"[model] groq/{GROQ_MODEL_ID} (direct)")
+        # retry=False is the important one. smolagents wraps every model call in its
+        # own exponential-backoff retryer (smolagents/utils.py Retrying) that catches
+        # rate-limit errors and sleeps silently with growing delays — no log line, no
+        # traceback, just a process using zero CPU. That is what every "hang" today
+        # actually was. client_kwargs disables the OpenAI SDK's separate retry layer.
+        return OpenAIServerModel(
+            model_id=GROQ_MODEL_ID,
+            api_base="https://api.groq.com/openai/v1",
+            api_key=os.environ["GROQ_API_KEY"],
+            temperature=0.2,
+            retry=False,
+            client_kwargs={"max_retries": 0, "timeout": 45},
+        )
     if os.getenv("GEMINI_API_KEY"):
-        print(f"[model] {GEMINI_MODEL}")
-        return LiteLLMModel(model_id=GEMINI_MODEL, temperature=0.2)
+        # Google exposes an OpenAI-compatible endpoint, so Gemini gets the same fast
+        # path as Groq — no LiteLLM, no retry loop. Measured 0.9s per call and 1.4s
+        # for a full agent run. Gemini was never slow; the retry wrapper was.
+        print(f"[model] {GEMINI_MODEL_ID} (openai-compat)")
+        return OpenAIServerModel(
+            model_id=GEMINI_MODEL_ID,
+            api_base="https://generativelanguage.googleapis.com/v1beta/openai/",
+            api_key=os.environ["GEMINI_API_KEY"],
+            temperature=0.2,
+            # gemini-3.5-flash allows only 5 requests/minute. It is much stronger than
+            # flash-lite, so pace to the limit rather than dropping to a weaker model:
+            # smolagents' own limiter spaces calls out instead of firing into a 429.
+            requests_per_minute=5,
+            retry=False,
+            client_kwargs={"max_retries": 0, "timeout": 60},
+        )
     print(f"[model] {HF_MODEL_ID} (HF free tier — expect quota errors)")
     return InferenceClientModel(model_id=HF_MODEL_ID, max_tokens=2048, temperature=0.2)
 
@@ -47,6 +104,14 @@ Rules for your final answer:
 - If asked for a name, give just the name.
 
 Work step by step and use your tools before answering. Never guess if a tool can check.
+
+Read the question for the exact sense of its words. "Botanically a vegetable" is not
+the same as "sold as a vegetable" — a green bean, zucchini, pepper and tomato are all
+botanically fruits. "Studio album" excludes live albums and compilations. Where a
+question names a definition or a cutoff date, apply it literally.
+
+Be economical: prefer one targeted search over several broad ones, and stop as soon
+as you can support an answer.
 """
 
 
@@ -90,6 +155,53 @@ def wikipedia_page(title: str) -> str:
     return f"No Wikipedia article found for '{title}'."
 
 
+@tool
+def wikipedia_tables(title: str) -> str:
+    """Returns the TABLES from an English Wikipedia article as text. Use this whenever
+    a question needs data held in a table — discographies, rosters, medal counts,
+    award winners, years. wikipedia_page strips tables out and will show the section
+    heading with nothing under it, so reach for this tool instead.
+
+    Args:
+        title: The article title, e.g. 'Mercedes Sosa'.
+    """
+    import pandas as pd
+    import requests
+
+    try:
+        r = requests.get(
+            "https://en.wikipedia.org/w/api.php",
+            params={
+                "action": "parse",
+                "page": title,
+                "prop": "text",
+                "format": "json",
+                "redirects": 1,
+            },
+            headers={"User-Agent": "gaia-agent/1.0 (course exercise)"},
+            timeout=45,
+        )
+        html = r.json()["parse"]["text"]["*"]
+    except Exception as e:
+        return f"Could not fetch tables for '{title}': {e}"
+
+    try:
+        from io import StringIO
+
+        tables = pd.read_html(StringIO(html))
+    except ValueError:
+        return f"No tables found in the article '{title}'."
+    except Exception as e:
+        return f"Could not parse tables for '{title}': {e}"
+
+    out = []
+    for i, df in enumerate(tables[:12]):
+        if df.shape[0] < 2:  # skip infoboxes and one-row layout tables
+            continue
+        out.append(f"--- table {i} ({df.shape[0]} rows) ---\n{df.to_string(max_rows=60)}")
+    return "\n\n".join(out)[:25000] or f"No usable tables in '{title}'."
+
+
 def build_agent(max_steps: int = 8):
     """Constructs the agent. Add tools here to handle more question types.
 
@@ -102,25 +214,34 @@ def build_agent(max_steps: int = 8):
     from gaia_tools import ALL_TOOLS
 
     model = make_model()
+    # Tool output was capped hard when Groq's 100K/day was the binding constraint.
+    # On Gemini it isn't, and starving the model of search results costs accuracy on
+    # the multi-hop questions — so these are set for quality, not frugality.
     tools = [
-        DuckDuckGoSearchTool(),
-        VisitWebpageTool(),
+        DuckDuckGoSearchTool(max_results=6),
+        VisitWebpageTool(max_output_length=10000),
         PythonInterpreterTool(),
         reverse_text,
         wikipedia_page,
+        wikipedia_tables,
         *ALL_TOOLS,
     ]
 
-    if os.getenv("GEMINI_API_KEY") and not os.getenv("GROQ_API_KEY"):
+    # flash-lite is too weak to write reliable Python for CodeAgent — it returns raw
+    # code as its final answer. It handles JSON tool calls fine, so it gets
+    # ToolCallingAgent. Groq's llama-3.3-70b is strong enough for CodeAgent.
+    if os.getenv("NVIDIA_API_KEY") or (
+        os.getenv("GEMINI_API_KEY") and not os.getenv("GROQ_API_KEY")
+    ):
         return ToolCallingAgent(
-            model=model, tools=tools, max_steps=max_steps, verbosity_level=1
+            model=model, tools=tools, max_steps=max_steps, verbosity_level=0
         )
 
     return CodeAgent(
         model=model,
         tools=tools,
         max_steps=max_steps,
-        verbosity_level=1,
+        verbosity_level=0,
         additional_authorized_imports=["json", "re", "math", "datetime", "itertools"],
     )
 
@@ -128,7 +249,8 @@ def build_agent(max_steps: int = 8):
 class GaiaAgent:
     """Wrapper matching the interface app.py expects."""
 
-    def __init__(self):
+    def __init__(self, max_attempts: int = 2):
+        self.max_attempts = max_attempts
         self.agent = build_agent()
 
     def __call__(self, question: str, task_id: str = "", file_name: str = "") -> str:
@@ -149,8 +271,41 @@ class GaiaAgent:
                 f"then pass the returned path to {reader}."
             )
 
-        try:
-            return str(self.agent.run(task)).strip()
-        except Exception as e:
-            print(f"  [agent error] {type(e).__name__}: {e}")
-            return ""
+        # A single attempt returning "" is a guaranteed zero, so retry: transient
+        # rate limits and one-off tool failures are the usual cause, and a fresh
+        # agent with cleared memory often succeeds where the first run stalled.
+        last_error = ""
+        for attempt in range(1, self.max_attempts + 1):
+            try:
+                result = str(self.agent.run(task)).strip()
+                # smolagents returns internal state as the "answer" in several failure
+                # modes: raw tool-call text when max_steps runs out, and a serialised
+                # error object when the final generation fails. Both would be submitted
+                # verbatim and score zero, so treat them as failures and retry.
+                # The leaked shapes seen in practice: "Calling tools:" prose, a list of
+                # typed content dicts, a bare tool-call object, and a serialised error.
+                # Anything that looks like JSON or mentions a tool name is not an answer.
+                junk = (
+                    result.startswith("Calling tools:")
+                    or result.startswith(("[{", "{'", '{"'))
+                    or "'function':" in result
+                    or '"function"' in result
+                    or '"tool":' in result
+                    or "Error in generating" in result
+                )
+                if junk:
+                    last_error = f"agent returned internal state on attempt {attempt}"
+                elif result and result.lower() not in {"none", "unknown", "n/a"}:
+                    return result
+                else:
+                    last_error = f"empty result on attempt {attempt}"
+            except Exception as e:
+                last_error = f"{type(e).__name__}: {e}"
+
+            if attempt < self.max_attempts:
+                print(f"  [retry {attempt}/{self.max_attempts - 1}] {last_error[:120]}")
+                time.sleep(5 * attempt)
+                self.agent = build_agent()  # fresh memory for the next attempt
+
+        print(f"  [gave up] {last_error[:160]}")
+        return ""

gemini_model.py

@@ -0,0 +1,109 @@
+"""A smolagents Model that talks to the Gemini REST API directly.
+
+LiteLLM's Gemini path hangs for minutes per call once tools are attached, while the
+same request over plain HTTP returns in seconds. This class skips LiteLLM entirely.
+
+It targets CodeAgent-style usage: it returns plain text and lets smolagents parse the
+code block, rather than using Gemini's native function-calling schema.
+"""
+
+import os
+
+import requests
+from smolagents.models import ChatMessage, Model
+
+ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/models"
+
+
+def _text_of(content) -> str:
+    """smolagents passes content as either a string or a list of typed parts."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(
+            part.get("text", "") for part in content if isinstance(part, dict)
+        )
+    return str(content or "")
+
+
+class GeminiRestModel(Model):
+    """Direct Gemini REST client. Set GEMINI_API_KEY in the environment."""
+
+    def __init__(
+        self,
+        model_id: str = "gemini-flash-lite-latest",
+        temperature: float = 0.2,
+        max_output_tokens: int = 2048,
+        timeout: int = 120,
+        **kwargs,
+    ):
+        super().__init__(model_id=model_id, **kwargs)
+        self.temperature = temperature
+        self.max_output_tokens = max_output_tokens
+        self.timeout = timeout
+
+    def generate(
+        self,
+        messages,
+        stop_sequences=None,
+        response_format=None,
+        tools_to_call_from=None,
+        **kwargs,
+    ) -> ChatMessage:
+        key = os.getenv("GEMINI_API_KEY")
+        if not key:
+            raise RuntimeError("GEMINI_API_KEY is not set")
+
+        # Gemini has no system role: fold system turns into the first user turn,
+        # and collapse consecutive same-role turns, which it also rejects.
+        system_text, contents = [], []
+        for m in messages:
+            role = m.role if hasattr(m, "role") else m.get("role")
+            role = getattr(role, "value", role)
+            text = _text_of(m.content if hasattr(m, "content") else m.get("content"))
+            if not text:
+                continue
+            if role == "system":
+                system_text.append(text)
+                continue
+            gem_role = "model" if role == "assistant" else "user"
+            if contents and contents[-1]["role"] == gem_role:
+                contents[-1]["parts"][0]["text"] += "\n\n" + text
+            else:
+                contents.append({"role": gem_role, "parts": [{"text": text}]})
+
+        if system_text and contents:
+            contents[0]["parts"][0]["text"] = (
+                "\n\n".join(system_text) + "\n\n" + contents[0]["parts"][0]["text"]
+            )
+        elif system_text:
+            contents = [{"role": "user", "parts": [{"text": "\n\n".join(system_text)}]}]
+
+        config = {
+            "temperature": self.temperature,
+            "maxOutputTokens": self.max_output_tokens,
+        }
+        if stop_sequences:
+            config["stopSequences"] = list(stop_sequences)[:5]  # Gemini allows max 5
+
+        r = requests.post(
+            f"{ENDPOINT}/{self.model_id}:generateContent?key={key}",
+            json={"contents": contents, "generationConfig": config},
+            timeout=self.timeout,
+        )
+        if r.status_code != 200:
+            raise RuntimeError(f"Gemini {r.status_code}: {r.text[:300]}")
+
+        data = r.json()
+        try:
+            parts = data["candidates"][0]["content"]["parts"]
+            text = "".join(p.get("text", "") for p in parts)
+        except (KeyError, IndexError):
+            finish = data.get("candidates", [{}])[0].get("finishReason", "?")
+            raise RuntimeError(f"Gemini returned no text (finishReason={finish})")
+
+        usage = data.get("usageMetadata", {})
+        self.last_input_token_count = usage.get("promptTokenCount", 0)
+        self.last_output_token_count = usage.get("candidatesTokenCount", 0)
+
+        return ChatMessage(role="assistant", content=text, raw=data)

run_local.py

@@ -28,7 +28,13 @@ def main():
         questions = [q for q in questions if not q.get("file_name")]
     else:
         print("(including questions with file attachments)")
-    if limit:
+
+    # --only 3,5,6 keeps just those 1-indexed questions, in the order given, so a
+    # limited token budget goes to the questions most likely to land.
+    if "--only" in args:
+        picks = [int(n) for n in args[args.index("--only") + 1].split(",")]
+        questions = [questions[i - 1] for i in picks if 1 <= i <= len(questions)]
+    elif limit:
         questions = questions[:limit]
 
     print(f"Running {len(questions)} question(s)\n")

run_parallel.py

@@ -0,0 +1,90 @@
+"""Run GAIA questions concurrently. Each question gets its own agent instance.
+
+Wall-clock is set by the slowest single question rather than the sum of all of
+them, so 15 questions take about as long as the worst one — a few minutes instead
+of an hour.
+
+Usage:
+    python run_parallel.py              # all questions without attachments
+    python run_parallel.py --workers 8  # tune concurrency
+    python run_parallel.py --all        # include file-attachment questions
+"""
+
+import json
+import sys
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from agent import GaiaAgent
+
+HERE = Path(__file__).parent
+QUESTIONS = HERE / "gaia_questions.json"
+ANSWERS = HERE / "answers.json"
+
+_print_lock = threading.Lock()
+
+
+def log(msg: str):
+    with _print_lock:
+        print(msg, flush=True)
+
+
+def answer_one(index: int, total: int, q: dict) -> dict:
+    """Runs one question on its own agent. Never raises — a failure returns ''."""
+    label = q["question"][:60].replace("\n", " ")
+    start = time.time()
+    try:
+        agent = GaiaAgent()
+        answer = agent(
+            q["question"], task_id=q["task_id"], file_name=q.get("file_name", "")
+        )
+    except Exception as e:
+        log(f"[{index}/{total}] FAILED {type(e).__name__}: {label}")
+        answer = ""
+
+    log(f"[{index}/{total}] {time.time() - start:5.0f}s  {answer!r:<40} | {label}")
+    return {"task_id": q["task_id"], "submitted_answer": answer}
+
+
+def main():
+    args = sys.argv[1:]
+    include_files = "--all" in args
+    workers = 6
+    if "--workers" in args:
+        workers = int(args[args.index("--workers") + 1])
+
+    questions = json.loads(QUESTIONS.read_text())
+    if not include_files:
+        questions = [q for q in questions if not q.get("file_name")]
+
+    # --only 3,5,6 keeps just those 1-indexed questions, in the order given.
+    # Free-tier token budgets are small, so spending them on the questions most
+    # likely to land beats spreading them evenly across ones that cannot.
+    if "--only" in args:
+        picks = [int(n) for n in args[args.index("--only") + 1].split(",")]
+        questions = [questions[i - 1] for i in picks if 1 <= i <= len(questions)]
+
+    total = len(questions)
+    log(f"Running {total} questions with {workers} workers\n")
+    started = time.time()
+
+    results = []
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {
+            pool.submit(answer_one, i, total, q): q
+            for i, q in enumerate(questions, 1)
+        }
+        for fut in as_completed(futures):
+            results.append(fut.result())
+            ANSWERS.write_text(json.dumps(results, indent=2))
+
+    answered = sum(1 for r in results if r["submitted_answer"])
+    log(f"\n{'=' * 70}")
+    log(f"Answered {answered}/{total} in {time.time() - started:.0f}s → {ANSWERS.name}")
+    log("Verify by hand before submitting — GAIA grades on exact match.")
+
+
+if __name__ == "__main__":
+    main()