Gemini vision (solves chess), solver plain-text fallback, pandas in python_repl

Vision provider configurable, default Gemini (reads the chess board -> Rd5). Solver falls back to plain-text when structured output returns empty (was dropping the Excel question). python_repl preloads pandas as pd and prompts push numeric work through it. HF_TOKEN documented.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -11,6 +11,9 @@ HF_TOKEN=
 # "openai_compatible" (freellmapi gateway, best for rate limits), "gemini", "groq".
 LLM_PROVIDER=gemini
 GEMINI_TEXT_MODEL=gemini-2.5-flash
+# Image understanding: "gemini" (best), "groq", or "openai_compatible"
+VISION_PROVIDER=gemini
+GEMINI_VISION_MODEL=gemini-2.0-flash
 GROQ_TEXT_MODEL=llama-3.1-8b-instant
 # Global pace across all text-LLM calls (req/sec). 0.15 = ~9/min (<10 RPM free).
 # Ignored when LLM_PROVIDER=openai_compatible (gateway manages its own limits).

gaia_agent/config.py

@@ -28,6 +28,10 @@ class Settings(BaseSettings):
     # Vision + Whisper always use Groq.
     llm_provider: str = "gemini"
 
+    # --- Vision (image understanding) provider: "gemini", "groq", or "openai_compatible" ---
+    vision_provider: str = "gemini"
+    gemini_vision_model: str = "gemini-2.0-flash"
+
     # --- Models (override via env) ---
     gemini_text_model: str = "gemini-2.5-flash"
     groq_text_model: str = "llama-3.1-8b-instant"

gaia_agent/llm.py

@@ -87,14 +87,40 @@ def get_structured_llm(schema, temperature: float = 0.0):
 
 @lru_cache(maxsize=1)
 def get_vision_llm():
-    """Return the Groq multimodal model used for image understanding."""
-    from langchain_groq import ChatGroq
+    """Return the multimodal model used for image understanding.
 
+    Defaults to Gemini (strongest vision); falls back to Groq if no Google key.
+    Image questions are rare (1-2/run) so a low-quota key is fine here.
+    """
     s = get_settings()
+    vp = s.vision_provider.lower()
+
+    if vp == "gemini" and s.google_api_key:
+        from langchain_google_genai import ChatGoogleGenerativeAI
+
+        return ChatGoogleGenerativeAI(
+            model=s.gemini_vision_model,
+            google_api_key=s.google_api_key,
+            temperature=0.0,
+            max_retries=2,
+        )
+
+    if vp == "openai_compatible" and s.openai_compatible_base_url:
+        from langchain_openai import ChatOpenAI
+
+        return ChatOpenAI(
+            model=s.openai_compatible_model,
+            base_url=s.openai_compatible_base_url,
+            api_key=s.openai_compatible_api_key,
+            temperature=0.0,
+            max_retries=2,
+        )
+
+    from langchain_groq import ChatGroq
+
     return ChatGroq(
         model=s.groq_vision_model,
         api_key=s.groq_api_key,
         temperature=0.0,
         max_retries=2,
-        rate_limiter=_rate_limiter(),
     )

gaia_agent/nodes.py

@@ -144,18 +144,27 @@ def route_after_research(state: GraphState) -> str:
 
 def solver(state: GraphState) -> dict:
     """Synthesize the research conversation + context into one Candidate answer."""
+    system = SOLVER_PROMPT.format(
+        rules=GAIA_RULES,
+        question=state["question"],
+        context_notes=state.get("context_notes", "") or "(none)",
+    )
+    msgs = [SystemMessage(system), *state.get("messages", [])]
     try:
-        llm = get_structured_llm(Candidate)
-        system = SOLVER_PROMPT.format(
-            rules=GAIA_RULES,
-            question=state["question"],
-            context_notes=state.get("context_notes", "") or "(none)",
-        )
-        cand = llm.invoke([SystemMessage(system), *state.get("messages", [])])
+        cand = get_structured_llm(Candidate).invoke(msgs)
     except Exception:  # noqa: BLE001
         cand = None
-    if cand is None:  # structured output can return None when the model emits no call
-        cand = Candidate(answer="", justification="solver-fallback")
+
+    # Structured output sometimes returns None/empty (model emits no function call).
+    # Fall back to a plain-text answer so we never drop a solvable question.
+    if cand is None or not (cand.answer or "").strip():
+        try:
+            txt = get_text_llm().invoke(
+                msgs + [HumanMessage("Output ONLY the final answer, nothing else.")]
+            ).content
+            cand = Candidate(answer=(txt or "").strip(), justification="plain-fallback")
+        except Exception:  # noqa: BLE001
+            cand = Candidate(answer="", justification="solver-fallback")
     return {"candidate": cand}
 
 

gaia_agent/prompts.py

@@ -32,8 +32,10 @@ file, attachment, image, audio, spreadsheet, table, or document, set needs_file=
 
 RESEARCH_PROMPT = """\
 You are the RESEARCHER. Gather the facts needed to answer the question precisely.
-Use the available tools. Prefer authoritative sources. Do exact arithmetic / string
-work with python_repl rather than guessing. Stop calling tools once you have enough.
+Use the available tools. Prefer authoritative sources. NEVER do arithmetic in your head
+-- for ANY counting, summing, or numeric work call python_repl (pandas is available as
+`pd`; read attached spreadsheets with pd.read_excel(path) using the file path in context).
+Stop calling tools once you have enough.
 
 {rules}
 

gaia_agent/tools/python_tool.py

@@ -23,9 +23,11 @@ _SAFE_BUILTINS = {
 def python_repl(code: str) -> str:
     """Execute a short Python snippet and return its stdout (and `result` if set).
 
-    Use for exact arithmetic, string manipulation, sorting, and date math. The
-    ``math``, ``statistics``, ``datetime``, ``re``, ``itertools``, and ``collections``
-    modules are available. Assign to a variable named ``result`` to return a value.
+    Use for exact arithmetic, string manipulation, sorting, and date math. ALWAYS use
+    this for numeric work instead of computing in your head. ``math``, ``statistics``,
+    ``datetime``, ``re``, ``itertools``, ``collections`` and ``pd`` (pandas) are
+    available -- e.g. read a spreadsheet with ``pd.read_excel(path)``. Assign to a
+    variable named ``result`` to return a value.
 
     Args:
         code: Python source to execute.
@@ -45,6 +47,14 @@ def python_repl(code: str) -> str:
         "itertools": itertools,
         "collections": collections,
     }
+    # Preload pandas as `pd` so spreadsheet math is exact, e.g.
+    #   df = pd.read_excel(path); result = df[["Burgers","Fries"]].sum().sum()
+    try:
+        import pandas as pd
+
+        env["pd"] = pd
+    except Exception:  # noqa: BLE001
+        pass
     buf = io.StringIO()
     try:
         with contextlib.redirect_stdout(buf):