Refactor LLM provider invocation and add new providers

- Updated `invoke_llm` function to improve provider fallback logic and error handling.
- Added new LLM providers: OpenRouter, Together, ZAI, HF Inference, and Opencode Zen.
- Modified provider order to include new providers and adjusted model retrieval logic.
- Enhanced error handling for rate limits and other exceptions during invocation.
- Added spreadsheet parsing functionality to read Excel and CSV files.
- Improved web search tool to use DDGS for better results.
- Introduced a new tool for fetching full Wikipedia page content.
- Updated local run script to handle AI message extraction more robustly.
- Added environment variable checks in `test_env.py` for better debugging.

agent.py

@@ -1,5 +1,6 @@
 import os
 import re
+from collections import Counter
 from typing import TypedDict, List, Union
 
 from dotenv import load_dotenv
@@ -15,15 +16,15 @@ load_dotenv()
 class AgentState(TypedDict):
     messages: List[Union[HumanMessage, AIMessage, SystemMessage, ToolMessage]]
     reflection_count: int
+    tool_call_count: int
+    tool_call_history: List[str]
 
 
 def _invoke_llm_with_tools(messages, fallback_count=0):
-    """Invoke LLM with provider fallback."""
     return invoke_llm(messages, tools, fallback_count)
 
-# --- Helper Functions ---
+
 def is_reversed_text(question: str) -> bool:
-    """Check if text appears to be reversed."""
     words = question.split()
     if len(words) < 3:
         return False
@@ -35,75 +36,64 @@ def is_reversed_text(question: str) -> bool:
     orig_valid = len([w for w in orig_words if w in common_words])
     return rev_valid > orig_valid
 
-# --- Graph Nodes ---
+
+SYSTEM_PROMPT = """Answer with FINAL ANSWER: <value>.
+Use tools to research. Read full Wikipedia pages (browse_url) rather than just searching.
+Never repeat the same tool call."""
+
+
 def call_model(state: AgentState):
-    messages = state["messages"]
-    
-    # Pre-processing: Detect and handle reversed text in the first message
+    messages = list(state["messages"])
+
     if len(messages) == 1 and isinstance(messages[0], HumanMessage):
         user_msg = messages[0].content
         if is_reversed_text(user_msg):
             fixed_msg = user_msg[::-1]
             messages = [HumanMessage(content=f"The following message was detected as reversed. I have reversed it back for you:\n{fixed_msg}")]
-    
-    # Add System Message if not present
+
     if not any(isinstance(m, SystemMessage) for m in messages):
-        system_prompt = """You are a highly capable General AI Assistant (GAIA). Your goal is to solve complex, multi-step tasks.
-
-Your thought process MUST be methodical:
-1. THINK: 
-    - Analyze the question deeply. Identify the core goal and ALL constraints (units, date formats, precision, etc.).
-    - If the task involves an image or video, describe the visual elements before attempting to solve.
-    - Plan your steps. Break the problem into smaller sub-problems.
-2. ACT (Python-First): 
-    - Use `python_repl` for ANY task involving: math, counting, data analysis, list filtering (e.g., botany), or verifying logic (e.g., commutativity). DO NOT do these manually.
-    - Use `web_search` for initial discovery and `browse_url` to verify details from the source.
-3. OBSERVE: Carefully review tool outputs. If a result is ambiguous, search for a second source to triangulate.
-4. REFINE: Question your assumptions. If the answer seems too simple for a complex GAIA task, you likely missed a constraint.
-5. VERIFY: Before finalizing, double-check units and precision.
-6. FINALIZE: Provide the result in the exact format: FINAL ANSWER: <answer>.
-
-Guidelines:
-- [Attached Files]: Always use `read_file` for local files.
-- Research: Don't trust a single snippet; browse the full page if the answer is buried.
-- Constraints: If the question says 'alphabetize' or 'comma-separated', use Python to ensure it is perfect.
-- Final Output: Return ONLY the final answer in the requested format.
-"""
-        messages = [SystemMessage(content=system_prompt)] + messages
+        messages = [SystemMessage(content=SYSTEM_PROMPT)] + messages
 
     response = _invoke_llm_with_tools(messages)
-    return {"messages": [response]}
+    return {"messages": state["messages"] + [response]}
+
 
 def reflect(state: AgentState):
-    """Node to reflect on the final answer and verify correctness."""
     messages = state["messages"]
     last_message = messages[-1]
-    
+
     if "FINAL ANSWER:" not in last_message.content:
-        return {"messages": []} # Should not happen based on routing
+        return {"messages": []}
 
     reflection_prompt = (
-        "You have provided a FINAL ANSWER. Before we finish, please perform a final a self-critique:\n"
+        "Before finalizing, double-check:\n"
         "1. Did you miss any constraints from the original question?\n"
-        "2. Are the units and precision exactly as requested?\n"
-        "3. Is there any step in your reasoning that could be flawed?\n"
-        "If the answer is correct, simply repeat the FINAL ANSWER: <answer> exactly as before.\n"
-        "If you find an error, explain it and provide a corrected FINAL ANSWER: <answer>."
+        "2. Are units and precision exactly as requested?\n"
+        "3. Could any step in reasoning be flawed?\n"
+        "If correct, repeat FINAL ANSWER: <answer> exactly.\n"
+        "If wrong, explain and provide corrected FINAL ANSWER: <answer>."
     )
-    
-    # We add the reflection prompt as a human message to trigger a new response
+
     response = _invoke_llm_with_tools(messages + [HumanMessage(content=reflection_prompt)])
-    return {"messages": [response], "reflection_count": state.get("reflection_count", 0) + 1}
+    return {"messages": state["messages"] + [response], "reflection_count": state.get("reflection_count", 0) + 1}
+
 
 def call_tool(state: AgentState):
     messages = state["messages"]
     last_message = messages[-1]
-    
+
+    tool_call_history = state.get("tool_call_history", [])
+    tool_call_count = state.get("tool_call_count", 0)
+
     tool_outputs = []
-    for tool_call in last_message.tool_calls:
+    # Limit 5 tool calls per response
+    for tool_call in last_message.tool_calls[:5]:
         tool_name = tool_call["name"]
         tool_args = tool_call["args"]
-        
+        key = f"{tool_name}({tool_args})"
+        tool_call_history.append(key)
+        tool_call_count += 1
+
         if tool_name not in tools_by_name:
             tool_outputs.append(ToolMessage(
                 content=f"Error: Tool {tool_name} not found.",
@@ -111,13 +101,16 @@ def call_tool(state: AgentState):
                 name=tool_name
             ))
             continue
-            
+
         tool = tools_by_name[tool_name]
-        print(f"Calling tool: {tool_name} with args: {tool_args}")
+        print(f"Calling tool: {tool_name} with args: {tool_args}", flush=True)
         try:
             output = tool.invoke(tool_args)
+            output_str = str(output)
+            if len(output_str) > 15000:
+                output_str = output_str[:15000] + "\n...[truncated]"
             tool_outputs.append(ToolMessage(
-                content=str(output),
+                content=output_str,
                 tool_call_id=tool_call["id"],
                 name=tool_name
             ))
@@ -127,18 +120,34 @@ def call_tool(state: AgentState):
                 tool_call_id=tool_call["id"],
                 name=tool_name
             ))
-    return {"messages": tool_outputs}
+    return {"messages": state["messages"] + tool_outputs, "tool_call_count": tool_call_count, "tool_call_history": tool_call_history}
+
 
 def should_continue(state: AgentState):
     messages = state["messages"]
     last_message = messages[-1]
+    tool_call_count = state.get("tool_call_count", 0)
+    tool_call_history = state.get("tool_call_history", [])
+    reflection_count = state.get("reflection_count", 0)
+
+    # Max 8 tool calls (128K context handles it)
+    if tool_call_count >= 8:
+        return END
+
+    # Detect loop: same tool name called 4+ times
+    if len(tool_call_history) >= 4:
+        tool_names = [h.split("(")[0] for h in tool_call_history]
+        if any(tool_names.count(n) >= 4 for n in set(tool_names)):
+            return END
+
     if hasattr(last_message, "tool_calls") and last_message.tool_calls:
         return "action"
-    if "FINAL ANSWER:" in last_message.content and state.get("reflection_count", 0) == 0:
+    content = getattr(last_message, "content", "") or ""
+    if "FINAL ANSWER:" in content and reflection_count == 0:
         return "reflect"
     return END
 
-# --- Graph Construction ---
+
 def build_graph():
     workflow = StateGraph(AgentState)
     workflow.add_node("agent", call_model)

analyze_results.py

@@ -0,0 +1,132 @@
+"""
+Analyze existing gaia_results.json and produce a diagnostic report.
+"""
+import json
+import re
+import sys
+
+# Fix Windows console encoding issues
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+
+with open("gaia_results.json", "r") as f:
+    data = json.load(f)
+
+results = data["results"]
+
+# Categorize failures
+categories = {
+    "groq_rate_limit_tpm": [],
+    "recursion_limit": [],
+    "tool_call_format_error": [],
+    "wrong_output_format": [],
+    "other_error": [],
+    "correct": [],
+}
+
+for r in results:
+    ans = r["submitted_answer"]
+    q = r["question"][:80]
+    tid = r["task_id"][:8]
+    gt = r["ground_truth"]
+    is_correct = r["correct"]
+
+    if is_correct:
+        categories["correct"].append(r)
+    elif "413" in ans and "tokens per minute" in ans.lower():
+        categories["groq_rate_limit_tpm"].append(r)
+    elif "Recursion limit" in ans:
+        categories["recursion_limit"].append(r)
+    elif "tool_use_failed" in ans or "tool call validation" in ans:
+        categories["tool_call_format_error"].append(r)
+    elif "<|python_tag|>" in ans or ("AGENT ERROR" in ans and "tool" in ans.lower()):
+        categories["wrong_output_format"].append(r)
+    else:
+        categories["other_error"].append(r)
+
+print("=" * 70)
+print("GAIA BENCHMARK - FAILURE ANALYSIS REPORT")
+print(f"Score: {data['correct']}/{data['total']} = {data['score']:.0f}%")
+print("=" * 70)
+
+print("\n## CATEGORY BREAKDOWN")
+for cat, items in categories.items():
+    print(f"  {cat}: {len(items)} questions")
+
+print("\n" + "=" * 70)
+
+for cat, label in [
+    ("groq_rate_limit_tpm", "[RATE_LIMIT] GROQ TPM RATE LIMIT (request too large for fallback model)"),
+    ("recursion_limit", "[RECURSION] RECURSION LIMIT (agent stuck in tool loop, no answer found)"),
+    ("tool_call_format_error", "[FORMAT_ERR] TOOL CALL FORMAT ERROR (LLM generated malformed tool invocations)"),
+    ("wrong_output_format", "[WRONG_OUT] WRONG OUTPUT FORMAT (agent returned tool calls as text, not answer)"),
+    ("other_error", "[OTHER] OTHER ERROR"),
+]:
+    items = categories[cat]
+    if not items:
+        continue
+    print(f"\n### {label} ({len(items)} questions)")
+    for r in items:
+        print(f"  - [{r['task_id'][:8]}] GT={r['ground_truth'][:40]!r}")
+        print(f"    Q: {r['question'][:100]}")
+        # Classify what tool/skill would solve it
+        q_lower = r["question"].lower()
+        skills = []
+        if "youtube.com" in r["question"]:
+            skills.append("YouTube Transcript / Video Analysis")
+        if "mp3" in q_lower or "audio" in q_lower or "voice memo" in q_lower or "recording" in q_lower:
+            skills.append("Audio Transcription (Whisper)")
+        if "image" in q_lower or "chess" in q_lower:
+            skills.append("Image Analysis (Vision LLM)")
+        if "excel" in q_lower or ".xlsx" in q_lower:
+            skills.append("Excel/File Reading")
+        if "wikipedia" in q_lower or "wiki" in q_lower:
+            skills.append("Wikipedia Search")
+        if "paper" in q_lower or "article" in q_lower:
+            skills.append("Web Browsing/Research")
+        if "python code" in q_lower or "code" in q_lower:
+            skills.append("Python REPL execution")
+        if "table" in q_lower or "commutative" in q_lower:
+            skills.append("Python REPL (logic check)")
+        if not skills:
+            skills.append("Web Search")
+        print(f"    Needed: {', '.join(skills)}")
+        print()
+
+print("\n" + "=" * 70)
+print("## PRIORITIZED IMPROVEMENT AREAS")
+print("""
+1. CRITICAL - PROVIDER FALLBACK (affects 8 questions):
+   - Groq falls back to llama-3.1-8b-instant (6000 TPM limit)
+   - Gemini API quota exhausted (free tier daily limit hit)
+   - Fix: Use gemini-1.5-flash or gemini-2.5-flash as PRIMARY provider
+   - Fix: Add proper provider rotation that skips quota-exhausted models
+
+2. CRITICAL - RECURSION LIMIT (affects 8 questions):
+   - Agent loops indefinitely (25 steps) without providing an answer
+   - Causes: Tool keeps failing or returning unhelpful results
+   - Fix: Add a MAX_TOOL_CALLS guard and force FINAL ANSWER after N iterations
+
+3. HIGH - TOOL CALL FORMAT ERRORS (affects 3 questions):
+   - LLM generates tool calls not matching the registered tool names/schema
+   - wiki_search called with wrong JSON format
+   - web_search called with 'keywords' instead of 'query' parameter
+   - Fix: Add function signature validation / tool schema alignment
+
+4. HIGH - AUDIO/VIDEO QUESTIONS (affects 3 questions):
+   - YouTube video analysis requires video frames, not just transcript
+   - Audio transcription (mp3) failing due to context overflow
+   - Fix: Ensure transcribe_audio + get_youtube_transcript work reliably
+
+5. HIGH - IMAGE/CHESS QUESTIONS (affects 1 question):
+   - Chess position from image requires multimodal vision model
+   - Current setup can't directly process images
+   - Fix: Pass image URL to Gemini vision model
+
+6. MEDIUM - CONTEXT OVERFLOW on multi-step research questions:
+   - Long Wikipedia/web searches fill context window before finding answer
+   - Fix: Summarize intermediate tool results before appending to messages
+""")

app.py

@@ -24,7 +24,10 @@ class BasicAgent:
     def __call__(self, question: str) -> str:
         print(f"Agent received question (first 50 chars): {question[:50]}...")
         messages = [HumanMessage(content=question)]
-        result = self.graph.invoke({"messages": messages})
+        result = self.graph.invoke(
+            {"messages": messages, "tool_call_count": 0, "reflection_count": 0, "tool_call_history": []},
+            config={"recursion_limit": 50},
+        )
         answer = result['messages'][-1].content
         print(f"Agent returning answer: {answer}")
         return answer

gaia_results.csv

@@ -1,11 +1,9 @@
 task_id,question,submitted_answer,ground_truth,correct
-8e867cd7-cff9-4e6c-867a-ff5ddc2550be,How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.,"ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': 'tool call validation failed: attempted to call tool \'wiki_search{""query"": ""Mercedes Sosa discography""}\' which was not in request.tools', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=wiki_search{""query"": ""Mercedes Sosa discography""}></function>'}}",3,False
-a1e91b78-d3d8-4675-bb8d-62741b4b68a6,"In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?","ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 29530, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",3,False
-2d83110e-a098-4ebb-9987-066c06fa42d0,".rewsna eht sa ""tfel"" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI","ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': ""Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details."", 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=reverse_text>{""text"": ""The following message was detected as reversed. I have reversed it back for you: If you understand this sentence, write the opposite of the word ""left"" as the answer.""}</function>'}}",Right,False
-cca530fc-4052-43b2-b130-b30968d8aa44,Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.,"ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",Rd5,False
-4fc2f1ae-8625-45b5-ab34-ad4433bc21f8,Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?,"ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",FunkMonk,False
+8e867cd7-cff9-4e6c-867a-ff5ddc2550be,How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.,,3,False
+a1e91b78-d3d8-4675-bb8d-62741b4b68a6,"In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",,3,False
+2d83110e-a098-4ebb-9987-066c06fa42d0,".rewsna eht sa ""tfel"" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",left,Right,False
+cca530fc-4052-43b2-b130-b30968d8aa44,Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.,,Rd5,False
+4fc2f1ae-8625-45b5-ab34-ad4433bc21f8,Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?,,FunkMonk,False
 6f37996b-2ac7-44b0-8e68-6d28256631b4,"Given this table defining * on the set S = {a, b, c, d, e}
 
 |*|a|b|c|d|e|
@@ -16,37 +14,30 @@ For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/er
 |d|b|e|b|e|d|
 |e|d|b|a|d|c|
 
-provide the subset of S involved in any possible counter-examples that prove * is not commutative. Provide your answer as a comma separated list of the elements in the set in alphabetical order.","ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 9367, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}","b, e",False
+provide the subset of S involved in any possible counter-examples that prove * is not commutative. Provide your answer as a comma separated list of the elements in the set in alphabetical order.","b,e","b, e",False
 9d191bce-651d-4746-be2d-7ef8ecadb9c2,"Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.
 
-What does Teal'c say in response to the question ""Isn't that hot?""","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",Extremely,False
-cabe07ed-9eca-40ea-8ead-410ef5e83f91,What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?,"ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': ""Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details."", 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=web_search {""keywords"": ""equine veterinarian surname CK-12 license LibreText Introductory Chemistry materials Marisa Alviar-Agnew Henry Agnew""} </function>'}}",Louvrier,False
+What does Teal'c say in response to the question ""Isn't that hot?""",,Extremely,False
+cabe07ed-9eca-40ea-8ead-410ef5e83f91,What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?,,Louvrier,False
 3cef3a44-215e-4aed-8e3b-b1e3f08063b7,"I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:
 
 milk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts
 
-I need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT","broccoli, celery, fresh basil, lettuce, sweet potatoes",False
+I need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.","broccoli, celery, fresh basil, lettuce, sweet potatoes","broccoli, celery, fresh basil, lettuce, sweet potatoes",True
 99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3,"Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.
 
 In your response, please only list the ingredients, not any measurements. So if the recipe calls for ""a pinch of salt"" or ""two cups of ripe strawberries"" the ingredients on the list would be ""salt"" and ""ripe strawberries"".
 
-Please format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.","ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 8415, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}","cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",False
-305ac316-eef6-4446-960a-92d80d542f82,Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.,"ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",Wojciech,False
-f918266a-b3e0-4914-865d-4faa564f1aef,What is the final numeric output from the attached Python code?,"<|python_tag|>web_search{""keywords"": ""definition of artificial intelligence""}; browse_url{""url"": ""https://www.example.com/what-is-ai""}; browse_url{""url"": ""https://www.example.com/ai-definition""}; python_repl{""code"": ""print('Artificial Intelligence (AI) is a field of computer science that focuses on creating intelligent machines that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, and decision-making.')""}",0,False
-3f57289b-8c60-48be-bd80-01f8099ca449,How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?,"ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 6414, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",519,False
+Please format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.",,"cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",False
+305ac316-eef6-4446-960a-92d80d542f82,Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.,,Wojciech,False
+f918266a-b3e0-4914-865d-4faa564f1aef,What is the final numeric output from the attached Python code?,,0,False
+3f57289b-8c60-48be-bd80-01f8099ca449,How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?,519,519,True
 1f975693-876d-457b-a649-393859e79bf3,"Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(
 
-Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT","132, 133, 134, 197, 245",False
-840bfca7-4f7b-481a-8794-c560c340185d,"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",80GSFC21M0002,False
-bda648d7-d618-4883-88f4-3466eabd860e,Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.,"ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 7915, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",Saint Petersburg,False
-cf106601-ab4f-4af9-b045-5295fe67b37d,"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",CUB,False
-a0c07678-e491-4bbc-8f0b-07405144218f,"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.","ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.
-For troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT","Yoshida, Uehara",False
-7bd855d8-463d-4ed5-93ca-5fe35145f733,The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.,"ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 14486, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",89706.00,False
-5a0c1adf-205e-4841-a666-7c3ef95def9d,What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,"ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 23154, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",Claus,False
+Could you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",,"132, 133, 134, 197, 245",False
+840bfca7-4f7b-481a-8794-c560c340185d,"On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",,80GSFC21M0002,False
+bda648d7-d618-4883-88f4-3466eabd860e,Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.,Saint Petersburg,Saint Petersburg,True
+cf106601-ab4f-4af9-b045-5295fe67b37d,"What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",,CUB,False
+a0c07678-e491-4bbc-8f0b-07405144218f,"Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",,"Yoshida, Uehara",False
+7bd855d8-463d-4ed5-93ca-5fe35145f733,The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.,89706.00,89706.00,True
+5a0c1adf-205e-4841-a666-7c3ef95def9d,What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?,,Claus,False

gaia_results.json

@@ -1,145 +1,145 @@
 {
-  "score": 0.0,
-  "correct": 0,
+  "score": 20.0,
+  "correct": 4,
   "total": 20,
   "results": [
     {
       "task_id": "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",
       "question": "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': 'tool call validation failed: attempted to call tool \\'wiki_search{\"query\": \"Mercedes Sosa discography\"}\\' which was not in request.tools', 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=wiki_search{\"query\": \"Mercedes Sosa discography\"}></function>'}}",
+      "submitted_answer": "",
       "ground_truth": "3",
       "correct": false
     },
     {
       "task_id": "a1e91b78-d3d8-4675-bb8d-62741b4b68a6",
       "question": "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 29530, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "",
       "ground_truth": "3",
       "correct": false
     },
     {
       "task_id": "2d83110e-a098-4ebb-9987-066c06fa42d0",
       "question": ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': \"Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details.\", 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=reverse_text>{\"text\": \"The following message was detected as reversed. I have reversed it back for you: If you understand this sentence, write the opposite of the word \"left\" as the answer.\"}</function>'}}",
+      "submitted_answer": "left",
       "ground_truth": "Right",
       "correct": false
     },
     {
       "task_id": "cca530fc-4052-43b2-b130-b30968d8aa44",
       "question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "Rd5",
       "correct": false
     },
     {
       "task_id": "4fc2f1ae-8625-45b5-ab34-ad4433bc21f8",
       "question": "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "FunkMonk",
       "correct": false
     },
     {
       "task_id": "6f37996b-2ac7-44b0-8e68-6d28256631b4",
       "question": "Given this table defining * on the set S = {a, b, c, d, e}\n\n|*|a|b|c|d|e|\n|---|---|---|---|---|---|\n|a|a|b|c|b|d|\n|b|b|c|a|e|c|\n|c|c|a|b|b|a|\n|d|b|e|b|e|d|\n|e|d|b|a|d|c|\n\nprovide the subset of S involved in any possible counter-examples that prove * is not commutative. Provide your answer as a comma separated list of the elements in the set in alphabetical order.",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 9367, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "b,e",
       "ground_truth": "b, e",
       "correct": false
     },
     {
       "task_id": "9d191bce-651d-4746-be2d-7ef8ecadb9c2",
       "question": "Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.\n\nWhat does Teal'c say in response to the question \"Isn't that hot?\"",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "Extremely",
       "correct": false
     },
     {
       "task_id": "cabe07ed-9eca-40ea-8ead-410ef5e83f91",
       "question": "What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 400 - {'error': {'message': \"Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details.\", 'type': 'invalid_request_error', 'code': 'tool_use_failed', 'failed_generation': '<function=web_search {\"keywords\": \"equine veterinarian surname CK-12 license LibreText Introductory Chemistry materials Marisa Alviar-Agnew Henry Agnew\"} </function>'}}",
+      "submitted_answer": "",
       "ground_truth": "Louvrier",
       "correct": false
     },
     {
       "task_id": "3cef3a44-215e-4aed-8e3b-b1e3f08063b7",
       "question": "I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\n\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\n\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "broccoli, celery, fresh basil, lettuce, sweet potatoes",
       "ground_truth": "broccoli, celery, fresh basil, lettuce, sweet potatoes",
-      "correct": false
+      "correct": true
     },
     {
       "task_id": "99c9cc74-fdc8-46c6-8f8d-3ce2d3bfeea3",
       "question": "Hi, I'm making a pie but I could use some help with my shopping list. I have everything I need for the crust, but I'm not sure about the filling. I got the recipe from my friend Aditi, but she left it as a voice memo and the speaker on my phone is buzzing so I can't quite make out what she's saying. Could you please listen to the recipe and list all of the ingredients that my friend described? I only want the ingredients for the filling, as I have everything I need to make my favorite pie crust. I've attached the recipe as Strawberry pie.mp3.\n\nIn your response, please only list the ingredients, not any measurements. So if the recipe calls for \"a pinch of salt\" or \"two cups of ripe strawberries\" the ingredients on the list would be \"salt\" and \"ripe strawberries\".\n\nPlease format your response as a comma separated list of ingredients. Also, please alphabetize the ingredients.",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 8415, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "",
       "ground_truth": "cornstarch, freshly squeezed lemon juice, granulated sugar, pure vanilla extract, ripe strawberries",
       "correct": false
     },
     {
       "task_id": "305ac316-eef6-4446-960a-92d80d542f82",
       "question": "Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "Wojciech",
       "correct": false
     },
     {
       "task_id": "f918266a-b3e0-4914-865d-4faa564f1aef",
       "question": "What is the final numeric output from the attached Python code?",
-      "submitted_answer": "<|python_tag|>web_search{\"keywords\": \"definition of artificial intelligence\"}; browse_url{\"url\": \"https://www.example.com/what-is-ai\"}; browse_url{\"url\": \"https://www.example.com/ai-definition\"}; python_repl{\"code\": \"print('Artificial Intelligence (AI) is a field of computer science that focuses on creating intelligent machines that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, and decision-making.')\"}",
+      "submitted_answer": "",
       "ground_truth": "0",
       "correct": false
     },
     {
       "task_id": "3f57289b-8c60-48be-bd80-01f8099ca449",
       "question": "How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 6414, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "519",
       "ground_truth": "519",
-      "correct": false
+      "correct": true
     },
     {
       "task_id": "1f975693-876d-457b-a649-393859e79bf3",
       "question": "Hi, I was out sick from my classes on Friday, so I'm trying to figure out what I need to study for my Calculus mid-term next week. My friend from class sent me an audio recording of Professor Willowbrook giving out the recommended reading for the test, but my headphones are broken :(\n\nCould you please listen to the recording for me and tell me the page numbers I'm supposed to go over? I've attached a file called Homework.mp3 that has the recording. Please provide just the page numbers as a comma-delimited list. And please provide the list in ascending order.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "132, 133, 134, 197, 245",
       "correct": false
     },
     {
       "task_id": "840bfca7-4f7b-481a-8794-c560c340185d",
       "question": "On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "80GSFC21M0002",
       "correct": false
     },
     {
       "task_id": "bda648d7-d618-4883-88f4-3466eabd860e",
       "question": "Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 7915, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "Saint Petersburg",
       "ground_truth": "Saint Petersburg",
-      "correct": false
+      "correct": true
     },
     {
       "task_id": "cf106601-ab4f-4af9-b045-5295fe67b37d",
       "question": "What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "CUB",
       "correct": false
     },
     {
       "task_id": "a0c07678-e491-4bbc-8f0b-07405144218f",
       "question": "Who are the pitchers with the number before and after Taish\u014d Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.",
-      "submitted_answer": "ERROR: Recursion limit of 25 reached without hitting a stop condition. You can increase the limit by setting the `recursion_limit` config key.\nFor troubleshooting, visit: https://python.langchain.com/docs/troubleshooting/errors/GRAPH_RECURSION_LIMIT",
+      "submitted_answer": "",
       "ground_truth": "Yoshida, Uehara",
       "correct": false
     },
     {
       "task_id": "7bd855d8-463d-4ed5-93ca-5fe35145f733",
       "question": "The attached Excel file contains the sales of menu items for a local fast-food chain. What were the total sales that the chain made from food (not including drinks)? Express your answer in USD with two decimal places.",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 14486, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "89706.00",
       "ground_truth": "89706.00",
-      "correct": false
+      "correct": true
     },
     {
       "task_id": "5a0c1adf-205e-4841-a666-7c3ef95def9d",
       "question": "What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?",
-      "submitted_answer": "ERROR: LLM invocation failed: Error code: 413 - {'error': {'message': 'Request too large for model `llama-3.1-8b-instant` in organization `org_01kk9kwrt4fcxtaxyjc5ze79eq` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 23154, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}",
+      "submitted_answer": "",
       "ground_truth": "Claus",
       "correct": false
     }

llm/client.py

@@ -1,66 +1,50 @@
 import os
-from typing import List
+import time
 
 from langchain_core.messages import AIMessage
 from llm.providers import PROVIDERS
 
-PROVIDER_ORDER = os.getenv("LLM_PROVIDER_ORDER", "groq, gemini, gemini_gemma").split(",")
+PROVIDER_ORDER = [p.strip() for p in os.getenv("LLM_PROVIDER_ORDER", "opencode_zen, groq").split(",")]
 
-_degraded_providers = {}
 
+def invoke_llm(messages, tools, fallback_count=0, _degraded=None):
+    if _degraded is None:
+        _degraded = {}
 
-def _get_next_provider():
-    """Get next available provider in priority order."""
-    for name in PROVIDER_ORDER:
-        if name not in _degraded_providers:
-            yield name
+    for provider_name in PROVIDER_ORDER:
+        if provider_name in _degraded:
+            continue
 
+        provider = PROVIDERS.get(provider_name)
+        if not provider:
+            continue
 
-def invoke_llm(messages: List, tools: List, fallback_count: int = 0) -> AIMessage:
-    """Invoke LLM with provider fallback.
-
-    Args:
-        messages: Chat messages to send to LLM
-        tools: List of tools to bind
-        fallback_count: Current retry attempt
-
-    Returns:
-        AIMessage response from successful provider
-    """
-    provider_name = None
-    provider = None
-
-    for name in _get_next_provider():
-        provider_name = name
-        provider = PROVIDERS.get(name)
-        if provider:
-            break
-
-    if not provider:
-        return AIMessage(content="ERROR: No available LLM providers")
-
-    try:
         models = provider.get_models()
-        model_index = min(fallback_count // 3, len(models) - 1)
-        model_name = models[model_index]
-
-        print(f"Invoking {provider_name} with model {model_name}")
-        return provider.invoke(messages, tools, model_name)
-
-    except Exception as e:
-        error_msg = str(e).lower()
-
-        if "rate limit" in error_msg or "429" in error_msg or "quota" in error_msg:
-            print(f"{provider_name} rate limit hit. Waiting before retry...")
-            import time
-            wait_time = 10 * (fallback_count + 1)
-            time.sleep(wait_time)
-            _degraded_providers[provider_name] = True
-        else:
-            print(f"{provider_name} error: {e}. Trying next provider.")
-
-        remaining = [n for n in PROVIDER_ORDER if n not in _degraded_providers]
-        if remaining:
-            return invoke_llm(messages, tools, fallback_count + 1)
-
-        return AIMessage(content=f"ERROR: All LLM providers failed: {e}")
+        model_attempts = 0
+
+        while model_attempts < len(models):
+            model_name = models[model_attempts]
+            print(f"Invoking {provider_name} with {model_name}", flush=True)
+
+            retries = 0
+            while retries < 2:
+                try:
+                    return provider.invoke(messages, tools, model_name)
+                except Exception as e:
+                    err_str = str(e)
+                    err = err_str.lower()
+                    if any(x in err for x in ("rate limit", "429", "quota", "resource ex")):
+                        print(f"{provider_name}/{model_name} rate limited, waiting...", flush=True)
+                        time.sleep(65)
+                        retries += 1
+                    elif any(x in err for x in ("payment required", "402", "tool_use_failed", "model_not_found", "too large", "413")):
+                        print(f"{provider_name}/{model_name} skip, trying next", flush=True)
+                        break
+                    else:
+                        print(f"{provider_name}/{model_name} error: {type(e).__name__}: {err_str[:150]}", flush=True)
+                        break
+            model_attempts += 1
+
+        _degraded[provider_name] = True
+
+    return AIMessage(content="ERROR: All LLM providers failed")

llm/providers/__init__.py

@@ -1,9 +1,14 @@
-from llm.providers import gemini, gemini_gemma, groq
+from llm.providers import gemini, gemini_gemma, groq, openrouter, together, zai, hf_inference, opencode_zen
 
 PROVIDERS = {
     "gemini": gemini,
     "gemini_gemma": gemini_gemma,
     "groq": groq,
+    "openrouter": openrouter,
+    "together": together,
+    "zai": zai,
+    "hf_inference": hf_inference,
+    "opencode_zen": opencode_zen,
 }
 
-__all__ = ["PROVIDERS", "gemini", "gemini_gemma", "groq"]
+__all__ = ["PROVIDERS", "gemini", "gemini_gemma", "groq", "openrouter", "together", "zai", "hf_inference", "opencode_zen"]

llm/providers/groq.py

@@ -10,4 +10,4 @@ def invoke(messages, tools, model_name: str = "llama-3.3-70b-versatile"):
 
 def get_models():
     """List available Groq models for fallback."""
-    return ["llama-3.3-70b-versatile", "llama-3.1-8b-instant"]
+    return ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"]

llm/providers/hf_inference.py

@@ -0,0 +1,97 @@
+import os
+import json
+from dotenv import load_dotenv
+from huggingface_hub import InferenceClient
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+
+load_dotenv()
+
+token = os.getenv("HF_TOKEN")
+client = InferenceClient(token=token)
+
+
+def _convert_message(msg):
+    role_map = {
+        "HumanMessage": "user",
+        "AIMessage": "assistant",
+        "SystemMessage": "system",
+        "ToolMessage": "tool",
+    }
+    role = role_map.get(type(msg).__name__, "user")
+    d = {"role": role, "content": msg.content if msg.content else ""}
+
+    if role == "tool":
+        d["tool_call_id"] = getattr(msg, "tool_call_id", "")
+        d["name"] = getattr(msg, "name", "")
+
+    if role == "assistant" and hasattr(msg, "tool_calls") and msg.tool_calls:
+        d["tool_calls"] = []
+        for tc in msg.tool_calls:
+            d["tool_calls"].append({
+                "id": tc.get("id", ""),
+                "type": "function",
+                "function": {
+                    "name": tc["name"],
+                    "arguments": json.dumps(tc["args"]) if isinstance(tc["args"], dict) else tc["args"],
+                },
+            })
+    return d
+
+
+def _convert_tools(tools):
+    result = []
+    for t in tools:
+        result.append({
+            "type": "function",
+            "function": {
+                "name": t.name,
+                "description": t.description,
+                "parameters": t.args_schema.schema() if hasattr(t, "args_schema") and t.args_schema else {},
+            },
+        })
+    return result
+
+
+def invoke(messages, tools, model_name: str = "deepseek-ai/DeepSeek-V3-0324"):
+    hf_messages = [_convert_message(m) for m in messages]
+    hf_tools = _convert_tools(tools) if tools else None
+
+    resp = client.chat_completion(
+        model=model_name,
+        messages=hf_messages,
+        tools=hf_tools,
+        tool_choice="auto" if hf_tools else None,
+        max_tokens=2048,
+        temperature=0,
+    )
+
+    choice = resp.choices[0]
+    msg = choice.message
+
+    response_kwargs = {"content": msg.content or ""}
+
+    if msg.tool_calls:
+        tool_calls = []
+        for tc in msg.tool_calls:
+            tool_calls.append({
+                "id": tc.id,
+                "name": tc.function.name,
+                "args": json.loads(tc.function.arguments) if tc.function.arguments else {},
+            })
+        response_kwargs["tool_calls"] = tool_calls
+        response_kwargs["additional_kwargs"] = {
+            "tool_calls": [
+                {
+                    "id": tc.id,
+                    "type": "function",
+                    "function": {"name": tc.function.name, "arguments": tc.function.arguments},
+                }
+                for tc in msg.tool_calls
+            ]
+        }
+
+    return AIMessage(**response_kwargs)
+
+
+def get_models():
+    return ["deepseek-ai/DeepSeek-V3-0324"]

llm/providers/opencode_zen.py

@@ -0,0 +1,124 @@
+import os
+import json
+import requests
+from dotenv import load_dotenv
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+
+load_dotenv()
+
+API_KEY = os.getenv("OPENCODE_ZEN_API_KEY", "sk-CEgFM8zjmQxtbByFEGNMBTr0bisvxSQvyjhKJEppQfoDjD7922P2Ljtupey6XQji")
+BASE_URL = "https://opencode.ai/zen/v1"
+HEADERS = {
+    "Authorization": f"Bearer {API_KEY}",
+    "Content-Type": "application/json",
+}
+
+
+def _convert_message(msg):
+    role_map = {
+        "HumanMessage": "user",
+        "AIMessage": "assistant",
+        "SystemMessage": "system",
+        "ToolMessage": "tool",
+    }
+    role = role_map.get(type(msg).__name__, "user")
+    d = {"role": role, "content": msg.content if msg.content else ""}
+
+    if role == "tool":
+        d["tool_call_id"] = getattr(msg, "tool_call_id", "")
+        d["name"] = getattr(msg, "name", "")
+
+    if role == "assistant":
+        rc = None
+        if hasattr(msg, "additional_kwargs") and msg.additional_kwargs:
+            rc = msg.additional_kwargs.get("reasoning_content")
+        if rc:
+            d["reasoning_content"] = rc
+
+        if hasattr(msg, "tool_calls") and msg.tool_calls:
+            d["tool_calls"] = []
+            for tc in msg.tool_calls:
+                d["tool_calls"].append({
+                    "id": tc.get("id", ""),
+                    "type": "function",
+                    "function": {
+                        "name": tc["name"],
+                        "arguments": json.dumps(tc["args"]) if isinstance(tc["args"], dict) else tc["args"],
+                    },
+                })
+    return d
+
+
+def _convert_tools(tools):
+    result = []
+    for t in tools:
+        result.append({
+            "type": "function",
+            "function": {
+                "name": t.name,
+                "description": t.description,
+                "parameters": t.args_schema.schema() if hasattr(t, "args_schema") and t.args_schema else {},
+            },
+        })
+    return result
+
+
+def invoke(messages, tools, model_name: str = None):
+    if model_name is None:
+        model_name = "deepseek-v4-flash-free"
+
+    hf_messages = [_convert_message(m) for m in messages]
+    hf_tools = _convert_tools(tools) if tools else None
+
+    data = {
+        "model": model_name,
+        "messages": hf_messages,
+        "max_tokens": 4096,
+        "temperature": 0,
+    }
+    if hf_tools:
+        data["tools"] = hf_tools
+        data["tool_choice"] = "auto"
+
+    resp = requests.post(f"{BASE_URL}/chat/completions", headers=HEADERS, json=data, timeout=120)
+    if resp.status_code != 200:
+        print(f"opencode_zen 400 body: {resp.text[:300]}", flush=True)
+        print(f"opencode_zen request model={model_name} tools={bool(hf_tools)} msgs={len(hf_messages)}", flush=True)
+    resp.raise_for_status()
+    choice = resp.json()["choices"][0]
+    msg = choice["message"]
+
+    response_kwargs = {"content": msg.get("content") or ""}
+    additional_kwargs = {}
+
+    reasoning = msg.get("reasoning_content")
+    if reasoning:
+        additional_kwargs["reasoning_content"] = reasoning
+
+    tool_calls_data = msg.get("tool_calls")
+    if tool_calls_data:
+        tool_calls = []
+        for tc in tool_calls_data:
+            tool_calls.append({
+                "id": tc["id"],
+                "name": tc["function"]["name"],
+                "args": json.loads(tc["function"]["arguments"]) if tc["function"].get("arguments") else {},
+            })
+        response_kwargs["tool_calls"] = tool_calls
+        additional_kwargs["tool_calls"] = [
+            {
+                "id": tc["id"],
+                "type": "function",
+                "function": {"name": tc["function"]["name"], "arguments": tc["function"]["arguments"]},
+            }
+            for tc in tool_calls_data
+        ]
+
+    if additional_kwargs:
+        response_kwargs["additional_kwargs"] = additional_kwargs
+
+    return AIMessage(**response_kwargs)
+
+
+def get_models():
+    return ["deepseek-v4-flash-free", "nemotron-3-super-free", "big-pickle"]

llm/providers/openrouter.py

@@ -0,0 +1,22 @@
+import os
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+
+load_dotenv()
+
+
+def invoke(messages, tools, model_name: str = "deepseek/deepseek-chat"):
+    """Invoke OpenRouter model."""
+    model = ChatOpenAI(
+        model=model_name,
+        temperature=0,
+        base_url="https://openrouter.ai/api/v1",
+        api_key=os.getenv("OPENROUTER_API_KEY"),
+    )
+    model_with_tools = model.bind_tools(tools)
+    return model_with_tools.invoke(messages)
+
+
+def get_models():
+    """Free models on OpenRouter."""
+    return ["deepseek/deepseek-chat", "meta-llama/llama-3.2-3b-instruct"]

llm/providers/together.py

@@ -0,0 +1,25 @@
+import os
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+
+load_dotenv()
+
+
+def invoke(messages, tools, model_name: str = "meta-llama/Llama-4-Scout-17B-16E-Instruct"):
+    """Invoke Together AI model."""
+    model = ChatOpenAI(
+        model=model_name,
+        temperature=0,
+        base_url="https://api.together.xyz/v1",
+        api_key=os.getenv("TOGETHER_API_KEY"),
+    )
+    model_with_tools = model.bind_tools(tools)
+    return model_with_tools.invoke(messages)
+
+
+def get_models():
+    """Free models on Together AI."""
+    return [
+        "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+        "mistralai/Mixtral-8x7B-Instruct-v0.1",
+    ]

llm/providers/zai.py

@@ -0,0 +1,22 @@
+import os
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI
+
+load_dotenv()
+
+
+def invoke(messages, tools, model_name: str = "z-ai/glm-5"):
+    """Invoke ZAI model."""
+    model = ChatOpenAI(
+        model=model_name,
+        temperature=0,
+        base_url="https://api.z.ai/api/paas/v4",
+        api_key=os.getenv("ZAI_API_KEY"),
+    )
+    model_with_tools = model.bind_tools(tools)
+    return model_with_tools.invoke(messages)
+
+
+def get_models():
+    """Available models on ZAI."""
+    return ["z-ai/glm-5", "z-ai/glm-5.1"]

run_local.py

@@ -16,10 +16,26 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 def extract_answer(content) -> str:
     if isinstance(content, str):
-        match = re.search(r'FINAL ANSWER:\s*(.+?)(?:\n|$)', content, re.IGNORECASE)
+        cleaned = content.strip()
+        if not cleaned:
+            return ""
+        # Try FINAL ANSWER: pattern (most specific first)
+        match = re.search(r'FINAL ANSWER:\s*(.+?)(?:\n|$)', cleaned, re.IGNORECASE)
         if match:
             return match.group(1).strip()
-        return content.strip()
+        # Try "Answer:" pattern
+        match = re.search(r'Answer:\s*(.+?)(?:\n|$)', cleaned, re.IGNORECASE)
+        if match:
+            return match.group(1).strip()
+        # Try "answer is" pattern
+        match = re.search(r'(?:the\s+)?answer\s+is\s*:?\s*(.+?)(?:\.|$)', cleaned, re.IGNORECASE)
+        if match:
+            return match.group(1).strip()
+        # Use last non-empty line
+        lines = [l.strip() for l in cleaned.split('\n') if l.strip()]
+        if lines:
+            return lines[-1]
+        return cleaned
     return str(content)
 
 class BasicAgent:
@@ -29,9 +45,16 @@ class BasicAgent:
 
     def __call__(self, question: str) -> str:
         messages = [HumanMessage(content=question)]
-        result = self.graph.invoke({"messages": messages})
-        answer = result['messages'][-1].content
-        return extract_answer(answer)
+        result = self.graph.invoke(
+            {"messages": messages, "tool_call_count": 0, "reflection_count": 0, "tool_call_history": []},
+            config={"recursion_limit": 50},
+        )
+        # Find last AIMessage with content (skip ToolMessages and tool-call-only AIMessages)
+        for m in reversed(result['messages']):
+            cls = type(m).__name__
+            if cls == 'AIMessage' and m.content:
+                return extract_answer(m.content)
+        return ""
 
 def file_extract(local_file_path, task_id):
     if not local_file_path:
@@ -107,8 +130,9 @@ def main():
         })
         
         status = "OK" if is_correct else "FAIL"
-        print(f"   {status} Submitted: {str(answer)[:40]}")
-        print(f"      Ground:   {str(ground_truth)[:40]}")
+        def safe(s): return str(s).encode('utf-8', errors='replace').decode('utf-8', errors='replace')[:40]
+        print(f"   {status} Submitted: {safe(answer)}")
+        print(f"      Ground:   {safe(ground_truth)}")
         
         time.sleep(1.5)
 

test_env.py

@@ -0,0 +1,8 @@
+import os
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+print("GROQ_API_KEY exists:", "GROQ_API_KEY" in os.environ)
+print("GOOGLE_API_KEY exists:", "GOOGLE_API_KEY" in os.environ)
+print("HF_TOKEN exists:", "HF_TOKEN" in os.environ)
+print("LLM_PROVIDER_ORDER:", os.getenv("LLM_PROVIDER_ORDER"))

test_gemini.py

@@ -0,0 +1,16 @@
+import os
+from dotenv import load_dotenv
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_core.messages import HumanMessage
+
+load_dotenv(override=True)
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+
+print("GOOGLE_API_KEY:", GOOGLE_API_KEY[:10] + "..." if GOOGLE_API_KEY else None)
+
+try:
+    model = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0, google_api_key=GOOGLE_API_KEY)
+    response = model.invoke([HumanMessage(content="Hello, who are you?")])
+    print("Gemini response:", response.content)
+except Exception as e:
+    print("Gemini failed:", e)

tools/__init__.py

@@ -1,19 +1,21 @@
 from tools.web.search import web_search
 from tools.web.wiki import wiki_search
+from tools.web.wiki_page import wiki_page
 from tools.web.browse import browse_url
 from tools.file.reader import read_file
+from tools.file.spreadsheet import parse_spreadsheet
 from tools.python import python_repl
-from tools.reverse import reverse_text
 from tools.youtube import get_youtube_transcript
 from tools.audio import transcribe_audio
 
 __all__ = [
     web_search,
     wiki_search,
+    wiki_page,
     browse_url,
     read_file,
+    parse_spreadsheet,
     python_repl,
-    reverse_text,
     get_youtube_transcript,
     transcribe_audio,
 ]

tools/file/reader.py

@@ -20,6 +20,15 @@ def read_file(path: str) -> str:
             loader = UnstructuredImageLoader(path)
             docs = loader.load()
             content = "\n\n".join([doc.page_content for doc in docs])
+        elif ext in (".xlsx", ".xls", ".csv"):
+            import pandas as pd
+            df = pd.read_excel(path) if ext != ".csv" else pd.read_csv(path)
+            buf = [f"Rows: {len(df)}, Columns: {list(df.columns)}"]
+            buf.append("  |  ".join(str(c) for c in df.columns))
+            buf.append("-" * min(200, 10 + 12 * len(df.columns)))
+            for i, (_, row) in enumerate(df.iterrows()):
+                buf.append(f"{i} |  " + "  |  ".join(str(v) if pd.notna(v) else "" for v in row))
+            content = "\n".join(buf)
         elif ext == ".pdf":
             try:
                 doc = fitz.open(path)

tools/file/spreadsheet.py

@@ -0,0 +1,37 @@
+from langchain_core.tools import tool
+
+
+@tool
+def parse_spreadsheet(path: str) -> str:
+    """Read an Excel (.xlsx) or CSV file and return its contents as a formatted text table.
+    Use this instead of read_file for spreadsheet files to get properly structured data.
+    Returns all rows with column headers and row numbers."""
+    try:
+        import pandas as pd
+        import os
+
+        if not os.path.exists(path):
+            return f"FILE_NOT_FOUND: {path}"
+
+        ext = os.path.splitext(path)[1].lower()
+        if ext == ".csv":
+            df = pd.read_csv(path)
+        elif ext in (".xlsx", ".xls"):
+            df = pd.read_excel(path, engine="openpyxl" if ext == ".xlsx" else "xlrd")
+        else:
+            return f"UNSUPPORTED_FORMAT: {ext}"
+
+        lines = [f"Sheet: {os.path.basename(path)}  |  Rows: {len(df)}  |  Columns: {len(df.columns)}"]
+        lines.append("  |  " + "  |  ".join(str(c) for c in df.columns))
+        lines.append("-" * min(200, 10 + 12 * len(df.columns)))
+        for i, (_, row) in enumerate(df.iterrows()):
+            vals = [str(v) if pd.notna(v) else "" for v in row]
+            lines.append(f"{i} |  " + "  |  ".join(vals))
+
+        result = "\n".join(lines)
+        if len(result) > 25000:
+            result = result[:25000] + "\n... [TRUNCATED]"
+
+        return result
+    except Exception as e:
+        return f"SPREADSHEET_ERROR: {e}"

tools/python.py

@@ -8,6 +8,7 @@ def python_repl(code: str) -> str:
     """Execute python code and return the output. Use this for calculations, data analysis, or processing files.
     The code should be a valid python script that prints the final result.
     You can use libraries like pandas, numpy, PIL, etc.
+    IMPORTANT: Variables persist between calls to this tool (same Python process). You can define a variable in one call and use it in the next.
     Example: print(df.head()) or print(2 + 2)"""
     try:
         old_stdout = sys.stdout

tools/web/search.py

@@ -1,18 +1,20 @@
-from langchain_tavily import TavilySearch
 from langchain_core.tools import tool
 
 
 @tool
 def web_search(keywords: str) -> str:
-    """Search the web using Tavily. This tool performs a concise, focused search to answer factual questions or gather brief information snippets.
-    For deeper research or browsing specific URLs, additional tools may be required.
-    """
+    """Search the web. Use this for finding information, facts, URLs, and general web content. Returns title, URL, and snippet for each result."""
     try:
-        tavily = TavilySearch(max_results=5)
-        results = tavily.invoke(keywords)
-        formatted_results = []
+        from ddgs import DDGS
+        results = list(DDGS().text(keywords, max_results=5))
+        if not results:
+            return "NO_RESULTS"
+        formatted = []
         for r in results:
-            formatted_results.append(f"Title: {r['title']}\nURL: {r['url']}\nContent: {r['content'][:300]}")
-        return "\n".join(formatted_results) or "NO_RESULTS"
+            title = r.get("title", "")
+            url = r.get("href", "")
+            body = r.get("body", "")[:300]
+            formatted.append(f"Title: {title}\nURL: {url}\nContent: {body}")
+        return "\n\n".join(formatted)
     except Exception as e:
         return f"SEARCH_ERROR: {e}"

tools/web/wiki_page.py

@@ -0,0 +1,42 @@
+from langchain_core.tools import tool
+
+
+@tool
+def wiki_page(title: str) -> str:
+    """Fetch the full text content of a Wikipedia page by its exact title.
+    Use this when wiki_search snippets are insufficient and you need the complete article.
+    Provide the exact Wikipedia page title (case-sensitive, spaces allowed).
+    Returns the first 25000 characters of the article."""
+    try:
+        import requests
+
+        params = {
+            "action": "query",
+            "format": "json",
+            "titles": title,
+            "prop": "extracts",
+            "explaintext": True,
+            "exlimit": 1,
+        }
+        resp = requests.get(
+            "https://en.wikipedia.org/w/api.php",
+            params=params,
+            headers={"User-Agent": "GAIA-Benchmark-Agent/1.0"},
+            timeout=15,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        pages = data.get("query", {}).get("pages", {})
+        if not pages:
+            return "NO_RESULTS"
+        page_id = list(pages.keys())[0]
+        if page_id == "-1":
+            return "PAGE_NOT_FOUND"
+        extract = pages[page_id].get("extract", "")
+        if not extract:
+            return "NO_CONTENT"
+        if len(extract) > 25000:
+            extract = extract[:25000] + "\n... [TRUNCATED]"
+        return extract
+    except Exception as e:
+        return f"WIKI_PAGE_ERROR: {e}"