Update app.py

app.py

@@ -35,6 +35,7 @@ ATTACHMENT_CACHE = {}
 WEBPAGE_CACHE = {}
 SEARCH_CACHE = {}
 SEARCH_LOCK = threading.Lock()
+RUN_STATE = threading.local()
 
 
 def compact_error(exc: Exception) -> str:
@@ -42,6 +43,67 @@ def compact_error(exc: Exception) -> str:
     return message or repr(exc)
 
 
+def reset_run_evidence():
+    RUN_STATE.evidence = []
+
+
+def record_run_evidence(tool_name: str, value) -> str:
+    text = str(value or "").strip()
+    if text:
+        evidence = getattr(RUN_STATE, "evidence", None)
+        if evidence is None:
+            evidence = []
+            RUN_STATE.evidence = evidence
+        evidence.append(
+            {
+                "tool": str(tool_name),
+                "text": text[:5_000],
+            }
+        )
+    return value
+
+
+def current_run_evidence(max_chars: int = 14_000) -> str:
+    items = getattr(RUN_STATE, "evidence", []) or []
+    blocks = []
+    used = 0
+    for index, item in enumerate(items, start=1):
+        block = f"[{index}. {item['tool']}]\n{item['text']}"
+        remaining = max_chars - used
+        if remaining <= 100:
+            break
+        blocks.append(block[:remaining])
+        used += len(blocks[-1]) + 2
+    return "\n\n".join(blocks)
+
+
+def instrument_tool(tool: Tool) -> Tool:
+    """Records tool observations for deterministic review and failover."""
+    original_forward = tool.forward
+
+    def recorded_forward(*args, **kwargs):
+        result = original_forward(*args, **kwargs)
+        return record_run_evidence(tool.name, result)
+
+    tool.forward = recorded_forward
+    return tool
+
+
+def concise_query_from_question(question: str, max_terms: int = 14) -> str:
+    stopwords = {
+        "about", "answer", "attached", "could", "from", "give", "have",
+        "into", "just", "number", "please", "provide", "question", "same",
+        "that", "their", "there", "these", "this", "under", "what", "when",
+        "where", "which", "with", "would", "your",
+    }
+    terms = [
+        token
+        for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9'._-]{2,}", question or "")
+        if token.casefold() not in stopwords
+    ]
+    return " ".join(terms[:max_terms])
+
+
 def focus_text(text: str, query: str, max_chars: int = 5_000) -> str:
     """Selects high-signal passages locally, before text reaches the LLM."""
     text = re.sub(r"\r\n?", "\n", str(text or ""))
@@ -1464,6 +1526,8 @@ class BasicAgent:
             AnalyzeGaiaImageTool(),
             CalculatorTool(),
         ]
+        agent_tools = [instrument_tool(tool) for tool in agent_tools]
+        self.tools_by_name = {tool.name: tool for tool in agent_tools}
 
         # GLM returns native tool calls. ToolCallingAgent handles that
         # structured format without parsing generated Python code.
@@ -1502,6 +1566,9 @@ TOOL ROUTING POLICY:
 Research carefully, prefer primary or official sources, and cross-check
 uncertain facts. A search snippet alone is insufficient when the source page
 can be opened. Never invent a tool, use subprocess, or use shell commands.
+When CONTROLLER-PRECOLLECTED EVIDENCE is present, use it first and do not
+repeat its exact tool call. If it directly answers the task, immediately call
+final_answer instead of researching again.
 Do not repeat nearly identical searches. Stop as soon as primary evidence
 answers the exact question. Never pass an entire task as a webpage query; use
 only names, identifiers, and the target fact. Use at most four tool calls, then
@@ -1558,7 +1625,232 @@ include reasoning, explanations, labels, Markdown, citations, or the words
             )
             print(f"Fallback Gemini habilitado: {fallback_name}")
 
-    def _run_with_failover(self, task_context: str):
+    def _precollect_deterministic_evidence(
+        self,
+        question: str,
+        task_id: str | None,
+        attachment_name: str = "",
+    ) -> str:
+        """Runs mandatory/specialized tools by rule before an LLM can choose."""
+        suffix = Path(attachment_name).suffix.lower()
+        query = concise_query_from_question(question)
+        route = ""
+
+        if task_id and suffix in {
+            ".mp3", ".wav", ".flac", ".m4a", ".ogg", ".webm", ".mp4"
+        }:
+            self.tools_by_name["transcribe_gaia_audio"].forward(task_id=task_id)
+            route = "audio attachment -> transcribe_gaia_audio"
+        elif task_id and suffix in {
+            ".png", ".jpg", ".jpeg", ".webp", ".gif"
+        }:
+            self.tools_by_name["analyze_gaia_image"].forward(
+                task_id=task_id,
+                question=question,
+            )
+            route = "image attachment -> analyze_gaia_image"
+        elif task_id and suffix in {".xlsx", ".xlsm", ".csv", ".tsv"}:
+            self.tools_by_name["query_gaia_spreadsheet"].forward(
+                task_id=task_id,
+                operation="describe",
+                sheet="",
+                column="",
+                filters="",
+            )
+            route = "spreadsheet attachment -> query_gaia_spreadsheet(describe)"
+        elif task_id and attachment_name:
+            self.tools_by_name["inspect_gaia_attachment"].forward(
+                task_id=task_id,
+                query=query,
+            )
+            route = "document attachment -> inspect_gaia_attachment"
+
+        youtube_match = re.search(
+            r"https?://(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)"
+            r"[A-Za-z0-9_-]{11}",
+            question,
+            flags=re.I,
+        )
+        spoken_cues = (
+            "say", "said", "says", "speak", "spoken", "quote", "transcript",
+            "according to the video", "what does", "what did",
+        )
+        if (
+            not route
+            and youtube_match
+            and any(cue in question.lower() for cue in spoken_cues)
+        ):
+            self.tools_by_name["youtube_transcript"].forward(
+                url=youtube_match.group(0),
+                query=query,
+            )
+            route = "spoken YouTube question -> youtube_transcript"
+
+        team_aliases = {
+            "yankee": "Yankees", "red sox": "Red Sox",
+            "oriole": "Orioles", "ray": "Rays", "blue jay": "Blue Jays",
+            "white sox": "White Sox", "guardian": "Guardians",
+            "cleveland indian": "Indians", "tiger": "Tigers",
+            "royal": "Royals", "twin": "Twins", "astro": "Astros",
+            "angel": "Angels", "athletic": "Athletics", "mariner": "Mariners",
+            "ranger": "Rangers", "brave": "Braves", "marlin": "Marlins",
+            "met": "Mets", "phillie": "Phillies", "national": "Nationals",
+            "cub": "Cubs", "red": "Reds", "brewer": "Brewers",
+            "pirate": "Pirates", "cardinal": "Cardinals",
+            "diamondback": "Diamondbacks", "rockie": "Rockies",
+            "dodger": "Dodgers", "padre": "Padres", "giant": "Giants",
+        }
+        lower_question = question.lower()
+        years = re.findall(r"\b(?:18|19|20)\d{2}\b", question)
+        matched_team = next(
+            (
+                canonical
+                for alias, canonical in team_aliases.items()
+                if re.search(rf"\b{re.escape(alias)}s?\b", lower_question)
+            ),
+            None,
+        )
+        stat_cues = [
+            ("most walks", "walks"), ("least walks", "walks"),
+            ("walks", "walks"), ("at bats", "at bats"),
+            ("home runs", "home runs"), ("stolen bases", "stolen bases"),
+            ("strikeouts", "strikeouts"), ("hits", "hits"),
+            ("runs batted", "rbi"), ("rbi", "rbi"), ("ops", "ops"),
+            ("obp", "obp"), ("slugging", "slg"), ("average", "average"),
+        ]
+        matched_stat = next(
+            (stat for cue, stat in stat_cues if cue in lower_question),
+            None,
+        )
+        if not route and matched_team and years and matched_stat:
+            self.tools_by_name["mlb_stats"].forward(
+                team=matched_team,
+                season=int(years[0]),
+                sort_stat=matched_stat,
+            )
+            route = "MLB statistics question -> official MLB Stats API"
+
+        if route:
+            print(f"Rota determinística: {route}")
+        return route
+
+    @staticmethod
+    def _invalid_candidate(candidate: str) -> bool:
+        value = str(candidate or "").strip()
+        lowered = value.lower()
+        if not value or lowered in {"none", "null", "n/a"}:
+            return True
+        if len(value) > 1_000:
+            return True
+        tool_syntax = (
+            bool(re.search(
+                r"""["']type["']\s*:\s*["']function["']""",
+                value,
+                flags=re.I,
+            ))
+            and bool(re.search(
+                r"""["']arguments["']\s*:""",
+                value,
+                flags=re.I,
+            ))
+        )
+        unfinished_cues = (
+            "call: ", "calling tool", "let's search", "lets search",
+            "let's do a search", "web_search(", "wikipedia_search(",
+            "visit_webpage(", "query_web_table(", "i need to search",
+        )
+        return tool_syntax or any(cue in lowered for cue in unfinished_cues)
+
+    def _gemini_answer_from_evidence(
+        self,
+        question: str,
+        evidence: str,
+        task_id: str | None = None,
+    ) -> str:
+        """Produces one answer from existing evidence without running tools."""
+        gemini_api_key = os.getenv("GEMINI_API_KEY")
+        if not gemini_api_key:
+            raise RuntimeError(
+                "GEMINI_API_KEY não está configurada para o fallback."
+            )
+        model = os.getenv(
+            "GAIA_GEMINI_FALLBACK_MODEL", DEFAULT_GEMINI_REVIEW_MODEL
+        )
+        if not model.lower().startswith("gemini-"):
+            model = DEFAULT_GEMINI_REVIEW_MODEL
+        prompt = f"""
+Answer this GAIA task using ONLY the collected evidence below. Do not call or
+suggest tools and do not perform another search. If the evidence is sufficient,
+return only the exact requested value in final_answer. If it is insufficient,
+set sufficient_evidence to false and leave final_answer empty.
+
+Task ID: {task_id or "test"}
+Question: {question}
+
+COLLECTED EVIDENCE:
+{evidence}
+""".strip()
+        response = requests.post(
+            (
+                "https://generativelanguage.googleapis.com/v1beta/models/"
+                f"{model}:generateContent"
+            ),
+            headers={
+                "x-goog-api-key": gemini_api_key,
+                "Content-Type": "application/json",
+            },
+            json={
+                "contents": [
+                    {"role": "user", "parts": [{"text": prompt}]}
+                ],
+                "generationConfig": {
+                    "maxOutputTokens": 512,
+                    "thinkingConfig": {"thinkingLevel": "minimal"},
+                    "responseMimeType": "application/json",
+                    "responseJsonSchema": {
+                        "type": "object",
+                        "properties": {
+                            "final_answer": {"type": "string"},
+                            "sufficient_evidence": {"type": "boolean"},
+                        },
+                        "required": [
+                            "final_answer",
+                            "sufficient_evidence",
+                        ],
+                        "additionalProperties": False,
+                    },
+                },
+            },
+            timeout=(WEBPAGE_CONNECT_TIMEOUT, 45),
+        )
+        response.raise_for_status()
+        payload = response.json()
+        parts = payload["candidates"][0]["content"]["parts"]
+        content = "".join(
+            str(part.get("text") or "") for part in parts
+        ).strip()
+        data = json.loads(content)
+        if not data.get("sufficient_evidence"):
+            raise RuntimeError(
+                "O Gemini informou que as evidências coletadas ainda são "
+                "insuficientes. A resposta não foi salva."
+            )
+        answer = self.enforce_direct_answer(
+            question, str(data.get("final_answer") or "")
+        )
+        if self._invalid_candidate(answer):
+            raise RuntimeError(
+                "O fallback Gemini não produziu uma resposta final válida."
+            )
+        print(f"Gemini respondeu usando as evidências existentes: {answer}")
+        return answer
+
+    def _run_with_failover(
+        self,
+        task_context: str,
+        question: str,
+        task_id: str | None,
+    ):
         try:
             return self.agent.run(task_context, reset=True), False
         except Exception as exc:
@@ -1580,10 +1872,20 @@ include reasoning, explanations, labels, Markdown, citations, or the words
                 )
             )
             if retryable_provider_error and self.gemini_fallback_agent:
+                evidence = current_run_evidence()
                 print(
                     "Cerebras indisponível; executando a questão com o "
                     f"fallback Gemini. Motivo: {compact_error(exc)}"
                 )
+                if evidence:
+                    return (
+                        self._gemini_answer_from_evidence(
+                            question=question,
+                            evidence=evidence,
+                            task_id=task_id,
+                        ),
+                        True,
+                    )
                 return (
                     self.gemini_fallback_agent.run(task_context, reset=True),
                     True,
@@ -1601,6 +1903,8 @@ include reasoning, explanations, labels, Markdown, citations, or the words
         if not question:
             raise ValueError("Digite uma pergunta para testar o agente.")
         SEARCH_CACHE.clear()
+        reset_run_evidence()
+        attachment_name = ""
 
         if task_id:
             attachment_name = get_task_file_name(task_id)
@@ -1640,8 +1944,26 @@ include reasoning, explanations, labels, Markdown, citations, or the words
             )
         else:
             task_context = question
+        route = self._precollect_deterministic_evidence(
+            question=question,
+            task_id=task_id,
+            attachment_name=attachment_name,
+        )
+        precollected = current_run_evidence()
+        if precollected:
+            task_context += (
+                "\n\nCONTROLLER-PRECOLLECTED EVIDENCE:\n"
+                f"{precollected}\n\n"
+                f"Controller route: {route}. Do not repeat the same tool call; "
+                "a new spreadsheet calculation is allowed after describe. "
+                "If this evidence answers the question, call final_answer now."
+            )
         try:
-            result, used_gemini_fallback = self._run_with_failover(task_context)
+            result, used_gemini_fallback = self._run_with_failover(
+                task_context=task_context,
+                question=question,
+                task_id=task_id,
+            )
         except Exception as exc:
             error_text = str(exc)
             if (
@@ -1672,6 +1994,9 @@ include reasoning, explanations, labels, Markdown, citations, or the words
                 ))
             )
         )
+        invalid_candidate = (
+            invalid_candidate or self._invalid_candidate(candidate)
+        )
         if (
             invalid_candidate
             and self.gemini_fallback_agent
@@ -1681,9 +2006,21 @@ include reasoning, explanations, labels, Markdown, citations, or the words
                 "Cerebras terminou sem resposta final; repetindo a questão "
                 "com o fallback Gemini."
             )
-            result = self.gemini_fallback_agent.run(task_context, reset=True)
+            evidence = current_run_evidence()
             used_gemini_fallback = True
-            candidate = self.enforce_direct_answer(question, str(result))
+            if evidence:
+                candidate = self._gemini_answer_from_evidence(
+                    question=question,
+                    evidence=evidence,
+                    task_id=task_id,
+                )
+            else:
+                result = self.gemini_fallback_agent.run(
+                    task_context, reset=True
+                )
+                candidate = self.enforce_direct_answer(
+                    question, str(result)
+                )
             invalid_candidate = (
                 not candidate
                 or candidate.strip().lower() in {"none", "null"}
@@ -1700,6 +2037,9 @@ include reasoning, explanations, labels, Markdown, citations, or the words
                     ))
                 )
             )
+            invalid_candidate = (
+                invalid_candidate or self._invalid_candidate(candidate)
+            )
         if invalid_candidate:
             raise RuntimeError(
                 "O agente esgotou as etapas sem produzir uma resposta final. "
@@ -1712,6 +2052,7 @@ include reasoning, explanations, labels, Markdown, citations, or the words
             question=question,
             candidate=candidate,
             task_id=task_id,
+            evidence=current_run_evidence(),
         )
 
     @staticmethod
@@ -1957,7 +2298,11 @@ include reasoning, explanations, labels, Markdown, citations, or the words
         return self.deterministic_answer_cleanup(cleaned)
 
     def review_answer_with_gemini(
-        self, question: str, candidate: str, task_id: str | None = None
+        self,
+        question: str,
+        candidate: str,
+        task_id: str | None = None,
+        evidence: str = "",
     ) -> str:
         """Revisa com Gemini e sempre preserva a resposta primária se falhar."""
         gemini_api_key = os.getenv("GEMINI_API_KEY")
@@ -1977,9 +2322,13 @@ Review this GAIA exact-match candidate. Preserve it unless a correction is
 clearly necessary. The final_answer must contain only the requested value,
 without explanation, label, Markdown, or citation. Respect requested numeric,
 currency, name, list separator/order, quote, or chess notation formats.
+Use only the collected evidence when it is present. Do not invent facts and do
+not suggest or call another tool.
 Task ID: {task_id or "test"}
 Question: {question}
 Candidate: {candidate}
+Collected evidence:
+{evidence or "(none; format-check the candidate only)"}
 """.strip()
 
         try:
@@ -2331,7 +2680,10 @@ def run_all_evaluation_questions(
         try:
             updated[task_id] = agent(question, task_id)
         except Exception as exc:
-            updated[task_id] = f"ERROR: {exc}"
+            print(
+                f"Questão {task_id} falhou e não foi salva: "
+                f"{compact_error(exc)}"
+            )
             failures += 1
 
     dataframe = review_dataframe(questions, updated)
@@ -2433,7 +2785,11 @@ def run_agent_only(profile: gr.OAuthProfile | None):
         try:
             answer = agent(question, task_id)
         except Exception as exc:
-            answer = f"ERROR: {exc}"
+            print(
+                f"Questão {task_id} falhou e foi omitida da avaliação local: "
+                f"{compact_error(exc)}"
+            )
+            continue
 
         results.append(
             {