Update app.py

app.py

@@ -24,11 +24,13 @@ GROQ_MODELS = [
     m.strip()
     for m in os.getenv(
         "GROQ_MODELS",
-        # 70b first — much better reasoning. Trim history aggressively to fit 6K TPM.
-        "llama-3.3-70b-versatile,llama-3.1-8b-instant",
+        # 8b only — 30K TPM is plenty. 70b's 6K TPM is too tight for tool-calling agents.
+        "llama-3.1-8b-instant",
     ).split(",")
     if m.strip()
 ]
+# Smarter model used ONLY for the final formatting/synthesis pass (one short call -> fits in TPM).
+GROQ_FINAL_MODEL = os.getenv("GROQ_FINAL_MODEL", "llama-3.3-70b-versatile")
 GROQ_VISION_MODEL = os.getenv("GROQ_VISION_MODEL", "meta-llama/llama-4-scout-17b-16e-instruct")
 GROQ_WHISPER_MODEL = os.getenv("GROQ_WHISPER_MODEL", "whisper-large-v3-turbo")
 
@@ -533,15 +535,17 @@ class GroqAgent:
         self.exhausted_models: set[str] = set()
         print(f"GroqAgent initialized with models={self.models}")
 
-    def _chat(self, messages, use_tools: bool = True, max_tokens: int = 800):
+    def _chat(self, messages, use_tools: bool = True, max_tokens: int = 800, model: str | None = None):
+        """Try the configured models in order. Handles 429 (retry), 413 (trim & next), TPD (skip)."""
         last_error: Exception | None = None
-        for model in self.models:
-            if model in self.exhausted_models:
+        models = [model] if model else self.models
+        for m in models:
+            if m in self.exhausted_models:
                 continue
             for attempt in range(3):
                 try:
                     kwargs = dict(
-                        model=model,
+                        model=m,
                         messages=messages,
                         temperature=0.0,
                         max_tokens=max_tokens,
@@ -557,24 +561,23 @@ class GroqAgent:
                     is_413 = "413" in msg or "too large" in msg.lower()
                     is_tpd = "per day" in msg.lower() or "tpd" in msg.lower()
                     if is_413:
-                        # Request is too big for this model's TPM bucket.
-                        # Trim history aggressively and try the next model.
-                        print(f"[{model}] 413 too large; will trim and try next model.")
-                        last_error = e
+                        print(f"[{m}] 413 too large; trying next model.")
                         break
                     if is_429 and is_tpd:
-                        print(f"[{model}] daily token limit exhausted; switching model.")
-                        self.exhausted_models.add(model)
+                        print(f"[{m}] daily token limit exhausted; switching model.")
+                        self.exhausted_models.add(m)
                         break
                     if is_429:
                         wait = self._parse_retry_seconds(msg)
                         wait = min(max(wait, 2), 30)
-                        print(f"[{model}] 429; sleeping {wait}s (attempt {attempt + 1}/3)")
+                        print(f"[{m}] 429; sleeping {wait}s (attempt {attempt + 1}/3)")
                         time.sleep(wait)
                         continue
-                    print(f"[{model}] API error: {e}")
+                    print(f"[{m}] API error: {e}")
                     break
-        raise RuntimeError(f"All Groq models failed. Last error: {last_error}")
+        # Use repr() so empty exception messages still show useful info.
+        err_str = repr(last_error) if last_error else "no error captured"
+        raise RuntimeError(f"All Groq models failed. {err_str}")
 
     @staticmethod
     def _parse_retry_seconds(error_msg: str) -> float:
@@ -617,27 +620,30 @@ class GroqAgent:
             {"role": "user", "content": user_content},
         ]
 
+        # Track tool outputs to feed into the synthesis pass even if loop fails.
+        collected_facts: list[str] = []
+
         for step in range(MAX_TOOL_ITERATIONS):
             try:
                 resp = self._chat(self._trim_messages(messages), use_tools=True, max_tokens=800)
             except Exception as e:
-                # Try one more time with no tools, heavily trimmed, to at least get a guess.
-                print(f"chat failed: {e} — trying no-tool fallback.")
+                print(f"chat iteration {step} failed: {e} — trimming and retrying once.")
+                # Aggressive trim: keep only system + user + last 2 messages.
+                short_msgs = [messages[0], messages[1]]
+                if len(messages) > 2:
+                    short_msgs += messages[-2:]
                 try:
-                    resp = self._chat(
-                        [messages[0], messages[1]],
-                        use_tools=False,
-                        max_tokens=200,
-                    )
+                    resp = self._chat(short_msgs, use_tools=True, max_tokens=600)
                 except Exception as e2:
-                    return f"AGENT ERROR: {e2}"
+                    print(f"retry also failed: {e2}; falling through to synthesis.")
+                    break
 
             msg = resp.choices[0].message
             tool_calls = getattr(msg, "tool_calls", None)
 
             if not tool_calls:
                 answer = (msg.content or "").strip()
-                return self._finalize(answer, question)
+                return self._finalize(answer, question, collected_facts)
 
             messages.append(
                 {
@@ -678,6 +684,9 @@ class GroqAgent:
                 if len(result) > TOOL_RESULT_MAX_CHARS:
                     result = result[:TOOL_RESULT_MAX_CHARS] + "\n...[truncated]"
 
+                # Save to facts (cap each at 800 chars for synthesis pass).
+                collected_facts.append(f"[{name}] {result[:800]}")
+
                 messages.append(
                     {
                         "role": "tool",
@@ -687,31 +696,68 @@ class GroqAgent:
                     }
                 )
 
-        # Out of tool iterations: ask for a final, no-tool answer.
-        messages.append(
+        # Loop ended (either ran out of iterations OR chat repeatedly failed).
+        # Do a final synthesis pass on a SHORT context using the smarter model.
+        return self._synthesize(question, collected_facts)
+
+    def _synthesize(self, question: str, facts: list[str]) -> str:
+        """Final answer pass on a short context. Uses smarter model if available."""
+        # Keep total facts under ~3500 chars to be safe with TPM on 70b.
+        joined = "\n\n".join(facts[-6:])  # last 6 tool outputs
+        if len(joined) > 3500:
+            joined = joined[-3500:]
+
+        synth_messages = [
+            {
+                "role": "system",
+                "content": (
+                    "You are a strict GAIA answer formatter. Read the question and the "
+                    "research notes below, then output ONLY the final answer string. "
+                    "No preamble, no labels, no explanation, no quotes, no trailing period. "
+                    "Match the question's required format exactly (number-only / IOC code / "
+                    "first name only / two-decimal currency / comma-space list / etc.)."
+                ),
+            },
             {
                 "role": "user",
-                "content": "Stop using tools. Reply with ONLY the final answer string per the formatting rules.",
-            }
-        )
-        try:
-            resp = self._chat(self._trim_messages(messages), use_tools=False, max_tokens=200)
-            return self._finalize(
-                (resp.choices[0].message.content or "").strip(), question
-            )
-        except Exception as e:
-            return f"AGENT ERROR: {e}"
+                "content": (
+                    f"Question:\n{question}\n\n"
+                    f"Research notes:\n{joined or '(no notes)'}\n\n"
+                    f"Final answer:"
+                ),
+            },
+        ]
+        # Try the smarter final model first; fall back to the regular pool.
+        for model_choice in (GROQ_FINAL_MODEL, *self.models):
+            try:
+                resp = self._chat(synth_messages, use_tools=False, max_tokens=120, model=model_choice)
+                ans = (resp.choices[0].message.content or "").strip()
+                ans = self._postprocess_answer(ans, question)
+                if ans:
+                    return ans
+            except Exception as e:
+                print(f"synth with {model_choice} failed: {e}")
+                continue
+        return ""
 
-    def _finalize(self, raw: str, question: str) -> str:
+    def _finalize(self, raw: str, question: str, facts: list[str] | None = None) -> str:
         """Post-process and, if the answer still looks like a sentence, ask the model to reformat."""
         cleaned = self._postprocess_answer(raw, question)
         if not cleaned:
+            # Empty answer? Try synthesis from collected facts.
+            if facts:
+                return self._synthesize(question, facts)
             return cleaned
         # If the cleaned answer is suspiciously long or contains explanation-y patterns,
         # do a single tiny reformat pass.
         looks_sentence = (
             len(cleaned.split()) > 12
-            or re.search(r"\b(because|received|grant|seems|unable|sorry|cannot|provides|indicating)\b", cleaned, re.IGNORECASE)
+            or re.search(
+                r"\b(because|received|grant|seems|unable|sorry|cannot|provides|indicating|"
+                r"web_search|youtube_transcript|fetch_url|task_id)\b",
+                cleaned,
+                re.IGNORECASE,
+            )
         )
         if looks_sentence:
             try: