Update tools.py

tools.py

@@ -1,4 +1,3 @@
-cat > /home/claude/tools.py << 'ENDOFFILE'
 # tools.py
 import os
 import re
@@ -37,17 +36,15 @@ def _save_cache(cache: dict) -> None:
 # Helpers used by agent.py (no LLM, no @tool)
 # ──────────────────────────────────────────────────────────────────────────────
 
-# Question-type classifier ─────────────────────────────────────────────────────
 def classify_question(question: str) -> str:
     """
-    Return one of: 'reasoning', 'youtube', 'image', 'wikipedia_log', 'web'
+    Route to one of: 'reasoning', 'youtube', 'image', 'wikipedia_log', 'web'.
     Checked in order; first match wins.
     """
     q = question.lower()
 
-    # Pure reasoning — no external lookup needed
     reasoning_patterns = [
-        r"\btable\b.*\bset\b.*\{",           # math/operation table
+        r"\btable\b.*\bset\b.*\{",
         r"\boperation\b.*\bset\b",
         r"grocery list",
         r"\bbotany\b",
@@ -56,7 +53,7 @@ def classify_question(question: str) -> str:
         r"\bcommutativ",
         r"\bassociativ",
         r"making a pie",
-        r"shopping list.*recipe",
+        r"shopping list.*(?:recipe|ingredient|pie)",
         r"recipe.*ingredient",
         r"\bconvert\b.*\bunits?\b",
         r"\bcalculat",
@@ -65,65 +62,84 @@ def classify_question(question: str) -> str:
         if re.search(pat, q):
             return "reasoning"
 
-    # YouTube video
     if "youtube.com/watch" in q or "youtu.be/" in q:
         return "youtube"
 
-    # Image / chess
-    if re.search(r"\bimage\b|\bchess\b|\bboard\b.*\bposition\b|\bpicture\b|\bphoto\b|\bscreenshot\b", q):
+    if re.search(
+        r"\bimage\b|\bchess\b|\bboard\b.*\bposition\b|\bpicture\b|\bphoto\b|\bscreenshot\b",
+        q
+    ):
         return "image"
 
-    # Wikipedia featured article log — needs the exact archive URL
-    if re.search(r"featured article.*wikipedia.*nominated|nominated.*featured article.*wikipedia", q):
-        return "wikipedia_log"
-    if re.search(r"featured log|featured article.*promoted.*\d{4}|promoted.*featured article.*\d{4}", q):
+    if re.search(
+        r"featured article.*wikipedia.*nominated|nominated.*featured article.*wikipedia"
+        r"|featured log|featured article.*promoted.*\d{4}|promoted.*featured article.*\d{4}",
+        q
+    ):
         return "wikipedia_log"
 
     return "web"
 
 
-# Search-query builder ─────────────────────────────────────────────────────────
 def build_search_query(question: str) -> str:
     """
-    Condense a verbose GAIA question into a tight DDG query (4-7 words).
-    Always appends 'wikipedia' so DDG surfaces the right article first.
+    Turn a verbose GAIA question into a tight 4-8 word DDG query.
+    Always appends 'wikipedia' to surface the right article first.
     """
     q = question.strip()
-    q = re.sub(r"\(.*?\)", "", q).strip()   # drop parenthetical hints
+    # Remove parenthetical hints
+    q = re.sub(r"\(.*?\)", "", q).strip()
 
-    # Drop question starters
+    # Drop question-word starters
     q = re.sub(
         r"^(how many|what (is|was|are|were)|who (is|was)|when (did|was|is)|"
         r"which|where (is|was)|why|tell me|find|give me|list)\s+",
         "", q, flags=re.I,
     )
-    # Drop filler
-    for filler in [
-        r"studio albums? were published by",
-        r"albums? were released by",
-        r"were published by",
-        r"was born in",
-        r"between \d{4} and \d{4}.*",
-        r"you can use .*",
-        r"the latest \d{4} version.*",
-        r"surname of the",
-        r"mentioned in .*libretexts.*",
-    ]:
-        q = re.sub(filler, "", q, flags=re.I).strip()
+
+    # Drop known filler — order matters (longer patterns first)
+    fillers = [
+        r"studio albums? (?:were )?published by\s*",
+        r"albums? (?:were )?released by\s*",
+        r"were published by\s*",
+        r"was born in\s*",
+        r"between \d{4} and \d{4}[^.]*",
+        r"you can use [^.]*",
+        r"the latest \d{4} version[^.]*",
+        r"surname of (?:the)?\s*",
+        r"(?:licensed|compiled) by .*",          # drop "licensed by X …"
+        r"from the chemistry materials?.*",
+        r"in \d+\.\w+ exercises?.*",
+        r"under the ck-12 .*",
+        r"libretexts.*",
+        r"mentioned in .*(?:exercises?|materials?)",
+    ]
+    for filler in fillers:
+        q = re.sub(filler, " ", q, flags=re.I).strip()
 
     q = re.sub(r"\s+", " ", q).strip().rstrip("?.,;:")
 
+    # Cap at 8 words so DDG returns precise results
+    words = q.split()
+    if len(words) > 8:
+        q = " ".join(words[:8])
+
     if "wikipedia" not in q.lower():
         q += " wikipedia"
     return q
 
 
-# URL scorer ──────────────���────────────────────────────────────────────────────
 def extract_best_url(search_output: str, question: str = "") -> str | None:
     """
-    Pick the most relevant URL from web_search output without an LLM call.
-    Scores by keyword overlap between the URL+snippet and the question.
+    Score URLs by keyword overlap with the question.
+    Avoids known useless domains; returns None if nothing looks good.
     """
+    BAD_DOMAINS = {
+        "youtube.com", "reddit.com", "facebook.com", "chegg.com",
+        "studyx.ai", "lespac.com", "fandom.com", "quora.com",
+        "answers.com", "yahoo.com",
+    }
+
     blocks = re.split(r"\n\n+", search_output)
     candidates: list[tuple[str, str]] = []
     for block in blocks:
@@ -140,6 +156,7 @@ def extract_best_url(search_output: str, question: str = "") -> str | None:
         "by","to","and","or","you","can","use","between","included","latest",
         "version","english","wikipedia","published","released","studio","albums",
         "surname","mentioned","exercises","chemistry","licensed","compiled",
+        "materials","introductory","ck12","libretexts",
     }
     keywords = [
         w.lower() for w in re.findall(r"[A-Za-z]{3,}", question)
@@ -153,9 +170,7 @@ def extract_best_url(search_output: str, question: str = "") -> str | None:
             s += 3
         if "disambiguation" in ul or "disambiguation" in ctx:
             s -= 2
-        # penalise obvious mismatches (YouTube, Facebook, Reddit, Chegg…)
-        for bad in ["youtube.com", "reddit.com", "facebook.com",
-                    "chegg.com", "studyx.ai", "lespac.com", "fandom.com"]:
+        for bad in BAD_DOMAINS:
             if bad in ul:
                 s -= 5
         for kw in keywords:
@@ -166,14 +181,12 @@ def extract_best_url(search_output: str, question: str = "") -> str | None:
         return s
 
     ranked = sorted(candidates, key=lambda x: score(x[0], x[1]), reverse=True)
-    best = ranked[0][0]
-    # Never return a score-negative URL (all bad sources) — return None instead
-    if score(best, ranked[0][1]) < 0:
+    best_url, best_ctx = ranked[0]
+    if score(best_url, best_ctx) < 0:
         return None
-    return best
+    return best_url
 
 
-# YouTube transcript helper ────────────────────────────────────────────────────
 def _extract_youtube_id(text: str) -> str | None:
     m = re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]{11})", text)
     return m.group(1) if m else None
@@ -184,7 +197,7 @@ def _extract_youtube_id(text: str) -> str | None:
 # ──────────────────────────────────────────────────────────────────────────────
 @tool
 def web_search(query: str) -> str:
-    """Search the web. Pass a SHORT query (4-7 words), never the full question.
+    """Search the web. Pass a SHORT query (4-8 words), never the full question.
 
     Args:
         query: Short search query, e.g. 'Mercedes Sosa discography wikipedia'
@@ -251,7 +264,7 @@ def visit_webpage(url: str) -> str:
 @tool
 def get_youtube_transcript(video_url: str) -> str:
     """Fetch the auto-generated transcript of a YouTube video.
-    Use this for any question that asks about audio/dialogue in a YouTube video.
+    Use this for any question that asks about spoken dialogue or audio in a video.
 
     Args:
         video_url: Full YouTube URL, e.g. 'https://www.youtube.com/watch?v=1htKBjuUWec'
@@ -260,25 +273,25 @@ def get_youtube_transcript(video_url: str) -> str:
     if not vid_id:
         return f"Could not extract video ID from: {video_url}"
 
-    # Try youtube-transcript-api (pip install youtube-transcript-api)
+    # Primary: youtube-transcript-api (pip install youtube-transcript-api)
     try:
         from youtube_transcript_api import YouTubeTranscriptApi
-        transcript_list = YouTubeTranscriptApi.get_transcript(vid_id)
-        text = " ".join(entry["text"] for entry in transcript_list)
+        entries = YouTubeTranscriptApi.get_transcript(vid_id)
+        text = " ".join(e["text"] for e in entries)
         return text[:8000]
-    except Exception as e:
-        pass  # fall through to scrape fallback
+    except Exception:
+        pass
 
-    # Fallback: scrape the timedtext API
+    # Fallback: scrape caption track from page source
     try:
-        url = f"https://www.youtube.com/watch?v={vid_id}"
-        resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=12)
-        # Extract caption track URL from page source
+        resp = requests.get(
+            f"https://www.youtube.com/watch?v={vid_id}",
+            headers={"User-Agent": "Mozilla/5.0"}, timeout=12
+        )
         cap_match = re.search(r'"captionTracks":\[.*?"baseUrl":"(.*?)"', resp.text)
         if cap_match:
             cap_url = cap_match.group(1).replace("\\u0026", "&")
             cap_resp = requests.get(cap_url, timeout=10)
-            # Strip XML tags from caption XML
             text = re.sub(r"<[^>]+>", " ", cap_resp.text)
             text = re.sub(r"\s+", " ", text).strip()
             return text[:8000]
@@ -306,5 +319,4 @@ def read_pdf(filepath: str) -> str:
             return "PDF appears to be empty or image-only (no extractable text)."
         return text[:15000]
     except Exception as e:
-        return f"PDF error: {e}"
-ENDOFFILE
+        return f"PDF error: {e}"