feat: implement robust web fetcher with redirect handling and add LLM-based answer formatting pipeline

src/lilith_agent/tools/search.py

@@ -4,6 +4,12 @@ from ddgs import DDGS
 from tavily import TavilyClient, UsageLimitExceededError
 from tavily.errors import ForbiddenError
 
+_SEARCH_RECOVERY_HINT = (
+    "SEARCH_HINT: no useful results. Try: (1) rephrase with alternative keywords or narrower scope; "
+    "(2) for academic/scientific queries, use arxiv_search or crossref_search; "
+    "(3) for removed or historical pages, search via https://web.archive.org."
+)
+
 
 def _ddg_search(query: str, max_results: int) -> str:
     results = DDGS().text(query, max_results=max_results)
@@ -26,17 +32,19 @@ def web_search(query: str, api_key: str, max_results: int = 5) -> str:
 
     # 2. Fallback to Tavily if DDG failed or returned nothing
     if not api_key:
-        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback skipped: no key]"
-        
+        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback skipped: no key]\n{_SEARCH_RECOVERY_HINT}"
+
     client = TavilyClient(api_key=api_key)
     try:
         res = client.search(query=query, max_results=max_results)
         lines: list[str] = []
         for item in res.get("results", []):
             lines.append(f"- {item['title']} ({item['url']})\n  {item['content']}")
-        tavily_results = "\n".join(lines) if lines else "No results."
+        if not lines:
+            return f"[Source: Tavily (DDG fallback: {ddg_results})]\nNo results.\n{_SEARCH_RECOVERY_HINT}"
+        tavily_results = "\n".join(lines)
         return f"[Source: Tavily (DDG fallback: {ddg_results})]\n{tavily_results}"
     except (UsageLimitExceededError, ForbiddenError) as e:
-        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback failed: limit/forbidden: {e}]"
+        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback failed: limit/forbidden: {e}]\n{_SEARCH_RECOVERY_HINT}"
     except Exception as e:
-        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback failed: {e}]"
+        return f"[Source: DuckDuckGo (Empty)]\n{ddg_results}\n[Tavily fallback failed: {e}]\n{_SEARCH_RECOVERY_HINT}"

src/lilith_agent/tools/web.py

@@ -1,11 +1,44 @@
 import io
 import ipaddress
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 
 import httpx
 import trafilatura
 from pypdf import PdfReader
 
+_FETCH_RECOVERY_HINT = (
+    "Try: (1) web_search for third-party citations or mirrors of this content; "
+    "(2) fetch the archived version at https://web.archive.org/web/<url>; "
+    "(3) use the snippet from your prior web_search result instead of fetching the page."
+)
+
+_MAX_REDIRECTS = 5
+
+
+def _safe_get(url: str, **kwargs) -> httpx.Response:
+    """GET with bounded manual redirect following; SSRF-rechecks every hop.
+
+    The initial `_is_safe_http_url` check only validates the first URL; a
+    server can still 302 to a private/metadata address. This helper follows
+    redirects one hop at a time, re-running the static SSRF check before each
+    request, and raises `httpx.HTTPError` if any hop resolves to an unsafe URL
+    or the redirect budget is exceeded.
+    """
+    kwargs.pop("follow_redirects", None)
+    current = url
+    for _ in range(_MAX_REDIRECTS + 1):
+        ok, reason = _is_safe_http_url(current)
+        if not ok:
+            raise httpx.HTTPError(f"redirect to unsafe URL blocked ({reason})")
+        resp = httpx.get(current, follow_redirects=False, **kwargs)
+        if not resp.is_redirect:
+            return resp
+        location = resp.headers.get("Location")
+        if not location:
+            return resp
+        current = urljoin(current, location)
+    raise httpx.HTTPError(f"too many redirects (> {_MAX_REDIRECTS})")
+
 
 def _is_safe_http_url(url: str) -> tuple[bool, str]:
     """SSRF guard: allow only http/https to public hosts.
@@ -63,13 +96,13 @@ def fetch_url(url: str, max_chars: int = 8000, timeout: float = 60.0) -> str:
     # 1. Check for PDF - Local extraction is always more reliable for PDFs
     if url.lower().endswith(".pdf"):
         try:
-            resp = httpx.get(url, timeout=timeout, follow_redirects=True)
+            resp = _safe_get(url, timeout=timeout)
             resp.raise_for_status()
             reader = PdfReader(io.BytesIO(resp.content))
             text = "\n".join((page.extract_text() or "") for page in reader.pages)
             return text[:max_chars] if len(text) > max_chars else text
         except Exception as e:
-            return f"PDF Fetch Failed: {e}"
+            return f"WEB_FETCH_ERROR: PDF fetch failed for {url}: {e}. " + _FETCH_RECOVERY_HINT
 
     # 2. Try Jina Reader (r.jina.ai) — It's specifically built to bypass anti-bot and return clean markdown
     jina_url = f"https://r.jina.ai/{url}"
@@ -81,7 +114,7 @@ def fetch_url(url: str, max_chars: int = 8000, timeout: float = 60.0) -> str:
     try:
         # Use a slightly shorter timeout for Jina to allow for fallback
         jina_timeout = min(timeout, 40.0)
-        resp = httpx.get(jina_url, timeout=jina_timeout, follow_redirects=True, headers=headers)
+        resp = _safe_get(jina_url, timeout=jina_timeout, headers=headers)
         if resp.status_code == 200 and len(resp.text) > 50:
             text = resp.text
             return text[:max_chars] if len(text) > max_chars else text
@@ -91,10 +124,9 @@ def fetch_url(url: str, max_chars: int = 8000, timeout: float = 60.0) -> str:
 
     # 3. Fallback: Normal httpx + trafilatura
     try:
-        resp = httpx.get(
+        resp = _safe_get(
             url,
             timeout=timeout,
-            follow_redirects=True,
             headers={
                 "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -120,11 +152,11 @@ def fetch_url(url: str, max_chars: int = 8000, timeout: float = 60.0) -> str:
         if any(marker in text_lower for marker in captcha_markers):
             return (
                 "WEB_FETCH_ERROR: CAPTCHA/Bot detection encountered. This URL is currently inaccessible via automated tools. "
-                "DO NOT retry this exact URL. Try a different source or use search snippets to answer."
+                "DO NOT retry this exact URL. " + _FETCH_RECOVERY_HINT
             )
 
         if len(text) > max_chars:
             text = text[:max_chars] + "\n...[truncated]"
         return text
     except Exception as e:
-        return f"All Fetch Attempts Failed: {e}"
+        return f"WEB_FETCH_ERROR: all fetch attempts failed for {url}: {e}. " + _FETCH_RECOVERY_HINT

tests/test_formatter.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+
+@dataclass(frozen=True)
+class Case:
+    id: str
+    raw: str
+    expected_deterministic: str
+    needs_llm: bool
+
+
+# Regression table: 17 real-shape GAIA answer fixtures.
+# CI runs assertions against the deterministic layer + gating layer only.
+# No network / LLM calls — the LLM path is exercised with a mock further down.
+CASES: list[Case] = [
+    # ---- Already clean, short — must pass through untouched and skip LLM ----
+    Case("bare_number",      "42",                  "42",                  False),
+    Case("bare_word",        "Paris",               "Paris",               False),
+    Case("year",             "1969",                "1969",                False),
+    Case("short_phrase",     "Mount Everest",       "Mount Everest",       False),
+    Case("abbrev_period",    "Mr.",                 "Mr.",                 False),
+    Case("abbrev_dots",      "U.S.",                "U.S.",                False),
+    Case("scene_descriptor", "INT. OFFICE - DAY",   "INT. OFFICE - DAY",   False),
+    Case("large_number",     "3000",                "3000",                False),
+
+    # ---- Clean after safe wrapper-strip — must skip LLM ----
+    Case("markdown_bold",    "**42**",              "42",                  False),
+    Case("double_quoted",    '"Paris"',             "Paris",               False),
+    Case("backticked",       "`code`",              "code",                False),
+    Case("final_answer",     "Final Answer: 42",    "42",                  False),
+    Case("answer_colon",     "Answer: Paris",       "Paris",               False),
+    Case("bold_and_prefix",  "**Final Answer: 42**","42",                  False),
+
+    # ---- Verbose — deterministic leaves alone; gate must route to LLM ----
+    Case("verbose_filler",   "The answer is 42",
+         "The answer is 42", True),
+    Case("verbose_approx",
+         "Based on my calculations, the answer is approximately 1500",
+         "Based on my calculations, the answer is approximately 1500",
+         True),
+    Case("long_sentence",
+         "I found that the total number of papers published was 127 in 2019",
+         "I found that the total number of papers published was 127 in 2019",
+         True),
+]
+
+_CASE_IDS = [c.id for c in CASES]
+
+
+@pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
+def test_deterministic_format_table(case: Case):
+    from lilith_agent.runner import _deterministic_format
+
+    assert _deterministic_format(case.raw) == case.expected_deterministic
+
+
+@pytest.mark.parametrize("case", CASES, ids=_CASE_IDS)
+def test_gating_table(case: Case):
+    from lilith_agent.runner import _needs_llm_formatter
+
+    assert _needs_llm_formatter(case.expected_deterministic) is case.needs_llm
+
+
+class _RaiseIfCalled:
+    def __init__(self):
+        self.called = False
+
+    def invoke(self, _messages):  # pragma: no cover - should never run
+        self.called = True
+        raise AssertionError("LLM formatter was called when it shouldn't have been")
+
+
+class _FakeModel:
+    def __init__(self, response: str):
+        self.response = response
+        self.called = False
+
+    def invoke(self, _messages):
+        self.called = True
+
+        class _Resp:
+            pass
+
+        r = _Resp()
+        r.content = self.response
+        return r
+
+
+def test_short_clean_answer_bypasses_llm():
+    from lilith_agent.runner import _final_formatting_cleanup
+
+    model = _RaiseIfCalled()
+    out = _final_formatting_cleanup(model, "What is 6*7?", "42")
+    assert out == "42"
+    assert model.called is False
+
+
+def test_verbose_answer_invokes_llm_by_default():
+    from lilith_agent.runner import _final_formatting_cleanup
+
+    model = _FakeModel(response="42")
+    out = _final_formatting_cleanup(
+        model,
+        "What is 6*7?",
+        "Based on my calculations, the answer is approximately 42",
+    )
+    assert model.called is True
+    assert out == "42"
+
+
+def test_llm_formatter_disabled_returns_deterministic_only():
+    from lilith_agent.runner import _final_formatting_cleanup
+
+    model = _RaiseIfCalled()
+    out = _final_formatting_cleanup(
+        model,
+        "What is 6*7?",
+        "Based on my calculations, the answer is approximately 42",
+        llm_formatter_enabled=False,
+    )
+    assert model.called is False
+    # deterministic is a no-op on verbose text — raw is returned unchanged
+    assert out == "Based on my calculations, the answer is approximately 42"
+
+
+def test_llm_formatter_disabled_still_applies_deterministic_strip():
+    from lilith_agent.runner import _final_formatting_cleanup
+
+    model = _RaiseIfCalled()
+    out = _final_formatting_cleanup(
+        model,
+        "Q",
+        "**Final Answer: 42**",
+        llm_formatter_enabled=False,
+    )
+    assert model.called is False
+    assert out == "42"
+
+
+def test_config_llm_formatter_enabled_defaults_on_and_is_env_overridable(monkeypatch):
+    from lilith_agent.config import Config
+
+    monkeypatch.delenv("GAIA_LLM_FORMATTER_ENABLED", raising=False)
+    assert Config.from_env().llm_formatter_enabled is True
+
+    monkeypatch.setenv("GAIA_LLM_FORMATTER_ENABLED", "false")
+    assert Config.from_env().llm_formatter_enabled is False

tests/test_search.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from lilith_agent.tools import search as search_mod
+
+
+def test_web_search_empty_without_tavily_key_includes_recovery_hint(monkeypatch):
+    monkeypatch.setattr(search_mod, "_ddg_search", lambda q, n: "No results.")
+
+    out = search_mod.web_search("obscure unlikely query", api_key="")
+
+    lower = out.lower()
+    assert "no results" in lower
+    assert "rephrase" in lower or "alternative keywords" in lower
+    assert "archive.org" in lower or "arxiv" in lower or "crossref" in lower
+
+
+def test_web_search_empty_with_tavily_failure_includes_recovery_hint(monkeypatch):
+    monkeypatch.setattr(search_mod, "_ddg_search", lambda q, n: "No results.")
+
+    class _BoomClient:
+        def __init__(self, api_key):
+            pass
+
+        def search(self, **_kw):
+            raise RuntimeError("tavily offline")
+
+    monkeypatch.setattr(search_mod, "TavilyClient", _BoomClient)
+
+    out = search_mod.web_search("obscure unlikely query", api_key="fake-key")
+
+    lower = out.lower()
+    assert "rephrase" in lower or "alternative keywords" in lower
+    assert "archive.org" in lower or "arxiv" in lower or "crossref" in lower

tests/test_web.py

@@ -69,3 +69,150 @@ def test_fetch_url_rejects_unsafe_scheme_without_network_call():
 def test_fetch_url_rejects_private_address_without_network_call():
     out = fetch_url("http://169.254.169.254/latest/meta-data/")
     assert out.startswith("WEB_FETCH_ERROR")
+
+
+class _FakeResp:
+    def __init__(self, status_code=200, text="", headers=None):
+        import httpx as _httpx
+
+        self.status_code = status_code
+        self.text = text
+        self.content = text.encode("utf-8")
+        self.headers = headers or {"Content-Type": "text/html"}
+        self._httpx = _httpx
+
+    @property
+    def is_redirect(self):
+        return self.status_code in (301, 302, 303, 307, 308)
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise self._httpx.HTTPStatusError(
+                f"{self.status_code} error", request=None, response=None
+            )
+
+
+def test_fetch_url_captcha_response_includes_recovery_hints(monkeypatch):
+    import httpx as _httpx
+
+    def fake_get(url, *args, **kwargs):
+        # Jina path fails (non-200), fallback path returns CAPTCHA page
+        if url.startswith("https://r.jina.ai/"):
+            return _FakeResp(status_code=403, text="")
+        return _FakeResp(
+            status_code=200,
+            text="<html><body>Please complete the captcha to continue</body></html>",
+        )
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/blocked")
+    lower = out.lower()
+    assert "captcha" in lower or "bot detection" in lower
+    assert "third-party" in lower or "citation" in lower
+    assert "archive.org" in lower
+    assert "snippet" in lower
+
+
+def test_fetch_url_exception_path_includes_recovery_hints(monkeypatch):
+    import httpx as _httpx
+
+    def fake_get(url, *args, **kwargs):
+        raise _httpx.HTTPError("403 Forbidden")
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/blocked")
+    assert out.startswith("WEB_FETCH_ERROR")
+    lower = out.lower()
+    assert "third-party" in lower or "citation" in lower
+    assert "archive.org" in lower
+    assert "snippet" in lower
+
+
+def test_fetch_url_pdf_failure_includes_recovery_hints(monkeypatch):
+    import httpx as _httpx
+
+    def fake_get(url, *args, **kwargs):
+        raise _httpx.HTTPError("403 Forbidden")
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/paper.pdf")
+    assert out.startswith("WEB_FETCH_ERROR")
+    lower = out.lower()
+    assert "third-party" in lower or "citation" in lower
+    assert "archive.org" in lower
+    assert "snippet" in lower
+
+
+def test_fetch_url_blocks_redirect_to_private_ip(monkeypatch):
+    """Post-redirect SSRF re-check: server sends 302 to link-local metadata IP."""
+    import httpx as _httpx
+
+    visited: list[str] = []
+
+    def fake_get(url, *args, **kwargs):
+        visited.append(url)
+        # Redirect every request to the AWS metadata IP.
+        return _FakeResp(
+            status_code=302,
+            headers={"Location": "http://169.254.169.254/latest/meta-data/iam/"},
+        )
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/bounce")
+    assert out.startswith("WEB_FETCH_ERROR")
+    # Agent must never have actually requested the private IP.
+    assert not any("169.254.169.254" in u for u in visited)
+
+
+def test_fetch_url_allows_redirect_to_public_hostname(monkeypatch):
+    """A legitimate 302 to a public hostname must still work end-to-end."""
+    import httpx as _httpx
+
+    def fake_get(url, *args, **kwargs):
+        # Jina path: fail so fallback path exercises the redirect logic.
+        if url.startswith("https://r.jina.ai/"):
+            return _FakeResp(status_code=403, text="")
+        if url == "https://example.com/start":
+            return _FakeResp(
+                status_code=302,
+                headers={"Location": "https://www.example.com/final"},
+            )
+        if url == "https://www.example.com/final":
+            return _FakeResp(
+                status_code=200,
+                text="<html><body>hello world from final page</body></html>",
+            )
+        return _FakeResp(status_code=404, text="")
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/start")
+    assert not out.startswith("WEB_FETCH_ERROR")
+    assert "hello world" in out.lower() or "final page" in out.lower()
+
+
+def test_fetch_url_blocks_redirect_loop(monkeypatch):
+    """Bounded redirect follower stops instead of looping forever."""
+    import httpx as _httpx
+
+    calls = {"n": 0}
+
+    def fake_get(url, *args, **kwargs):
+        if url.startswith("https://r.jina.ai/"):
+            return _FakeResp(status_code=403, text="")
+        calls["n"] += 1
+        return _FakeResp(
+            status_code=302,
+            headers={"Location": "https://example.com/next"},
+        )
+
+    monkeypatch.setattr(_httpx, "get", fake_get)
+
+    out = fetch_url("https://example.com/start")
+    assert out.startswith("WEB_FETCH_ERROR")
+    # Must have stopped — not burning arbitrary calls.
+    assert calls["n"] <= 10