| from tools.web import fetch_url | |
| class Response: | |
| url = "https://example.test/final" | |
| headers = {"content-type": "text/html; charset=utf-8"} | |
| encoding = "utf-8" | |
| apparent_encoding = "utf-8" | |
| text = "<html><nav>menu</nav><main><h1>Title</h1><p>Primary evidence.</p></main><script>bad()</script></html>" | |
| content = text.encode() | |
| def raise_for_status(self): | |
| pass | |
| def test_html_fetch_follows_cleaning_and_limits(monkeypatch): | |
| observed = {} | |
| def fake_get(url, **kwargs): | |
| observed.update(kwargs) | |
| return Response() | |
| monkeypatch.setattr("tools.web.requests.get", fake_get) | |
| text = fetch_url("https://example.test", max_chars=10) | |
| assert "Title" in text and "bad()" not in text and "menu" not in text | |
| assert "CONTENT TRUNCATED" in text | |
| assert observed["allow_redirects"] is True | |
| assert observed["headers"]["User-Agent"].startswith("GAIA-Level1-Agent") | |
| def test_pdf_fetch_uses_page_extraction(monkeypatch): | |
| class PdfResponse(Response): | |
| headers = {"content-type": "application/pdf"} | |
| content = b"fake-pdf" | |
| url = "https://example.test/paper.pdf" | |
| class Page: | |
| def extract_text(self): | |
| return "Primary paper evidence" | |
| class Reader: | |
| pages = [Page()] | |
| monkeypatch.setattr("tools.web.requests.get", lambda *args, **kwargs: PdfResponse()) | |
| monkeypatch.setattr("tools.web.PdfReader", lambda stream: Reader()) | |
| assert fetch_url("https://example.test/paper.pdf") == "Primary paper evidence" | |