File size: 1,525 Bytes
c641d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | from tools.web import fetch_url
class Response:
url = "https://example.test/final"
headers = {"content-type": "text/html; charset=utf-8"}
encoding = "utf-8"
apparent_encoding = "utf-8"
text = "<html><nav>menu</nav><main><h1>Title</h1><p>Primary evidence.</p></main><script>bad()</script></html>"
content = text.encode()
def raise_for_status(self):
pass
def test_html_fetch_follows_cleaning_and_limits(monkeypatch):
observed = {}
def fake_get(url, **kwargs):
observed.update(kwargs)
return Response()
monkeypatch.setattr("tools.web.requests.get", fake_get)
text = fetch_url("https://example.test", max_chars=10)
assert "Title" in text and "bad()" not in text and "menu" not in text
assert "CONTENT TRUNCATED" in text
assert observed["allow_redirects"] is True
assert observed["headers"]["User-Agent"].startswith("GAIA-Level1-Agent")
def test_pdf_fetch_uses_page_extraction(monkeypatch):
class PdfResponse(Response):
headers = {"content-type": "application/pdf"}
content = b"fake-pdf"
url = "https://example.test/paper.pdf"
class Page:
def extract_text(self):
return "Primary paper evidence"
class Reader:
pages = [Page()]
monkeypatch.setattr("tools.web.requests.get", lambda *args, **kwargs: PdfResponse())
monkeypatch.setattr("tools.web.PdfReader", lambda stream: Reader())
assert fetch_url("https://example.test/paper.pdf") == "Primary paper evidence"
|