Title

from tools.web import fetch_url


class Response:
    url = "https://example.test/final"
    headers = {"content-type": "text/html; charset=utf-8"}
    encoding = "utf-8"
    apparent_encoding = "utf-8"
    text = "<html><nav>menu</nav><main><h1>Title</h1><p>Primary evidence.</p></main><script>bad()</script></html>"
    content = text.encode()

    def raise_for_status(self):
        pass


def test_html_fetch_follows_cleaning_and_limits(monkeypatch):
    observed = {}

    def fake_get(url, **kwargs):
        observed.update(kwargs)
        return Response()

    monkeypatch.setattr("tools.web.requests.get", fake_get)
    text = fetch_url("https://example.test", max_chars=10)
    assert "Title" in text and "bad()" not in text and "menu" not in text
    assert "CONTENT TRUNCATED" in text
    assert observed["allow_redirects"] is True
    assert observed["headers"]["User-Agent"].startswith("GAIA-Level1-Agent")


def test_pdf_fetch_uses_page_extraction(monkeypatch):
    class PdfResponse(Response):
        headers = {"content-type": "application/pdf"}
        content = b"fake-pdf"
        url = "https://example.test/paper.pdf"

    class Page:
        def extract_text(self):
            return "Primary paper evidence"

    class Reader:
        pages = [Page()]

    monkeypatch.setattr("tools.web.requests.get", lambda *args, **kwargs: PdfResponse())
    monkeypatch.setattr("tools.web.PdfReader", lambda stream: Reader())
    assert fetch_url("https://example.test/paper.pdf") == "Primary paper evidence"