scripts/fetch_web_sources.py

"""Scrape web tutorial sources and save as plain-text files in data/web/.

Uses requests + BeautifulSoup to extract readable text from each URL defined
as a tutorial/course source in CANONICAL_TEXT_SOURCES. Skips files that already
exist — safe to re-run.

Requires: pip install beautifulsoup4 (added to requirements/pyproject.toml separately)

Usage:
    uv run python scripts/fetch_web_sources.py
"""
from __future__ import annotations

import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from rich.console import Console
from rich.table import Table

from researchpath.corpus import CANONICAL_TEXT_SOURCES

console = Console()
ROOT = Path(__file__).resolve().parents[1]
WEB_DIR = ROOT / "data" / "web"
WEB_DIR.mkdir(parents=True, exist_ok=True)

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (compatible; ResearchPath/1.0; educational corpus builder)"
    )
}
TIMEOUT = 30


def _fetch_text(url: str) -> str:
    """Fetch URL and return cleaned plain text via BeautifulSoup."""
    import requests
    from bs4 import BeautifulSoup

    resp = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
    resp.raise_for_status()
    # Force UTF-8 — prevents ¶ artifacts from latin-1 mis-detection
    resp.encoding = "utf-8"

    soup = BeautifulSoup(resp.text, "html.parser")

    # Remove nav, footer, header, scripts, style blocks
    for tag in soup.find_all(["nav", "footer", "header", "script", "style", "aside"]):
        tag.decompose()

    # Prefer main content areas in priority order
    for selector in ["article", "main", '[role="main"]', ".content", "#content", "body"]:
        container = soup.select_one(selector)
        if container:
            break

    if container is None:
        container = soup

    lines = []
    for element in container.find_all(["h1", "h2", "h3", "h4", "p", "li", "pre", "code"]):
        text = element.get_text(" ", strip=True)
        if text:
            lines.append(text)

    return "\n\n".join(lines)


def main() -> int:
    try:
        import requests  # noqa: F401
        from bs4 import BeautifulSoup  # noqa: F401
    except ImportError:
        console.print(
            "[red]Missing dependencies. Run:[/red]\n"
            "  uv add beautifulsoup4 requests"
        )
        return 1

    web_sources = [
        s for s in CANONICAL_TEXT_SOURCES
        if s.filename.endswith(".txt")
    ]

    table = Table(title="Web scrape results")
    table.add_column("Source ID", style="bold")
    table.add_column("Status")
    table.add_column("Detail")

    ok = 0
    for source in web_sources:
        dest = WEB_DIR / source.filename
        if dest.exists():
            chars = dest.stat().st_size
            table.add_row(source.source_id, "[green]OK[/green]", f"already exists ({chars:,} chars)")
            ok += 1
            continue
        try:
            text = _fetch_text(source.url)
            dest.write_text(text, encoding="utf-8")
            table.add_row(source.source_id, "[green]OK[/green]", f"scraped {len(text):,} chars")
            ok += 1
        except Exception as e:
            table.add_row(source.source_id, "[red]FAIL[/red]", str(e)[:80])
        time.sleep(2)  # polite crawl delay

    console.print(table)
    console.print(
        f"\n[bold]{'[green]' if ok == len(web_sources) else '[yellow]'}"
        f"{ok}/{len(web_sources)} web sources saved to data/web/[/bold]"
    )
    return 0 if ok > 0 else 1


if __name__ == "__main__":
    sys.exit(main())