scripts/scrape_wisdomlib.py

"""
ByteAstra — Wisdomlib.org Ayurveda Text Scraper

Scrapes Charaka Samhita, Sushruta Samhita, and Ashtanga Hridayam chapters
from wisdomlib.org, which hosts full public domain English translations
with proper citation structure (book > chapter > verse).

Usage:
    python scripts/scrape_wisdomlib.py --output-dir ./data/ayurveda/scraped

Output:
    One .md file per chapter, with YAML frontmatter containing:
      source, book, chapter, section, url
"""
from __future__ import annotations
import argparse
import logging
import re
import sys
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

logging.basicConfig(level=logging.INFO, format="%(levelname)s | %(message)s")
logger = logging.getLogger(__name__)

BASE_URL = "https://www.wisdomlib.org"

# Target book index URLs on wisdomlib.org
BOOKS = [
    {
        "name": "Charaka Samhita",
        "slug": "charaka-samhita",
        "url": "https://www.wisdomlib.org/hinduism/book/charaka-samhita-english",
        "key": "charaka",
    },
    {
        "name": "Sushruta Samhita",
        "slug": "sushruta-samhita",
        "url": "https://www.wisdomlib.org/hinduism/book/sushruta-samhita-volume-1-sutrasthana",
        "key": "sushruta",
    },
    {
        "name": "Ashtanga Hridayam",
        "slug": "ashtanga-hridayam",
        "url": "https://www.wisdomlib.org/hinduism/book/ashtanga-hridaya-samhita-sanskrit",
        "key": "ashtanga",
    },
]

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/120.0.0.0 Safari/537.36"
    )
}
REQUEST_DELAY = 1.0  # seconds between requests — be polite to the server


def get_page(url: str, retries: int = 3) -> BeautifulSoup | None:
    for attempt in range(retries):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=20)
            resp.raise_for_status()
            return BeautifulSoup(resp.text, "html.parser")
        except Exception as e:
            logger.warning("Attempt %d failed for %s: %s", attempt + 1, url, e)
            if attempt < retries - 1:
                time.sleep(REQUEST_DELAY * 2)
    return None


def extract_chapter_links(book_url: str) -> list[dict]:
    """Scrape the book index page and return list of chapter link dicts."""
    logger.info("Fetching book index: %s", book_url)
    soup = get_page(book_url)
    if not soup:
        return []

    links = []
    # Wisdomlib chapter links are typically in .chapter-list or <ol><li><a>
    for a in soup.find_all("a", href=True):
        href = a["href"]
        text = a.get_text(strip=True)
        if not text or len(text) < 5:
            continue
        # Look for chapter-like paths or text
        is_chapter = False
        if "/chapter/" in href or "chapter" in href.lower() or "chapter" in text.lower():
            is_chapter = True
        elif "/d/doc" in href and ("section" in text.lower() or "chapter" in text.lower() or text.strip().startswith("Chapter")):
            is_chapter = True
        elif "/d/doc" in href and len(text) > 8 and any(char.isdigit() for char in text):
            # Match document links containing numbers (e.g. Chapter titles, verses)
            is_chapter = True

        if is_chapter:
            full_url = href if href.startswith("http") else urljoin(BASE_URL, href)
            links.append({"title": text, "url": full_url})

    # Deduplicate by URL
    seen = set()
    unique = []
    for link in links:
        if link["url"] not in seen:
            seen.add(link["url"])
            unique.append(link)

    logger.info("Found %d chapter links", len(unique))
    return unique


def scrape_chapter(url: str, book_name: str, chapter_title: str) -> str | None:
    """Scrape a single chapter page and return cleaned markdown text."""
    soup = get_page(url)
    if not soup:
        return None

    # Try common content selectors on wisdomlib
    content_div = (
        soup.find("div", class_="article-content")
        or soup.find("div", class_="content")
        or soup.find("article")
        or soup.find("main")
    )
    if not content_div:
        logger.warning("No content div found at %s", url)
        return None

    # Remove navigation, ads, footnote sections
    for tag in content_div.find_all(["nav", "script", "style", "aside", "footer"]):
        tag.decompose()

    # Extract heading and body
    lines = []
    for elem in content_div.find_all(["h1", "h2", "h3", "h4", "p", "li", "blockquote"]):
        text = elem.get_text(separator=" ", strip=True)
        text = re.sub(r"\s+", " ", text).strip()
        if not text or len(text) < 10:
            continue
        tag = elem.name
        if tag in ("h1", "h2"):
            lines.append(f"\n## {text}\n")
        elif tag in ("h3", "h4"):
            lines.append(f"\n### {text}\n")
        elif tag == "blockquote":
            lines.append(f"\n> {text}\n")
        elif tag == "li":
            lines.append(f"- {text}")
        else:
            lines.append(text)

    body = "\n".join(lines).strip()
    if len(body) < 200:
        logger.warning("Very short content (%d chars) at %s", len(body), url)
        return None

    return body


def slugify(text: str) -> str:
    text = text.lower().strip()
    text = re.sub(r"[^\w\s-]", "", text)
    text = re.sub(r"[\s_]+", "_", text)
    return text[:60]


def scrape_book(book: dict, output_dir: Path, max_chapters: int = 50):
    """Scrape all chapters of a book and save as markdown files."""
    book_dir = output_dir / book["key"]
    book_dir.mkdir(parents=True, exist_ok=True)

    chapter_links = extract_chapter_links(book["url"])
    if not chapter_links:
        logger.error("No chapters found for %s", book["name"])
        return 0

    saved = 0
    for i, link in enumerate(chapter_links[:max_chapters]):
        logger.info("[%d/%d] Scraping: %s", i + 1, len(chapter_links[:max_chapters]), link["title"])
        time.sleep(REQUEST_DELAY)

        body = scrape_chapter(link["url"], book["name"], link["title"])
        if not body:
            continue

        # Build frontmatter
        frontmatter = (
            f"---\n"
            f"source: {book['name']}\n"
            f"chapter: {link['title']}\n"
            f"section: \"\"\n"
            f"url: {link['url']}\n"
            f"---\n\n"
        )

        filename = f"{slugify(link['title'])}.md"
        out_path = book_dir / filename
        out_path.write_text(frontmatter + body, encoding="utf-8")
        saved += 1
        logger.info("  ✓ Saved: %s (%d chars)", filename, len(body))

    logger.info("Book '%s' done — %d chapters saved.", book["name"], saved)
    return saved


def main():
    parser = argparse.ArgumentParser(description="Scrape Ayurveda texts from wisdomlib.org")
    parser.add_argument("--output-dir", default="./data/ayurveda/scraped", type=Path)
    parser.add_argument("--books", nargs="+", default=["charaka", "sushruta", "ashtanga"],
                        help="Book keys to scrape (charaka/sushruta/ashtanga)")
    parser.add_argument("--max-chapters", type=int, default=30,
                        help="Max chapters to scrape per book (default 30 for testing)")
    args = parser.parse_args()

    args.output_dir.mkdir(parents=True, exist_ok=True)

    total = 0
    for book in BOOKS:
        if book["key"] not in args.books:
            continue
        count = scrape_book(book, args.output_dir, max_chapters=args.max_chapters)
        total += count

    logger.info("✓ Scraping complete. Total chapters: %d", total)
    logger.info("Output at: %s", args.output_dir.resolve())
    logger.info("Next step: python scripts/generate_qa_pairs.py --input-dir %s", args.output_dir)


if __name__ == "__main__":
    main()