scraper/extractor.py

"""
Content extraction: multi-strategy with CSS scoring, text density, readability fallback.
"""
from __future__ import annotations

import logging
import re
from typing import Optional

from bs4 import BeautifulSoup, NavigableString, Tag

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Noise patterns
# ---------------------------------------------------------------------------

_NOISE_CLASSES = re.compile(
    r"ad|ads|advert|advertisement|banner|popup|modal|cookie|gdpr|"
    r"nav|navigation|header|footer|sidebar|widget|comment|disqus|"
    r"share|social|related|recommend|newsletter|subscribe|signup|"
    r"author-bio|breadcrumb|pagination|next-chapter|prev-chapter|"
    r"chapter-nav|bookmark|rating|review|buy|purchase|unlock",
    re.IGNORECASE,
)

_NOISE_IDS = _NOISE_CLASSES  # same pattern

_NOISE_TEXT_PATTERNS = [
    re.compile(p, re.IGNORECASE) for p in [
        r"if you (?:find|enjoy|like) this (novel|book|story)",
        r"please (support|visit|check out|subscribe)",
        r"read this (novel|chapter|story) at",
        r"this chapter was (stolen|taken|scrapped)",
        r"visit\s+\w+\.(com|net|org)\s+for",
        r"translator'?s?\s+note",
        r"\[TL[:\s]",
        r"^\s*\*+\s*$",
        r"chapter\s+\d+\s+has\s+been\s+stolen",
        r"this\s+content\s+is\s+taken\s+from",
        r"you\s+can\s+find\s+the\s+latest\s+chapters",
        r"join\s+our\s+discord",
        r"patreon\.com",
        r"ko-fi\.com",
        r"support\s+the\s+author",
    ]
]

_LOCKED_PATTERNS = [
    re.compile(p, re.IGNORECASE) for p in [
        r"this chapter is locked",
        r"premium chapter",
        r"vip (chapter|content)",
        r"unlock (with|using) coins?",
        r"purchase to (read|unlock)",
        r"subscribe to (read|access)",
    ]
]

# ---------------------------------------------------------------------------
# Content container candidates
# ---------------------------------------------------------------------------

_CONTENT_SELECTORS = [
    # Novelhall.com specific
    "div#htmlContent",
    # High-confidence specific selectors
    "div.chapter-content",
    "div.chapter_content",
    "div#chapter-content",
    "div#chapter_content",
    "div.text-chapter",
    "div.reading-content",
    "div#reading-content",
    "article.chapter",
    "div.content-text",
    "div.novel-content",
    "div.post-content",
    "article.post-content",
    "div.entry-content",
    "div#content",
    "article",
    "main",
    "div.content",
    "div.text",
]

_TITLE_SELECTORS = [
    # Novelhall.com specific
    "div.single-header h1",
    "h1.chapter-title",
    "h2.chapter-title",
    "h1.entry-title",
    "h1",
    "h2",
    ".chapter-title",
    "#chapter-title",
    "title",
]


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _remove_noise_elements(soup: BeautifulSoup) -> None:
    """In-place removal of known noisy elements."""
    # Remove by tag
    for tag in soup.find_all(["script", "style", "noscript", "iframe", "form",
                               "button", "input", "select", "textarea", "canvas"]):
        tag.decompose()

    # Remove by class/id
    for element in soup.find_all(True):
        cls = " ".join(element.get("class", []))
        id_ = element.get("id", "")
        if _NOISE_CLASSES.search(cls) or _NOISE_IDS.search(id_):
            element.decompose()


def _is_noise_paragraph(text: str) -> bool:
    """Return True if paragraph is spam/noise."""
    text = text.strip()
    if not text or len(text) < 5:
        return True
    for pat in _NOISE_TEXT_PATTERNS:
        if pat.search(text):
            return True
    return False


def _text_density(element: Tag) -> float:
    """Ratio of text length to tag count."""
    text_len = len(element.get_text())
    tags = len(element.find_all(True))
    if tags == 0:
        return float(text_len)
    return text_len / (tags + 1)


def _extract_paragraphs(element: Tag) -> list[str]:
    """Extract clean paragraphs from a container element."""
    paragraphs: list[str] = []

    # Replace <br> tags with newlines (v1 approach — many novel sites use <br> not <p>)
    for br in element.find_all("br"):
        br.replace_with("\n")

    # Prefer <p> tags
    p_tags = element.find_all("p")
    if p_tags:
        for p in p_tags:
            text = p.get_text(separator=" ", strip=True)
            if not _is_noise_paragraph(text):
                paragraphs.append(text)
    else:
        # Fall back to br-delimited text
        for line in element.get_text(separator="\n").splitlines():
            line = line.strip()
            if line and not _is_noise_paragraph(line) and len(line) > 20:
                paragraphs.append(line)

    # Deduplicate consecutive identical paragraphs
    deduped: list[str] = []
    prev = ""
    for p in paragraphs:
        if p != prev:
            deduped.append(p)
            prev = p

    return deduped


# ---------------------------------------------------------------------------
# Strategy 1: CSS selector scoring
# ---------------------------------------------------------------------------

def _strategy_selectors(soup: BeautifulSoup) -> Optional[list[str]]:
    for sel in _CONTENT_SELECTORS:
        try:
            el = soup.select_one(sel)
            if el:
                paras = _extract_paragraphs(el)
                if len(paras) >= 1 and sum(len(p) for p in paras) > 50:
                    return paras
        except Exception:
            continue
    return None


# ---------------------------------------------------------------------------
# Strategy 2: Text density analysis
# ---------------------------------------------------------------------------

def _strategy_density(soup: BeautifulSoup) -> Optional[list[str]]:
    candidates: list[tuple[float, Tag]] = []
    for tag in soup.find_all(["div", "article", "section", "main"]):
        d = _text_density(tag)
        if d > 50:
            candidates.append((d, tag))

    candidates.sort(reverse=True, key=lambda x: x[0])

    for _, el in candidates[:3]:
        paras = _extract_paragraphs(el)
        if len(paras) >= 3 and sum(len(p) for p in paras) > 300:
            return paras

    return None


# ---------------------------------------------------------------------------
# Strategy 3: Readability / trafilatura
# ---------------------------------------------------------------------------

def _strategy_readability(html: str) -> Optional[list[str]]:
    # Try trafilatura first
    try:
        import trafilatura
        extracted = trafilatura.extract(html, include_comments=False, include_tables=False)
        if extracted and len(extracted) > 300:
            lines = [l.strip() for l in extracted.splitlines() if l.strip() and len(l.strip()) > 20]
            if lines:
                return lines
    except ImportError:
        pass

    # Try readability-lxml
    try:
        from readability import Document
        doc = Document(html)
        content_html = doc.summary()
        sub_soup = BeautifulSoup(content_html, "lxml")
        paras = _extract_paragraphs(sub_soup)
        if paras and sum(len(p) for p in paras) > 200:
            return paras
    except ImportError:
        pass

    return None


# ---------------------------------------------------------------------------
# Strategy 4: Largest text block heuristic
# ---------------------------------------------------------------------------

def _strategy_largest_block(soup: BeautifulSoup) -> Optional[list[str]]:
    best: tuple[int, Optional[list[str]]] = (0, None)
    for tag in soup.find_all(["div", "p", "article", "section"]):
        text = tag.get_text(separator="\n")
        paras = [l.strip() for l in text.splitlines() if len(l.strip()) > 30]
        total = sum(len(p) for p in paras)
        if total > best[0]:
            best = (total, paras)

    if best[1] and best[0] > 500:
        # Clean noise
        return [p for p in best[1] if not _is_noise_paragraph(p)]
    return None


# ---------------------------------------------------------------------------
# Title extraction
# ---------------------------------------------------------------------------

def extract_title(soup: BeautifulSoup, fallback: str = "") -> str:
    for sel in _TITLE_SELECTORS:
        try:
            el = soup.select_one(sel)
            if el:
                text = el.get_text(strip=True)
                if text and len(text) < 200:
                    return text
        except Exception:
            continue
    return fallback


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------

def is_locked_chapter(html: str) -> bool:
    for pat in _LOCKED_PATTERNS:
        if pat.search(html):
            return True
    return False


def extract_chapter(html: str, fallback_title: str = "") -> tuple[str, str, list[str]]:
    """
    Extract chapter from raw HTML.
    Returns (title, content_text, paragraphs).
    """
    soup = BeautifulSoup(html, "lxml")
    title = extract_title(soup, fallback=fallback_title)

    # Remove noise before content extraction
    _remove_noise_elements(soup)

    # Try strategies in order
    paragraphs: Optional[list[str]] = None

    paragraphs = _strategy_selectors(soup)
    if not paragraphs:
        logger.debug("CSS selectors found nothing, trying density...")
        paragraphs = _strategy_density(soup)

    if not paragraphs:
        logger.debug("Density failed, trying readability...")
        paragraphs = _strategy_readability(html)

    if not paragraphs:
        logger.debug("Readability failed, trying largest block...")
        paragraphs = _strategy_largest_block(soup)

    if not paragraphs:
        paragraphs = ["[Content extraction failed for this chapter]"]

    content_text = "\n\n".join(paragraphs)
    return title, content_text, paragraphs