scraper/discovery.py

"""
Chapter link discovery — inspired by v1's direct approach:
fetch the index page, find the container holding the most same-domain
links, treat those as the chapter list. Pagination handled if needed.
"""
from __future__ import annotations

import json
import logging
import re
from typing import Callable, Optional
from urllib.parse import urljoin, urlparse, parse_qs, urlunparse

from bs4 import BeautifulSoup

from .engine import Chapter, smart_fetch, FetchMode, _is_cloudflare
import aiohttp

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# URLs / text that are never chapter links
# ---------------------------------------------------------------------------

_BAD_URL_FRAGMENTS = [
    "/tag/", "/category/", "/author/", "/search", "/login", "/register",
    "/contact", "/about", "/privacy", "/sitemap", "/feed", "/cdn-cgi/",
    "javascript:", "mailto:", "#", "?s=", "/wp-", "/page/1",
]

_BAD_TEXT_PATTERNS = re.compile(
    r"^(home|about|contact|privacy|terms|login|register|sign.?in|sign.?up|"
    r"search|tags?|categor|author|admin|next|prev|previous|back|forward|"
    r"more|see more|load more|show more|menu|nav)$",
    re.IGNORECASE,
)

# ---------------------------------------------------------------------------
# Chapter number detector — used for sorting and validation
# ---------------------------------------------------------------------------

_CH_NUM_RE = re.compile(
    r"(?:chapter|ch\.?|ep\.?|episode|part|vol\.?|volume)?\s*(\d+(?:\.\d+)?)",
    re.IGNORECASE,
)
_SIDE_STORY_RE = re.compile(
    r"side\s*story|extra|bonus|omake|special|ss\s*\d", re.IGNORECASE
)


def _chapter_sort_key(ch: Chapter) -> tuple[float, float, str]:
    is_side = 1.0 if _SIDE_STORY_RE.search(ch.title) else 0.0
    m = _CH_NUM_RE.search(ch.title)
    num = float(m.group(1)) if m else 0.0
    return (is_side, num, ch.title.lower())


def sort_chapters(chapters: list[Chapter]) -> list[Chapter]:
    sorted_chs = sorted(chapters, key=_chapter_sort_key)
    for i, ch in enumerate(sorted_chs, start=1):
        ch.index = i
    return sorted_chs


# ---------------------------------------------------------------------------
# Core helper: extract all plausible chapter links from a page
# ---------------------------------------------------------------------------

def _is_bad_url(href: str) -> bool:
    low = href.lower()
    return any(frag in low for frag in _BAD_URL_FRAGMENTS)


def _is_bad_text(text: str) -> bool:
    text = text.strip()
    if not text or len(text) > 200:
        return True
    return bool(_BAD_TEXT_PATTERNS.match(text))


def _extract_links_from_html(html: str, base_url: str) -> list[tuple[str, str]]:
    """
    V1-inspired approach: parse the page, collect every same-domain <a> link
    that isn't obviously navigation/noise. Return [(url, title), ...] in
    document order, deduplicated.
    """
    base_domain = urlparse(base_url).netloc
    soup = BeautifulSoup(html, "lxml")

    # Strip script/style/nav noise up front
    for tag in soup.find_all(["script", "style", "noscript", "iframe",
                               "header", "footer", "nav"]):
        tag.decompose()

    seen: dict[str, str] = {}

    for a in soup.find_all("a", href=True):
        href = a.get("href", "").strip()
        if not href:
            continue

        full_url = urljoin(base_url, href)
        parsed = urlparse(full_url)

        # Same domain only
        if parsed.netloc and parsed.netloc != base_domain:
            continue

        if _is_bad_url(full_url):
            continue

        title = a.get_text(strip=True)
        if _is_bad_text(title):
            continue

        if full_url not in seen:
            seen[full_url] = title

    return list(seen.items())


# ---------------------------------------------------------------------------
# Container-based strategy (v1's key insight)
#
# Rather than scoring individual links, find the HTML container that holds
# the MOST links to the same domain. That container is almost always the
# chapter list. This matches exactly how v1 worked for novelhall — it just
# happened to know the specific container ID. Here we discover it
# automatically by counting links per container.
# ---------------------------------------------------------------------------

def _find_chapter_list_container(html: str, base_url: str) -> list[tuple[str, str]]:
    """
    Find the single container element with the most same-domain links.
    Returns [(url, title), ...] from that container.
    """
    base_domain = urlparse(base_url).netloc
    soup = BeautifulSoup(html, "lxml")

    for tag in soup.find_all(["script", "style", "noscript", "iframe",
                               "header", "footer", "nav"]):
        tag.decompose()

    best_container = None
    best_count = 0

    # Walk all block-level containers, find the one with most valid links
    for el in soup.find_all(["div", "ul", "ol", "section", "article", "main", "table"]):
        links = []
        for a in el.find_all("a", href=True, recursive=True):
            href = a.get("href", "").strip()
            if not href:
                continue
            full_url = urljoin(base_url, href)
            parsed = urlparse(full_url)
            if parsed.netloc and parsed.netloc != base_domain:
                continue
            if _is_bad_url(full_url):
                continue
            title = a.get_text(strip=True)
            if _is_bad_text(title):
                continue
            links.append((full_url, title))

        # Prefer the container with the most unique links
        unique = len(dict(links))
        if unique > best_count:
            best_count = unique
            best_container = links

    if best_container and best_count >= 3:
        # Deduplicate preserving order
        seen: dict[str, str] = {}
        for url, title in best_container:
            if url not in seen:
                seen[url] = title
        return list(seen.items())

    return []


# ---------------------------------------------------------------------------
# Pagination
# ---------------------------------------------------------------------------

def _next_page_url(html: str, current_url: str) -> str | None:
    """Find a 'next page' link for paginated chapter lists."""
    soup = BeautifulSoup(html, "lxml")
    base_domain = urlparse(current_url).netloc

    # Look for explicit next-page links
    for a in soup.find_all("a", href=True):
        text = a.get_text(strip=True).lower()
        rel = a.get("rel", [])
        if text in ("next", "next page", "›", "»", ">") or "next" in rel:
            href = urljoin(current_url, a["href"])
            if urlparse(href).netloc == base_domain and href != current_url:
                return href

    return None


def _build_page_urls(base_url: str, max_pages: int = 50) -> list[str]:
    """Generate ?page=N and /page/N variants."""
    parsed = urlparse(base_url)
    candidates = []
    for i in range(2, max_pages + 1):
        qs = parse_qs(parsed.query)
        qs["page"] = [str(i)]
        params = "&".join(f"{k}={v[0]}" for k, v in qs.items())
        candidates.append(urlunparse(parsed._replace(query=params)))

        path_paged = parsed.path.rstrip("/") + f"/page/{i}"
        candidates.append(urlunparse(parsed._replace(path=path_paged, query="")))
    return candidates


# ---------------------------------------------------------------------------
# TOC URL variants to try
# ---------------------------------------------------------------------------

_TOC_SUFFIXES = [
    "",
    "/chapter-list",
    "/chapters",
    "/chapter",
    "/contents",
    "/toc",
    "/episodes",
    "/all-chapters",
    "/table-of-contents",
]


def _toc_candidates(url: str) -> list[str]:
    parsed = urlparse(url)
    base = f"{parsed.scheme}://{parsed.netloc}{parsed.path.rstrip('/')}"
    return [base + s for s in _TOC_SUFFIXES]


# ---------------------------------------------------------------------------
# Main discover function
# ---------------------------------------------------------------------------

async def discover_chapters(
    session: aiohttp.ClientSession,
    novel_url: str,
    mode: FetchMode = FetchMode.BALANCED,
    cookies: dict | None = None,
    log_cb: Callable[[str], None] | None = None,
) -> list[Chapter]:
    """
    Discover all chapter links. Uses v1's core insight: find the page
    container with the most links — that's the chapter list.
    """

    def log(msg: str):
        if log_cb:
            log_cb(msg)
        logger.info(msg)

    async def fetch(url: str) -> str | None:
        try:
            html, status = await smart_fetch(
                session, url, mode=mode, cookies=cookies, log_cb=log_cb
            )
            if _is_cloudflare(html, status):
                log(f"🛑 Cloudflare wall at {url}")
                return None
            return html
        except Exception as exc:
            log(f"⚠️  Fetch failed for {url}: {exc}")
            return None

    log(f"🔍 Discovering chapters for: {novel_url}")

    all_links: dict[str, str] = {}  # url -> title

    # ── Step 1: Try TOC URL variants, use the one with most chapter links ──
    best_html = None
    best_url = None
    best_count = 0

    for toc_url in _toc_candidates(novel_url):
        html = await fetch(toc_url)
        if not html:
            continue
        links = _find_chapter_list_container(html, toc_url)
        log(f"  {toc_url} → {len(links)} links")
        if len(links) > best_count:
            best_count = len(links)
            best_html = html
            best_url = toc_url
        # If the base URL already has plenty, stop trying variants
        if toc_url == _toc_candidates(novel_url)[0] and best_count >= 10:
            break

    if best_html and best_count >= 3:
        log(f"✅ Best TOC page: {best_url} ({best_count} links)")
        for url, title in _find_chapter_list_container(best_html, best_url):
            if url not in all_links:
                all_links[url] = title

    # ── Step 2: Follow pagination if the list seems incomplete ────────────
    if best_html and best_url:
        current_html = best_html
        current_url = best_url
        page = 2

        while page <= 100:
            # Try explicit next-page link first
            next_url = _next_page_url(current_html, current_url)
            if not next_url:
                # Try ?page=N pattern
                parsed = urlparse(current_url)
                qs = parse_qs(parsed.query)
                qs["page"] = [str(page)]
                params = "&".join(f"{k}={v[0]}" for k, v in qs.items())
                next_url = urlunparse(parsed._replace(query=params))

            html = await fetch(next_url)
            if not html:
                break

            new_links = _find_chapter_list_container(html, next_url)
            before = len(all_links)
            for url, title in new_links:
                if url not in all_links:
                    all_links[url] = title

            added = len(all_links) - before
            if added == 0:
                break  # No new links on this page — stop paginating

            log(f"  Page {page}: +{added} new links ({len(all_links)} total)")
            current_html = html
            current_url = next_url
            page += 1

    if not all_links:
        log("⚠️  Container method found nothing — falling back to flat link scan")
        html = await fetch(novel_url)
        if html:
            for url, title in _extract_links_from_html(html, novel_url):
                all_links[url] = title

    log(f"✅ Discovered {len(all_links)} chapter links total")

    chapters: list[Chapter] = []
    for i, (href, title) in enumerate(all_links.items(), start=1):
        if not title:
            title = f"Chapter {i}"
        chapters.append(Chapter(index=i, title=title, url=href))

    return sort_chapters(chapters)