Update app.py

app.py

@@ -1,542 +1,452 @@
 """
-Novel Scraper — novelbin.com + webnovel.com
-Runs on HuggingFace Spaces (Gradio UI).
-Saves title + description of every novel to CSV and JSONL.
-Fully resumable — safe to restart if interrupted.
+novelhall_scraper.py
+====================
+Scrapes EVERY novel title + description from https://www.novelhall.com
+and saves them to novelhall_novels.json + novelhall_novels.csv
+
+HOW TO RUN ON HUGGINGFACE SPACES
+---------------------------------
+1. Create a new Space → "Blank" template (SDK: Gradio or Static — doesn't matter)
+2. Upload this file as `app.py`  (rename it)  OR add it to your repo and run via terminal
+3. In your Space's "Files" tab, open a terminal (or use the Jupyter-style cell runner)
+   and execute:  python novelhall_scraper.py
+4. Output files will appear in the repo root:
+      novelhall_novels.json
+      novelhall_novels.csv
+
+DEPENDENCIES (all pre-installed on HuggingFace Spaces):
+   requests, beautifulsoup4, lxml, tqdm
 """
 
-# ── audioop stub ──────────────────────────────────────────────────────────────
-# Python 3.13 removed the `audioop` stdlib module.
-# Gradio 4.x pulls in pydub which tries to import it — and crashes.
-# We don't use any audio features, so a no-op stub fixes the crash entirely.
-import sys as _sys
-import types as _types
-if "audioop" not in _sys.modules:
-    _audioop_stub = _types.ModuleType("audioop")
-    _sys.modules["audioop"]   = _audioop_stub
-    _sys.modules["pyaudioop"] = _audioop_stub   # pydub tries this name too
-# ─────────────────────────────────────────────────────────────────────────────
-
-import os
-import csv
+import requests
+from bs4 import BeautifulSoup
 import json
+import csv
 import time
-import random
+import sys
 import logging
-import threading
-import traceback
-from datetime import datetime
-from pathlib import Path
-
-import requests
-from bs4 import BeautifulSoup
-import gradio as gr
-
-# ─────────────────────────────────────────────
-# CONFIG
-# ─────────────────────────────────────────────
-OUTPUT_DIR = Path("output")
-OUTPUT_DIR.mkdir(exist_ok=True)
-
-LOG_FILE         = OUTPUT_DIR / "scraper.log"
-NOVELBIN_CSV     = OUTPUT_DIR / "novelbin_novels.csv"
-NOVELBIN_JSONL   = OUTPUT_DIR / "novelbin_novels.jsonl"
-WEBNOVEL_CSV     = OUTPUT_DIR / "webnovel_novels.csv"
-WEBNOVEL_JSONL   = OUTPUT_DIR / "webnovel_novels.jsonl"
-PROGRESS_FILE    = OUTPUT_DIR / "progress.json"
-
-NOVELBIN_BASE    = "https://novelbin.com"
-NOVELBIN_LIST    = "https://novelbin.com/sort/updated?page={page}"
-
-# webnovel public category API — returns JSON, no JS needed
-WEBNOVEL_API     = (
-    "https://www.webnovel.com/go/pcm/category/categoryAjax"
-    "?_csrfToken=&categoryId=0&pageIndex={page}&orderBy=1&bookType=1"
-)
+import random
+from urllib.parse import urljoin, urlparse
+from tqdm import tqdm
+
+# ─────────────────────────── CONFIGURATION ──────────────────────────────────
+BASE_URL        = "https://www.novelhall.com"
+LIST_URL        = "https://www.novelhall.com/all.html"   # full catalogue page
+OUTPUT_JSON     = "novelhall_novels.json"
+OUTPUT_CSV      = "novelhall_novels.csv"
+REQUEST_TIMEOUT = 30          # seconds per request
+DELAY_MIN       = 0.4         # minimum sleep between requests (seconds)
+DELAY_MAX       = 1.0         # maximum sleep between requests (seconds)
+MAX_RETRIES     = 5           # retries per failed request
+RETRY_BACKOFF   = 2.0         # exponential-back-off multiplier
+LOG_EVERY       = 50          # print progress every N novels
+# ─────────────────────────────────────────────────────────────────────────────
 
-REQUEST_TIMEOUT  = 30       # seconds per request
-RETRY_MAX        = 5        # retries per failed request
-RETRY_BACKOFF    = [2, 5, 10, 20, 40]  # seconds between retries
-PAGE_DELAY       = (1.0, 2.5)  # random sleep between pages (min, max)
-
-HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/124.0.0.0 Safari/537.36"
-    ),
-    "Accept-Language": "en-US,en;q=0.9",
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-}
-
-# ─────────────────────────────────────────────
-# LOGGING
-# ────────────────��────────────────────────────
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s [%(levelname)s] %(message)s",
-    handlers=[
-        logging.FileHandler(LOG_FILE, encoding="utf-8"),
-        logging.StreamHandler(),
-    ],
+    handlers=[logging.StreamHandler(sys.stdout)],
 )
 log = logging.getLogger(__name__)
 
-# ─────────────────────────────────────────────
-# SHARED STATE  (thread-safe via GIL for simple types)
-# ─────────────────────────────────────────────
-state = {
-    "running": False,
-    "status": "Idle",
-    "novelbin_done": 0,
-    "webnovel_done": 0,
-    "errors": 0,
-    "log_lines": [],          # last N lines shown in UI
-    "stop_flag": False,
-}
-
-def _log(msg: str, level: str = "info"):
-    getattr(log, level)(msg)
-    line = f"[{datetime.now().strftime('%H:%M:%S')}] {msg}"
-    state["log_lines"].append(line)
-    if len(state["log_lines"]) > 300:
-        state["log_lines"] = state["log_lines"][-300:]
-
-# ─────────────────────────────────────────────
-# PROGRESS PERSISTENCE
-# ─────────────────────────────────────────────
-def load_progress() -> dict:
-    if PROGRESS_FILE.exists():
-        try:
-            return json.loads(PROGRESS_FILE.read_text())
-        except Exception:
-            pass
-    return {"novelbin_page": 1, "webnovel_page": 1,
-            "novelbin_seen": [], "webnovel_seen": []}
-
-def save_progress(prog: dict):
-    PROGRESS_FILE.write_text(json.dumps(prog, indent=2))
-
-# ─────────────────────────────────────────────
-# CSV / JSONL HELPERS
-# ─────────────────────────────────────────────
-def ensure_csv(path: Path, fieldnames: list):
-    if not path.exists():
-        with open(path, "w", newline="", encoding="utf-8") as f:
-            writer = csv.DictWriter(f, fieldnames=fieldnames)
-            writer.writeheader()
-
-def append_record(csv_path: Path, jsonl_path: Path, record: dict, fieldnames: list):
-    with open(csv_path, "a", newline="", encoding="utf-8") as f:
-        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
-        writer.writerow(record)
-    with open(jsonl_path, "a", encoding="utf-8") as f:
-        f.write(json.dumps(record, ensure_ascii=False) + "\n")
-
-def load_seen_ids(jsonl_path: Path, id_key: str) -> set:
-    """Load already-scraped IDs from JSONL so we never duplicate."""
-    seen = set()
-    if jsonl_path.exists():
-        for line in jsonl_path.read_text(encoding="utf-8").splitlines():
-            try:
-                obj = json.loads(line)
-                seen.add(obj[id_key])
-            except Exception:
-                pass
-    return seen
-
-# ─────────────────────────────────────────────
-# HTTP WITH RETRY
-# ─────────────────────────────────────────────
-def get(url: str, session: requests.Session, *, is_json=False, extra_headers=None):
-    """GET with exponential-backoff retry. Returns (content, ok)."""
-    hdrs = dict(HEADERS)
-    if extra_headers:
-        hdrs.update(extra_headers)
-    for attempt in range(RETRY_MAX):
-        if state["stop_flag"]:
-            return None, False
-        try:
-            r = session.get(url, headers=hdrs, timeout=REQUEST_TIMEOUT)
-            if r.status_code == 429:
-                wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)] * 2
-                _log(f"Rate-limited on {url} — waiting {wait}s", "warning")
-                time.sleep(wait)
-                continue
-            if r.status_code != 200:
-                _log(f"HTTP {r.status_code} on {url} (attempt {attempt+1})", "warning")
-                time.sleep(RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)])
-                continue
-            return (r.json() if is_json else r.text), True
-        except requests.exceptions.RequestException as e:
-            wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)]
-            _log(f"Request error on {url}: {e} — retry in {wait}s", "warning")
-            time.sleep(wait)
-    _log(f"FAILED after {RETRY_MAX} attempts: {url}", "error")
-    state["errors"] += 1
-    return None, False
-
-# ─────────────────────────────────────────────
-# NOVELBIN SCRAPER
-# ─────────────────────────────��───────────────
-NB_FIELDS = ["id", "title", "url", "description", "author", "genres", "status", "scraped_at"]
-
-def scrape_novelbin_page(page: int, session: requests.Session, seen: set) -> tuple[list, bool]:
-    """Scrape one listing page. Returns (records, has_more)."""
-    url = NOVELBIN_LIST.format(page=page)
-    html, ok = get(url, session)
-    if not ok or not html:
-        return [], True  # treat as transient error, not end-of-pages
-
-    soup = BeautifulSoup(html, "html.parser")
-
-    # Check if page has any novels at all
-    novel_items = soup.select("div.col-novel-main .row.novel-item, li.novel-item, .list-novel .row")
-    if not novel_items:
-        # Try alternate selectors
-        novel_items = soup.select(".col-novel-main .col-xs-12")
-
-    if not novel_items:
-        _log(f"novelbin page {page}: no items found — assuming end of listing", "warning")
-        return [], False  # signal end
-
-    records = []
-    for item in novel_items:
-        try:
-            # Title + link
-            a_tag = item.select_one("h3.novel-title a, .novel-title a, a.novel-title")
-            if not a_tag:
-                continue
-            title = a_tag.get_text(strip=True)
-            href  = a_tag.get("href", "")
-            if not href:
-                continue
-            novel_url = href if href.startswith("http") else NOVELBIN_BASE + href
-
-            novel_id = href.rstrip("/").split("/")[-1]
-            if novel_id in seen:
-                continue
-
-            # Fetch detail page for description
-            detail_html, dok = get(novel_url, session)
-            desc = author = genres = status = ""
-            if dok and detail_html:
-                dsoup = BeautifulSoup(detail_html, "html.parser")
-
-                # Description — multiple possible selectors
-                for sel in ["div.desc-text", "div#tab-description", ".summary__content", ".description"]:
-                    el = dsoup.select_one(sel)
-                    if el:
-                        desc = el.get_text(" ", strip=True)[:4000]
-                        break
-
-                # Author
-                for sel in ["a[href*='author']", "span.author", ".author a"]:
-                    el = dsoup.select_one(sel)
-                    if el:
-                        author = el.get_text(strip=True)
-                        break
-
-                # Genres
-                genre_els = dsoup.select("div.categories a, .genres a, a.genre")
-                genres = ", ".join(g.get_text(strip=True) for g in genre_els)
-
-                # Status
-                for sel in [".header-stats span.ongoing, .header-stats span.completed", ".info-item .completed, .info-item .ongoing"]:
-                    el = dsoup.select_one(sel)
-                    if el:
-                        status = el.get_text(strip=True)
-                        break
-
-            record = {
-                "id": novel_id,
-                "title": title,
-                "url": novel_url,
-                "description": desc,
-                "author": author,
-                "genres": genres,
-                "status": status,
-                "scraped_at": datetime.utcnow().isoformat(),
-            }
-            records.append(record)
-            seen.add(novel_id)
-
-            time.sleep(random.uniform(0.3, 0.8))  # polite delay per detail page
-
-        except Exception as e:
-            _log(f"novelbin item parse error: {e}", "error")
-            state["errors"] += 1
-
-    return records, True
-
-def run_novelbin(prog: dict, session: requests.Session):
-    _log("═══ Starting novelbin.com scrape ═══")
-    state["status"] = "Scraping novelbin.com..."
-    seen = load_seen_ids(NOVELBIN_JSONL, "id")
-    _log(f"novelbin: {len(seen)} novels already saved, resuming from page {prog['novelbin_page']}")
-    ensure_csv(NOVELBIN_CSV, NB_FIELDS)
-
-    page = prog["novelbin_page"]
-    consecutive_empty = 0
-
-    while not state["stop_flag"]:
-        _log(f"novelbin page {page}...")
-        records, has_more = scrape_novelbin_page(page, session, seen)
-
-        for r in records:
-            append_record(NOVELBIN_CSV, NOVELBIN_JSONL, r, NB_FIELDS)
-            state["novelbin_done"] += 1
-
-        _log(f"novelbin page {page}: +{len(records)} novels (total {state['novelbin_done']})")
-
-        if not records:
-            consecutive_empty += 1
-            if consecutive_empty >= 3:
-                _log("novelbin: 3 consecutive empty pages — scrape complete.")
-                break
-        else:
-            consecutive_empty = 0
-
-        if not has_more:
-            _log("novelbin: reached end of listing.")
-            break
-
-        page += 1
-        prog["novelbin_page"] = page
-        save_progress(prog)
-
-        time.sleep(random.uniform(*PAGE_DELAY))
-
-    _log(f"═══ novelbin done — {state['novelbin_done']} novels saved ═══")
+# Rotate between a handful of realistic user-agent strings so the server
+# does not treat every request as a bot.
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
+]
+
+
+def make_session() -> requests.Session:
+    """Return a requests.Session with shared headers."""
+    session = requests.Session()
+    session.headers.update(
+        {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Referer": BASE_URL,
+        }
+    )
+    return session
 
-# ─────────────────────────────────────────────
-# WEBNOVEL SCRAPER  (uses public JSON API)
-# ─────────────────────────────────────────────
-WN_FIELDS = ["id", "title", "url", "description", "author", "genres", "status",
-             "total_chapters", "rating", "scraped_at"]
 
-def scrape_webnovel_page(page: int, session: requests.Session, seen: set) -> tuple[list, bool]:
-    """One page of webnovel category API. Returns (records, has_more)."""
-    url = WEBNOVEL_API.format(page=page)
-    extra = {
-        "Referer": "https://www.webnovel.com/",
-        "Accept": "application/json, text/javascript, */*; q=0.01",
-        "X-Requested-With": "XMLHttpRequest",
-    }
-    data, ok = get(url, session, is_json=True, extra_headers=extra)
-    if not ok or not data:
-        return [], True
-
-    try:
-        code = data.get("code", -1)
-        if code != 0:
-            _log(f"webnovel API non-zero code {code} on page {page}", "warning")
-            return [], True
-
-        items = (data.get("data") or {}).get("items") or []
-        if not items:
-            return [], False  # end of pages
-
-    except Exception as e:
-        _log(f"webnovel page {page} parse error: {e}", "error")
-        return [], True
-
-    records = []
-    for item in items:
+def fetch(session: requests.Session, url: str) -> str | None:
+    """
+    GET a URL with retry + exponential back-off.
+    Returns the response text, or None if all retries fail.
+    """
+    delay = RETRY_BACKOFF
+    for attempt in range(1, MAX_RETRIES + 1):
         try:
-            novel_id = str(item.get("bookId") or item.get("id") or "")
-            if not novel_id or novel_id in seen:
-                continue
-
-            title    = item.get("bookName") or item.get("name") or ""
-            author   = item.get("authorName") or ""
-            status   = "Completed" if item.get("isComplete") else "Ongoing"
-            chapters = str(item.get("totalChapterNum") or item.get("chapterNum") or "")
-            rating   = str(item.get("totalScore") or item.get("reviewTotal") or "")
-            cat_list = item.get("categoryItems") or item.get("categoryList") or []
-            genres   = ", ".join(c.get("categoryName", "") for c in cat_list if isinstance(c, dict))
-            novel_url = f"https://www.webnovel.com/book/{novel_id}"
-
-            # Description is sometimes in the listing, sometimes needs a detail call
-            desc = item.get("description") or item.get("intro") or ""
-            if not desc:
-                desc = fetch_webnovel_description(novel_id, session)
-
-            record = {
-                "id": novel_id,
-                "title": title,
-                "url": novel_url,
-                "description": desc[:4000],
-                "author": author,
-                "genres": genres,
-                "status": status,
-                "total_chapters": chapters,
-                "rating": rating,
-                "scraped_at": datetime.utcnow().isoformat(),
-            }
-            records.append(record)
-            seen.add(novel_id)
-
-        except Exception as e:
-            _log(f"webnovel item parse error: {e}", "error")
-            state["errors"] += 1
-
-    return records, True
-
-def fetch_webnovel_description(novel_id: str, session: requests.Session) -> str:
-    """Fallback: fetch book detail API for description."""
-    url = (
-        f"https://www.webnovel.com/go/pcm/book/getBookDetail"
-        f"?_csrfToken=&bookId={novel_id}"
-    )
-    extra = {
-        "Referer": f"https://www.webnovel.com/book/{novel_id}",
-        "Accept": "application/json, */*; q=0.01",
-        "X-Requested-With": "XMLHttpRequest",
-    }
-    data, ok = get(url, session, is_json=True, extra_headers=extra)
-    if not ok or not data:
-        return ""
-    try:
-        return (data.get("data") or {}).get("bookInfo", {}).get("description", "")
-    except Exception:
-        return ""
-
-def run_webnovel(prog: dict, session: requests.Session):
-    _log("═══ Starting webnovel.com scrape ═══")
-    state["status"] = "Scraping webnovel.com..."
-    seen = load_seen_ids(WEBNOVEL_JSONL, "id")
-    _log(f"webnovel: {len(seen)} novels already saved, resuming from page {prog['webnovel_page']}")
-    ensure_csv(WEBNOVEL_CSV, WN_FIELDS)
-
-    page = prog["webnovel_page"]
+            # Rotate user-agent on every retry
+            session.headers["User-Agent"] = random.choice(USER_AGENTS)
+            resp = session.get(url, timeout=REQUEST_TIMEOUT)
+            resp.raise_for_status()
+            return resp.text
+        except requests.exceptions.HTTPError as exc:
+            status = exc.response.status_code if exc.response is not None else "?"
+            if status == 404:
+                log.warning("404 for %s – skipping", url)
+                return None
+            log.warning("HTTP %s on %s (attempt %d/%d)", status, url, attempt, MAX_RETRIES)
+        except requests.exceptions.RequestException as exc:
+            log.warning("Request error on %s (attempt %d/%d): %s", url, attempt, MAX_RETRIES, exc)
+
+        if attempt < MAX_RETRIES:
+            sleep_time = delay + random.uniform(0, 0.5)
+            log.info("  → Retrying in %.1f s …", sleep_time)
+            time.sleep(sleep_time)
+            delay *= RETRY_BACKOFF
+
+    log.error("All retries exhausted for %s", url)
+    return None
+
+
+# ──────────────────────── PHASE 1: collect novel links ───────────────────────
+
+def discover_novel_links(session: requests.Session) -> list[str]:
+    """
+    Crawl the novelhall catalogue and return a de-duplicated list of
+    novel detail-page URLs.
+
+    Strategy
+    --------
+    novelhall.com typically exposes novels through:
+      • /all.html  (or paginated /all-1.html, /all-2.html …)
+      • Genre / tag listing pages
+
+    We start from /all.html and follow pagination.  If that yields nothing
+    (site structure may vary) we fall back to the sitemap or the home page.
+    """
+    log.info("=== PHASE 1: Discovering novel links ===")
+
+    novel_links: set[str] = set()
+
+    # ── Try paginated /all.html first ─────────────────────────────────────
+    page_num = 1
     consecutive_empty = 0
 
-    while not state["stop_flag"]:
-        _log(f"webnovel page {page}...")
-        records, has_more = scrape_webnovel_page(page, session, seen)
+    while True:
+        if page_num == 1:
+            url = LIST_URL
+        else:
+            # Common patterns: /all-2.html  or  /all.html?page=2
+            url = f"https://www.novelhall.com/all-{page_num}.html"
+
+        log.info("Fetching catalogue page %d: %s", page_num, url)
+        html = fetch(session, url)
 
-        for r in records:
-            append_record(WEBNOVEL_CSV, WEBNOVEL_JSONL, r, WN_FIELDS)
-            state["webnovel_done"] += 1
+        if html is None:
+            log.info("  Page %d returned nothing – stopping pagination.", page_num)
+            break
 
-        _log(f"webnovel page {page}: +{len(records)} novels (total {state['webnovel_done']})")
+        soup = BeautifulSoup(html, "lxml")
+        found_on_page = _extract_novel_links_from_soup(soup)
 
-        if not records:
+        if not found_on_page:
             consecutive_empty += 1
-            if consecutive_empty >= 3:
-                _log("webnovel: 3 consecutive empty pages — scrape complete.")
+            log.info("  No new novel links on page %d (empty streak: %d)", page_num, consecutive_empty)
+            if consecutive_empty >= 2:
                 break
         else:
             consecutive_empty = 0
+            novel_links.update(found_on_page)
+            log.info("  Found %d novel links on page %d (total so far: %d)",
+                     len(found_on_page), page_num, len(novel_links))
+
+        page_num += 1
+        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
+
+    # ── Fallback: scrape home page for genre/tag links, then crawl those ──
+    if len(novel_links) == 0:
+        log.info("Catalogue pages yielded nothing – falling back to home-page crawl")
+        novel_links.update(_crawl_home_categories(session))
+
+    # ── Second fallback: /sitemap.xml ─────────────────────────────────────
+    if len(novel_links) == 0:
+        log.info("Home-page crawl also empty – trying sitemap.xml")
+        novel_links.update(_crawl_sitemap(session))
+
+    result = sorted(novel_links)
+    log.info("=== Phase 1 complete: %d unique novel links discovered ===", len(result))
+    return result
+
+
+def _extract_novel_links_from_soup(soup: BeautifulSoup) -> list[str]:
+    """
+    Extract novel detail-page hrefs from any listing page soup.
+    Returns absolute URLs.
+    """
+    links = []
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        # novelhall detail pages look like  /Novel-Title-123/  or /123/
+        # We accept any path that is NOT a category/genre/page index link
+        if not href or href in ("#", "/"):
+            continue
+        abs_url = urljoin(BASE_URL, href)
+        parsed = urlparse(abs_url)
+        # Must be on the same domain
+        if parsed.netloc not in ("www.novelhall.com", "novelhall.com"):
+            continue
+        # Filter out obvious non-novel pages
+        path = parsed.path.rstrip("/")
+        skip_patterns = (
+            "/all", "/genre", "/tag", "/category", "/search",
+            "/login", "/register", "/user", "/sitemap", "/cdn-cgi",
+        )
+        if any(path.startswith(p) for p in skip_patterns):
+            continue
+        # Novel pages typically have at least one path segment of substance
+        segments = [s for s in path.split("/") if s]
+        if len(segments) == 1 and segments[0].isdigit():
+            # Pattern like /12345 → probably a novel
+            links.append(abs_url)
+        elif len(segments) == 1 and "-" in segments[0]:
+            # Pattern like /novel-title → probably a novel
+            links.append(abs_url)
+        elif len(segments) >= 1 and not segments[0].startswith("all"):
+            links.append(abs_url)
+    return links
+
+
+def _crawl_home_categories(session: requests.Session) -> list[str]:
+    """Grab the home page, find genre/category links, crawl each."""
+    log.info("Crawling home page for category links …")
+    html = fetch(session, BASE_URL)
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "lxml")
+    links: set[str] = set()
+
+    # Collect category links first
+    cat_links = set()
+    for a in soup.find_all("a", href=True):
+        href = a["href"].strip()
+        abs_url = urljoin(BASE_URL, href)
+        parsed = urlparse(abs_url)
+        if parsed.netloc in ("www.novelhall.com", "novelhall.com"):
+            path = parsed.path.rstrip("/")
+            if any(kw in path for kw in ("/genre", "/tag", "/category", "/type", "/list")):
+                cat_links.add(abs_url)
+
+    log.info("  Found %d category links", len(cat_links))
+
+    for cat_url in sorted(cat_links):
+        log.info("  Crawling category: %s", cat_url)
+        cat_html = fetch(session, cat_url)
+        if not cat_html:
+            continue
+        cat_soup = BeautifulSoup(cat_html, "lxml")
+        found = _extract_novel_links_from_soup(cat_soup)
+        links.update(found)
+        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
+
+    return list(links)
+
+
+def _crawl_sitemap(session: requests.Session) -> list[str]:
+    """Try to parse /sitemap.xml for novel URLs."""
+    sitemap_url = f"{BASE_URL}/sitemap.xml"
+    log.info("Fetching %s", sitemap_url)
+    xml = fetch(session, sitemap_url)
+    if not xml:
+        return []
+    soup = BeautifulSoup(xml, "lxml-xml")
+    links = []
+    for loc in soup.find_all("loc"):
+        url = loc.get_text(strip=True)
+        parsed = urlparse(url)
+        if parsed.netloc in ("www.novelhall.com", "novelhall.com"):
+            path = parsed.path.rstrip("/")
+            segments = [s for s in path.split("/") if s]
+            if segments:
+                links.append(url)
+    log.info("  Sitemap gave %d candidate links", len(links))
+    return links
+
+
+# ──────────────────────── PHASE 2: scrape each detail page ───────────────────
+
+def scrape_novel_detail(session: requests.Session, url: str) -> dict | None:
+    """
+    Fetch a novel's detail page and extract:
+      - title
+      - description / synopsis
+      - url  (canonical)
+    Returns None if the page cannot be fetched or is not a novel page.
+    """
+    html = fetch(session, url)
+    if not html:
+        return None
+
+    soup = BeautifulSoup(html, "lxml")
+
+    # ── Title ──────────────────────────────────────────────────────────────
+    title = _extract_title(soup)
+    if not title:
+        # Not a real novel page (could be a category/error page)
+        return None
+
+    # ── Description ────────────────────────────────────────────────────────
+    description = _extract_description(soup)
+
+    return {
+        "title": title,
+        "description": description,
+        "url": url,
+    }
 
-        if not has_more:
-            _log("webnovel: reached end of listing.")
-            break
-
-        page += 1
-        prog["webnovel_page"] = page
-        save_progress(prog)
-
-        time.sleep(random.uniform(*PAGE_DELAY))
-
-    _log(f"═══ webnovel done — {state['webnovel_done']} novels saved ═══")
-
-# ────────────────────────��────────────────────
-# MASTER RUNNER
-# ─────────────────────────────────────────────
-def master_run():
-    state["running"]    = True
-    state["stop_flag"]  = False
-    state["errors"]     = 0
-
-    # Restore saved counts from existing JSONL files
-    state["novelbin_done"] = sum(1 for _ in NOVELBIN_JSONL.open(encoding="utf-8")) if NOVELBIN_JSONL.exists() else 0
-    state["webnovel_done"] = sum(1 for _ in WEBNOVEL_JSONL.open(encoding="utf-8")) if WEBNOVEL_JSONL.exists() else 0
-
-    prog = load_progress()
-
-    session = requests.Session()
-    session.headers.update(HEADERS)
-
-    try:
-        run_novelbin(prog, session)
-        if not state["stop_flag"]:
-            run_webnovel(prog, session)
-    except Exception as e:
-        _log(f"FATAL: {e}\n{traceback.format_exc()}", "error")
-    finally:
-        state["running"] = False
-        state["status"]  = "Finished ✅" if not state["stop_flag"] else "Stopped ⛔"
-        _log(f"Run complete. novelbin={state['novelbin_done']}  webnovel={state['webnovel_done']}  errors={state['errors']}")
-        session.close()
-
-# ─────────────────────────────────────────────
-# GRADIO UI
-# ─────────────────────────────────────────────
-def ui_start():
-    if state["running"]:
-        return "⚠️ Already running!"
-    state["log_lines"] = []
-    thread = threading.Thread(target=master_run, daemon=True)
-    thread.start()
-    return "🚀 Scraper started!"
-
-def ui_stop():
-    state["stop_flag"] = True
-    return "⛔ Stop signal sent — finishing current page..."
-
-def ui_status():
-    lines = "\n".join(state["log_lines"][-80:])
-    nb_size = f"{NOVELBIN_JSONL.stat().st_size/1024:.1f} KB" if NOVELBIN_JSONL.exists() else "—"
-    wn_size = f"{WEBNOVEL_JSONL.stat().st_size/1024:.1f} KB" if WEBNOVEL_JSONL.exists() else "—"
-    info = (
-        f"Status : {state['status']}\n"
-        f"novelbin: {state['novelbin_done']} novels  ({nb_size})\n"
-        f"webnovel: {state['webnovel_done']} novels  ({wn_size})\n"
-        f"Errors  : {state['errors']}\n"
-        f"{'─'*40}\n"
-        f"{lines}"
-    )
-    return info
-
-def ui_files():
-    files = []
-    for p in [NOVELBIN_CSV, NOVELBIN_JSONL, WEBNOVEL_CSV, WEBNOVEL_JSONL, LOG_FILE, PROGRESS_FILE]:
-        if p.exists():
-            files.append(str(p))
-    return files if files else None
-
-with gr.Blocks(title="Novel Scraper") as demo:
-    gr.Markdown(
-        "# 📚 Novel Scraper\n"
-        "Scrapes **title + description** of every novel on **novelbin.com** and **webnovel.com**.\n"
-        "Output is saved as **CSV + JSONL** in the `output/` folder. Fully resumable."
-    )
-
-    with gr.Row():
-        btn_start   = gr.Button("▶ Start Scraper", variant="primary")
-        btn_stop    = gr.Button("⏹ Stop", variant="stop")
-        btn_refresh = gr.Button("🔄 Refresh Status")
-
-    status_box = gr.Textbox(
-        label="Live Log",
-        lines=25,
-        max_lines=25,
-        interactive=False,
-        value="Press ▶ Start Scraper to begin.",
-    )
-
-    file_out = gr.File(label="Download Output Files", file_count="multiple")
 
-    btn_start.click(ui_start,  outputs=status_box)
-    btn_stop.click(ui_stop,    outputs=status_box)
-    btn_refresh.click(ui_status, outputs=status_box)
-    btn_refresh.click(ui_files,  outputs=file_out)
+def _extract_title(soup: BeautifulSoup) -> str:
+    """Try multiple selectors to find the novel title."""
+    # 1. <h1> inside common novel-info wrappers
+    for selector in (
+        "div.book-info h1",
+        "div.novel-info h1",
+        "div.info h1",
+        "div.header-wrapper h1",
+        "div.title h1",
+        "article h1",
+        "h1.novel-title",
+        "h1",
+    ):
+        tag = soup.select_one(selector)
+        if tag:
+            text = tag.get_text(strip=True)
+            if text:
+                return text
+
+    # 2. <title> tag, strip " - Novelhall" suffix
+    title_tag = soup.find("title")
+    if title_tag:
+        text = title_tag.get_text(strip=True)
+        for suffix in (" - Novelhall", " - Novel Hall", " | Novelhall", " | Novel Hall"):
+            if text.lower().endswith(suffix.lower()):
+                text = text[: -len(suffix)].strip()
+        if text:
+            return text
+
+    return ""
+
+
+def _extract_description(soup: BeautifulSoup) -> str:
+    """Try multiple selectors to find the synopsis / description."""
+    # 1. Common class names used by novel sites
+    for selector in (
+        "div.book-intro",
+        "div.novel-intro",
+        "div.synopsis",
+        "div.description",
+        "div.intro",
+        "div.summary",
+        "div.novel-desc",
+        "div.book-desc",
+        "section.synopsis",
+        "div#bookIntro",
+        "div#intro",
+        "div#synopsis",
+        "div#description",
+        "div.content > p",          # generic fallback
+    ):
+        tag = soup.select_one(selector)
+        if tag:
+            text = tag.get_text(separator=" ", strip=True)
+            if len(text) > 20:        # ignore tiny/empty hits
+                return text
+
+    # 2. meta description
+    meta = soup.find("meta", attrs={"name": "description"})
+    if meta and meta.get("content"):
+        return meta["content"].strip()
+
+    return ""
+
+
+# ───────────────────────────── PHASE 3: save ─────────────────────────────────
+
+def save_results(novels: list[dict]) -> None:
+    """Write results to JSON and CSV."""
+    # JSON
+    with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
+        json.dump(novels, f, ensure_ascii=False, indent=2)
+    log.info("Saved %d novels to %s", len(novels), OUTPUT_JSON)
+
+    # CSV
+    fieldnames = ["title", "description", "url"]
+    with open(OUTPUT_CSV, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(novels)
+    log.info("Saved %d novels to %s", len(novels), OUTPUT_CSV)
+
+
+# ─────────────────────────────── MAIN ────────────────────────────────────────
+
+def main() -> None:
+    log.info("╔══════════════════════════════════════════╗")
+    log.info("║  Novelhall.com Full Catalogue Scraper    ║")
+    log.info("╚══════════════════════════════════════════╝")
+
+    session = make_session()
+
+    # ── Phase 1: discover all novel URLs ──────────────────────────────────
+    novel_urls = discover_novel_links(session)
+
+    if not novel_urls:
+        log.error("No novel links found at all – aborting.")
+        sys.exit(1)
+
+    # ── Phase 2: scrape each novel's detail page ───────────────────────────
+    log.info("=== PHASE 2: Scraping %d novel detail pages ===", len(novel_urls))
+    novels: list[dict] = []
+    failed: list[str] = []
+
+    for idx, url in enumerate(tqdm(novel_urls, desc="Scraping novels", unit="novel"), start=1):
+        result = scrape_novel_detail(session, url)
+
+        if result:
+            novels.append(result)
+        else:
+            failed.append(url)
+            log.warning("  Skipped (no data): %s", url)
+
+        if idx % LOG_EVERY == 0:
+            log.info("  Progress: %d / %d scraped, %d ok, %d failed",
+                     idx, len(novel_urls), len(novels), len(failed))
+
+        # Polite crawl delay
+        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
+
+    # ── Phase 3: save ─────────────────────────────────────────────────────
+    log.info("=== PHASE 3: Saving results ===")
+    save_results(novels)
+
+    # ── Summary ────────────────────────────────────────────────────────────
+    log.info("╔══════════════════════════════════════════╗")
+    log.info("║  DONE                                    ║")
+    log.info("║  Total discovered : %-5d                ║", len(novel_urls))
+    log.info("║  Successfully saved: %-5d               ║", len(novels))
+    log.info("║  Failed / skipped  : %-5d               ║", len(failed))
+    log.info("║  Output files:                           ║")
+    log.info("║    • %-36s ║", OUTPUT_JSON)
+    log.info("║    • %-36s ║", OUTPUT_CSV)
+    log.info("╚══════════════════════════════════════════╝")
+
+    if failed:
+        failed_path = "novelhall_failed.txt"
+        with open(failed_path, "w", encoding="utf-8") as f:
+            f.write("\n".join(failed))
+        log.info("URLs that could not be scraped written to: %s", failed_path)
 
-    # Gradio 4.x auto-poll — fires every N seconds while the tab is open
-    demo.load(ui_status, outputs=status_box, every=5)
-    demo.load(ui_files,  outputs=file_out,   every=10)
 
 if __name__ == "__main__":
-    demo.launch()
+    main()