Update app.py

app.py

@@ -1,452 +1,321 @@
-"""
-novelhall_scraper.py
-====================
-Scrapes EVERY novel title + description from https://www.novelhall.com
-and saves them to novelhall_novels.json + novelhall_novels.csv
-
-HOW TO RUN ON HUGGINGFACE SPACES
----------------------------------
-1. Create a new Space → "Blank" template (SDK: Gradio or Static — doesn't matter)
-2. Upload this file as `app.py`  (rename it)  OR add it to your repo and run via terminal
-3. In your Space's "Files" tab, open a terminal (or use the Jupyter-style cell runner)
-   and execute:  python novelhall_scraper.py
-4. Output files will appear in the repo root:
-      novelhall_novels.json
-      novelhall_novels.csv
-
-DEPENDENCIES (all pre-installed on HuggingFace Spaces):
-   requests, beautifulsoup4, lxml, tqdm
-"""
-
+import gradio as gr
 import requests
 from bs4 import BeautifulSoup
-import json
-import csv
+import re
 import time
-import sys
-import logging
-import random
-from urllib.parse import urljoin, urlparse
-from tqdm import tqdm
-
-# ─────────────────────────── CONFIGURATION ──────────────────────────────────
-BASE_URL        = "https://www.novelhall.com"
-LIST_URL        = "https://www.novelhall.com/all.html"   # full catalogue page
-OUTPUT_JSON     = "novelhall_novels.json"
-OUTPUT_CSV      = "novelhall_novels.csv"
-REQUEST_TIMEOUT = 30          # seconds per request
-DELAY_MIN       = 0.4         # minimum sleep between requests (seconds)
-DELAY_MAX       = 1.0         # maximum sleep between requests (seconds)
-MAX_RETRIES     = 5           # retries per failed request
-RETRY_BACKOFF   = 2.0         # exponential-back-off multiplier
-LOG_EVERY       = 50          # print progress every N novels
-# ─────────────────────────────────────────────────────────────────────────────
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [%(levelname)s] %(message)s",
-    handlers=[logging.StreamHandler(sys.stdout)],
-)
-log = logging.getLogger(__name__)
-
-# Rotate between a handful of realistic user-agent strings so the server
-# does not treat every request as a bot.
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
-]
-
-
-def make_session() -> requests.Session:
-    """Return a requests.Session with shared headers."""
-    session = requests.Session()
-    session.headers.update(
-        {
-            "User-Agent": random.choice(USER_AGENTS),
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.9",
-            "Accept-Encoding": "gzip, deflate, br",
-            "Connection": "keep-alive",
-            "Referer": BASE_URL,
-        }
-    )
-    return session
-
-
-def fetch(session: requests.Session, url: str) -> str | None:
-    """
-    GET a URL with retry + exponential back-off.
-    Returns the response text, or None if all retries fail.
-    """
-    delay = RETRY_BACKOFF
-    for attempt in range(1, MAX_RETRIES + 1):
+import os
+import json
+from datetime import datetime
+from pathlib import Path
+
+# ─────────────────────────────────────────────
+#  CONFIG
+# ─────────────────────────────────────────────
+BASE_URL      = "https://www.novelhall.com"
+NOVEL_RE      = re.compile(r"https://www\.novelhall\.com/[a-z0-9-]+-\d+/$")
+OUTPUT_DIR    = Path("outputs")
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+}
+
+SESSION = requests.Session()
+SESSION.headers.update(HEADERS)
+
+# ─────────────────────────────────────────────
+#  HELPERS
+# ─────────────────────────────────────────────
+def fetch_page(url: str, retries: int = 3, delay: float = 2.0) -> str | None:
+    """Fetch HTML from a URL with retries."""
+    for attempt in range(1, retries + 1):
         try:
-            # Rotate user-agent on every retry
-            session.headers["User-Agent"] = random.choice(USER_AGENTS)
-            resp = session.get(url, timeout=REQUEST_TIMEOUT)
+            resp = SESSION.get(url, timeout=20)
             resp.raise_for_status()
             return resp.text
-        except requests.exceptions.HTTPError as exc:
-            status = exc.response.status_code if exc.response is not None else "?"
-            if status == 404:
-                log.warning("404 for %s – skipping", url)
+        except Exception as exc:
+            if attempt < retries:
+                time.sleep(delay * attempt)
+            else:
                 return None
-            log.warning("HTTP %s on %s (attempt %d/%d)", status, url, attempt, MAX_RETRIES)
-        except requests.exceptions.RequestException as exc:
-            log.warning("Request error on %s (attempt %d/%d): %s", url, attempt, MAX_RETRIES, exc)
-
-        if attempt < MAX_RETRIES:
-            sleep_time = delay + random.uniform(0, 0.5)
-            log.info("  → Retrying in %.1f s …", sleep_time)
-            time.sleep(sleep_time)
-            delay *= RETRY_BACKOFF
-
-    log.error("All retries exhausted for %s", url)
-    return None
-
-
-# ──────────────────────── PHASE 1: collect novel links ───────────────────────
 
-def discover_novel_links(session: requests.Session) -> list[str]:
-    """
-    Crawl the novelhall catalogue and return a de-duplicated list of
-    novel detail-page URLs.
-
-    Strategy
-    --------
-    novelhall.com typically exposes novels through:
-      • /all.html  (or paginated /all-1.html, /all-2.html …)
-      • Genre / tag listing pages
-
-    We start from /all.html and follow pagination.  If that yields nothing
-    (site structure may vary) we fall back to the sitemap or the home page.
-    """
-    log.info("=== PHASE 1: Discovering novel links ===")
 
-    novel_links: set[str] = set()
-
-    # ── Try paginated /all.html first ─────────────────────────────────────
-    page_num = 1
-    consecutive_empty = 0
-
-    while True:
-        if page_num == 1:
-            url = LIST_URL
-        else:
-            # Common patterns: /all-2.html  or  /all.html?page=2
-            url = f"https://www.novelhall.com/all-{page_num}.html"
-
-        log.info("Fetching catalogue page %d: %s", page_num, url)
-        html = fetch(session, url)
-
-        if html is None:
-            log.info("  Page %d returned nothing – stopping pagination.", page_num)
-            break
-
-        soup = BeautifulSoup(html, "lxml")
-        found_on_page = _extract_novel_links_from_soup(soup)
-
-        if not found_on_page:
-            consecutive_empty += 1
-            log.info("  No new novel links on page %d (empty streak: %d)", page_num, consecutive_empty)
-            if consecutive_empty >= 2:
-                break
-        else:
-            consecutive_empty = 0
-            novel_links.update(found_on_page)
-            log.info("  Found %d novel links on page %d (total so far: %d)",
-                     len(found_on_page), page_num, len(novel_links))
-
-        page_num += 1
-        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
-
-    # ── Fallback: scrape home page for genre/tag links, then crawl those ──
-    if len(novel_links) == 0:
-        log.info("Catalogue pages yielded nothing – falling back to home-page crawl")
-        novel_links.update(_crawl_home_categories(session))
-
-    # ── Second fallback: /sitemap.xml ─────────────────────────────────────
-    if len(novel_links) == 0:
-        log.info("Home-page crawl also empty – trying sitemap.xml")
-        novel_links.update(_crawl_sitemap(session))
-
-    result = sorted(novel_links)
-    log.info("=== Phase 1 complete: %d unique novel links discovered ===", len(result))
-    return result
-
-
-def _extract_novel_links_from_soup(soup: BeautifulSoup) -> list[str]:
-    """
-    Extract novel detail-page hrefs from any listing page soup.
-    Returns absolute URLs.
-    """
+def extract_novel_links(html: str) -> list[str]:
+    """Extract all novel detail-page links from a listing page."""
+    soup = BeautifulSoup(html, "html.parser")
     links = []
-    for a in soup.find_all("a", href=True):
-        href = a["href"].strip()
-        # novelhall detail pages look like  /Novel-Title-123/  or /123/
-        # We accept any path that is NOT a category/genre/page index link
-        if not href or href in ("#", "/"):
-            continue
-        abs_url = urljoin(BASE_URL, href)
-        parsed = urlparse(abs_url)
-        # Must be on the same domain
-        if parsed.netloc not in ("www.novelhall.com", "novelhall.com"):
-            continue
-        # Filter out obvious non-novel pages
-        path = parsed.path.rstrip("/")
-        skip_patterns = (
-            "/all", "/genre", "/tag", "/category", "/search",
-            "/login", "/register", "/user", "/sitemap", "/cdn-cgi",
-        )
-        if any(path.startswith(p) for p in skip_patterns):
-            continue
-        # Novel pages typically have at least one path segment of substance
-        segments = [s for s in path.split("/") if s]
-        if len(segments) == 1 and segments[0].isdigit():
-            # Pattern like /12345 → probably a novel
-            links.append(abs_url)
-        elif len(segments) == 1 and "-" in segments[0]:
-            # Pattern like /novel-title → probably a novel
-            links.append(abs_url)
-        elif len(segments) >= 1 and not segments[0].startswith("all"):
-            links.append(abs_url)
-    return links
-
-
-def _crawl_home_categories(session: requests.Session) -> list[str]:
-    """Grab the home page, find genre/category links, crawl each."""
-    log.info("Crawling home page for category links …")
-    html = fetch(session, BASE_URL)
-    if not html:
-        return []
-    soup = BeautifulSoup(html, "lxml")
-    links: set[str] = set()
-
-    # Collect category links first
-    cat_links = set()
-    for a in soup.find_all("a", href=True):
-        href = a["href"].strip()
-        abs_url = urljoin(BASE_URL, href)
-        parsed = urlparse(abs_url)
-        if parsed.netloc in ("www.novelhall.com", "novelhall.com"):
-            path = parsed.path.rstrip("/")
-            if any(kw in path for kw in ("/genre", "/tag", "/category", "/type", "/list")):
-                cat_links.add(abs_url)
-
-    log.info("  Found %d category links", len(cat_links))
-
-    for cat_url in sorted(cat_links):
-        log.info("  Crawling category: %s", cat_url)
-        cat_html = fetch(session, cat_url)
-        if not cat_html:
-            continue
-        cat_soup = BeautifulSoup(cat_html, "lxml")
-        found = _extract_novel_links_from_soup(cat_soup)
-        links.update(found)
-        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
-
-    return list(links)
-
-
-def _crawl_sitemap(session: requests.Session) -> list[str]:
-    """Try to parse /sitemap.xml for novel URLs."""
-    sitemap_url = f"{BASE_URL}/sitemap.xml"
-    log.info("Fetching %s", sitemap_url)
-    xml = fetch(session, sitemap_url)
-    if not xml:
-        return []
-    soup = BeautifulSoup(xml, "lxml-xml")
-    links = []
-    for loc in soup.find_all("loc"):
-        url = loc.get_text(strip=True)
-        parsed = urlparse(url)
-        if parsed.netloc in ("www.novelhall.com", "novelhall.com"):
-            path = parsed.path.rstrip("/")
-            segments = [s for s in path.split("/") if s]
-            if segments:
-                links.append(url)
-    log.info("  Sitemap gave %d candidate links", len(links))
+    seen = set()
+    for a in soup.find_all("a", href=NOVEL_RE):
+        href = a["href"]
+        if href not in seen:
+            seen.add(href)
+            links.append(href)
     return links
 
 
-# ──────────────────────── PHASE 2: scrape each detail page ───────────────────
+def extract_novel_info(html: str, url: str) -> dict:
+    """Extract title and description from a novel detail page."""
+    soup = BeautifulSoup(html, "html.parser")
 
-def scrape_novel_detail(session: requests.Session, url: str) -> dict | None:
-    """
-    Fetch a novel's detail page and extract:
-      - title
-      - description / synopsis
-      - url  (canonical)
-    Returns None if the page cannot be fetched or is not a novel page.
-    """
-    html = fetch(session, url)
-    if not html:
-        return None
-
-    soup = BeautifulSoup(html, "lxml")
-
-    # ── Title ──────────────────────────────────────────────────────────────
-    title = _extract_title(soup)
+    # Title – prefer h1 inside book-info, fall back to <title>
+    title = ""
+    h1 = soup.find("h1")
+    if h1:
+        title = h1.get_text(strip=True)
     if not title:
-        # Not a real novel page (could be a category/error page)
-        return None
-
-    # ── Description ────────────────────────────────────────────────────────
-    description = _extract_description(soup)
+        t = soup.find("title")
+        if t:
+            title = t.get_text(strip=True)
+
+    # Description – div.intro contains synopsis text
+    description = ""
+    intro_div = soup.find("div", class_="intro")
+    if intro_div:
+        # Remove the "...more>>" button text and clean duplicates
+        for tag in intro_div.find_all(["a", "span", "button"]):
+            tag.decompose()
+        raw = intro_div.get_text(separator=" ", strip=True)
+        # The site sometimes duplicates the synopsis; take the first half
+        half = len(raw) // 2
+        if half > 50 and raw[:half].strip() == raw[half:].strip():
+            raw = raw[:half].strip()
+        # Strip leading "Synopsis" label if present (may be fused: "SynopsisAt")
+        raw = re.sub(r"^Synopsis", "", raw, flags=re.IGNORECASE).strip()
+        description = raw
+
+    # Fall back to meta description
+    if not description:
+        meta = soup.find("meta", {"name": "description"})
+        if meta:
+            description = meta.get("content", "").strip()
+            description = re.sub(r"\.\.\.$", "", description).strip()
 
     return {
+        "url": url,
         "title": title,
         "description": description,
-        "url": url,
     }
 
 
-def _extract_title(soup: BeautifulSoup) -> str:
-    """Try multiple selectors to find the novel title."""
-    # 1. <h1> inside common novel-info wrappers
-    for selector in (
-        "div.book-info h1",
-        "div.novel-info h1",
-        "div.info h1",
-        "div.header-wrapper h1",
-        "div.title h1",
-        "article h1",
-        "h1.novel-title",
-        "h1",
-    ):
-        tag = soup.select_one(selector)
-        if tag:
-            text = tag.get_text(strip=True)
-            if text:
-                return text
-
-    # 2. <title> tag, strip " - Novelhall" suffix
-    title_tag = soup.find("title")
-    if title_tag:
-        text = title_tag.get_text(strip=True)
-        for suffix in (" - Novelhall", " - Novel Hall", " | Novelhall", " | Novel Hall"):
-            if text.lower().endswith(suffix.lower()):
-                text = text[: -len(suffix)].strip()
-        if text:
-            return text
-
-    return ""
-
-
-def _extract_description(soup: BeautifulSoup) -> str:
-    """Try multiple selectors to find the synopsis / description."""
-    # 1. Common class names used by novel sites
-    for selector in (
-        "div.book-intro",
-        "div.novel-intro",
-        "div.synopsis",
-        "div.description",
-        "div.intro",
-        "div.summary",
-        "div.novel-desc",
-        "div.book-desc",
-        "section.synopsis",
-        "div#bookIntro",
-        "div#intro",
-        "div#synopsis",
-        "div#description",
-        "div.content > p",          # generic fallback
+def build_list_url(base_pattern: str, page_num: int) -> str:
+    """
+    Construct a page URL given a pattern string.
+    Detects 'all2022-N' style URLs and replaces the number.
+    Falls back to appending ?page=N.
+    """
+    # Pattern: ends with -N.html or -N/ or -N
+    m = re.search(r"(-\d+)(\.html)?$", base_pattern)
+    if m:
+        prefix = base_pattern[: m.start()]
+        suffix = m.group(2) or ""
+        return f"{prefix}-{page_num}{suffix}"
+    # Fallback: append ?page=N
+    sep = "&" if "?" in base_pattern else "?"
+    return f"{base_pattern}{sep}page={page_num}"
+
+
+# ─────────────────────────────────────────────
+#  MAIN SCRAPER
+# ─────────────────────────────────────────────
+def scrape(
+    start_url: str,
+    end_page: int,
+    delay_sec: float,
+    save_html: bool,
+    progress=gr.Progress(track_tqdm=True),
+):
+    """
+    Core scraping function called by Gradio.
+    Returns (log_text, output_file_path).
+    """
+    start_url  = start_url.strip().rstrip("/")
+    logs       = []
+    all_novels = []
+    html_dir   = OUTPUT_DIR / "html_pages"
+    if save_html:
+        html_dir.mkdir(exist_ok=True)
+
+    def log(msg: str):
+        ts = datetime.now().strftime("%H:%M:%S")
+        line = f"[{ts}] {msg}"
+        logs.append(line)
+        return line
+
+    # Detect starting page number
+    m = re.search(r"-(\d+)(\.html)?$", start_url)
+    start_page = int(m.group(1)) if m else 1
+
+    log(f"Starting from: {start_url}")
+    log(f"Page range   : {start_page} → {end_page}")
+    log("─" * 60)
+
+    total_pages  = end_page - start_page + 1
+    novels_done  = 0
+
+    for page_idx, page_num in enumerate(
+        range(start_page, end_page + 1), start=1
     ):
-        tag = soup.select_one(selector)
-        if tag:
-            text = tag.get_text(separator=" ", strip=True)
-            if len(text) > 20:        # ignore tiny/empty hits
-                return text
-
-    # 2. meta description
-    meta = soup.find("meta", attrs={"name": "description"})
-    if meta and meta.get("content"):
-        return meta["content"].strip()
-
-    return ""
-
-
-# ───────────────────────────── PHASE 3: save ─────────────────────────────────
-
-def save_results(novels: list[dict]) -> None:
-    """Write results to JSON and CSV."""
-    # JSON
-    with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
-        json.dump(novels, f, ensure_ascii=False, indent=2)
-    log.info("Saved %d novels to %s", len(novels), OUTPUT_JSON)
-
-    # CSV
-    fieldnames = ["title", "description", "url"]
-    with open(OUTPUT_CSV, "w", encoding="utf-8", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=fieldnames)
-        writer.writeheader()
-        writer.writerows(novels)
-    log.info("Saved %d novels to %s", len(novels), OUTPUT_CSV)
+        list_url = build_list_url(start_url, page_num)
+        yield "\n".join(logs + [log(f"📄 LIST PAGE {page_num}/{end_page} → {list_url}")]), None
 
+        html = fetch_page(list_url)
+        if html is None:
+            log(f"  ⚠️  Could not fetch listing page {page_num} – skipping")
+            yield "\n".join(logs), None
+            continue
 
-# ─────────────────────────────── MAIN ────────────────────────────────────────
-
-def main() -> None:
-    log.info("╔══════════════════════════════════════════╗")
-    log.info("║  Novelhall.com Full Catalogue Scraper    ║")
-    log.info("╚══════════════════════════════════════════╝")
-
-    session = make_session()
-
-    # ── Phase 1: discover all novel URLs ──────────────────────────────────
-    novel_urls = discover_novel_links(session)
-
-    if not novel_urls:
-        log.error("No novel links found at all – aborting.")
-        sys.exit(1)
-
-    # ── Phase 2: scrape each novel's detail page ───────────────────────────
-    log.info("=== PHASE 2: Scraping %d novel detail pages ===", len(novel_urls))
-    novels: list[dict] = []
-    failed: list[str] = []
-
-    for idx, url in enumerate(tqdm(novel_urls, desc="Scraping novels", unit="novel"), start=1):
-        result = scrape_novel_detail(session, url)
-
-        if result:
-            novels.append(result)
-        else:
-            failed.append(url)
-            log.warning("  Skipped (no data): %s", url)
+        novel_urls = extract_novel_links(html)
+        log(f"  Found {len(novel_urls)} novels on this page")
+
+        for idx, novel_url in enumerate(novel_urls, start=1):
+            yield "\n".join(
+                logs + [f"    [{idx}/{len(novel_urls)}] Fetching: {novel_url}"]
+            ), None
+
+            novel_html = fetch_page(novel_url)
+            if novel_html is None:
+                log(f"    ⚠️  Failed to fetch {novel_url}")
+                all_novels.append({
+                    "url": novel_url,
+                    "title": "FETCH_ERROR",
+                    "description": "",
+                })
+                yield "\n".join(logs), None
+                continue
+
+            info = extract_novel_info(novel_html, novel_url)
+            all_novels.append(info)
+            log(
+                f"    ✅ [{novels_done + 1}] {info['title'][:60]}"
+            )
+            novels_done += 1
+
+            # Optionally save the raw HTML
+            if save_html:
+                slug = re.sub(r"[^a-z0-9]+", "_", novel_url.split("/")[-2].lower())
+                html_path = html_dir / f"{slug}.html"
+                html_path.write_text(novel_html, encoding="utf-8")
+
+            time.sleep(delay_sec)
+            yield "\n".join(logs), None
+
+        time.sleep(delay_sec)  # polite pause between listing pages
+
+    # ── Write combined TXT output ──────────────────────────────────────
+    out_path = OUTPUT_DIR / f"novels_{start_page}_to_{end_page}.txt"
+    with out_path.open("w", encoding="utf-8") as fout:
+        fout.write(
+            f"Novelhall Novel Scrape — Pages {start_page}–{end_page}\n"
+            f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
+            f"Total novels: {len(all_novels)}\n"
+            f"{'=' * 70}\n\n"
+        )
+        for i, novel in enumerate(all_novels, start=1):
+            fout.write(f"[{i}]\n")
+            fout.write(f"TITLE      : {novel['title']}\n")
+            fout.write(f"URL        : {novel['url']}\n")
+            fout.write(f"DESCRIPTION:\n{novel['description']}\n")
+            fout.write(f"{'─' * 70}\n\n")
+
+    log("─" * 60)
+    log(f"✅ Done! {len(all_novels)} novels scraped.")
+    log(f"📁 Output saved → {out_path}")
+
+    yield "\n".join(logs), str(out_path)
+
+
+# ─────────────────────────────────────────────
+#  GRADIO UI
+# ─────────────────────────────────────────────
+CSS = """
+#log_box textarea { font-family: monospace; font-size: 12px; }
+.status-bar { font-size: 13px; color: #555; }
+"""
 
-        if idx % LOG_EVERY == 0:
-            log.info("  Progress: %d / %d scraped, %d ok, %d failed",
-                     idx, len(novel_urls), len(novels), len(failed))
+with gr.Blocks(title="Novelhall Scraper", css=CSS) as demo:
+    gr.Markdown(
+        """
+# 📚 Novelhall Novel Scraper
+Scrapes every novel on Novelhall listing pages, extracts **title** and **description**,
+and saves a combined `.txt` file.
+
+**How it works:**
+1. Paste a listing-page URL like `https://www.novelhall.com/all2022-1.html`
+2. Set the **End Page** number (e.g. `337` for pages 1–337)
+3. Click **Start Scraping**
+        """
+    )
 
-        # Polite crawl delay
-        time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
+    with gr.Row():
+        with gr.Column(scale=3):
+            start_url_box = gr.Textbox(
+                label="Start URL (first listing page)",
+                value="https://www.novelhall.com/all2022-1.html",
+                placeholder="https://www.novelhall.com/all2022-1.html",
+            )
+        with gr.Column(scale=1):
+            end_page_box = gr.Number(
+                label="End Page Number",
+                value=5,
+                minimum=1,
+                maximum=500,
+                step=1,
+            )
+
+    with gr.Row():
+        delay_slider = gr.Slider(
+            label="Delay between requests (seconds)",
+            minimum=0.5,
+            maximum=5.0,
+            value=1.5,
+            step=0.5,
+        )
+        save_html_cb = gr.Checkbox(
+            label="Also save raw HTML of each novel page",
+            value=False,
+        )
 
-    # ── Phase 3: save ─────────────────────────────────────────────────────
-    log.info("=== PHASE 3: Saving results ===")
-    save_results(novels)
+    with gr.Row():
+        run_btn  = gr.Button("🚀 Start Scraping", variant="primary", scale=3)
+        stop_btn = gr.Button("⛔ Stop", variant="stop", scale=1)
 
-    # ── Summary ────────────────────────────────────────────────────────────
-    log.info("╔══════════════════════════════════════════╗")
-    log.info("║  DONE                                    ║")
-    log.info("║  Total discovered : %-5d                ║", len(novel_urls))
-    log.info("║  Successfully saved: %-5d               ║", len(novels))
-    log.info("║  Failed / skipped  : %-5d               ║", len(failed))
-    log.info("║  Output files:                           ║")
-    log.info("║    • %-36s ║", OUTPUT_JSON)
-    log.info("║    • %-36s ║", OUTPUT_CSV)
-    log.info("╚══════════════════════════════════════════╝")
+    log_box = gr.Textbox(
+        label="Live Log",
+        lines=20,
+        max_lines=40,
+        interactive=False,
+        elem_id="log_box",
+    )
 
-    if failed:
-        failed_path = "novelhall_failed.txt"
-        with open(failed_path, "w", encoding="utf-8") as f:
-            f.write("\n".join(failed))
-        log.info("URLs that could not be scraped written to: %s", failed_path)
+    out_file = gr.File(label="📥 Download Combined TXT", interactive=False)
 
+    # Wire up
+    scrape_event = run_btn.click(
+        fn=scrape,
+        inputs=[start_url_box, end_page_box, delay_slider, save_html_cb],
+        outputs=[log_box, out_file],
+    )
+    stop_btn.click(fn=None, cancels=[scrape_event])
+
+    gr.Markdown(
+        """
+---
+### 📌 Notes
+- **URL pattern**: The scraper auto-detects `all2022-N` style pagination.
+  Pages `all2022-1.html` through `all2022-337.html` are all supported.
+- **Rate limiting**: Default 1.5 s delay is polite and avoids blocks.
+  Increase it if you get 429 errors.
+- **Output**: One `.txt` file with title, URL, and full description per novel.
+- **HTML saving**: Optional — saves each novel's main page as `<slug>.html`
+  in `outputs/html_pages/`.
+        """
+    )
 
 if __name__ == "__main__":
-    main()
+    demo.launch()