Update app.py

app.py

@@ -1,92 +1,530 @@
 """
-Webnovel.com NTR keyword scraper
-Searches Webnovel's public search API for titles/descriptions
-containing "netorase", "netorare", or "ntr" and saves results to CSV.
-
-Run with: python webnovel_ntr_scraper.py
-Requires: pip install requests
+Novel Scraper — novelbin.com + webnovel.com
+Runs on HuggingFace Spaces (Gradio UI).
+Saves title + description of every novel to CSV and JSONL.
+Fully resumable — safe to restart if interrupted.
 """
 
-import requests
+import os
 import csv
-import time
 import json
+import time
+import random
+import logging
+import threading
+import traceback
+from datetime import datetime
+from pathlib import Path
+
+import requests
+from bs4 import BeautifulSoup
+import gradio as gr
+
+# ─────────────────────────────────────────────
+# CONFIG
+# ─────────────────────────────────────────────
+OUTPUT_DIR = Path("output")
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+LOG_FILE         = OUTPUT_DIR / "scraper.log"
+NOVELBIN_CSV     = OUTPUT_DIR / "novelbin_novels.csv"
+NOVELBIN_JSONL   = OUTPUT_DIR / "novelbin_novels.jsonl"
+WEBNOVEL_CSV     = OUTPUT_DIR / "webnovel_novels.csv"
+WEBNOVEL_JSONL   = OUTPUT_DIR / "webnovel_novels.jsonl"
+PROGRESS_FILE    = OUTPUT_DIR / "progress.json"
 
-KEYWORDS = ["netorase", "netorare", "ntr"]
-OUTPUT_FILE = "webnovel_ntr_results.csv"
+NOVELBIN_BASE    = "https://novelbin.com"
+NOVELBIN_LIST    = "https://novelbin.com/sort/updated?page={page}"
 
-# Webnovel's internal search API (subject to change)
-SEARCH_URL = "https://www.webnovel.com/go/pcm/search/result"
+# webnovel public category API — returns JSON, no JS needed
+WEBNOVEL_API     = (
+    "https://www.webnovel.com/go/pcm/category/categoryAjax"
+    "?_csrfToken=&categoryId=0&pageIndex={page}&orderBy=1&bookType=1"
+)
+
+REQUEST_TIMEOUT  = 30       # seconds per request
+RETRY_MAX        = 5        # retries per failed request
+RETRY_BACKOFF    = [2, 5, 10, 20, 40]  # seconds between retries
+PAGE_DELAY       = (1.0, 2.5)  # random sleep between pages (min, max)
 
 HEADERS = {
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-                  "(KHTML, like Gecko) Chrome/124.0 Safari/537.36",
-    "Accept": "application/json, text/plain, */*",
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 }
 
-def search_keyword(keyword, page_size=20, max_pages=20, delay=1.5):
-    """Query Webnovel's search endpoint for a keyword, paginating through results."""
-    results = []
-    for page in range(1, max_pages + 1):
-        params = {
-            "keywords": keyword,
-            "pageIndex": page,
-            "pageSize": page_size,
-        }
+# ─────────────────────────────────────────────
+# LOGGING
+# ─────────────────────────────────────────────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.FileHandler(LOG_FILE, encoding="utf-8"),
+        logging.StreamHandler(),
+    ],
+)
+log = logging.getLogger(__name__)
+
+# ─────────────────────────────────────────────
+# SHARED STATE  (thread-safe via GIL for simple types)
+# ─────────────────────────────────────────────
+state = {
+    "running": False,
+    "status": "Idle",
+    "novelbin_done": 0,
+    "webnovel_done": 0,
+    "errors": 0,
+    "log_lines": [],          # last N lines shown in UI
+    "stop_flag": False,
+}
+
+def _log(msg: str, level: str = "info"):
+    getattr(log, level)(msg)
+    line = f"[{datetime.now().strftime('%H:%M:%S')}] {msg}"
+    state["log_lines"].append(line)
+    if len(state["log_lines"]) > 300:
+        state["log_lines"] = state["log_lines"][-300:]
+
+# ─────────────────────────────────────────────
+# PROGRESS PERSISTENCE
+# ─────────────────────────────────────────────
+def load_progress() -> dict:
+    if PROGRESS_FILE.exists():
+        try:
+            return json.loads(PROGRESS_FILE.read_text())
+        except Exception:
+            pass
+    return {"novelbin_page": 1, "webnovel_page": 1,
+            "novelbin_seen": [], "webnovel_seen": []}
+
+def save_progress(prog: dict):
+    PROGRESS_FILE.write_text(json.dumps(prog, indent=2))
+
+# ─────────────────────────────────────────────
+# CSV / JSONL HELPERS
+# ─────────────────────────────────────────────
+def ensure_csv(path: Path, fieldnames: list):
+    if not path.exists():
+        with open(path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+
+def append_record(csv_path: Path, jsonl_path: Path, record: dict, fieldnames: list):
+    with open(csv_path, "a", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
+        writer.writerow(record)
+    with open(jsonl_path, "a", encoding="utf-8") as f:
+        f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+def load_seen_ids(jsonl_path: Path, id_key: str) -> set:
+    """Load already-scraped IDs from JSONL so we never duplicate."""
+    seen = set()
+    if jsonl_path.exists():
+        for line in jsonl_path.read_text(encoding="utf-8").splitlines():
+            try:
+                obj = json.loads(line)
+                seen.add(obj[id_key])
+            except Exception:
+                pass
+    return seen
+
+# ─────────────────────────────────────────────
+# HTTP WITH RETRY
+# ─────────────────────────────────────────────
+def get(url: str, session: requests.Session, *, is_json=False, extra_headers=None):
+    """GET with exponential-backoff retry. Returns (content, ok)."""
+    hdrs = dict(HEADERS)
+    if extra_headers:
+        hdrs.update(extra_headers)
+    for attempt in range(RETRY_MAX):
+        if state["stop_flag"]:
+            return None, False
         try:
-            resp = requests.get(SEARCH_URL, params=params, headers=HEADERS, timeout=10)
-            resp.raise_for_status()
-            data = resp.json()
-        except (requests.RequestException, json.JSONDecodeError) as e:
-            print(f"  [!] Error on page {page} for '{keyword}': {e}")
+            r = session.get(url, headers=hdrs, timeout=REQUEST_TIMEOUT)
+            if r.status_code == 429:
+                wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)] * 2
+                _log(f"Rate-limited on {url} — waiting {wait}s", "warning")
+                time.sleep(wait)
+                continue
+            if r.status_code != 200:
+                _log(f"HTTP {r.status_code} on {url} (attempt {attempt+1})", "warning")
+                time.sleep(RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)])
+                continue
+            return (r.json() if is_json else r.text), True
+        except requests.exceptions.RequestException as e:
+            wait = RETRY_BACKOFF[min(attempt, len(RETRY_BACKOFF)-1)]
+            _log(f"Request error on {url}: {e} — retry in {wait}s", "warning")
+            time.sleep(wait)
+    _log(f"FAILED after {RETRY_MAX} attempts: {url}", "error")
+    state["errors"] += 1
+    return None, False
+
+# ─────────────────────────────────────────────
+# NOVELBIN SCRAPER
+# ─────────────────────────────────────────────
+NB_FIELDS = ["id", "title", "url", "description", "author", "genres", "status", "scraped_at"]
+
+def scrape_novelbin_page(page: int, session: requests.Session, seen: set) -> tuple[list, bool]:
+    """Scrape one listing page. Returns (records, has_more)."""
+    url = NOVELBIN_LIST.format(page=page)
+    html, ok = get(url, session)
+    if not ok or not html:
+        return [], True  # treat as transient error, not end-of-pages
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Check if page has any novels at all
+    novel_items = soup.select("div.col-novel-main .row.novel-item, li.novel-item, .list-novel .row")
+    if not novel_items:
+        # Try alternate selectors
+        novel_items = soup.select(".col-novel-main .col-xs-12")
+
+    if not novel_items:
+        _log(f"novelbin page {page}: no items found — assuming end of listing", "warning")
+        return [], False  # signal end
+
+    records = []
+    for item in novel_items:
+        try:
+            # Title + link
+            a_tag = item.select_one("h3.novel-title a, .novel-title a, a.novel-title")
+            if not a_tag:
+                continue
+            title = a_tag.get_text(strip=True)
+            href  = a_tag.get("href", "")
+            if not href:
+                continue
+            novel_url = href if href.startswith("http") else NOVELBIN_BASE + href
+
+            novel_id = href.rstrip("/").split("/")[-1]
+            if novel_id in seen:
+                continue
+
+            # Fetch detail page for description
+            detail_html, dok = get(novel_url, session)
+            desc = author = genres = status = ""
+            if dok and detail_html:
+                dsoup = BeautifulSoup(detail_html, "html.parser")
+
+                # Description — multiple possible selectors
+                for sel in ["div.desc-text", "div#tab-description", ".summary__content", ".description"]:
+                    el = dsoup.select_one(sel)
+                    if el:
+                        desc = el.get_text(" ", strip=True)[:4000]
+                        break
+
+                # Author
+                for sel in ["a[href*='author']", "span.author", ".author a"]:
+                    el = dsoup.select_one(sel)
+                    if el:
+                        author = el.get_text(strip=True)
+                        break
+
+                # Genres
+                genre_els = dsoup.select("div.categories a, .genres a, a.genre")
+                genres = ", ".join(g.get_text(strip=True) for g in genre_els)
+
+                # Status
+                for sel in [".header-stats span.ongoing, .header-stats span.completed", ".info-item .completed, .info-item .ongoing"]:
+                    el = dsoup.select_one(sel)
+                    if el:
+                        status = el.get_text(strip=True)
+                        break
+
+            record = {
+                "id": novel_id,
+                "title": title,
+                "url": novel_url,
+                "description": desc,
+                "author": author,
+                "genres": genres,
+                "status": status,
+                "scraped_at": datetime.utcnow().isoformat(),
+            }
+            records.append(record)
+            seen.add(novel_id)
+
+            time.sleep(random.uniform(0.3, 0.8))  # polite delay per detail page
+
+        except Exception as e:
+            _log(f"novelbin item parse error: {e}", "error")
+            state["errors"] += 1
+
+    return records, True
+
+def run_novelbin(prog: dict, session: requests.Session):
+    _log("═══ Starting novelbin.com scrape ═══")
+    state["status"] = "Scraping novelbin.com..."
+    seen = load_seen_ids(NOVELBIN_JSONL, "id")
+    _log(f"novelbin: {len(seen)} novels already saved, resuming from page {prog['novelbin_page']}")
+    ensure_csv(NOVELBIN_CSV, NB_FIELDS)
+
+    page = prog["novelbin_page"]
+    consecutive_empty = 0
+
+    while not state["stop_flag"]:
+        _log(f"novelbin page {page}...")
+        records, has_more = scrape_novelbin_page(page, session, seen)
+
+        for r in records:
+            append_record(NOVELBIN_CSV, NOVELBIN_JSONL, r, NB_FIELDS)
+            state["novelbin_done"] += 1
+
+        _log(f"novelbin page {page}: +{len(records)} novels (total {state['novelbin_done']})")
+
+        if not records:
+            consecutive_empty += 1
+            if consecutive_empty >= 3:
+                _log("novelbin: 3 consecutive empty pages — scrape complete.")
+                break
+        else:
+            consecutive_empty = 0
+
+        if not has_more:
+            _log("novelbin: reached end of listing.")
             break
 
-        # Adjust this path based on actual API response structure
-        items = (
-            data.get("data", {}).get("items")
-            or data.get("data", {}).get("bookItems")
-            or []
-        )
+        page += 1
+        prog["novelbin_page"] = page
+        save_progress(prog)
+
+        time.sleep(random.uniform(*PAGE_DELAY))
+
+    _log(f"═══ novelbin done — {state['novelbin_done']} novels saved ═══")
+
+# ─────────────────────────────────────────────
+# WEBNOVEL SCRAPER  (uses public JSON API)
+# ─────────────────────────────────────────────
+WN_FIELDS = ["id", "title", "url", "description", "author", "genres", "status",
+             "total_chapters", "rating", "scraped_at"]
 
+def scrape_webnovel_page(page: int, session: requests.Session, seen: set) -> tuple[list, bool]:
+    """One page of webnovel category API. Returns (records, has_more)."""
+    url = WEBNOVEL_API.format(page=page)
+    extra = {
+        "Referer": "https://www.webnovel.com/",
+        "Accept": "application/json, text/javascript, */*; q=0.01",
+        "X-Requested-With": "XMLHttpRequest",
+    }
+    data, ok = get(url, session, is_json=True, extra_headers=extra)
+    if not ok or not data:
+        return [], True
+
+    try:
+        code = data.get("code", -1)
+        if code != 0:
+            _log(f"webnovel API non-zero code {code} on page {page}", "warning")
+            return [], True
+
+        items = (data.get("data") or {}).get("items") or []
         if not items:
+            return [], False  # end of pages
+
+    except Exception as e:
+        _log(f"webnovel page {page} parse error: {e}", "error")
+        return [], True
+
+    records = []
+    for item in items:
+        try:
+            novel_id = str(item.get("bookId") or item.get("id") or "")
+            if not novel_id or novel_id in seen:
+                continue
+
+            title    = item.get("bookName") or item.get("name") or ""
+            author   = item.get("authorName") or ""
+            status   = "Completed" if item.get("isComplete") else "Ongoing"
+            chapters = str(item.get("totalChapterNum") or item.get("chapterNum") or "")
+            rating   = str(item.get("totalScore") or item.get("reviewTotal") or "")
+            cat_list = item.get("categoryItems") or item.get("categoryList") or []
+            genres   = ", ".join(c.get("categoryName", "") for c in cat_list if isinstance(c, dict))
+            novel_url = f"https://www.webnovel.com/book/{novel_id}"
+
+            # Description is sometimes in the listing, sometimes needs a detail call
+            desc = item.get("description") or item.get("intro") or ""
+            if not desc:
+                desc = fetch_webnovel_description(novel_id, session)
+
+            record = {
+                "id": novel_id,
+                "title": title,
+                "url": novel_url,
+                "description": desc[:4000],
+                "author": author,
+                "genres": genres,
+                "status": status,
+                "total_chapters": chapters,
+                "rating": rating,
+                "scraped_at": datetime.utcnow().isoformat(),
+            }
+            records.append(record)
+            seen.add(novel_id)
+
+        except Exception as e:
+            _log(f"webnovel item parse error: {e}", "error")
+            state["errors"] += 1
+
+    return records, True
+
+def fetch_webnovel_description(novel_id: str, session: requests.Session) -> str:
+    """Fallback: fetch book detail API for description."""
+    url = (
+        f"https://www.webnovel.com/go/pcm/book/getBookDetail"
+        f"?_csrfToken=&bookId={novel_id}"
+    )
+    extra = {
+        "Referer": f"https://www.webnovel.com/book/{novel_id}",
+        "Accept": "application/json, */*; q=0.01",
+        "X-Requested-With": "XMLHttpRequest",
+    }
+    data, ok = get(url, session, is_json=True, extra_headers=extra)
+    if not ok or not data:
+        return ""
+    try:
+        return (data.get("data") or {}).get("bookInfo", {}).get("description", "")
+    except Exception:
+        return ""
+
+def run_webnovel(prog: dict, session: requests.Session):
+    _log("═══ Starting webnovel.com scrape ═══")
+    state["status"] = "Scraping webnovel.com..."
+    seen = load_seen_ids(WEBNOVEL_JSONL, "id")
+    _log(f"webnovel: {len(seen)} novels already saved, resuming from page {prog['webnovel_page']}")
+    ensure_csv(WEBNOVEL_CSV, WN_FIELDS)
+
+    page = prog["webnovel_page"]
+    consecutive_empty = 0
+
+    while not state["stop_flag"]:
+        _log(f"webnovel page {page}...")
+        records, has_more = scrape_webnovel_page(page, session, seen)
+
+        for r in records:
+            append_record(WEBNOVEL_CSV, WEBNOVEL_JSONL, r, WN_FIELDS)
+            state["webnovel_done"] += 1
+
+        _log(f"webnovel page {page}: +{len(records)} novels (total {state['webnovel_done']})")
+
+        if not records:
+            consecutive_empty += 1
+            if consecutive_empty >= 3:
+                _log("webnovel: 3 consecutive empty pages — scrape complete.")
+                break
+        else:
+            consecutive_empty = 0
+
+        if not has_more:
+            _log("webnovel: reached end of listing.")
             break
 
-        for item in items:
-            title = item.get("bookName") or item.get("title", "")
-            desc = item.get("description") or item.get("bookDesc", "")
-            book_id = item.get("bookId", "")
-            url = f"https://www.webnovel.com/book/{book_id}" if book_id else ""
-            results.append({"title": title, "description": desc, "url": url})
+        page += 1
+        prog["webnovel_page"] = page
+        save_progress(prog)
+
+        time.sleep(random.uniform(*PAGE_DELAY))
+
+    _log(f"═══ webnovel done — {state['webnovel_done']} novels saved ═══")
+
+# ─────────────────────────────────────────────
+# MASTER RUNNER
+# ─────────────────────────────────────────────
+def master_run():
+    state["running"]    = True
+    state["stop_flag"]  = False
+    state["errors"]     = 0
+
+    # Restore saved counts from existing JSONL files
+    state["novelbin_done"] = sum(1 for _ in NOVELBIN_JSONL.open(encoding="utf-8")) if NOVELBIN_JSONL.exists() else 0
+    state["webnovel_done"] = sum(1 for _ in WEBNOVEL_JSONL.open(encoding="utf-8")) if WEBNOVEL_JSONL.exists() else 0
+
+    prog = load_progress()
+
+    session = requests.Session()
+    session.headers.update(HEADERS)
+
+    try:
+        run_novelbin(prog, session)
+        if not state["stop_flag"]:
+            run_webnovel(prog, session)
+    except Exception as e:
+        _log(f"FATAL: {e}\n{traceback.format_exc()}", "error")
+    finally:
+        state["running"] = False
+        state["status"]  = "Finished ✅" if not state["stop_flag"] else "Stopped ⛔"
+        _log(f"Run complete. novelbin={state['novelbin_done']}  webnovel={state['webnovel_done']}  errors={state['errors']}")
+        session.close()
+
+# ─────────────────────────────────────────────
+# GRADIO UI
+# ─────────────────────────────────────────────
+def ui_start():
+    if state["running"]:
+        return "⚠️ Already running!"
+    state["log_lines"] = []
+    thread = threading.Thread(target=master_run, daemon=True)
+    thread.start()
+    return "🚀 Scraper started!"
+
+def ui_stop():
+    state["stop_flag"] = True
+    return "⛔ Stop signal sent — finishing current page..."
 
-        print(f"  Keyword '{keyword}' page {page}: {len(items)} items")
-        time.sleep(delay)
+def ui_status():
+    lines = "\n".join(state["log_lines"][-80:])
+    nb_size = f"{NOVELBIN_JSONL.stat().st_size/1024:.1f} KB" if NOVELBIN_JSONL.exists() else "—"
+    wn_size = f"{WEBNOVEL_JSONL.stat().st_size/1024:.1f} KB" if WEBNOVEL_JSONL.exists() else "—"
+    info = (
+        f"Status : {state['status']}\n"
+        f"novelbin: {state['novelbin_done']} novels  ({nb_size})\n"
+        f"webnovel: {state['webnovel_done']} novels  ({wn_size})\n"
+        f"Errors  : {state['errors']}\n"
+        f"{'─'*40}\n"
+        f"{lines}"
+    )
+    return info
 
-    return results
+def ui_files():
+    files = []
+    for p in [NOVELBIN_CSV, NOVELBIN_JSONL, WEBNOVEL_CSV, WEBNOVEL_JSONL, LOG_FILE, PROGRESS_FILE]:
+        if p.exists():
+            files.append(str(p))
+    return files if files else None
 
-def matches_ntr(title, description, keywords):
-    text = f"{title} {description}".lower()
-    return any(k.lower() in text for k in keywords)
+with gr.Blocks(title="Novel Scraper") as demo:
+    gr.Markdown(
+        "# 📚 Novel Scraper\n"
+        "Scrapes **title + description** of every novel on **novelbin.com** and **webnovel.com**.\n"
+        "Output is saved as **CSV + JSONL** in the `output/` folder. Fully resumable."
+    )
 
-def main():
-    all_matches = {}
+    with gr.Row():
+        btn_start = gr.Button("▶ Start Scraper", variant="primary")
+        btn_stop  = gr.Button("⏹ Stop", variant="stop")
+        btn_refresh = gr.Button("🔄 Refresh Status")
 
-    for kw in KEYWORDS:
-        print(f"Searching for keyword: {kw}")
-        items = search_keyword(kw)
-        for item in items:
-            if matches_ntr(item["title"], item["description"], KEYWORDS):
-                key = item["url"] or item["title"]
-                all_matches[key] = item
+    status_box = gr.Textbox(
+        label="Live Log",
+        lines=25,
+        max_lines=25,
+        interactive=False,
+        value="Press ▶ Start Scraper to begin.",
+    )
 
-    print(f"\nFound {len(all_matches)} unique matching novels.")
+    file_out = gr.File(label="Download Output Files", file_count="multiple")
 
-    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f:
-        writer = csv.DictWriter(f, fieldnames=["title", "description", "url"])
-        writer.writeheader()
-        for item in all_matches.values():
-            writer.writerow(item)
+    btn_start.click(ui_start, outputs=status_box)
+    btn_stop.click(ui_stop,  outputs=status_box)
+    btn_refresh.click(ui_status, outputs=status_box)
+    btn_refresh.click(ui_files,  outputs=file_out)
 
-    print(f"Saved results to {OUTPUT_FILE}")
+    # Auto-poll every 5 seconds while visible
+    demo.load(ui_status, outputs=status_box, every=5)
+    demo.load(ui_files,  outputs=file_out,   every=10)
 
 if __name__ == "__main__":
-    main()
+    demo.launch()