Spaces:
Sleeping
Sleeping
| """ | |
| Novel Scraper — Gradio UI | |
| Universal adaptive web novel downloader. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import io | |
| import json | |
| import os | |
| import shutil | |
| import threading | |
| import time | |
| import zipfile | |
| from pathlib import Path | |
| from typing import Optional | |
| import gradio as gr | |
| from scraper.engine import ChapterStatus, FetchMode, NovelJob | |
| from scraper.manager import DownloadConfig, DownloadManager, load_state | |
| from utils.logging_setup import AppLogger | |
| from utils.range_parser import parse_chapter_range, format_range_hint | |
| # --------------------------------------------------------------------------- | |
| # Global state | |
| # --------------------------------------------------------------------------- | |
| OUTPUT_DIR = Path("output") | |
| OUTPUT_DIR.mkdir(parents=True, exist_ok=True) | |
| app_logger = AppLogger.get(OUTPUT_DIR) | |
| # Active download managers keyed by job URL | |
| _managers: dict[str, DownloadManager] = {} | |
| _jobs: dict[str, NovelJob] = {} | |
| _threads: dict[str, threading.Thread] = {} | |
| _results: dict[str, Optional[Path]] = {} | |
| _loop: Optional[asyncio.AbstractEventLoop] = None | |
| def _get_loop() -> asyncio.AbstractEventLoop: | |
| global _loop | |
| if _loop is None or _loop.is_closed(): | |
| _loop = asyncio.new_event_loop() | |
| threading.Thread(target=_loop.run_forever, daemon=True).start() | |
| return _loop | |
| def _run_async(coro): | |
| loop = _get_loop() | |
| future = asyncio.run_coroutine_threadsafe(coro, loop) | |
| return future | |
| # --------------------------------------------------------------------------- | |
| # Helpers | |
| # --------------------------------------------------------------------------- | |
| def _parse_cookies(cookie_str: str) -> dict: | |
| """Parse 'key=value; key2=value2' or JSON cookie string.""" | |
| cookie_str = cookie_str.strip() | |
| if not cookie_str: | |
| return {} | |
| if cookie_str.startswith("{") or cookie_str.startswith("["): | |
| try: | |
| data = json.loads(cookie_str) | |
| if isinstance(data, list): | |
| return {item["name"]: item["value"] for item in data if "name" in item} | |
| return data | |
| except Exception: | |
| return {} | |
| cookies = {} | |
| for part in cookie_str.split(";"): | |
| kv = part.strip().split("=", 1) | |
| if len(kv) == 2: | |
| cookies[kv[0].strip()] = kv[1].strip() | |
| return cookies | |
| def _build_progress_table(job: NovelJob) -> str: | |
| if not job.chapters: | |
| return "No chapters discovered yet." | |
| done = job.done_count | |
| failed = job.failed_count | |
| locked = sum(1 for c in job.chapters if c.status == ChapterStatus.LOCKED) | |
| skipped = sum(1 for c in job.chapters if c.status == ChapterStatus.SKIPPED) | |
| pct = f"{job.progress*100:.1f}" | |
| elapsed = "" | |
| if job.started_at: | |
| e = time.time() - job.started_at | |
| elapsed = f"{int(e//60)}m {int(e%60)}s" | |
| eta = "" | |
| if job.started_at and done > 0 and job.progress < 1: | |
| rate = done / max(time.time() - job.started_at, 1) | |
| remaining = job.total - done | |
| eta_secs = remaining / rate | |
| eta = f"{int(eta_secs//60)}m {int(eta_secs%60)}s" | |
| return ( | |
| f"**{job.title or 'Novel'}**\n\n" | |
| f"| Stat | Value |\n|------|-------|\n" | |
| f"| Status | {job.status.upper()} |\n" | |
| f"| Progress | {done}/{job.total} ({pct}%) |\n" | |
| f"| Done | ✅ {done} |\n" | |
| f"| Failed | ❌ {failed} |\n" | |
| f"| Locked | 💰 {locked} |\n" | |
| f"| Skipped | ⏭️ {skipped} |\n" | |
| f"| Elapsed | {elapsed or '—'} |\n" | |
| f"| ETA | {eta or '—'} |\n" | |
| ) | |
| def _build_failed_list(job: NovelJob) -> str: | |
| failed = [c for c in job.chapters if c.status == ChapterStatus.FAILED] | |
| if not failed: | |
| return "No failed chapters." | |
| lines = [f"**{len(failed)} failed chapters:**\n"] | |
| for c in failed[:50]: | |
| lines.append(f"- [{c.title}]({c.url}) — {c.error}") | |
| if len(failed) > 50: | |
| lines.append(f"... and {len(failed)-50} more (see failed_chapters.txt)") | |
| return "\n".join(lines) | |
| def _zip_outputs(job: NovelJob) -> Optional[str]: | |
| """Create a ZIP of all output files for a job.""" | |
| safe_title = "".join(c if c.isalnum() or c in " -_" else "_" for c in (job.title or "novel"))[:60] | |
| zip_path = OUTPUT_DIR / f"{safe_title}_package.zip" | |
| with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: | |
| for f in OUTPUT_DIR.iterdir(): | |
| if f.is_file() and not f.name.startswith("."): | |
| zf.write(f, f.name) | |
| return str(zip_path) | |
| # --------------------------------------------------------------------------- | |
| # Core download entrypoint (runs in background thread) | |
| # --------------------------------------------------------------------------- | |
| def _start_download_thread( | |
| url: str, | |
| config: DownloadConfig, | |
| resume: bool = False, | |
| ) -> None: | |
| """Launch download in background thread.""" | |
| # Try to restore state | |
| job = None | |
| if resume: | |
| job = load_state(url, OUTPUT_DIR) | |
| if job: | |
| app_logger.ui_log(f"📂 Resumed from saved state: {job.title}") | |
| if job is None: | |
| job = NovelJob(url=url, output_dir=OUTPUT_DIR) | |
| job.start_chapter = config.__dict__.get("start_chapter", 1) | |
| job.end_chapter = config.__dict__.get("end_chapter", 0) | |
| job.specific_chapters = config.__dict__.get("specific_chapters", []) | |
| _jobs[url] = job | |
| def log_cb(msg: str) -> None: | |
| app_logger.ui_log(msg) | |
| def progress_cb(j: NovelJob) -> None: | |
| pass # polling-based UI | |
| manager = DownloadManager(job, config, log_cb=log_cb, progress_cb=progress_cb) | |
| _managers[url] = manager | |
| async def _run(): | |
| try: | |
| result = await manager.run() | |
| _results[url] = result | |
| except Exception as exc: | |
| app_logger.ui_log(f"❌ Download error: {exc}") | |
| _results[url] = None | |
| job.status = "failed" | |
| future = _run_async(_run()) | |
| _results[url] = None | |
| # --------------------------------------------------------------------------- | |
| # Gradio event handlers | |
| # --------------------------------------------------------------------------- | |
| def add_novels( | |
| urls_text: str, | |
| mode_str: str, | |
| rate: float, | |
| concurrency: int, | |
| retries: int, | |
| timeout: int, | |
| cookie_str: str, | |
| chapter_range: str, | |
| resume_prev: bool, | |
| ) -> tuple[str, str, str]: | |
| """Parse inputs and launch downloads.""" | |
| urls = [u.strip() for u in urls_text.strip().splitlines() if u.strip()] | |
| if not urls: | |
| return "⚠️ Please enter at least one URL.", "", "" | |
| mode_map = {"Fast": FetchMode.FAST, "Balanced": FetchMode.BALANCED, "Compatibility": FetchMode.COMPAT} | |
| mode = mode_map.get(mode_str, FetchMode.BALANCED) | |
| cookies = _parse_cookies(cookie_str) | |
| # Parse range | |
| specific = parse_chapter_range(chapter_range) | |
| start_ch = 1 | |
| end_ch = 0 | |
| if specific: | |
| start_ch = min(specific) | |
| end_ch = max(specific) | |
| config = DownloadConfig( | |
| mode=mode, | |
| max_concurrent=max(1, min(concurrency, 10)), | |
| rate_limit=max(0.1, 1.0 / max(rate, 0.1)), | |
| max_retries=max(1, retries), | |
| timeout=max(5, timeout), | |
| cookies=cookies, | |
| output_dir=OUTPUT_DIR, | |
| ) | |
| config.__dict__["start_chapter"] = start_ch | |
| config.__dict__["end_chapter"] = end_ch | |
| config.__dict__["specific_chapters"] = specific | |
| messages = [] | |
| for url in urls: | |
| if url in _jobs and _jobs[url].status == "running": | |
| messages.append(f"⚠️ Already running: {url}") | |
| continue | |
| app_logger.ui_log(f"🚀 Queuing: {url}") | |
| t = threading.Thread( | |
| target=_start_download_thread, | |
| args=(url, config, resume_prev), | |
| daemon=True, | |
| ) | |
| t.start() | |
| _threads[url] = t | |
| messages.append(f"✅ Started: {url}") | |
| return "\n".join(messages), "", app_logger.get_ui_text() | |
| def pause_job(url: str) -> str: | |
| url = url.strip() | |
| if url in _managers: | |
| _managers[url].pause() | |
| return f"⏸️ Paused: {url}" | |
| return "⚠️ No active download for that URL." | |
| def resume_job(url: str) -> str: | |
| url = url.strip() | |
| if url in _managers: | |
| _managers[url].resume() | |
| return f"▶️ Resumed: {url}" | |
| return "⚠️ No active download for that URL." | |
| def cancel_job(url: str) -> str: | |
| url = url.strip() | |
| if url in _managers: | |
| _managers[url].cancel() | |
| return f"🚫 Cancelled: {url}" | |
| return "⚠️ No active download for that URL." | |
| def retry_failed(url: str) -> str: | |
| url = url.strip() | |
| if url not in _jobs: | |
| return "⚠️ No job found for that URL." | |
| job = _jobs[url] | |
| for c in job.chapters: | |
| if c.status == ChapterStatus.FAILED: | |
| c.status = ChapterStatus.PENDING | |
| c.retries = 0 | |
| c.error = "" | |
| app_logger.ui_log(f"🔄 Reset failed chapters for retry: {url}") | |
| return f"✅ Failed chapters reset for retry." | |
| def get_status() -> tuple[str, str, str, gr.File]: | |
| """Poll status — called by timer.""" | |
| if not _jobs: | |
| return "No downloads started.", "", app_logger.get_ui_text(), None | |
| progress_parts = [] | |
| failed_parts = [] | |
| for url, job in _jobs.items(): | |
| progress_parts.append(_build_progress_table(job)) | |
| failed_parts.append(_build_failed_list(job)) | |
| # Find completed downloads for file output | |
| output_file = None | |
| for url, result_path in _results.items(): | |
| if result_path and Path(result_path).exists(): | |
| output_file = str(result_path) | |
| break | |
| return ( | |
| "\n\n---\n\n".join(progress_parts), | |
| "\n\n---\n\n".join(failed_parts), | |
| app_logger.get_ui_text(200), | |
| output_file, | |
| ) | |
| def get_download_files() -> list[str]: | |
| """Return list of available output files.""" | |
| files = [] | |
| for f in OUTPUT_DIR.iterdir(): | |
| if f.is_file() and not f.name.startswith(".") and f.suffix in (".txt", ".zip"): | |
| files.append(str(f)) | |
| return files | |
| def create_zip() -> Optional[str]: | |
| if not _jobs: | |
| return None | |
| # Use the most recent job | |
| job = list(_jobs.values())[-1] | |
| return _zip_outputs(job) | |
| def clear_logs() -> str: | |
| app_logger.ring.clear() | |
| return "" | |
| def preview_extraction(url: str, mode_str: str, cookie_str: str) -> str: | |
| """Preview extracted content from a URL without saving.""" | |
| url = url.strip() | |
| if not url: | |
| return "⚠️ Enter a URL to preview." | |
| mode_map = {"Fast": FetchMode.FAST, "Balanced": FetchMode.BALANCED, "Compatibility": FetchMode.COMPAT} | |
| mode = mode_map.get(mode_str, FetchMode.BALANCED) | |
| cookies = _parse_cookies(cookie_str) | |
| from scraper.engine import create_session, smart_fetch | |
| from scraper.extractor import extract_chapter | |
| async def _preview(): | |
| session = await create_session(cookies) | |
| try: | |
| html, status = await smart_fetch(session, url, mode=mode, cookies=cookies) | |
| title, content, paras = extract_chapter(html) | |
| preview = f"**Title:** {title}\n\n**Preview (first 500 chars):**\n\n{content[:500]}..." | |
| return preview | |
| finally: | |
| await session.close() | |
| try: | |
| future = _run_async(_preview()) | |
| result = future.result(timeout=30) | |
| return result | |
| except Exception as exc: | |
| return f"❌ Preview failed: {exc}" | |
| # --------------------------------------------------------------------------- | |
| # CSS theme — redesigned: light/dark toggle, mobile-first, clean typography | |
| # --------------------------------------------------------------------------- | |
| CUSTOM_CSS = """ | |
| @import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap'); | |
| /* ─── DESIGN TOKENS ─────────────────────────────────────────────────────── */ | |
| :root { | |
| /* Light theme (default) */ | |
| --bg: #f5f4f0; | |
| --bg-raised: #ffffff; | |
| --bg-sunken: #ece9e3; | |
| --bg-input: #ffffff; | |
| --surface: #f0ede7; | |
| --border: #d8d4cc; | |
| --border-focus:#2a2a2a; | |
| --text-1: #1a1a1a; | |
| --text-2: #5a5752; | |
| --text-3: #9a9590; | |
| --accent: #1a1a1a; | |
| --accent-bg: #1a1a1a; | |
| --accent-fg: #ffffff; | |
| --green: #1a7a4a; | |
| --green-bg: #edf7f1; | |
| --red: #c0392b; | |
| --red-bg: #fdf2f1; | |
| --amber: #b35c00; | |
| --amber-bg: #fef8ec; | |
| --log-bg: #1c1c1c; | |
| --log-text: #d4f0c0; | |
| --shadow-sm: 0 1px 3px rgba(0,0,0,.08), 0 1px 2px rgba(0,0,0,.05); | |
| --shadow-md: 0 4px 12px rgba(0,0,0,.10), 0 2px 4px rgba(0,0,0,.06); | |
| --r: 10px; | |
| --r-sm: 7px; | |
| --r-lg: 16px; | |
| --transition: all .18s cubic-bezier(.4,0,.2,1); | |
| } | |
| /* Dark theme */ | |
| [data-theme="dark"] { | |
| --bg: #111111; | |
| --bg-raised: #1a1a1a; | |
| --bg-sunken: #0d0d0d; | |
| --bg-input: #1e1e1e; | |
| --surface: #161616; | |
| --border: #2e2e2e; | |
| --border-focus:#ffffff; | |
| --text-1: #f0f0f0; | |
| --text-2: #888888; | |
| --text-3: #555555; | |
| --accent: #f0f0f0; | |
| --accent-bg: #f0f0f0; | |
| --accent-fg: #111111; | |
| --green: #4ade80; | |
| --green-bg: #0f2a1a; | |
| --red: #f87171; | |
| --red-bg: #2a0f0f; | |
| --amber: #fbbf24; | |
| --amber-bg: #2a1f05; | |
| --log-bg: #0a0a0a; | |
| --log-text: #7dde8a; | |
| --shadow-sm: 0 1px 3px rgba(0,0,0,.4); | |
| --shadow-md: 0 4px 12px rgba(0,0,0,.5); | |
| } | |
| /* ─── RESET & BASE ───────────────────────────────────────────────────────── */ | |
| *, *::before, *::after { box-sizing: border-box; margin: 0; } | |
| html { font-size: 16px; } | |
| body, .gradio-container { | |
| background: var(--bg) !important; | |
| color: var(--text-1) !important; | |
| font-family: 'DM Sans', system-ui, sans-serif !important; | |
| transition: background .25s, color .25s !important; | |
| -webkit-font-smoothing: antialiased !important; | |
| } | |
| .gradio-container { | |
| max-width: 900px !important; | |
| margin: 0 auto !important; | |
| padding: 0 16px 48px !important; | |
| } | |
| /* ─── SCROLLBAR ─────────────────────────────────────────────────────────── */ | |
| ::-webkit-scrollbar { width: 5px; height: 5px; } | |
| ::-webkit-scrollbar-track { background: transparent; } | |
| ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 99px; } | |
| /* ─── HEADER ─────────────────────────────────────────────────────────────── */ | |
| .ns-header { | |
| padding: 36px 0 28px; | |
| display: flex; | |
| align-items: center; | |
| justify-content: space-between; | |
| gap: 16px; | |
| border-bottom: 1px solid var(--border); | |
| margin-bottom: 28px; | |
| flex-wrap: wrap; | |
| } | |
| .ns-brand { display: flex; flex-direction: column; gap: 4px; } | |
| .ns-title { | |
| font-family: 'Instrument Serif', Georgia, serif; | |
| font-size: clamp(1.6rem, 5vw, 2.2rem); | |
| font-weight: 400; | |
| font-style: italic; | |
| color: var(--text-1); | |
| line-height: 1; | |
| letter-spacing: -0.3px; | |
| } | |
| .ns-sub { | |
| font-size: 0.78rem; | |
| font-weight: 400; | |
| color: var(--text-3); | |
| letter-spacing: 0.4px; | |
| } | |
| /* ─── THEME TOGGLE ───────────────────────────────────────────────────────── */ | |
| .theme-toggle-wrap { display: flex; align-items: center; gap: 8px; flex-shrink: 0; } | |
| .theme-toggle { | |
| position: relative; | |
| width: 48px; | |
| height: 26px; | |
| flex-shrink: 0; | |
| } | |
| .theme-toggle input { opacity: 0; width: 0; height: 0; position: absolute; } | |
| .theme-toggle-track { | |
| position: absolute; | |
| inset: 0; | |
| background: var(--surface); | |
| border: 1.5px solid var(--border); | |
| border-radius: 99px; | |
| cursor: pointer; | |
| transition: var(--transition); | |
| } | |
| .theme-toggle-track::after { | |
| content: ''; | |
| position: absolute; | |
| top: 3px; left: 3px; | |
| width: 18px; height: 18px; | |
| border-radius: 50%; | |
| background: var(--text-2); | |
| transition: var(--transition); | |
| } | |
| .theme-toggle input:checked + .theme-toggle-track { background: var(--text-1); border-color: var(--text-1); } | |
| .theme-toggle input:checked + .theme-toggle-track::after { transform: translateX(22px); background: var(--bg); } | |
| .theme-label { font-size: 0.78rem; color: var(--text-3); user-select: none; min-width: 30px; } | |
| /* ─── TABS ───────────────────────────────────────────────────────────────── */ | |
| .tabs { margin-top: 0 !important; } | |
| .tab-nav { | |
| display: flex !important; | |
| gap: 4px !important; | |
| background: var(--surface) !important; | |
| border: 1px solid var(--border) !important; | |
| border-radius: var(--r) !important; | |
| padding: 4px !important; | |
| overflow-x: auto !important; | |
| overflow-y: hidden !important; | |
| scrollbar-width: none !important; | |
| margin-bottom: 20px !important; | |
| flex-wrap: nowrap !important; | |
| } | |
| .tab-nav::-webkit-scrollbar { display: none; } | |
| .tab-nav button { | |
| background: transparent !important; | |
| border: none !important; | |
| border-radius: var(--r-sm) !important; | |
| color: var(--text-2) !important; | |
| font-family: 'DM Sans', sans-serif !important; | |
| font-size: 0.82rem !important; | |
| font-weight: 500 !important; | |
| padding: 7px 14px !important; | |
| white-space: nowrap !important; | |
| transition: var(--transition) !important; | |
| cursor: pointer !important; | |
| flex-shrink: 0 !important; | |
| } | |
| .tab-nav button:hover { | |
| color: var(--text-1) !important; | |
| background: var(--bg-raised) !important; | |
| } | |
| .tab-nav button.selected { | |
| background: var(--bg-raised) !important; | |
| color: var(--text-1) !important; | |
| font-weight: 600 !important; | |
| box-shadow: var(--shadow-sm) !important; | |
| } | |
| /* ─── BLOCKS / CARDS ─────────────────────────────────────────────────────── */ | |
| .gr-group, .gradio-group { | |
| background: var(--bg-raised) !important; | |
| border: 1px solid var(--border) !important; | |
| border-radius: var(--r-lg) !important; | |
| box-shadow: var(--shadow-sm) !important; | |
| padding: 20px !important; | |
| margin-bottom: 14px !important; | |
| } | |
| /* ─── LABELS ─────────────────────────────────────────────────────────────── */ | |
| label span, .label-wrap span, .gr-form label { | |
| font-family: 'DM Sans', sans-serif !important; | |
| font-size: 0.75rem !important; | |
| font-weight: 600 !important; | |
| letter-spacing: 0.5px !important; | |
| text-transform: uppercase !important; | |
| color: var(--text-2) !important; | |
| } | |
| /* ─── INPUTS ─────────────────────────────────────────────────────────────── */ | |
| textarea, | |
| input[type="text"], | |
| input[type="number"], | |
| input[type="search"], | |
| select, | |
| .gr-input, | |
| .gr-textarea { | |
| background: var(--bg-input) !important; | |
| border: 1.5px solid var(--border) !important; | |
| border-radius: var(--r-sm) !important; | |
| color: var(--text-1) !important; | |
| font-family: 'DM Sans', sans-serif !important; | |
| font-size: 0.88rem !important; | |
| padding: 10px 12px !important; | |
| transition: var(--transition) !important; | |
| box-shadow: none !important; | |
| } | |
| textarea:focus, | |
| input[type="text"]:focus, | |
| input[type="number"]:focus, | |
| select:focus { | |
| border-color: var(--border-focus) !important; | |
| box-shadow: 0 0 0 3px rgba(0,0,0,0.06) !important; | |
| outline: none !important; | |
| } | |
| [data-theme="dark"] textarea:focus, | |
| [data-theme="dark"] input:focus { | |
| box-shadow: 0 0 0 3px rgba(255,255,255,0.06) !important; | |
| } | |
| /* placeholder */ | |
| textarea::placeholder, input::placeholder { color: var(--text-3) !important; } | |
| /* ─── SLIDERS ─────────────────────────────────────────────────────────────── */ | |
| input[type="range"] { | |
| accent-color: var(--text-1) !important; | |
| height: 4px !important; | |
| } | |
| /* ─── BUTTONS ─────────────────────────────────────────────────────────────── */ | |
| button, .gr-button { | |
| font-family: 'DM Sans', sans-serif !important; | |
| font-size: 0.85rem !important; | |
| font-weight: 600 !important; | |
| border-radius: var(--r-sm) !important; | |
| padding: 10px 18px !important; | |
| transition: var(--transition) !important; | |
| cursor: pointer !important; | |
| letter-spacing: 0.1px !important; | |
| } | |
| /* Primary */ | |
| button.gr-button-primary, .gr-button-primary { | |
| background: var(--accent-bg) !important; | |
| color: var(--accent-fg) !important; | |
| border: 1.5px solid var(--accent-bg) !important; | |
| box-shadow: var(--shadow-sm) !important; | |
| } | |
| button.gr-button-primary:hover { | |
| opacity: 0.88 !important; | |
| box-shadow: var(--shadow-md) !important; | |
| transform: translateY(-1px) !important; | |
| } | |
| button.gr-button-primary:active { transform: translateY(0) !important; } | |
| /* Secondary */ | |
| button.gr-button-secondary, .gr-button-secondary { | |
| background: var(--bg-raised) !important; | |
| color: var(--text-1) !important; | |
| border: 1.5px solid var(--border) !important; | |
| } | |
| button.gr-button-secondary:hover { | |
| border-color: var(--text-1) !important; | |
| background: var(--surface) !important; | |
| } | |
| /* Icon-only small buttons */ | |
| .btn-icon { | |
| padding: 8px 12px !important; | |
| font-size: 0.8rem !important; | |
| } | |
| /* ─── CHECKBOX ───────────────────────────────────────────────────────────── */ | |
| input[type="checkbox"] { accent-color: var(--text-1) !important; } | |
| /* ─── DROPDOWN ───────────────────────────────────────────────────────────── */ | |
| .gr-dropdown > div { | |
| background: var(--bg-input) !important; | |
| border: 1.5px solid var(--border) !important; | |
| border-radius: var(--r-sm) !important; | |
| color: var(--text-1) !important; | |
| } | |
| /* ─── ACCORDION ──────────────────────────────────────────────────────────── */ | |
| .gr-accordion { | |
| background: var(--surface) !important; | |
| border: 1px solid var(--border) !important; | |
| border-radius: var(--r) !important; | |
| } | |
| .gr-accordion > .label-wrap { | |
| padding: 12px 16px !important; | |
| } | |
| /* ─── MARKDOWN ───────────────────────────────────────────────────────────── */ | |
| .gr-markdown, .output-markdown, .prose { | |
| color: var(--text-1) !important; | |
| font-family: 'DM Sans', sans-serif !important; | |
| font-size: 0.88rem !important; | |
| line-height: 1.75 !important; | |
| } | |
| .gr-markdown h1, .gr-markdown h2, .gr-markdown h3 { | |
| font-family: 'Instrument Serif', serif !important; | |
| font-style: italic !important; | |
| font-weight: 400 !important; | |
| color: var(--text-1) !important; | |
| margin: 1.2em 0 0.5em !important; | |
| } | |
| .gr-markdown h2 { font-size: 1.25rem !important; } | |
| .gr-markdown h3 { font-size: 1.05rem !important; } | |
| .gr-markdown table { | |
| width: 100% !important; | |
| border-collapse: collapse !important; | |
| font-size: 0.83rem !important; | |
| border-radius: var(--r-sm) !important; | |
| overflow: hidden !important; | |
| } | |
| .gr-markdown th { | |
| background: var(--surface) !important; | |
| color: var(--text-2) !important; | |
| padding: 8px 12px !important; | |
| font-size: 0.72rem !important; | |
| font-weight: 600 !important; | |
| letter-spacing: 0.5px !important; | |
| text-transform: uppercase !important; | |
| text-align: left !important; | |
| } | |
| .gr-markdown td { | |
| padding: 8px 12px !important; | |
| border-bottom: 1px solid var(--border) !important; | |
| color: var(--text-1) !important; | |
| vertical-align: top !important; | |
| } | |
| .gr-markdown tr:last-child td { border-bottom: none !important; } | |
| .gr-markdown code { | |
| background: var(--surface) !important; | |
| color: var(--text-1) !important; | |
| padding: 1px 6px !important; | |
| border-radius: 4px !important; | |
| font-family: 'DM Mono', monospace !important; | |
| font-size: 0.8rem !important; | |
| border: 1px solid var(--border) !important; | |
| } | |
| .gr-markdown pre { | |
| background: var(--bg-sunken) !important; | |
| border: 1px solid var(--border) !important; | |
| border-radius: var(--r-sm) !important; | |
| padding: 14px !important; | |
| overflow-x: auto !important; | |
| } | |
| .gr-markdown a { color: var(--text-1) !important; text-underline-offset: 3px !important; } | |
| /* ─── LOGS ───────────────────────────────────────────────────────────────── */ | |
| .logs-area textarea { | |
| font-family: 'DM Mono', monospace !important; | |
| font-size: 0.77rem !important; | |
| line-height: 1.65 !important; | |
| color: var(--log-text) !important; | |
| background: var(--log-bg) !important; | |
| border: 1px solid #2a2a2a !important; | |
| border-radius: var(--r) !important; | |
| padding: 14px !important; | |
| } | |
| [data-theme="dark"] .logs-area textarea { | |
| background: #080808 !important; | |
| } | |
| /* ─── STATUS PILLS ───────────────────────────────────────────────────────── */ | |
| .pill { | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 5px; | |
| padding: 3px 10px; | |
| border-radius: 99px; | |
| font-size: 0.72rem; | |
| font-weight: 600; | |
| letter-spacing: 0.3px; | |
| } | |
| .pill-green { background: var(--green-bg); color: var(--green); } | |
| .pill-red { background: var(--red-bg); color: var(--red); } | |
| .pill-amber { background: var(--amber-bg); color: var(--amber); } | |
| /* ─── FILE DOWNLOAD ──────────────────────────────────────────────────────── */ | |
| .file-zone { | |
| background: var(--surface) !important; | |
| border: 1.5px dashed var(--border) !important; | |
| border-radius: var(--r) !important; | |
| } | |
| /* ─── STATUS TEXTBOX ─────────────────────────────────────────────────────── */ | |
| .status-box textarea { | |
| background: var(--surface) !important; | |
| border: 1px solid var(--border) !important; | |
| font-size: 0.83rem !important; | |
| color: var(--text-1) !important; | |
| } | |
| /* ─── SECTION DIVIDERS ───────────────────────────────────────────────────── */ | |
| .section-label { | |
| font-size: 0.72rem; | |
| font-weight: 700; | |
| letter-spacing: 0.8px; | |
| text-transform: uppercase; | |
| color: var(--text-3); | |
| margin: 20px 0 10px; | |
| padding-bottom: 6px; | |
| border-bottom: 1px solid var(--border); | |
| } | |
| /* ─── INFO TEXT ──────────────────────────────────────────────────────────── */ | |
| .gr-form .info, .description { color: var(--text-3) !important; font-size: 0.77rem !important; } | |
| /* ─── ROW GAPS (mobile-friendly) ─────────────────────────────────────────── */ | |
| .gr-row { gap: 12px !important; } | |
| @media (max-width: 640px) { | |
| .gradio-container { padding: 0 12px 40px !important; } | |
| .ns-header { padding: 24px 0 20px; } | |
| .gr-row { flex-direction: column !important; } | |
| .tab-nav button { padding: 7px 10px !important; font-size: 0.78rem !important; } | |
| button, .gr-button { padding: 10px 14px !important; font-size: 0.83rem !important; } | |
| } | |
| /* ─── GRADIO INTERNAL OVERRIDES ──────────────────────────────────────────── */ | |
| .block { background: transparent !important; border: none !important; box-shadow: none !important; } | |
| .form { gap: 12px !important; } | |
| footer { display: none !important; } | |
| .gap { gap: 12px !important; } | |
| """ | |
| # --------------------------------------------------------------------------- | |
| # Build Gradio app — redesigned layout | |
| # --------------------------------------------------------------------------- | |
| def build_app() -> gr.Blocks: | |
| with gr.Blocks( | |
| title="Novel Scraper", | |
| css=CUSTOM_CSS, | |
| theme=gr.themes.Base( | |
| font=[gr.themes.GoogleFont("DM Sans"), "ui-sans-serif"], | |
| ), | |
| analytics_enabled=False, | |
| ) as demo: | |
| # ── Header with theme toggle ───────────────────────────────────── | |
| gr.HTML(""" | |
| <div class="ns-header" id="ns-header"> | |
| <div class="ns-brand"> | |
| <div class="ns-title">Novel Scraper</div> | |
| <div class="ns-sub">Universal adaptive web novel downloader</div> | |
| </div> | |
| <div class="theme-toggle-wrap"> | |
| <span class="theme-label" id="theme-lbl">Light</span> | |
| <label class="theme-toggle" title="Toggle dark mode"> | |
| <input type="checkbox" id="theme-chk" onchange="toggleTheme(this)"> | |
| <span class="theme-toggle-track"></span> | |
| </label> | |
| </div> | |
| </div> | |
| <script> | |
| function toggleTheme(chk) { | |
| const root = document.documentElement; | |
| const lbl = document.getElementById('theme-lbl'); | |
| if (chk.checked) { | |
| root.setAttribute('data-theme', 'dark'); | |
| lbl.textContent = 'Dark'; | |
| localStorage.setItem('ns-theme', 'dark'); | |
| } else { | |
| root.removeAttribute('data-theme'); | |
| lbl.textContent = 'Light'; | |
| localStorage.setItem('ns-theme', 'light'); | |
| } | |
| } | |
| // Restore saved preference | |
| (function() { | |
| const saved = localStorage.getItem('ns-theme'); | |
| if (saved === 'dark') { | |
| document.documentElement.setAttribute('data-theme', 'dark'); | |
| const chk = document.getElementById('theme-chk'); | |
| const lbl = document.getElementById('theme-lbl'); | |
| if (chk) { chk.checked = true; } | |
| if (lbl) { lbl.textContent = 'Dark'; } | |
| } | |
| })(); | |
| </script> | |
| """) | |
| with gr.Tabs(): | |
| # ── TAB 1: Download ────────────────────────────────────────── | |
| with gr.Tab("Download"): | |
| # ── URL Input ──────────────────────────────────────────── | |
| with gr.Group(): | |
| urls_input = gr.Textbox( | |
| label="Novel URLs", | |
| placeholder="https://novelsite.com/my-novel\nhttps://anothersite.com/story\n\nOne URL per line — supports multiple novels", | |
| lines=4, | |
| ) | |
| with gr.Row(): | |
| chapter_range = gr.Textbox( | |
| label="Chapter Range", | |
| placeholder="all / 1-100 / 1-50,70-90 / 5,10,15", | |
| lines=1, | |
| scale=3, | |
| ) | |
| resume_check = gr.Checkbox( | |
| label="Resume previous", | |
| value=True, | |
| scale=1, | |
| ) | |
| # ── Settings ───────────────────────────────────────────── | |
| with gr.Accordion("Settings", open=False): | |
| mode_select = gr.Dropdown( | |
| label="Fetch Mode", | |
| choices=["Fast", "Balanced", "Compatibility"], | |
| value="Balanced", | |
| info="Fast = aiohttp · Balanced = + cloudscraper · Compatibility = + Playwright browser", | |
| ) | |
| with gr.Row(): | |
| rate_slider = gr.Slider( | |
| label="Chapters per second", | |
| minimum=0.1, maximum=10.0, step=0.1, value=1.0, | |
| ) | |
| concurrency = gr.Slider( | |
| label="Concurrent workers", | |
| minimum=1, maximum=10, step=1, value=3, | |
| ) | |
| with gr.Row(): | |
| retries = gr.Slider( | |
| label="Max retries", | |
| minimum=1, maximum=30, step=1, value=10, | |
| ) | |
| timeout = gr.Slider( | |
| label="Timeout (sec)", | |
| minimum=5, maximum=120, step=5, value=30, | |
| ) | |
| # ── Cookies ────────────────────────────────────────────── | |
| with gr.Accordion("Cookies / Authentication", open=False): | |
| cookie_input = gr.Textbox( | |
| label="Browser Cookies", | |
| placeholder='Paste as: key=value; key2=value2\n\nOr as JSON: [{"name":"sid","value":"abc123"}]', | |
| lines=3, | |
| info="For age-gated or login-required sites. Export from browser DevTools → Application → Cookies.", | |
| ) | |
| # ── Actions ─────────────────────────────────────────────── | |
| with gr.Row(): | |
| start_btn = gr.Button("Start Download", variant="primary", scale=3) | |
| preview_btn = gr.Button("Preview", variant="secondary", scale=1) | |
| status_msg = gr.Textbox( | |
| label="Status", | |
| lines=2, | |
| interactive=False, | |
| elem_classes=["status-box"], | |
| show_label=True, | |
| ) | |
| # ── Queue Controls ──────────────────────────────────────── | |
| with gr.Accordion("Queue Controls (pause / cancel / retry)", open=False): | |
| control_url = gr.Textbox( | |
| label="Target URL", | |
| placeholder="Paste the novel URL to control", | |
| lines=1, | |
| ) | |
| with gr.Row(): | |
| pause_btn = gr.Button("⏸ Pause", variant="secondary") | |
| resume_btn = gr.Button("▶ Resume", variant="secondary") | |
| cancel_btn = gr.Button("✕ Cancel", variant="secondary") | |
| retry_btn = gr.Button("↺ Retry Failed", variant="secondary") | |
| control_msg = gr.Textbox( | |
| label="Output", | |
| lines=1, | |
| interactive=False, | |
| elem_classes=["status-box"], | |
| ) | |
| # ── Preview pane ────────────────────────────────────────── | |
| preview_out = gr.Markdown( | |
| value="", | |
| label="", | |
| visible=True, | |
| ) | |
| # ── TAB 2: Progress ────────────────────────────────────────── | |
| with gr.Tab("Progress"): | |
| with gr.Row(): | |
| refresh_btn = gr.Button("↻ Refresh", variant="secondary") | |
| with gr.Group(): | |
| progress_display = gr.Markdown( | |
| value="*Start a download to see progress here.*", | |
| ) | |
| with gr.Group(): | |
| gr.HTML('<div class="section-label">Failed Chapters</div>') | |
| failed_display = gr.Markdown(value="*None*") | |
| # ── TAB 3: Logs ─────────────────────────────────────────────── | |
| with gr.Tab("Logs"): | |
| with gr.Row(): | |
| refresh_logs_btn = gr.Button("↻ Refresh", variant="secondary") | |
| clear_logs_btn = gr.Button("Clear", variant="secondary") | |
| logs_out = gr.Textbox( | |
| label="", | |
| lines=22, | |
| interactive=False, | |
| max_lines=22, | |
| elem_classes=["logs-area"], | |
| value="", | |
| show_label=False, | |
| ) | |
| # ── TAB 4: Files ────────────────────────────────────────────── | |
| with gr.Tab("Files"): | |
| with gr.Row(): | |
| list_files_btn = gr.Button("List Files", variant="secondary") | |
| zip_btn = gr.Button("Create ZIP", variant="secondary") | |
| with gr.Group(): | |
| files_display = gr.Markdown(value="*Click 'List Files' to see available downloads.*") | |
| file_download = gr.File( | |
| label="Download", | |
| interactive=False, | |
| elem_classes=["file-zone"], | |
| ) | |
| # ── TAB 5: Help ─────────────────────────────────────────────── | |
| with gr.Tab("Help"): | |
| gr.Markdown(""" | |
| ## Quick Start | |
| 1. Paste one or more novel URLs (one per line) | |
| 2. Optionally set a chapter range — leave blank for all | |
| 3. Click **Start Download** | |
| 4. Watch progress in the **Progress** tab | |
| 5. Grab your `.txt` from the **Files** tab | |
| --- | |
| ## Chapter Ranges | |
| | Expression | Meaning | | |
| |------------|---------| | |
| | *(blank)* or `all` | Every chapter | | |
| | `1-100` | Chapters 1 to 100 | | |
| | `50-200` | Chapters 50 to 200 | | |
| | `1,5,9,20` | Specific chapters | | |
| | `1-50,70-90` | Combined ranges | | |
| --- | |
| ## Fetch Modes | |
| | Mode | Description | | |
| |------|-------------| | |
| | **Fast** | aiohttp only — fastest | | |
| | **Balanced** | + cloudscraper fallback *(recommended)* | | |
| | **Compatibility** | + Playwright browser — handles almost any site | | |
| --- | |
| ## Cookies | |
| For login-required or age-gated sites: | |
| 1. Log in to the site in your browser | |
| 2. Open DevTools (`F12`) → **Application** → **Cookies** | |
| 3. Copy as `key=value; key2=value2` | |
| 4. Or export as JSON from a cookie manager extension | |
| --- | |
| ## Output Format | |
| Clean plain text — no dividers, no noise: | |
| ``` | |
| Chapter 1: The Beginning | |
| The hero walked into the forest... | |
| Chapter 2: The Trial | |
| Dark clouds gathered overhead... | |
| ``` | |
| --- | |
| ## Tips | |
| - Start with **1 chapter/sec** and **3 workers** to avoid bans | |
| - Use **Compatibility** mode for 403/503 errors | |
| - Enable **Resume previous** to continue interrupted downloads | |
| - Large novels (5000+ chapters): use 2–3 workers | |
| - CAPTCHA sites: pause, solve in browser, paste fresh cookies | |
| --- | |
| ## Supported Site Structures | |
| - Direct chapter lists on the main page | |
| - TOC sub-pages: `/chapters`, `/toc`, `/episodes`, `/contents` | |
| - Paginated lists: `?page=N`, `/page/N` | |
| - JSON chapter data embedded in `<script>` tags | |
| - Cloudflare-protected sites (Compatibility mode) | |
| """) | |
| # ── Event wiring ───────────────────────────────────────────────── | |
| start_btn.click( | |
| fn=add_novels, | |
| inputs=[urls_input, mode_select, rate_slider, concurrency, | |
| retries, timeout, cookie_input, chapter_range, resume_check], | |
| outputs=[status_msg, control_msg, logs_out], | |
| ) | |
| preview_btn.click( | |
| fn=preview_extraction, | |
| inputs=[urls_input, mode_select, cookie_input], | |
| outputs=[preview_out], | |
| ) | |
| pause_btn.click(fn=pause_job, inputs=[control_url], outputs=[control_msg]) | |
| resume_btn.click(fn=resume_job, inputs=[control_url], outputs=[control_msg]) | |
| cancel_btn.click(fn=cancel_job, inputs=[control_url], outputs=[control_msg]) | |
| retry_btn.click(fn=retry_failed, inputs=[control_url], outputs=[control_msg]) | |
| def do_refresh(): | |
| prog, failed, logs, _ = get_status() | |
| return prog, failed, logs | |
| refresh_btn.click(fn=do_refresh, outputs=[progress_display, failed_display, logs_out]) | |
| refresh_logs_btn.click(fn=lambda: app_logger.get_ui_text(300), outputs=[logs_out]) | |
| clear_logs_btn.click(fn=clear_logs, outputs=[logs_out]) | |
| def list_files(): | |
| files = get_download_files() | |
| if not files: | |
| return "*No output files yet — start a download first.*", None | |
| lines = [f"- `{Path(f).name}` — {Path(f).stat().st_size // 1024} KB" for f in files] | |
| return "\n".join(lines), files[0] if files else None | |
| list_files_btn.click(fn=list_files, outputs=[files_display, file_download]) | |
| def make_zip(): | |
| p = create_zip() | |
| if p: | |
| return f"ZIP created: `{Path(p).name}`", p | |
| return "No files to ZIP yet.", None | |
| zip_btn.click(fn=make_zip, outputs=[files_display, file_download]) | |
| return demo | |
| # --------------------------------------------------------------------------- | |
| # Main entry point | |
| # --------------------------------------------------------------------------- | |
| if __name__ == "__main__": | |
| app = build_app() | |
| app.launch( | |
| server_name="0.0.0.0", | |
| server_port=int(os.environ.get("PORT", 7860)), | |
| share=False, | |
| show_error=True, | |
| ) | |