""" Novel Scraper β Gradio UI Universal adaptive web novel downloader. """ from __future__ import annotations import asyncio import io import json import os import shutil import threading import time import zipfile from pathlib import Path from typing import Optional import gradio as gr from scraper.engine import ChapterStatus, FetchMode, NovelJob from scraper.manager import DownloadConfig, DownloadManager, load_state from utils.logging_setup import AppLogger from utils.range_parser import parse_chapter_range, format_range_hint # --------------------------------------------------------------------------- # Global state # --------------------------------------------------------------------------- OUTPUT_DIR = Path("output") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) app_logger = AppLogger.get(OUTPUT_DIR) # Active download managers keyed by job URL _managers: dict[str, DownloadManager] = {} _jobs: dict[str, NovelJob] = {} _threads: dict[str, threading.Thread] = {} _results: dict[str, Optional[Path]] = {} _loop: Optional[asyncio.AbstractEventLoop] = None def _get_loop() -> asyncio.AbstractEventLoop: global _loop if _loop is None or _loop.is_closed(): _loop = asyncio.new_event_loop() threading.Thread(target=_loop.run_forever, daemon=True).start() return _loop def _run_async(coro): loop = _get_loop() future = asyncio.run_coroutine_threadsafe(coro, loop) return future # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _parse_cookies(cookie_str: str) -> dict: """Parse 'key=value; key2=value2' or JSON cookie string.""" cookie_str = cookie_str.strip() if not cookie_str: return {} if cookie_str.startswith("{") or cookie_str.startswith("["): try: data = json.loads(cookie_str) if isinstance(data, list): return {item["name"]: item["value"] for item in data if "name" in item} return data except Exception: return {} cookies = {} for part in cookie_str.split(";"): kv = part.strip().split("=", 1) if len(kv) == 2: cookies[kv[0].strip()] = kv[1].strip() return cookies def _build_progress_table(job: NovelJob) -> str: if not job.chapters: return "No chapters discovered yet." done = job.done_count failed = job.failed_count locked = sum(1 for c in job.chapters if c.status == ChapterStatus.LOCKED) skipped = sum(1 for c in job.chapters if c.status == ChapterStatus.SKIPPED) pct = f"{job.progress*100:.1f}" elapsed = "" if job.started_at: e = time.time() - job.started_at elapsed = f"{int(e//60)}m {int(e%60)}s" eta = "" if job.started_at and done > 0 and job.progress < 1: rate = done / max(time.time() - job.started_at, 1) remaining = job.total - done eta_secs = remaining / rate eta = f"{int(eta_secs//60)}m {int(eta_secs%60)}s" return ( f"**{job.title or 'Novel'}**\n\n" f"| Stat | Value |\n|------|-------|\n" f"| Status | {job.status.upper()} |\n" f"| Progress | {done}/{job.total} ({pct}%) |\n" f"| Done | β {done} |\n" f"| Failed | β {failed} |\n" f"| Locked | π° {locked} |\n" f"| Skipped | βοΈ {skipped} |\n" f"| Elapsed | {elapsed or 'β'} |\n" f"| ETA | {eta or 'β'} |\n" ) def _build_failed_list(job: NovelJob) -> str: failed = [c for c in job.chapters if c.status == ChapterStatus.FAILED] if not failed: return "No failed chapters." lines = [f"**{len(failed)} failed chapters:**\n"] for c in failed[:50]: lines.append(f"- [{c.title}]({c.url}) β {c.error}") if len(failed) > 50: lines.append(f"... and {len(failed)-50} more (see failed_chapters.txt)") return "\n".join(lines) def _zip_outputs(job: NovelJob) -> Optional[str]: """Create a ZIP of all output files for a job.""" safe_title = "".join(c if c.isalnum() or c in " -_" else "_" for c in (job.title or "novel"))[:60] zip_path = OUTPUT_DIR / f"{safe_title}_package.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for f in OUTPUT_DIR.iterdir(): if f.is_file() and not f.name.startswith("."): zf.write(f, f.name) return str(zip_path) # --------------------------------------------------------------------------- # Core download entrypoint (runs in background thread) # --------------------------------------------------------------------------- def _start_download_thread( url: str, config: DownloadConfig, resume: bool = False, ) -> None: """Launch download in background thread.""" # Try to restore state job = None if resume: job = load_state(url, OUTPUT_DIR) if job: app_logger.ui_log(f"π Resumed from saved state: {job.title}") if job is None: job = NovelJob(url=url, output_dir=OUTPUT_DIR) job.start_chapter = config.__dict__.get("start_chapter", 1) job.end_chapter = config.__dict__.get("end_chapter", 0) job.specific_chapters = config.__dict__.get("specific_chapters", []) _jobs[url] = job def log_cb(msg: str) -> None: app_logger.ui_log(msg) def progress_cb(j: NovelJob) -> None: pass # polling-based UI manager = DownloadManager(job, config, log_cb=log_cb, progress_cb=progress_cb) _managers[url] = manager async def _run(): try: result = await manager.run() _results[url] = result except Exception as exc: app_logger.ui_log(f"β Download error: {exc}") _results[url] = None job.status = "failed" future = _run_async(_run()) _results[url] = None # --------------------------------------------------------------------------- # Gradio event handlers # --------------------------------------------------------------------------- def add_novels( urls_text: str, mode_str: str, rate: float, concurrency: int, retries: int, timeout: int, cookie_str: str, chapter_range: str, resume_prev: bool, ) -> tuple[str, str, str]: """Parse inputs and launch downloads.""" urls = [u.strip() for u in urls_text.strip().splitlines() if u.strip()] if not urls: return "β οΈ Please enter at least one URL.", "", "" mode_map = {"Fast": FetchMode.FAST, "Balanced": FetchMode.BALANCED, "Compatibility": FetchMode.COMPAT} mode = mode_map.get(mode_str, FetchMode.BALANCED) cookies = _parse_cookies(cookie_str) # Parse range specific = parse_chapter_range(chapter_range) start_ch = 1 end_ch = 0 if specific: start_ch = min(specific) end_ch = max(specific) config = DownloadConfig( mode=mode, max_concurrent=max(1, min(concurrency, 10)), rate_limit=max(0.1, 1.0 / max(rate, 0.1)), max_retries=max(1, retries), timeout=max(5, timeout), cookies=cookies, output_dir=OUTPUT_DIR, ) config.__dict__["start_chapter"] = start_ch config.__dict__["end_chapter"] = end_ch config.__dict__["specific_chapters"] = specific messages = [] for url in urls: if url in _jobs and _jobs[url].status == "running": messages.append(f"β οΈ Already running: {url}") continue app_logger.ui_log(f"π Queuing: {url}") t = threading.Thread( target=_start_download_thread, args=(url, config, resume_prev), daemon=True, ) t.start() _threads[url] = t messages.append(f"β Started: {url}") return "\n".join(messages), "", app_logger.get_ui_text() def pause_job(url: str) -> str: url = url.strip() if url in _managers: _managers[url].pause() return f"βΈοΈ Paused: {url}" return "β οΈ No active download for that URL." def resume_job(url: str) -> str: url = url.strip() if url in _managers: _managers[url].resume() return f"βΆοΈ Resumed: {url}" return "β οΈ No active download for that URL." def cancel_job(url: str) -> str: url = url.strip() if url in _managers: _managers[url].cancel() return f"π« Cancelled: {url}" return "β οΈ No active download for that URL." def retry_failed(url: str) -> str: url = url.strip() if url not in _jobs: return "β οΈ No job found for that URL." job = _jobs[url] for c in job.chapters: if c.status == ChapterStatus.FAILED: c.status = ChapterStatus.PENDING c.retries = 0 c.error = "" app_logger.ui_log(f"π Reset failed chapters for retry: {url}") return f"β Failed chapters reset for retry." def get_status() -> tuple[str, str, str, gr.File]: """Poll status β called by timer.""" if not _jobs: return "No downloads started.", "", app_logger.get_ui_text(), None progress_parts = [] failed_parts = [] for url, job in _jobs.items(): progress_parts.append(_build_progress_table(job)) failed_parts.append(_build_failed_list(job)) # Find completed downloads for file output output_file = None for url, result_path in _results.items(): if result_path and Path(result_path).exists(): output_file = str(result_path) break return ( "\n\n---\n\n".join(progress_parts), "\n\n---\n\n".join(failed_parts), app_logger.get_ui_text(200), output_file, ) def get_download_files() -> list[str]: """Return list of available output files.""" files = [] for f in OUTPUT_DIR.iterdir(): if f.is_file() and not f.name.startswith(".") and f.suffix in (".txt", ".zip"): files.append(str(f)) return files def create_zip() -> Optional[str]: if not _jobs: return None # Use the most recent job job = list(_jobs.values())[-1] return _zip_outputs(job) def clear_logs() -> str: app_logger.ring.clear() return "" def preview_extraction(url: str, mode_str: str, cookie_str: str) -> str: """Preview extracted content from a URL without saving.""" url = url.strip() if not url: return "β οΈ Enter a URL to preview." mode_map = {"Fast": FetchMode.FAST, "Balanced": FetchMode.BALANCED, "Compatibility": FetchMode.COMPAT} mode = mode_map.get(mode_str, FetchMode.BALANCED) cookies = _parse_cookies(cookie_str) from scraper.engine import create_session, smart_fetch from scraper.extractor import extract_chapter async def _preview(): session = await create_session(cookies) try: html, status = await smart_fetch(session, url, mode=mode, cookies=cookies) title, content, paras = extract_chapter(html) preview = f"**Title:** {title}\n\n**Preview (first 500 chars):**\n\n{content[:500]}..." return preview finally: await session.close() try: future = _run_async(_preview()) result = future.result(timeout=30) return result except Exception as exc: return f"β Preview failed: {exc}" # --------------------------------------------------------------------------- # CSS theme β redesigned: light/dark toggle, mobile-first, clean typography # --------------------------------------------------------------------------- CUSTOM_CSS = """ @import url('https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&family=DM+Sans:wght@300;400;500;600&family=DM+Mono:wght@400;500&display=swap'); /* βββ DESIGN TOKENS βββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ :root { /* Light theme (default) */ --bg: #f5f4f0; --bg-raised: #ffffff; --bg-sunken: #ece9e3; --bg-input: #ffffff; --surface: #f0ede7; --border: #d8d4cc; --border-focus:#2a2a2a; --text-1: #1a1a1a; --text-2: #5a5752; --text-3: #9a9590; --accent: #1a1a1a; --accent-bg: #1a1a1a; --accent-fg: #ffffff; --green: #1a7a4a; --green-bg: #edf7f1; --red: #c0392b; --red-bg: #fdf2f1; --amber: #b35c00; --amber-bg: #fef8ec; --log-bg: #1c1c1c; --log-text: #d4f0c0; --shadow-sm: 0 1px 3px rgba(0,0,0,.08), 0 1px 2px rgba(0,0,0,.05); --shadow-md: 0 4px 12px rgba(0,0,0,.10), 0 2px 4px rgba(0,0,0,.06); --r: 10px; --r-sm: 7px; --r-lg: 16px; --transition: all .18s cubic-bezier(.4,0,.2,1); } /* Dark theme */ [data-theme="dark"] { --bg: #111111; --bg-raised: #1a1a1a; --bg-sunken: #0d0d0d; --bg-input: #1e1e1e; --surface: #161616; --border: #2e2e2e; --border-focus:#ffffff; --text-1: #f0f0f0; --text-2: #888888; --text-3: #555555; --accent: #f0f0f0; --accent-bg: #f0f0f0; --accent-fg: #111111; --green: #4ade80; --green-bg: #0f2a1a; --red: #f87171; --red-bg: #2a0f0f; --amber: #fbbf24; --amber-bg: #2a1f05; --log-bg: #0a0a0a; --log-text: #7dde8a; --shadow-sm: 0 1px 3px rgba(0,0,0,.4); --shadow-md: 0 4px 12px rgba(0,0,0,.5); } /* βββ RESET & BASE βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ *, *::before, *::after { box-sizing: border-box; margin: 0; } html { font-size: 16px; } body, .gradio-container { background: var(--bg) !important; color: var(--text-1) !important; font-family: 'DM Sans', system-ui, sans-serif !important; transition: background .25s, color .25s !important; -webkit-font-smoothing: antialiased !important; } .gradio-container { max-width: 900px !important; margin: 0 auto !important; padding: 0 16px 48px !important; } /* βββ SCROLLBAR βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ ::-webkit-scrollbar { width: 5px; height: 5px; } ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 99px; } /* βββ HEADER βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .ns-header { padding: 36px 0 28px; display: flex; align-items: center; justify-content: space-between; gap: 16px; border-bottom: 1px solid var(--border); margin-bottom: 28px; flex-wrap: wrap; } .ns-brand { display: flex; flex-direction: column; gap: 4px; } .ns-title { font-family: 'Instrument Serif', Georgia, serif; font-size: clamp(1.6rem, 5vw, 2.2rem); font-weight: 400; font-style: italic; color: var(--text-1); line-height: 1; letter-spacing: -0.3px; } .ns-sub { font-size: 0.78rem; font-weight: 400; color: var(--text-3); letter-spacing: 0.4px; } /* βββ THEME TOGGLE βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .theme-toggle-wrap { display: flex; align-items: center; gap: 8px; flex-shrink: 0; } .theme-toggle { position: relative; width: 48px; height: 26px; flex-shrink: 0; } .theme-toggle input { opacity: 0; width: 0; height: 0; position: absolute; } .theme-toggle-track { position: absolute; inset: 0; background: var(--surface); border: 1.5px solid var(--border); border-radius: 99px; cursor: pointer; transition: var(--transition); } .theme-toggle-track::after { content: ''; position: absolute; top: 3px; left: 3px; width: 18px; height: 18px; border-radius: 50%; background: var(--text-2); transition: var(--transition); } .theme-toggle input:checked + .theme-toggle-track { background: var(--text-1); border-color: var(--text-1); } .theme-toggle input:checked + .theme-toggle-track::after { transform: translateX(22px); background: var(--bg); } .theme-label { font-size: 0.78rem; color: var(--text-3); user-select: none; min-width: 30px; } /* βββ TABS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .tabs { margin-top: 0 !important; } .tab-nav { display: flex !important; gap: 4px !important; background: var(--surface) !important; border: 1px solid var(--border) !important; border-radius: var(--r) !important; padding: 4px !important; overflow-x: auto !important; overflow-y: hidden !important; scrollbar-width: none !important; margin-bottom: 20px !important; flex-wrap: nowrap !important; } .tab-nav::-webkit-scrollbar { display: none; } .tab-nav button { background: transparent !important; border: none !important; border-radius: var(--r-sm) !important; color: var(--text-2) !important; font-family: 'DM Sans', sans-serif !important; font-size: 0.82rem !important; font-weight: 500 !important; padding: 7px 14px !important; white-space: nowrap !important; transition: var(--transition) !important; cursor: pointer !important; flex-shrink: 0 !important; } .tab-nav button:hover { color: var(--text-1) !important; background: var(--bg-raised) !important; } .tab-nav button.selected { background: var(--bg-raised) !important; color: var(--text-1) !important; font-weight: 600 !important; box-shadow: var(--shadow-sm) !important; } /* βββ BLOCKS / CARDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .gr-group, .gradio-group { background: var(--bg-raised) !important; border: 1px solid var(--border) !important; border-radius: var(--r-lg) !important; box-shadow: var(--shadow-sm) !important; padding: 20px !important; margin-bottom: 14px !important; } /* βββ LABELS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ label span, .label-wrap span, .gr-form label { font-family: 'DM Sans', sans-serif !important; font-size: 0.75rem !important; font-weight: 600 !important; letter-spacing: 0.5px !important; text-transform: uppercase !important; color: var(--text-2) !important; } /* βββ INPUTS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ textarea, input[type="text"], input[type="number"], input[type="search"], select, .gr-input, .gr-textarea { background: var(--bg-input) !important; border: 1.5px solid var(--border) !important; border-radius: var(--r-sm) !important; color: var(--text-1) !important; font-family: 'DM Sans', sans-serif !important; font-size: 0.88rem !important; padding: 10px 12px !important; transition: var(--transition) !important; box-shadow: none !important; } textarea:focus, input[type="text"]:focus, input[type="number"]:focus, select:focus { border-color: var(--border-focus) !important; box-shadow: 0 0 0 3px rgba(0,0,0,0.06) !important; outline: none !important; } [data-theme="dark"] textarea:focus, [data-theme="dark"] input:focus { box-shadow: 0 0 0 3px rgba(255,255,255,0.06) !important; } /* placeholder */ textarea::placeholder, input::placeholder { color: var(--text-3) !important; } /* βββ SLIDERS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ input[type="range"] { accent-color: var(--text-1) !important; height: 4px !important; } /* βββ BUTTONS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ button, .gr-button { font-family: 'DM Sans', sans-serif !important; font-size: 0.85rem !important; font-weight: 600 !important; border-radius: var(--r-sm) !important; padding: 10px 18px !important; transition: var(--transition) !important; cursor: pointer !important; letter-spacing: 0.1px !important; } /* Primary */ button.gr-button-primary, .gr-button-primary { background: var(--accent-bg) !important; color: var(--accent-fg) !important; border: 1.5px solid var(--accent-bg) !important; box-shadow: var(--shadow-sm) !important; } button.gr-button-primary:hover { opacity: 0.88 !important; box-shadow: var(--shadow-md) !important; transform: translateY(-1px) !important; } button.gr-button-primary:active { transform: translateY(0) !important; } /* Secondary */ button.gr-button-secondary, .gr-button-secondary { background: var(--bg-raised) !important; color: var(--text-1) !important; border: 1.5px solid var(--border) !important; } button.gr-button-secondary:hover { border-color: var(--text-1) !important; background: var(--surface) !important; } /* Icon-only small buttons */ .btn-icon { padding: 8px 12px !important; font-size: 0.8rem !important; } /* βββ CHECKBOX βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ input[type="checkbox"] { accent-color: var(--text-1) !important; } /* βββ DROPDOWN βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .gr-dropdown > div { background: var(--bg-input) !important; border: 1.5px solid var(--border) !important; border-radius: var(--r-sm) !important; color: var(--text-1) !important; } /* βββ ACCORDION ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .gr-accordion { background: var(--surface) !important; border: 1px solid var(--border) !important; border-radius: var(--r) !important; } .gr-accordion > .label-wrap { padding: 12px 16px !important; } /* βββ MARKDOWN βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .gr-markdown, .output-markdown, .prose { color: var(--text-1) !important; font-family: 'DM Sans', sans-serif !important; font-size: 0.88rem !important; line-height: 1.75 !important; } .gr-markdown h1, .gr-markdown h2, .gr-markdown h3 { font-family: 'Instrument Serif', serif !important; font-style: italic !important; font-weight: 400 !important; color: var(--text-1) !important; margin: 1.2em 0 0.5em !important; } .gr-markdown h2 { font-size: 1.25rem !important; } .gr-markdown h3 { font-size: 1.05rem !important; } .gr-markdown table { width: 100% !important; border-collapse: collapse !important; font-size: 0.83rem !important; border-radius: var(--r-sm) !important; overflow: hidden !important; } .gr-markdown th { background: var(--surface) !important; color: var(--text-2) !important; padding: 8px 12px !important; font-size: 0.72rem !important; font-weight: 600 !important; letter-spacing: 0.5px !important; text-transform: uppercase !important; text-align: left !important; } .gr-markdown td { padding: 8px 12px !important; border-bottom: 1px solid var(--border) !important; color: var(--text-1) !important; vertical-align: top !important; } .gr-markdown tr:last-child td { border-bottom: none !important; } .gr-markdown code { background: var(--surface) !important; color: var(--text-1) !important; padding: 1px 6px !important; border-radius: 4px !important; font-family: 'DM Mono', monospace !important; font-size: 0.8rem !important; border: 1px solid var(--border) !important; } .gr-markdown pre { background: var(--bg-sunken) !important; border: 1px solid var(--border) !important; border-radius: var(--r-sm) !important; padding: 14px !important; overflow-x: auto !important; } .gr-markdown a { color: var(--text-1) !important; text-underline-offset: 3px !important; } /* βββ LOGS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .logs-area textarea { font-family: 'DM Mono', monospace !important; font-size: 0.77rem !important; line-height: 1.65 !important; color: var(--log-text) !important; background: var(--log-bg) !important; border: 1px solid #2a2a2a !important; border-radius: var(--r) !important; padding: 14px !important; } [data-theme="dark"] .logs-area textarea { background: #080808 !important; } /* βββ STATUS PILLS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .pill { display: inline-flex; align-items: center; gap: 5px; padding: 3px 10px; border-radius: 99px; font-size: 0.72rem; font-weight: 600; letter-spacing: 0.3px; } .pill-green { background: var(--green-bg); color: var(--green); } .pill-red { background: var(--red-bg); color: var(--red); } .pill-amber { background: var(--amber-bg); color: var(--amber); } /* βββ FILE DOWNLOAD ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .file-zone { background: var(--surface) !important; border: 1.5px dashed var(--border) !important; border-radius: var(--r) !important; } /* βββ STATUS TEXTBOX βββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .status-box textarea { background: var(--surface) !important; border: 1px solid var(--border) !important; font-size: 0.83rem !important; color: var(--text-1) !important; } /* βββ SECTION DIVIDERS βββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .section-label { font-size: 0.72rem; font-weight: 700; letter-spacing: 0.8px; text-transform: uppercase; color: var(--text-3); margin: 20px 0 10px; padding-bottom: 6px; border-bottom: 1px solid var(--border); } /* βββ INFO TEXT ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ */ .gr-form .info, .description { color: var(--text-3) !important; font-size: 0.77rem !important; } /* βββ ROW GAPS (mobile-friendly) βββββββββββββββββββββββββββββββββββββββββββ */ .gr-row { gap: 12px !important; } @media (max-width: 640px) { .gradio-container { padding: 0 12px 40px !important; } .ns-header { padding: 24px 0 20px; } .gr-row { flex-direction: column !important; } .tab-nav button { padding: 7px 10px !important; font-size: 0.78rem !important; } button, .gr-button { padding: 10px 14px !important; font-size: 0.83rem !important; } } /* βββ GRADIO INTERNAL OVERRIDES ββββββββββββββββββββββββββββββββββββββββββββ */ .block { background: transparent !important; border: none !important; box-shadow: none !important; } .form { gap: 12px !important; } footer { display: none !important; } .gap { gap: 12px !important; } """ # --------------------------------------------------------------------------- # Build Gradio app β redesigned layout # --------------------------------------------------------------------------- def build_app() -> gr.Blocks: with gr.Blocks( title="Novel Scraper", css=CUSTOM_CSS, theme=gr.themes.Base( font=[gr.themes.GoogleFont("DM Sans"), "ui-sans-serif"], ), analytics_enabled=False, ) as demo: # ββ Header with theme toggle βββββββββββββββββββββββββββββββββββββ gr.HTML("""