import gradio as gr import requests from bs4 import BeautifulSoup import re import time import os import tempfile import json from urllib.parse import urlparse, urljoin import urllib.robotparser # ───────────────────────────────────────────── # HEADERS & SESSION # ───────────────────────────────────────────── HEADERS = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0.0.0 Safari/537.36" ), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", } SESSION = requests.Session() SESSION.headers.update(HEADERS) # ───────────────────────────────────────────── # SITE PROFILES # Each profile tells the scraper HOW to parse # a specific site's HTML structure. # ───────────────────────────────────────────── SITE_PROFILES = { # ── RoyalRoad ──────────────────────────── "royalroad.com": { "name": "Royal Road", "novel_title": ("h1", {"class": "font-white"}), "novel_author": ("span", {"property": "name"}), "chapter_list": ("table", {"id": "chapters"}), "chapter_links": ("a", {"href": re.compile(r"/fiction/\d+/chapter/")}), "chapter_title": ("h1", {"class": "font-white"}), "content_sel": ("div", {"class": "chapter-content"}), "next_page": None, "delay": 1.2, }, # ── ScribbleHub ────────────────────────── "scribblehub.com": { "name": "ScribbleHub", "novel_title": ("div", {"class": "fic_title"}), "novel_author": ("span", {"class": "auth_name_fic"}), "chapter_list": None, # uses API "api_toc": "https://www.scribblehub.com/wp-admin/admin-ajax.php", "chapter_links": ("a", {"class": "toc_a"}), "chapter_title": ("div", {"class": "chapter-title"}), "content_sel": ("div", {"class": "chp_raw"}), "next_page": ("a", {"class": "btn-chapter-next"}), "delay": 1.5, }, # ── WuxiaWorld ─────────────────────────── "wuxiaworld.com": { "name": "WuxiaWorld", "novel_title": ("h3", {"class": "novel-title"}), "novel_author": ("span", {"class": "author"}), "chapter_list": ("div", {"class": "chapter-list"}), "chapter_links": ("a", {"href": re.compile(r"/novel/[^/]+/[^/]+-chapter")}), "chapter_title": ("h4", {}), "content_sel": ("div", {"class": "chapter-content"}), "next_page": None, "delay": 1.5, }, # ── NovelFull / NovelFull.com ───────────── "novelfull.com": { "name": "NovelFull", "novel_title": ("h3", {"class": "title"}), "novel_author": ("a", {"itemprop": "author"}), "chapter_list": ("ul", {"class": "list-chapter"}), "chapter_links": ("a", {"href": re.compile(r"\.html")}), "chapter_title": ("a", {"class": "chr-title"}), "content_sel": ("div", {"id": "chapter-content"}), "next_page": ("a", {"id": "next_chap"}), "delay": 1.0, }, # ── LightNovelPub ──────────────────────── "lightnovelpub.com": { "name": "LightNovelPub", "novel_title": ("h1", {"class": "novel-title"}), "novel_author": ("span", {"itemprop": "author"}), "chapter_list": ("ul", {"class": "chapter-list"}), "chapter_links": ("a", {"href": re.compile(r"/novel/.+/chapter-\d+")}), "chapter_title": ("span", {"class": "chapter-title"}), "content_sel": ("div", {"class": "chapter-content"}), "next_page": ("a", {"class": "btn-next"}), "delay": 1.2, }, # ── FreeWebNovel ───────────────────────── "freewebnovel.com": { "name": "FreeWebNovel", "novel_title": ("h1", {"class": "pc_title"}), "novel_author": ("a", {"class": "a1"}), "chapter_list": ("ul", {"class": "ul-list5"}), "chapter_links": ("a", {"href": re.compile(r"\.html")}), "chapter_title": ("h1", {"class": "chapter-title"}), "content_sel": ("div", {"class": "txt"}), "next_page": ("a", {"id": "idnext"}), "delay": 1.0, }, # ── NovelUpdates (info page, links out) ── "novelupdates.com": { "name": "NovelUpdates", "novel_title": ("div", {"class": "seriestitlenu"}), "novel_author": ("div", {"id": "authtag"}), "chapter_list": ("table", {"id": "myTable"}), "chapter_links": ("a", {"class": "chp-release"}), "chapter_title": None, "content_sel": None, "next_page": None, "delay": 1.5, "_external_links": True, # chapters hosted elsewhere }, # ── WebNovel.com ───────────────────────── "webnovel.com": { "name": "WebNovel", "novel_title": ("h2", {"class": "dib"}), "novel_author": ("address", {}), "chapter_list": ("ol", {"class": "c_list"}), "chapter_links": ("a", {"href": re.compile(r"/book/\d+/\d+")}), "chapter_title": ("h3", {"class": "cha-tit"}), "content_sel": ("div", {"class": "cha-words"}), "next_page": ("a", {"class": "_nextChapter"}), "delay": 2.0, }, # ── Wattpad ────────────────────────────── "wattpad.com": { "name": "Wattpad", "novel_title": ("h1", {"class": "story-title"}), "novel_author": ("strong", {"class": "author-info__username"}), "chapter_list": ("ul", {"class": "table-of-contents"}), "chapter_links": ("a", {"href": re.compile(r"/story/\d+.+")}), "chapter_title": ("h2", {"class": "part-title"}), "content_sel": ("pre", {"class": ""}), "next_page": ("a", {"class": "next-part-link"}), "delay": 1.5, }, # ── Moonquill ──────────────────────────── "moonquill.com": { "name": "Moonquill", "novel_title": ("h1", {"class": "title"}), "novel_author": ("a", {"class": "author"}), "chapter_list": ("div", {"class": "table-of-contents"}), "chapter_links": ("a", {"href": re.compile(r"/book/.+")}), "chapter_title": ("h1", {"class": "title"}), "content_sel": ("div", {"class": "chapter-body"}), "next_page": None, "delay": 1.0, }, } # ───────────────────────────────────────────── # GENERIC FALLBACK HEURISTICS # ───────────────────────────────────────────── GENERIC_CONTENT_SELECTORS = [ ("div", {"id": "chapter-content"}), ("div", {"class": "chapter-content"}), ("div", {"class": "content"}), ("div", {"class": "entry-content"}), ("div", {"id": "content"}), ("article", {}), ("div", {"class": "post-content"}), ("div", {"class": "text-left"}), ("div", {"class": "novel-content"}), ("div", {"class": "read-content"}), ] GENERIC_TITLE_SELECTORS = [ ("h1", {}), ("h2", {}), ("h3", {"class": "chapter-title"}), ("title", {}), ] # ───────────────────────────────────────────── # HELPER UTILITIES # ───────────────────────────────────────────── def get_domain(url: str) -> str: parsed = urlparse(url) domain = parsed.netloc.lower().replace("www.", "") return domain def safe_get(url: str, retries: int = 3, delay: float = 1.0): for attempt in range(retries): try: resp = SESSION.get(url, timeout=20) if resp.status_code == 200: return resp if resp.status_code == 429: time.sleep(delay * (attempt + 1) * 3) else: time.sleep(delay) except Exception as e: time.sleep(delay * (attempt + 1)) return None def extract_text_blocks(soup_element) -> str: """Extract clean paragraphs from a BS4 element.""" if soup_element is None: return "" # Remove script/style/ads for tag in soup_element(["script", "style", "noscript", "iframe", "ins", "aside"]): tag.decompose() # Build paragraphs paras = [] for p in soup_element.find_all(["p", "br"]): text = p.get_text(separator=" ", strip=True) if text: paras.append(text) if not paras: # fallback: just grab all text raw = soup_element.get_text(separator="\n", strip=True) paras = [line for line in raw.splitlines() if line.strip()] return "\n\n".join(paras) def clean_title(title: str) -> str: return re.sub(r'[<>:"/\\|?*\x00-\x1F]', '', title).strip() # ───────────────────────────────────────────── # CORE SCRAPER # ───────────────────────────────────────────── def detect_profile(domain: str): for key, profile in SITE_PROFILES.items(): if key in domain: return profile return None def get_toc_links(main_url: str, soup: BeautifulSoup, profile, log_fn) -> list: """Return list of (chapter_title, chapter_url).""" domain = get_domain(main_url) base = f"{urlparse(main_url).scheme}://{urlparse(main_url).netloc}" links = [] # ── ScribbleHub uses AJAX for TOC ──────── if profile and "api_toc" in profile and "scribblehub" in domain: fic_id_match = re.search(r"scribblehub\.com/series/(\d+)/", main_url) if fic_id_match: fic_id = fic_id_match.group(1) data = {"action": "wi_gettocchp", "strSID": fic_id, "strmypostid": "0", "strFic": "yes"} resp = SESSION.post(profile["api_toc"], data=data, timeout=20) if resp.ok: toc_soup = BeautifulSoup(resp.text, "html.parser") for a in reversed(toc_soup.find_all("a")): href = a.get("href", "") title = a.get_text(strip=True) if href and title: links.append((title, href)) log_fn(f" ✅ ScribbleHub AJAX TOC: {len(links)} chapters") return links # ── Standard: find chapter list container ─ container = None if profile and profile.get("chapter_list"): tag, attrs = profile["chapter_list"] container = soup.find(tag, attrs) if attrs else soup.find(tag) search_root = container if container else soup link_sel = (profile.get("chapter_links") if profile else None) or \ ("a", {"href": re.compile(r"chapter|chap|ch-\d+", re.I)}) tag, attrs = link_sel found = search_root.find_all(tag, attrs) seen = set() for a in found: href = a.get("href", "") if not href or href in seen: continue seen.add(href) full_url = href if href.startswith("http") else urljoin(base, href) title = a.get_text(strip=True) or f"Chapter {len(links)+1}" links.append((title, full_url)) log_fn(f" 📋 Found {len(links)} chapter links") return links def scrape_chapter(url: str, profile, log_fn) -> tuple: """Returns (chapter_title, chapter_text).""" resp = safe_get(url, delay=profile.get("delay", 1.2) if profile else 1.2) if not resp: log_fn(f" ⚠️ Failed to fetch: {url}") return None, None soup = BeautifulSoup(resp.text, "html.parser") # ── Title ── chapter_title = "" if profile and profile.get("chapter_title"): t, a = profile["chapter_title"] el = soup.find(t, a) if el: chapter_title = el.get_text(strip=True) if not chapter_title: for t, a in GENERIC_TITLE_SELECTORS: el = soup.find(t, a) if el: chapter_title = el.get_text(strip=True) break # ── Content ── content_el = None if profile and profile.get("content_sel"): t, a = profile["content_sel"] content_el = soup.find(t, a) if not content_el: for t, a in GENERIC_CONTENT_SELECTORS: content_el = soup.find(t, a) if content_el: break if not content_el: # Last resort: largest
by text length divs = soup.find_all("div") if divs: content_el = max(divs, key=lambda d: len(d.get_text())) text = extract_text_blocks(content_el) return chapter_title, text # ───────────────────────────────────────────── # MAIN ORCHESTRATOR # ───────────────────────────────────────────── def scrape_novel(url: str, max_chapters: int, include_metadata: bool, chapter_delay: float, progress=gr.Progress()): logs = [] chapters_data = [] def log(msg): logs.append(msg) return "\n".join(logs) url = url.strip() if not url.startswith("http"): url = "https://" + url domain = get_domain(url) profile = detect_profile(domain) site_name = profile["name"] if profile else domain yield (None, f"🔍 Detected site: **{site_name}**\n" f"🌐 URL: {url}\n" f"{'✅ Known site profile loaded' if profile else '⚠️ Unknown site — using generic heuristics'}\n") # ── Fetch main/TOC page ────────────────── log(f"\n📥 Fetching novel index page...") resp = safe_get(url) if not resp: yield (None, log("❌ Could not reach the URL. Check your link.")) return soup = BeautifulSoup(resp.text, "html.parser") # ── Novel title & author ───────────────── novel_title = "Unknown Novel" novel_author = "Unknown Author" if profile and profile.get("novel_title"): t, a = profile["novel_title"] el = soup.find(t, a) if el: novel_title = el.get_text(strip=True) if not novel_title or novel_title == "Unknown Novel": h1 = soup.find("h1") if h1: novel_title = h1.get_text(strip=True) if profile and profile.get("novel_author"): t, a = profile["novel_author"] el = soup.find(t, a) if el: novel_author = el.get_text(strip=True) novel_title = clean_title(novel_title) log(f"📖 Title: {novel_title}") log(f"✍️ Author: {novel_author}") yield (None, "\n".join(logs)) # ── Gather chapter links ───────────────── log("\n🔎 Collecting chapter list...") yield (None, "\n".join(logs)) toc_links = get_toc_links(url, soup, profile, log) yield (None, "\n".join(logs)) if not toc_links: # Maybe the URL IS already a chapter — scrape just that log("⚠️ No TOC found. Attempting to scrape URL as single chapter...") ch_title, ch_text = scrape_chapter(url, profile, log) if ch_text: toc_links = [(ch_title or "Chapter 1", url)] else: yield (None, log("❌ No chapters found and single-page scrape failed.")) return # ── Respect max_chapters cap ───────────── if max_chapters > 0: toc_links = toc_links[:max_chapters] log(f"🔢 Capped to {max_chapters} chapters (as requested).") total = len(toc_links) log(f"\n🚀 Starting scrape of {total} chapters...\n{'─'*40}") yield (None, "\n".join(logs)) delay = chapter_delay if chapter_delay > 0 else ( profile.get("delay", 1.2) if profile else 1.2 ) # ── Scrape each chapter ────────────────── for idx, (ch_link_title, ch_url) in enumerate(toc_links, 1): progress((idx - 1) / total, desc=f"Chapter {idx}/{total}") log(f"[{idx:>4}/{total}] {ch_link_title[:60]}") ch_title, ch_text = scrape_chapter(ch_url, profile, log) display_title = ch_title or ch_link_title or f"Chapter {idx}" if not ch_text: log(f" ⚠️ Empty content — skipping") chapters_data.append((display_title, "(Content unavailable)")) else: word_count = len(ch_text.split()) log(f" ✅ {word_count:,} words") chapters_data.append((display_title, ch_text)) yield (None, "\n".join(logs)) time.sleep(delay) # ── Build output TXT ───────────────────── log(f"\n{'─'*40}") log("📝 Building output file...") lines = [] if include_metadata: lines += [ "═" * 60, f" {novel_title}", f" by {novel_author}", "═" * 60, f" Source: {url}", f" Site: {site_name}", f" Chapters: {len(chapters_data)}", f" Total words: {sum(len(c[1].split()) for c in chapters_data):,}", "═" * 60, "", ] for i, (ch_title, ch_text) in enumerate(chapters_data, 1): lines += [ "", "─" * 60, f" Chapter {i}: {ch_title}", "─" * 60, "", ch_text, "", ] full_text = "\n".join(lines) # ── Save to temp file ──────────────────── safe_name = re.sub(r'\s+', '_', clean_title(novel_title))[:80] tmp = tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", suffix=".txt", prefix=f"{safe_name}_", delete=False ) tmp.write(full_text) tmp.close() total_words = sum(len(c[1].split()) for c in chapters_data) log(f"\n🎉 DONE!") log(f" 📚 {len(chapters_data)} chapters scraped") log(f" 📝 {total_words:,} total words") log(f" 💾 File ready for download!") yield (tmp.name, "\n".join(logs)) # ───────────────────────────────────────────── # GRADIO UI # ───────────────────────────────────────────── SUPPORTED_SITES = "\n".join( f"• {p['name']} ({k})" for k, p in SITE_PROFILES.items() ) THEME = gr.themes.Base( primary_hue="violet", secondary_hue="purple", neutral_hue="slate", font=[gr.themes.GoogleFont("Syne"), gr.themes.GoogleFont("JetBrains Mono")], ).set( body_background_fill="#0f0f13", body_text_color="#e8e6f0", block_background_fill="#1a1a24", block_border_color="#2d2d40", input_background_fill="#12121a", button_primary_background_fill="*primary_500", ) CSS = """ body { background: #0f0f13 !important; } .title-block { text-align: center; padding: 2rem 0 1rem; background: linear-gradient(135deg, #1a0530 0%, #0d1a3a 50%, #1a0530 100%); border-radius: 16px; margin-bottom: 1rem; border: 1px solid #2a1a4a; } .title-block h1 { font-size: 2.4rem; background: linear-gradient(90deg, #b06ef3, #6ea8f7, #b06ef3); background-size: 200%; -webkit-background-clip: text; -webkit-text-fill-color: transparent; animation: shimmer 3s linear infinite; font-family: 'Syne', sans-serif; font-weight: 800; letter-spacing: -1px; margin-bottom: 0.3rem; } @keyframes shimmer { 0%{background-position:0%} 100%{background-position:200%} } .title-block p { color: #9090b0; font-size: 0.95rem; margin: 0; } .log-box textarea { font-family: 'JetBrains Mono', monospace !important; font-size: 0.78rem !important; line-height: 1.6 !important; color: #a0e8a0 !important; background: #050a05 !important; } .site-list { background: #12121a; border: 1px solid #2a2a3a; border-radius: 8px; padding: 1rem 1.2rem; font-size: 0.82rem; color: #8080a0; line-height: 1.9; } .gr-button-primary { background: linear-gradient(135deg, #7c3aed, #3b82f6) !important; border: none !important; font-size: 1rem !important; font-weight: 700 !important; letter-spacing: 0.5px; transition: all 0.2s !important; } .gr-button-primary:hover { transform: translateY(-1px) !important; box-shadow: 0 4px 20px rgba(124, 58, 237, 0.5) !important; } """ with gr.Blocks(theme=THEME, css=CSS, title="📚 WebNovel Scraper") as demo: gr.HTML("""

📚 WebNovel Scraper

Paste any novel URL · Get a clean .txt file · Works on 10+ sites

""") with gr.Row(): with gr.Column(scale=3): url_input = gr.Textbox( label="🔗 Novel URL", placeholder="https://www.royalroad.com/fiction/12345/your-novel-title", lines=1, ) with gr.Row(): max_ch = gr.Slider( label="📦 Max Chapters (0 = all)", minimum=0, maximum=1000, step=1, value=0, ) delay_sl = gr.Slider( label="⏱️ Delay Between Requests (sec)", minimum=0.5, maximum=5.0, step=0.1, value=1.5, ) meta_cb = gr.Checkbox( label="📋 Include Metadata Header in Output", value=True, ) scrape_btn = gr.Button("⚡ SCRAPE NOVEL", variant="primary", size="lg") file_out = gr.File( label="📥 Download Scraped Novel (.txt)", visible=True, interactive=False, ) with gr.Column(scale=2): gr.HTML(f"""
✅ Supported Sites

{"
".join(f"• {p['name']} ({k})" for k, p in SITE_PROFILES.items())}

⚙️ Generic mode activates automatically for unlisted sites — heuristics detect chapters & content on most novel platforms.
""") log_out = gr.Textbox( label="📜 Scraping Log", lines=18, max_lines=30, interactive=False, elem_classes=["log-box"], placeholder="Logs will appear here once you start scraping...", ) gr.HTML("""
⚠️ Please respect each site's Terms of Service. Use responsibly and avoid hammering servers. Built with ❤️ using Python + Gradio + BeautifulSoup4.
""") scrape_btn.click( fn=scrape_novel, inputs=[url_input, max_ch, meta_cb, delay_sl], outputs=[file_out, log_out], ) if __name__ == "__main__": demo.launch()