Update app.py

app.py

@@ -1,121 +1,125 @@
-import gradio as gr
+"""
+WebNovel link scraper
+----------------------
+Crawls WebNovel's genre/tag/search listing pages and collects every unique
+novel ("/book/...") link it finds, then writes them all to a .txt file.
+
+Run on Hugging Face (Space / Notebook / Colab-style env) or locally:
+    pip install requests beautifulsoup4
+    python webnovel_link_scraper.py
+
+Notes:
+- WebNovel loads a lot of its catalog dynamically via JS/AJAX. This script
+  only sees server-rendered HTML, so it will pick up a large subset of
+  novels (everything reachable from listing/search pages) rather than
+  literally every title on the site.
+- Be a good citizen: there's a delay between requests, and you should check
+  https://www.webnovel.com/robots.txt and their Terms of Service before
+  running this at scale or pointing it at many pages.
+"""
+
+import re
+import time
+import random
+from urllib.parse import urljoin
+
 import requests
 from bs4 import BeautifulSoup
-import os
-import zipfile
-import shutil
-
-def scrape_novels(start_page, end_page, progress=gr.Progress()):
-    base_url = "https://www.novelhall.com/all2022-{}.html"
-    
-    # Reset and prepare the HTML directory for a fresh scrape
-    html_dir = "html_pages"
-    if os.path.exists(html_dir):
-        shutil.rmtree(html_dir)
-    os.makedirs(html_dir, exist_ok=True)
-    
-    combined_txt_path = "combined_novels.txt"
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
-    
-    with open(combined_txt_path, 'w', encoding='utf-8') as combined_file:
-        
-        # Iterate over the specified page range
-        for page in progress.tqdm(range(int(start_page), int(end_page) + 1), desc="Scraping List Pages"):
-            page_url = base_url.format(page)
-            try:
-                response = requests.get(page_url, headers=headers, timeout=15)
-                soup = BeautifulSoup(response.content, 'html.parser')
-                
-                # Extract all novel links from the current list page
-                links = []
-                for li in soup.find_all('li', class_='btm'):
-                    a_tag = li.find('a')
-                    if a_tag and 'href' in a_tag.attrs:
-                        href = a_tag['href']
-                        if href.startswith('/'):
-                            href = "https://www.novelhall.com" + href
-                        links.append(href)
-                
-                # Visit each novel link individually
-                for link in links:
-                    try:
-                        novel_response = requests.get(link, headers=headers, timeout=15)
-                        novel_soup = BeautifulSoup(novel_response.content, 'html.parser')
-                        
-                        # 1. Extract Title
-                        title = "Unknown Title"
-                        info_div = novel_soup.find('div', class_='book-info')
-                        if info_div and info_div.find('h1'):
-                            title = info_div.find('h1').text.strip()
-                        elif novel_soup.find('h1'):
-                            title = novel_soup.find('h1').text.strip()
-                            
-                        # 2. Extract Description (Ensuring we don't scrape the truncated version)
-                        description = "Unknown Description"
-                        intro_div = novel_soup.find('div', class_='intro')
-                        if intro_div:
-                            close_wrap = intro_div.find('span', class_='js-close-wrap')
-                            if close_wrap:
-                                # Remove the "back<<" span text
-                                back_span = close_wrap.find('span', class_='blue')
-                                if back_span:
-                                    back_span.decompose()
-                                description = close_wrap.text.strip()
-                            else:
-                                description = intro_div.text.strip()
-                        
-                        # 3. Save the HTML file locally
-                        # Sanitize title to ensure the filename is valid
-                        safe_title = "".join(c for c in title if c.isalnum() or c in " -_").rstrip()
-                        if not safe_title:
-                            safe_title = link.strip('/').split('/')[-1]
-                            
-                        html_filename = f"{html_dir}/{safe_title}.html"
-                        with open(html_filename, 'w', encoding='utf-8') as html_file:
-                            html_file.write(novel_response.text)
-                                
-                        # 4. Write data to the Combined TXT File
-                        combined_file.write(f"Title: {title}\n")
-                        combined_file.write(f"Link: {link}\n")
-                        combined_file.write(f"Description:\n{description}\n")
-                        combined_file.write("-" * 80 + "\n\n")
-                        
-                    except Exception as e:
-                        combined_file.write(f"Error scraping novel {link}: {e}\n\n")
-                        
-            except Exception as e:
-                 combined_file.write(f"Error scraping list page {page_url}: {e}\n\n")
-                 
-    # Compress the text file and HTML directories into a downloadable ZIP
-    zip_path = "scraped_data.zip"
-    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-        zipf.write(combined_txt_path, arcname=combined_txt_path)
-        for root, _, files in os.walk(html_dir):
-            for file in files:
-                file_path = os.path.join(root, file)
-                zipf.write(file_path, arcname=os.path.join(html_dir, file))
-                
-    return zip_path, f"Successfully processed pages {start_page} to {end_page}!"
-
-# Set up the UI logic
-with gr.Blocks(title="Novelhall Web Scraper") as demo:
-    gr.Markdown("# 📚 Novelhall Scraper Application")
-    gr.Markdown("Extracts individual novel links, saves their HTML structure, and compiles titles and full descriptions into a combined text file. **Note:** Large ranges take a long time to run; scrape in batches to prevent server timeouts.")
-    
-    with gr.Row():
-        start_pg = gr.Number(label="Start Page", value=1, precision=0)
-        end_pg = gr.Number(label="End Page", value=2, precision=0)
-        
-    scrape_btn = gr.Button("Start Scraping", variant="primary")
-    
-    output_msg = gr.Textbox(label="Status Tracker")
-    output_file = gr.File(label="Download Archive (Contains HTMLs & Combined TXT)")
-    
-    # Link the function
-    scrape_btn.click(
-        fn=scrape_novels, 
-        inputs=[start_pg, end_pg], 
-        outputs=[output_file, output_msg]
+
+BASE_URL = "https://www.webnovel.com"
+OUTPUT_FILE = "webnovel_links.txt"
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
     )
+}
+
+# A handful of starting points that list lots of novels.
+# Add/remove genre or tag slugs as you like.
+SEED_PATHS = [
+    "/category/all",
+    "/category/all?orderBy=0",          # popularity sort, if supported
+    "/category/all?orderBy=1",          # other sort options
+    "/tags/fantasy",
+    "/tags/romance",
+    "/tags/action",
+    "/tags/adventure",
+    "/tags/r18",
+    "/tags/system",
+]
+
+# Matches links like /book/some-novel-title_12345678905012345
+BOOK_LINK_RE = re.compile(r"^/book/[\w\-]+$")
+
+MAX_PAGES_PER_SEED = 20      # how many "page=" pages to walk per seed URL
+REQUEST_DELAY_RANGE = (1.0, 2.5)   # polite random delay between requests
+
+
+def fetch(url: str) -> str | None:
+    try:
+        resp = requests.get(url, headers=HEADERS, timeout=15)
+        if resp.status_code == 200:
+            return resp.text
+        print(f"  [!] {resp.status_code} for {url}")
+    except requests.RequestException as e:
+        print(f"  [!] request failed for {url}: {e}")
+    return None
+
+
+def extract_book_links(html: str) -> set[str]:
+    soup = BeautifulSoup(html, "html.parser")
+    found = set()
+    for a in soup.find_all("a", href=True):
+        href = a["href"]
+        # Normalize relative/absolute links pointing at /book/...
+        if href.startswith(BASE_URL):
+            href = href[len(BASE_URL):]
+        if BOOK_LINK_RE.match(href):
+            found.add(urljoin(BASE_URL, href))
+    return found
+
+
+def crawl():
+    all_links: set[str] = set()
+
+    for seed in SEED_PATHS:
+        print(f"\n=== Seed: {seed} ===")
+        for page in range(1, MAX_PAGES_PER_SEED + 1):
+            sep = "&" if "?" in seed else "?"
+            url = f"{BASE_URL}{seed}{sep}pageIndex={page}"
+            print(f"Fetching {url}")
+
+            html = fetch(url)
+            if not html:
+                break
+
+            new_links = extract_book_links(html)
+            before = len(all_links)
+            all_links.update(new_links)
+            added = len(all_links) - before
+            print(f"  found {len(new_links)} links on page, {added} new")
+
+            # Stop walking this seed once a page yields nothing new
+            if added == 0 and page > 1:
+                break
+
+            time.sleep(random.uniform(*REQUEST_DELAY_RANGE))
+
+    return all_links
+
+
+def main():
+    links = crawl()
+    print(f"\nTotal unique novel links collected: {len(links)}")
+
+    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
+        for link in sorted(links):
+            f.write(link + "\n")
+
+    print(f"Saved to {OUTPUT_FILE}")
+
 
-demo.launch()
+if __name__ == "__main__":
+    main()