Update app.py

app.py

@@ -1,321 +1,121 @@
 import gradio as gr
 import requests
 from bs4 import BeautifulSoup
-import re
-import time
 import os
-import json
-from datetime import datetime
-from pathlib import Path
-
-# ─────────────────────────────────────────────
-#  CONFIG
-# ─────────────────────────────────────────────
-BASE_URL      = "https://www.novelhall.com"
-NOVEL_RE      = re.compile(r"https://www\.novelhall\.com/[a-z0-9-]+-\d+/$")
-OUTPUT_DIR    = Path("outputs")
-OUTPUT_DIR.mkdir(exist_ok=True)
-
-HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/120.0.0.0 Safari/537.36"
-    ),
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "Accept-Language": "en-US,en;q=0.5",
-}
-
-SESSION = requests.Session()
-SESSION.headers.update(HEADERS)
-
-# ─────────────────────────────────────────────
-#  HELPERS
-# ─────────────────────────────────────────────
-def fetch_page(url: str, retries: int = 3, delay: float = 2.0) -> str | None:
-    """Fetch HTML from a URL with retries."""
-    for attempt in range(1, retries + 1):
-        try:
-            resp = SESSION.get(url, timeout=20)
-            resp.raise_for_status()
-            return resp.text
-        except Exception as exc:
-            if attempt < retries:
-                time.sleep(delay * attempt)
-            else:
-                return None
-
-
-def extract_novel_links(html: str) -> list[str]:
-    """Extract all novel detail-page links from a listing page."""
-    soup = BeautifulSoup(html, "html.parser")
-    links = []
-    seen = set()
-    for a in soup.find_all("a", href=NOVEL_RE):
-        href = a["href"]
-        if href not in seen:
-            seen.add(href)
-            links.append(href)
-    return links
-
-
-def extract_novel_info(html: str, url: str) -> dict:
-    """Extract title and description from a novel detail page."""
-    soup = BeautifulSoup(html, "html.parser")
-
-    # Title – prefer h1 inside book-info, fall back to <title>
-    title = ""
-    h1 = soup.find("h1")
-    if h1:
-        title = h1.get_text(strip=True)
-    if not title:
-        t = soup.find("title")
-        if t:
-            title = t.get_text(strip=True)
-
-    # Description – div.intro contains synopsis text
-    description = ""
-    intro_div = soup.find("div", class_="intro")
-    if intro_div:
-        # Remove the "...more>>" button text and clean duplicates
-        for tag in intro_div.find_all(["a", "span", "button"]):
-            tag.decompose()
-        raw = intro_div.get_text(separator=" ", strip=True)
-        # The site sometimes duplicates the synopsis; take the first half
-        half = len(raw) // 2
-        if half > 50 and raw[:half].strip() == raw[half:].strip():
-            raw = raw[:half].strip()
-        # Strip leading "Synopsis" label if present (may be fused: "SynopsisAt")
-        raw = re.sub(r"^Synopsis", "", raw, flags=re.IGNORECASE).strip()
-        description = raw
-
-    # Fall back to meta description
-    if not description:
-        meta = soup.find("meta", {"name": "description"})
-        if meta:
-            description = meta.get("content", "").strip()
-            description = re.sub(r"\.\.\.$", "", description).strip()
-
-    return {
-        "url": url,
-        "title": title,
-        "description": description,
-    }
-
-
-def build_list_url(base_pattern: str, page_num: int) -> str:
-    """
-    Construct a page URL given a pattern string.
-    Detects 'all2022-N' style URLs and replaces the number.
-    Falls back to appending ?page=N.
-    """
-    # Pattern: ends with -N.html or -N/ or -N
-    m = re.search(r"(-\d+)(\.html)?$", base_pattern)
-    if m:
-        prefix = base_pattern[: m.start()]
-        suffix = m.group(2) or ""
-        return f"{prefix}-{page_num}{suffix}"
-    # Fallback: append ?page=N
-    sep = "&" if "?" in base_pattern else "?"
-    return f"{base_pattern}{sep}page={page_num}"
-
-
-# ─────────────────────────────────────────────
-#  MAIN SCRAPER
-# ─────────────────────────────────────────────
-def scrape(
-    start_url: str,
-    end_page: int,
-    delay_sec: float,
-    save_html: bool,
-    progress=gr.Progress(track_tqdm=True),
-):
-    """
-    Core scraping function called by Gradio.
-    Returns (log_text, output_file_path).
-    """
-    start_url  = start_url.strip().rstrip("/")
-    logs       = []
-    all_novels = []
-    html_dir   = OUTPUT_DIR / "html_pages"
-    if save_html:
-        html_dir.mkdir(exist_ok=True)
-
-    def log(msg: str):
-        ts = datetime.now().strftime("%H:%M:%S")
-        line = f"[{ts}] {msg}"
-        logs.append(line)
-        return line
-
-    # Detect starting page number
-    m = re.search(r"-(\d+)(\.html)?$", start_url)
-    start_page = int(m.group(1)) if m else 1
-
-    log(f"Starting from: {start_url}")
-    log(f"Page range   : {start_page} → {end_page}")
-    log("─" * 60)
-
-    total_pages  = end_page - start_page + 1
-    novels_done  = 0
-
-    for page_idx, page_num in enumerate(
-        range(start_page, end_page + 1), start=1
-    ):
-        list_url = build_list_url(start_url, page_num)
-        yield "\n".join(logs + [log(f"📄 LIST PAGE {page_num}/{end_page} → {list_url}")]), None
-
-        html = fetch_page(list_url)
-        if html is None:
-            log(f"  ⚠️  Could not fetch listing page {page_num} – skipping")
-            yield "\n".join(logs), None
-            continue
-
-        novel_urls = extract_novel_links(html)
-        log(f"  Found {len(novel_urls)} novels on this page")
-
-        for idx, novel_url in enumerate(novel_urls, start=1):
-            yield "\n".join(
-                logs + [f"    [{idx}/{len(novel_urls)}] Fetching: {novel_url}"]
-            ), None
-
-            novel_html = fetch_page(novel_url)
-            if novel_html is None:
-                log(f"    ⚠️  Failed to fetch {novel_url}")
-                all_novels.append({
-                    "url": novel_url,
-                    "title": "FETCH_ERROR",
-                    "description": "",
-                })
-                yield "\n".join(logs), None
-                continue
-
-            info = extract_novel_info(novel_html, novel_url)
-            all_novels.append(info)
-            log(
-                f"    ✅ [{novels_done + 1}] {info['title'][:60]}"
-            )
-            novels_done += 1
-
-            # Optionally save the raw HTML
-            if save_html:
-                slug = re.sub(r"[^a-z0-9]+", "_", novel_url.split("/")[-2].lower())
-                html_path = html_dir / f"{slug}.html"
-                html_path.write_text(novel_html, encoding="utf-8")
-
-            time.sleep(delay_sec)
-            yield "\n".join(logs), None
-
-        time.sleep(delay_sec)  # polite pause between listing pages
-
-    # ── Write combined TXT output ──────────────────────────────────────
-    out_path = OUTPUT_DIR / f"novels_{start_page}_to_{end_page}.txt"
-    with out_path.open("w", encoding="utf-8") as fout:
-        fout.write(
-            f"Novelhall Novel Scrape — Pages {start_page}–{end_page}\n"
-            f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n"
-            f"Total novels: {len(all_novels)}\n"
-            f"{'=' * 70}\n\n"
-        )
-        for i, novel in enumerate(all_novels, start=1):
-            fout.write(f"[{i}]\n")
-            fout.write(f"TITLE      : {novel['title']}\n")
-            fout.write(f"URL        : {novel['url']}\n")
-            fout.write(f"DESCRIPTION:\n{novel['description']}\n")
-            fout.write(f"{'─' * 70}\n\n")
-
-    log("─" * 60)
-    log(f"✅ Done! {len(all_novels)} novels scraped.")
-    log(f"📁 Output saved → {out_path}")
-
-    yield "\n".join(logs), str(out_path)
-
-
-# ─────────────────────────────────────────────
-#  GRADIO UI
-# ─────────────────────────────────────────────
-CSS = """
-#log_box textarea { font-family: monospace; font-size: 12px; }
-.status-bar { font-size: 13px; color: #555; }
-"""
-
-with gr.Blocks(title="Novelhall Scraper", css=CSS) as demo:
-    gr.Markdown(
-        """
-# 📚 Novelhall Novel Scraper
-Scrapes every novel on Novelhall listing pages, extracts **title** and **description**,
-and saves a combined `.txt` file.
-
-**How it works:**
-1. Paste a listing-page URL like `https://www.novelhall.com/all2022-1.html`
-2. Set the **End Page** number (e.g. `337` for pages 1–337)
-3. Click **Start Scraping**
-        """
-    )
-
-    with gr.Row():
-        with gr.Column(scale=3):
-            start_url_box = gr.Textbox(
-                label="Start URL (first listing page)",
-                value="https://www.novelhall.com/all2022-1.html",
-                placeholder="https://www.novelhall.com/all2022-1.html",
-            )
-        with gr.Column(scale=1):
-            end_page_box = gr.Number(
-                label="End Page Number",
-                value=5,
-                minimum=1,
-                maximum=500,
-                step=1,
-            )
-
-    with gr.Row():
-        delay_slider = gr.Slider(
-            label="Delay between requests (seconds)",
-            minimum=0.5,
-            maximum=5.0,
-            value=1.5,
-            step=0.5,
-        )
-        save_html_cb = gr.Checkbox(
-            label="Also save raw HTML of each novel page",
-            value=False,
-        )
-
+import zipfile
+import shutil
+
+def scrape_novels(start_page, end_page, progress=gr.Progress()):
+    base_url = "https://www.novelhall.com/all2022-{}.html"
+    
+    # Reset and prepare the HTML directory for a fresh scrape
+    html_dir = "html_pages"
+    if os.path.exists(html_dir):
+        shutil.rmtree(html_dir)
+    os.makedirs(html_dir, exist_ok=True)
+    
+    combined_txt_path = "combined_novels.txt"
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+    
+    with open(combined_txt_path, 'w', encoding='utf-8') as combined_file:
+        
+        # Iterate over the specified page range
+        for page in progress.tqdm(range(int(start_page), int(end_page) + 1), desc="Scraping List Pages"):
+            page_url = base_url.format(page)
+            try:
+                response = requests.get(page_url, headers=headers, timeout=15)
+                soup = BeautifulSoup(response.content, 'html.parser')
+                
+                # Extract all novel links from the current list page
+                links = []
+                for li in soup.find_all('li', class_='btm'):
+                    a_tag = li.find('a')
+                    if a_tag and 'href' in a_tag.attrs:
+                        href = a_tag['href']
+                        if href.startswith('/'):
+                            href = "https://www.novelhall.com" + href
+                        links.append(href)
+                
+                # Visit each novel link individually
+                for link in links:
+                    try:
+                        novel_response = requests.get(link, headers=headers, timeout=15)
+                        novel_soup = BeautifulSoup(novel_response.content, 'html.parser')
+                        
+                        # 1. Extract Title
+                        title = "Unknown Title"
+                        info_div = novel_soup.find('div', class_='book-info')
+                        if info_div and info_div.find('h1'):
+                            title = info_div.find('h1').text.strip()
+                        elif novel_soup.find('h1'):
+                            title = novel_soup.find('h1').text.strip()
+                            
+                        # 2. Extract Description (Ensuring we don't scrape the truncated version)
+                        description = "Unknown Description"
+                        intro_div = novel_soup.find('div', class_='intro')
+                        if intro_div:
+                            close_wrap = intro_div.find('span', class_='js-close-wrap')
+                            if close_wrap:
+                                # Remove the "back<<" span text
+                                back_span = close_wrap.find('span', class_='blue')
+                                if back_span:
+                                    back_span.decompose()
+                                description = close_wrap.text.strip()
+                            else:
+                                description = intro_div.text.strip()
+                        
+                        # 3. Save the HTML file locally
+                        # Sanitize title to ensure the filename is valid
+                        safe_title = "".join(c for c in title if c.isalnum() or c in " -_").rstrip()
+                        if not safe_title:
+                            safe_title = link.strip('/').split('/')[-1]
+                            
+                        html_filename = f"{html_dir}/{safe_title}.html"
+                        with open(html_filename, 'w', encoding='utf-8') as html_file:
+                            html_file.write(novel_response.text)
+                                
+                        # 4. Write data to the Combined TXT File
+                        combined_file.write(f"Title: {title}\n")
+                        combined_file.write(f"Link: {link}\n")
+                        combined_file.write(f"Description:\n{description}\n")
+                        combined_file.write("-" * 80 + "\n\n")
+                        
+                    except Exception as e:
+                        combined_file.write(f"Error scraping novel {link}: {e}\n\n")
+                        
+            except Exception as e:
+                 combined_file.write(f"Error scraping list page {page_url}: {e}\n\n")
+                 
+    # Compress the text file and HTML directories into a downloadable ZIP
+    zip_path = "scraped_data.zip"
+    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        zipf.write(combined_txt_path, arcname=combined_txt_path)
+        for root, _, files in os.walk(html_dir):
+            for file in files:
+                file_path = os.path.join(root, file)
+                zipf.write(file_path, arcname=os.path.join(html_dir, file))
+                
+    return zip_path, f"Successfully processed pages {start_page} to {end_page}!"
+
+# Set up the UI logic
+with gr.Blocks(title="Novelhall Web Scraper") as demo:
+    gr.Markdown("# 📚 Novelhall Scraper Application")
+    gr.Markdown("Extracts individual novel links, saves their HTML structure, and compiles titles and full descriptions into a combined text file. **Note:** Large ranges take a long time to run; scrape in batches to prevent server timeouts.")
+    
     with gr.Row():
-        run_btn  = gr.Button("🚀 Start Scraping", variant="primary", scale=3)
-        stop_btn = gr.Button("⛔ Stop", variant="stop", scale=1)
-
-    log_box = gr.Textbox(
-        label="Live Log",
-        lines=20,
-        max_lines=40,
-        interactive=False,
-        elem_id="log_box",
-    )
-
-    out_file = gr.File(label="📥 Download Combined TXT", interactive=False)
-
-    # Wire up
-    scrape_event = run_btn.click(
-        fn=scrape,
-        inputs=[start_url_box, end_page_box, delay_slider, save_html_cb],
-        outputs=[log_box, out_file],
-    )
-    stop_btn.click(fn=None, cancels=[scrape_event])
-
-    gr.Markdown(
-        """
----
-### 📌 Notes
-- **URL pattern**: The scraper auto-detects `all2022-N` style pagination.
-  Pages `all2022-1.html` through `all2022-337.html` are all supported.
-- **Rate limiting**: Default 1.5 s delay is polite and avoids blocks.
-  Increase it if you get 429 errors.
-- **Output**: One `.txt` file with title, URL, and full description per novel.
-- **HTML saving**: Optional — saves each novel's main page as `<slug>.html`
-  in `outputs/html_pages/`.
-        """
+        start_pg = gr.Number(label="Start Page", value=1, precision=0)
+        end_pg = gr.Number(label="End Page", value=2, precision=0)
+        
+    scrape_btn = gr.Button("Start Scraping", variant="primary")
+    
+    output_msg = gr.Textbox(label="Status Tracker")
+    output_file = gr.File(label="Download Archive (Contains HTMLs & Combined TXT)")
+    
+    # Link the function
+    scrape_btn.click(
+        fn=scrape_novels, 
+        inputs=[start_pg, end_pg], 
+        outputs=[output_file, output_msg]
     )
 
-if __name__ == "__main__":
-    demo.launch()
+demo.launch()