Update app.py

app.py

@@ -1,125 +1,248 @@
 """
-WebNovel link scraper
-----------------------
-Crawls WebNovel's genre/tag/search listing pages and collects every unique
-novel ("/book/...") link it finds, then writes them all to a .txt file.
-
-Run on Hugging Face (Space / Notebook / Colab-style env) or locally:
-    pip install requests beautifulsoup4
-    python webnovel_link_scraper.py
-
-Notes:
-- WebNovel loads a lot of its catalog dynamically via JS/AJAX. This script
-  only sees server-rendered HTML, so it will pick up a large subset of
-  novels (everything reachable from listing/search pages) rather than
-  literally every title on the site.
-- Be a good citizen: there's a delay between requests, and you should check
-  https://www.webnovel.com/robots.txt and their Terms of Service before
-  running this at scale or pointing it at many pages.
+Link Scraper — Hugging Face Space
+----------------------------------
+Upload a .txt file containing one URL per line.
+The app will:
+  1. Visit each URL one by one (or with a few parallel workers)
+  2. Save the raw webpage as an .html file
+  3. Extract the page <title> and meta description
+  4. Produce one combined .txt file with: URL | Title | Description
+  5. Zip all the saved .html files for download
+
+Built for Gradio on Hugging Face Spaces.
 """
 
+import os
 import re
+import io
 import time
+import zipfile
 import random
-from urllib.parse import urljoin
+import concurrent.futures
+from urllib.parse import urlparse
 
 import requests
 from bs4 import BeautifulSoup
+import gradio as gr
 
-BASE_URL = "https://www.webnovel.com"
-OUTPUT_FILE = "webnovel_links.txt"
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
 
-HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-        "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
-    )
-}
-
-# A handful of starting points that list lots of novels.
-# Add/remove genre or tag slugs as you like.
-SEED_PATHS = [
-    "/category/all",
-    "/category/all?orderBy=0",          # popularity sort, if supported
-    "/category/all?orderBy=1",          # other sort options
-    "/tags/fantasy",
-    "/tags/romance",
-    "/tags/action",
-    "/tags/adventure",
-    "/tags/r18",
-    "/tags/system",
-]
-
-# Matches links like /book/some-novel-title_12345678905012345
-BOOK_LINK_RE = re.compile(r"^/book/[\w\-]+$")
-
-MAX_PAGES_PER_SEED = 20      # how many "page=" pages to walk per seed URL
-REQUEST_DELAY_RANGE = (1.0, 2.5)   # polite random delay between requests
-
-
-def fetch(url: str) -> str | None:
-    try:
-        resp = requests.get(url, headers=HEADERS, timeout=15)
-        if resp.status_code == 200:
-            return resp.text
-        print(f"  [!] {resp.status_code} for {url}")
-    except requests.RequestException as e:
-        print(f"  [!] request failed for {url}: {e}")
-    return None
-
-
-def extract_book_links(html: str) -> set[str]:
-    soup = BeautifulSoup(html, "html.parser")
-    found = set()
-    for a in soup.find_all("a", href=True):
-        href = a["href"]
-        # Normalize relative/absolute links pointing at /book/...
-        if href.startswith(BASE_URL):
-            href = href[len(BASE_URL):]
-        if BOOK_LINK_RE.match(href):
-            found.add(urljoin(BASE_URL, href))
-    return found
-
-
-def crawl():
-    all_links: set[str] = set()
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+)
 
-    for seed in SEED_PATHS:
-        print(f"\n=== Seed: {seed} ===")
-        for page in range(1, MAX_PAGES_PER_SEED + 1):
-            sep = "&" if "?" in seed else "?"
-            url = f"{BASE_URL}{seed}{sep}pageIndex={page}"
-            print(f"Fetching {url}")
+WORK_DIR = "/tmp/link_scraper"
+HTML_DIR = os.path.join(WORK_DIR, "html_pages")
+os.makedirs(HTML_DIR, exist_ok=True)
 
-            html = fetch(url)
-            if not html:
-                break
 
-            new_links = extract_book_links(html)
-            before = len(all_links)
-            all_links.update(new_links)
-            added = len(all_links) - before
-            print(f"  found {len(new_links)} links on page, {added} new")
+def slugify_for_filename(url: str, index: int) -> str:
+    """Turn a URL into a safe, short filename."""
+    parsed = urlparse(url)
+    path = (parsed.netloc + parsed.path).strip("/")
+    path = re.sub(r"[^a-zA-Z0-9._-]+", "_", path)
+    path = path[:120]  # keep filenames from getting too long
+    if not path:
+        path = "page"
+    return f"{index:05d}_{path}.html"
 
-            # Stop walking this seed once a page yields nothing new
-            if added == 0 and page > 1:
-                break
 
-            time.sleep(random.uniform(*REQUEST_DELAY_RANGE))
-
-    return all_links
+def extract_title_and_description(html: str) -> tuple[str, str]:
+    """Pull the best available title and description from a page's HTML."""
+    soup = BeautifulSoup(html, "html.parser")
 
+    title = ""
+    og_title = soup.find("meta", property="og:title")
+    if og_title and og_title.get("content"):
+        title = og_title["content"].strip()
+    if not title and soup.title and soup.title.string:
+        title = soup.title.string.strip()
+
+    description = ""
+    meta_desc = soup.find("meta", attrs={"name": "description"})
+    if meta_desc and meta_desc.get("content"):
+        description = meta_desc["content"].strip()
+    if not description:
+        og_desc = soup.find("meta", property="og:description")
+        if og_desc and og_desc.get("content"):
+            description = og_desc["content"].strip()
+
+    # Keep everything on a single line in the combined output
+    title = re.sub(r"\s+", " ", title)
+    description = re.sub(r"\s+", " ", description)
+    return title, description
+
+
+def fetch_one(index: int, url: str, save_html: bool, timeout: int):
+    """Fetch a single URL. Returns a result dict; never raises."""
+    result = {
+        "index": index,
+        "url": url,
+        "title": "",
+        "description": "",
+        "status": "",
+        "html_filename": "",
+    }
+    try:
+        resp = requests.get(
+            url,
+            headers={"User-Agent": USER_AGENT},
+            timeout=timeout,
+        )
+        result["status"] = f"HTTP {resp.status_code}"
+        resp.raise_for_status()
+        html = resp.text
+
+        if save_html:
+            filename = slugify_for_filename(url, index)
+            filepath = os.path.join(HTML_DIR, filename)
+            with open(filepath, "w", encoding="utf-8") as f:
+                f.write(html)
+            result["html_filename"] = filename
+
+        title, description = extract_title_and_description(html)
+        result["title"] = title
+        result["description"] = description
+
+    except Exception as e:  # noqa: BLE001
+        result["status"] = f"ERROR: {e}"
+
+    return result
+
+
+def zip_html_folder(zip_path: str):
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+        for fname in os.listdir(HTML_DIR):
+            zf.write(os.path.join(HTML_DIR, fname), arcname=fname)
+
+
+def run_scraper(
+    file_obj,
+    save_html: bool,
+    workers: int,
+    delay_seconds: float,
+    timeout_seconds: int,
+    progress=gr.Progress(track_tqdm=False),
+):
+    """Main generator driving the Gradio UI: yields (log_text, combined_file, zip_file)."""
+
+    if file_obj is None:
+        yield "Please upload a .txt file with one URL per line.", None, None
+        return
+
+    # Reset working folders for a clean run
+    if os.path.exists(HTML_DIR):
+        for f in os.listdir(HTML_DIR):
+            try:
+                os.remove(os.path.join(HTML_DIR, f))
+            except OSError:
+                pass
+    os.makedirs(HTML_DIR, exist_ok=True)
+
+    with open(file_obj, "r", encoding="utf-8", errors="ignore") as f:
+        urls = [line.strip() for line in f if line.strip()]
+
+    total = len(urls)
+    if total == 0:
+        yield "The uploaded file has no URLs in it.", None, None
+        return
+
+    log_lines = [f"Found {total} URLs. Starting...\n"]
+    yield "".join(log_lines), None, None
+
+    results = [None] * total
+    workers = max(1, min(int(workers), 16))
+
+    def worker_task(i, url):
+        if delay_seconds > 0:
+            time.sleep(delay_seconds * random.uniform(0.5, 1.5))
+        return fetch_one(i, url, save_html, int(timeout_seconds))
+
+    completed = 0
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = {
+            executor.submit(worker_task, i, url): i for i, url in enumerate(urls)
+        }
+        for future in concurrent.futures.as_completed(futures):
+            i = futures[future]
+            res = future.result()
+            results[i] = res
+            completed += 1
+
+            short_url = res["url"][:70]
+            log_lines.append(f"[{completed}/{total}] {res['status']} - {short_url}")
+            progress(completed / total, desc=f"{completed}/{total} pages done")
+
+            # Stream log updates periodically (every 5 items or near the end)
+            if completed % 5 == 0 or completed == total:
+                yield "\n".join(log_lines[-200:]), None, None
+
+    # Build combined output file
+    combined_path = os.path.join(WORK_DIR, "combined_results.txt")
+    with open(combined_path, "w", encoding="utf-8") as f:
+        for res in results:
+            f.write(f"URL: {res['url']}\n")
+            f.write(f"Title: {res['title']}\n")
+            f.write(f"Description: {res['description']}\n")
+            f.write(f"Status: {res['status']}\n")
+            f.write("-" * 60 + "\n")
+
+    zip_path = None
+    if save_html:
+        zip_path = os.path.join(WORK_DIR, "saved_html_pages.zip")
+        zip_html_folder(zip_path)
+
+    ok_count = sum(1 for r in results if r["status"].startswith("HTTP 2"))
+    log_lines.append(
+        f"\nDone. {ok_count}/{total} pages fetched successfully. "
+        f"Combined file and {'zip ' if save_html else ''}ready for download below."
+    )
 
-def main():
-    links = crawl()
-    print(f"\nTotal unique novel links collected: {len(links)}")
+    yield "\n".join(log_lines[-200:]), combined_path, zip_path
 
-    with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
-        for link in sorted(links):
-            f.write(link + "\n")
 
-    print(f"Saved to {OUTPUT_FILE}")
+with gr.Blocks(title="Link Scraper") as demo:
+    gr.Markdown(
+        "# Link Scraper\n"
+        "Upload a `.txt` file with one URL per line. The app visits each link, "
+        "saves the page as HTML, pulls out the title + description, and gives you "
+        "back one combined `.txt` file (plus a `.zip` of all the saved HTML pages)."
+    )
 
+    with gr.Row():
+        file_input = gr.File(label="Upload .txt file of URLs", file_types=[".txt"])
+
+    with gr.Row():
+        save_html_cb = gr.Checkbox(label="Save raw HTML files (zipped)", value=True)
+        workers_slider = gr.Slider(
+            label="Parallel workers (lower = safer, less likely to get blocked)",
+            minimum=1, maximum=16, step=1, value=4,
+        )
+        delay_slider = gr.Slider(
+            label="Delay between requests per worker (seconds)",
+            minimum=0, maximum=10, step=0.5, value=1,
+        )
+        timeout_slider = gr.Slider(
+            label="Timeout per request (seconds)",
+            minimum=5, maximum=60, step=5, value=20,
+        )
+
+    start_btn = gr.Button("Start Scraping", variant="primary")
+
+    log_box = gr.Textbox(label="Progress / Log", lines=20, max_lines=20)
+
+    with gr.Row():
+        combined_output = gr.File(label="Combined results (.txt)")
+        zip_output = gr.File(label="All saved pages (.zip)")
+
+    start_btn.click(
+        fn=run_scraper,
+        inputs=[file_input, save_html_cb, workers_slider, delay_slider, timeout_slider],
+        outputs=[log_box, combined_output, zip_output],
+    )
 
 if __name__ == "__main__":
-    main()
+    demo.queue().launch()