Update app.py

app.py

@@ -1,151 +1,399 @@
 import gradio as gr
 import requests
-from bs4 import BeautifulSoup
-import time
 import zipfile
-from urllib.parse import urlparse
+import os
 import re
+import time
+from datetime import datetime
+from urllib.parse import urlparse
+from bs4 import BeautifulSoup
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0 Safari/537.36"
+    ),
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+OUTPUT_DIR = "fetched_pages"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+
+def safe_filename(url: str, index: int) -> str:
+    """Turn a URL into a safe filename like 001_example_com_path.html"""
+    parsed = urlparse(url)
+    slug = re.sub(r"[^\w\-]", "_", parsed.netloc + parsed.path)
+    slug = re.sub(r"_+", "_", slug).strip("_")[:80]
+    return f"{index:03d}_{slug}.html"
+
+
+def fetch_url(url: str, timeout: int = 15) -> tuple[str, str]:
+    """
+    Fetch a URL and return (html_content, error_message).
+    error_message is empty string on success.
+    """
+    try:
+        r = requests.get(url, headers=HEADERS, timeout=timeout)
+        r.raise_for_status()
+        # normalise encoding
+        r.encoding = r.apparent_encoding or "utf-8"
+        return r.text, ""
+    except requests.exceptions.Timeout:
+        return "", f"Timed out after {timeout}s"
+    except requests.exceptions.HTTPError as e:
+        return "", f"HTTP {e.response.status_code}"
+    except Exception as e:
+        return "", str(e)
+
+
+def inject_base_tag(html: str, url: str) -> str:
+    """Inject a <base> tag so relative links resolve correctly in previews."""
+    parsed = urlparse(url)
+    base = f"{parsed.scheme}://{parsed.netloc}"
+    if "<head" in html.lower():
+        return re.sub(
+            r"(<head[^>]*>)",
+            rf'\1<base href="{base}">',
+            html,
+            count=1,
+            flags=re.IGNORECASE,
+        )
+    return f'<base href="{base}">' + html
+
+
+def make_combined(pages: list[dict], title: str = "Combined Pages") -> str:
+    """Build one big HTML file with a sidebar TOC and all pages in iframes."""
+    nav_items = ""
+    sections = ""
+
+    for i, page in enumerate(pages):
+        anchor = f"page_{i}"
+        label = page["title"] or page["url"]
+        status_badge = (
+            '<span style="color:#22c55e">✓</span>'
+            if page["ok"]
+            else '<span style="color:#ef4444">✗</span>'
+        )
+        nav_items += f"""
+        <li>
+          <a href="#{anchor}" class="nav-link" onclick="showSection('{anchor}')">
+            {status_badge} {i+1}. {label[:60]}
+          </a>
+        </li>"""
+
+        if page["ok"]:
+            # embed raw HTML in a sandboxed iframe via srcdoc
+            escaped = page["html"].replace("&", "&amp;").replace('"', "&quot;")
+            content_block = f'<iframe srcdoc="{escaped}" class="page-frame" sandbox="allow-same-origin allow-scripts"></iframe>'
+        else:
+            content_block = f'<div class="error-box">⚠ Could not fetch this page: {page["error"]}</div>'
+
+        sections += f"""
+        <section id="{anchor}" class="page-section">
+          <div class="page-header">
+            <span class="page-index">#{i+1}</span>
+            <a href="{page['url']}" target="_blank" class="page-url">{page['url']}</a>
+          </div>
+          {content_block}
+        </section>"""
+
+    fetched_at = datetime.now().strftime("%Y-%m-%d %H:%M")
+    total = len(pages)
+    ok_count = sum(1 for p in pages if p["ok"])
+
+    return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{title}</title>
+<style>
+  *, *::before, *::after {{ box-sizing: border-box; margin: 0; padding: 0; }}
+  :root {{
+    --bg: #0f1117;
+    --sidebar-bg: #161b22;
+    --surface: #1c2230;
+    --accent: #58a6ff;
+    --accent2: #3fb950;
+    --text: #e6edf3;
+    --muted: #8b949e;
+    --border: #30363d;
+    --radius: 8px;
+    --sidebar-w: 300px;
+  }}
+  body {{
+    font-family: 'Segoe UI', system-ui, sans-serif;
+    background: var(--bg);
+    color: var(--text);
+    display: flex;
+    height: 100vh;
+    overflow: hidden;
+  }}
+
+  /* ── Sidebar ── */
+  #sidebar {{
+    width: var(--sidebar-w);
+    min-width: var(--sidebar-w);
+    background: var(--sidebar-bg);
+    border-right: 1px solid var(--border);
+    display: flex;
+    flex-direction: column;
+    overflow: hidden;
+  }}
+  .sidebar-header {{
+    padding: 18px 16px 14px;
+    border-bottom: 1px solid var(--border);
+  }}
+  .sidebar-header h1 {{
+    font-size: 15px;
+    font-weight: 700;
+    color: var(--accent);
+    letter-spacing: .02em;
+  }}
+  .sidebar-meta {{
+    font-size: 11px;
+    color: var(--muted);
+    margin-top: 4px;
+  }}
+  #nav-list {{
+    list-style: none;
+    overflow-y: auto;
+    flex: 1;
+    padding: 8px 0;
+  }}
+  #nav-list li {{ padding: 2px 8px; }}
+  .nav-link {{
+    display: block;
+    padding: 7px 10px;
+    border-radius: var(--radius);
+    color: var(--muted);
+    text-decoration: none;
+    font-size: 12px;
+    line-height: 1.4;
+    transition: background .15s, color .15s;
+    word-break: break-all;
+  }}
+  .nav-link:hover, .nav-link.active {{
+    background: rgba(88,166,255,.12);
+    color: var(--text);
+  }}
+
+  /* ── Main ── */
+  #main {{
+    flex: 1;
+    overflow-y: auto;
+    padding: 24px 28px;
+  }}
+  .page-section {{
+    margin-bottom: 48px;
+  }}
+  .page-header {{
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    margin-bottom: 10px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid var(--border);
+  }}
+  .page-index {{
+    background: var(--accent);
+    color: #000;
+    font-size: 11px;
+    font-weight: 700;
+    padding: 2px 7px;
+    border-radius: 20px;
+    flex-shrink: 0;
+  }}
+  .page-url {{
+    color: var(--muted);
+    font-size: 12px;
+    word-break: break-all;
+    text-decoration: none;
+  }}
+  .page-url:hover {{ color: var(--accent); text-decoration: underline; }}
+  .page-frame {{
+    width: 100%;
+    height: 600px;
+    border: 1px solid var(--border);
+    border-radius: var(--radius);
+    background: #fff;
+  }}
+  .error-box {{
+    padding: 20px;
+    border: 1px solid #f8514933;
+    background: #f8514911;
+    border-radius: var(--radius);
+    color: #ff7b72;
+    font-size: 13px;
+  }}
+</style>
+</head>
+<body>
+
+<aside id="sidebar">
+  <div class="sidebar-header">
+    <h1>📄 {title}</h1>
+    <div class="sidebar-meta">
+      {ok_count}/{total} pages fetched &nbsp;·&nbsp; {fetched_at}
+    </div>
+  </div>
+  <ul id="nav-list">
+    {nav_items}
+  </ul>
+</aside>
+
+<main id="main">
+  {sections}
+</main>
 
-def scrape_kemono(links_text, delay_sec, max_retries):
-    # Smart URL extraction: find all kemono.cr links even if wrapped in HTML tags
-    raw_urls = re.findall(r'(https://kemono\.cr/[^\s"\'<>]+)', links_text)
-    
-    # Deduplicate while preserving order
-    seen = set()
-    links = [url for url in raw_urls if not (url in seen or seen.add(url))]
-    
-    if not links:
-        return None, "No valid kemono.cr links found in the input."
-    
-    combined_text_path = "combined_extracted_text.txt"
-    html_zip_path = "saved_raw_htmls.zip"
-    
-    extracted_data = []
-    
-    # Create a zip file to store all the individual HTML pages
-    with zipfile.ZipFile(html_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-        for i, link in enumerate(links):
+<script>
+  function showSection(id) {{
+    document.querySelectorAll('.nav-link').forEach(l => l.classList.remove('active'));
+    const link = document.querySelector(`a[href="#${{id}}"]`);
+    if (link) link.classList.add('active');
+  }}
+
+  // Highlight nav item on scroll
+  const observer = new IntersectionObserver(entries => {{
+    entries.forEach(e => {{
+      if (e.isIntersecting) {{
+        document.querySelectorAll('.nav-link').forEach(l => l.classList.remove('active'));
+        const link = document.querySelector(`a[href="#${{e.target.id}}"]`);
+        if (link) link.classList.add('active');
+      }}
+    }});
+  }}, {{ threshold: 0.3 }});
+  document.querySelectorAll('.page-section').forEach(s => observer.observe(s));
+</script>
+</body>
+</html>"""
+
+
+# ── Gradio logic ─────────────────────────────────────────────────────────────
+
+def process_urls(raw_input: str, delay: float, progress=gr.Progress(track_tqdm=True)):
+    """
+    Main processing function called by Gradio.
+    Returns (status_text, combined_html_path, zip_path)
+    """
+    lines = [l.strip() for l in raw_input.strip().splitlines() if l.strip()]
+    urls = [l for l in lines if l.startswith("http://") or l.startswith("https://")]
+
+    if not urls:
+        return "❌ No valid URLs found. Make sure each URL starts with http:// or https://", None, None
+
+    pages = []
+    log_lines = []
+
+    for i, url in enumerate(progress.tqdm(urls, desc="Fetching pages")):
+        progress((i + 1) / len(urls), desc=f"Fetching {i+1}/{len(urls)}: {url[:50]}…")
+        html, err = fetch_url(url)
+
+        if html:
+            html = inject_base_tag(html, url)
+            # try to extract a page title
             try:
-                # Request Delay Controller
-                if i > 0:
-                    time.sleep(delay_sec)
-                
-                headers = {
-                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-                }
-                
-                # Fetch page with retry mechanism
-                response = None
-                for attempt in range(int(max_retries)):
-                    response = requests.get(link, headers=headers, timeout=15)
-                    if response.status_code == 200:
-                        break
-                    time.sleep(2) # brief pause before retry
-                
-                if response is None or response.status_code != 200:
-                    extracted_data.append(f"--- FAILED TO FETCH: {link} (Status: {response.status_code if response else 'Timeout'}) ---")
-                    continue
-                
-                html_content = response.text
-                
-                # Save Raw HTML to the ZIP file
-                parsed_url = urlparse(link)
-                path_parts = [p for p in parsed_url.path.strip('/').split('/') if p]
-                # Create a safe filename using the last two parts of the URL (e.g., user ID and post ID)
-                safe_name = f"post_{i+1}_" + "_".join(path_parts[-2:]) + ".html"
-                zipf.writestr(safe_name, html_content)
-                
-                # Parse HTML for Text and Title extraction
-                soup = BeautifulSoup(html_content, 'html.parser')
-                
-                # Check for Cloudflare block
-                page_title = soup.find('title')
-                if page_title and "Just a moment" in page_title.get_text():
-                    extracted_data.append(f"--- BLOCKED BY CLOUDFLARE: {link} (Increase your delay!) ---\n")
-                    continue
-
-                # 1. EXTRACT TITLE (Better Fallbacks)
-                title_tag = soup.find('h1', class_='post__title')
-                if title_tag:
-                    title = title_tag.get_text(strip=True)
-                elif page_title:
-                    title = page_title.get_text(strip=True).replace(" | Kemono", "")
-                else:
-                    title = f"Untitled_Post_{i+1}"
-                
-                # 2. EXTRACT TEXT CONTENT
-                content_tag = soup.find('div', class_='post__content') or soup.find('div', class_='post__body')
-                text_content = content_tag.get_text(separator='\n', strip=True) if content_tag else ""
-                
-                # 3. EXTRACT ATTACHMENTS & IMAGES
-                media_links = []
-                # Look for attachments
-                attachments = soup.find_all('a', class_='post__attachment-link')
-                for atk in attachments:
-                    href = atk.get('href')
-                    if href:
-                        media_links.append("Attachment: " + (href if href.startswith('http') else f"https://kemono.cr{href}"))
-                
-                # Look for images inside the post
-                images = soup.find_all('img', class_='post__image')
-                for img in images:
-                    src = img.get('src')
-                    if src:
-                        media_links.append("Image: " + (src if src.startswith('http') else f"https://kemono.cr{src}"))
-                
-                # Build the final text block
-                post_text = f"========================================================================\n"
-                post_text += f"TITLE: {title}\n"
-                post_text += f"URL: {link}\n"
-                post_text += f"========================================================================\n\n"
-                
-                if not text_content and not media_links:
-                    post_text += "[No text or media content found. The post might be empty or locked.]\n\n"
-                else:
-                    if text_content:
-                        post_text += f"--- TEXT ---\n{text_content}\n\n"
-                    if media_links:
-                        post_text += f"--- MEDIA LINKS ---\n" + "\n".join(media_links) + "\n\n"
-                
-                post_text += "\n"
-                extracted_data.append(post_text)
-                
-            except Exception as e:
-                extracted_data.append(f"--- ERROR PROCESSING {link}: {str(e)} ---\n\n")
-
-    # Write the combined text file
-    with open(combined_text_path, 'w', encoding='utf-8') as f:
-        f.write("\n".join(extracted_data))
-        
-    return [combined_text_path, html_zip_path], f"Successfully processed {len(links)} links."
-
-# --- Gradio UI Layout ---
-with gr.Blocks(theme=gr.themes.Soft()) as app:
-    gr.Markdown("# 🦊 Kemono.cr Batch Scraper & Extractor")
-    gr.Markdown("Paste a list of links (or raw HTML tags from your `links.txt` file). The scraper will fetch each link, extract the title, text, and media links, save the raw HTMLs in a ZIP, and give you a combined TXT document.")
-    
+                soup = BeautifulSoup(html, "html.parser")
+                title = soup.title.string.strip() if soup.title else ""
+            except Exception:
+                title = ""
+
+            fname = safe_filename(url, i + 1)
+            fpath = os.path.join(OUTPUT_DIR, fname)
+            with open(fpath, "w", encoding="utf-8", errors="replace") as f:
+                f.write(html)
+
+            pages.append({"url": url, "html": html, "title": title, "ok": True, "file": fpath, "error": ""})
+            log_lines.append(f"✅ [{i+1}/{len(urls)}] {url}")
+        else:
+            pages.append({"url": url, "html": "", "title": "", "ok": False, "file": None, "error": err})
+            log_lines.append(f"❌ [{i+1}/{len(urls)}] {url}  →  {err}")
+
+        if delay > 0 and i < len(urls) - 1:
+            time.sleep(delay)
+
+    # Build combined HTML
+    combined_html = make_combined(pages, title="Fetched Pages")
+    combined_path = os.path.join(OUTPUT_DIR, "combined.html")
+    with open(combined_path, "w", encoding="utf-8", errors="replace") as f:
+        f.write(combined_html)
+
+    # Build ZIP of individual files
+    zip_path = os.path.join(OUTPUT_DIR, "all_pages.zip")
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+        for p in pages:
+            if p["file"]:
+                zf.write(p["file"], os.path.basename(p["file"]))
+        zf.write(combined_path, "combined.html")
+
+    ok_count = sum(1 for p in pages if p["ok"])
+    status = "\n".join(log_lines)
+    status += f"\n\n🎉 Done! {ok_count}/{len(urls)} pages saved."
+    return status, combined_path, zip_path
+
+
+# ── UI ───────────────────────────────────────────────────────────────────────
+
+with gr.Blocks(
+    title="URL → HTML Fetcher",
+    theme=gr.themes.Base(
+        primary_hue="blue",
+        neutral_hue="slate",
+    ),
+    css="""
+    #url-box textarea { font-family: monospace; font-size: 13px; }
+    #log-box textarea  { font-family: monospace; font-size: 12px; }
+    """
+) as demo:
+
+    gr.Markdown(
+        """
+# 🌐 URL → HTML Fetcher
+Paste URLs (one per line), click **Fetch**, and get:
+- Individual `.html` files for each page
+- One `combined.html` with a sidebar viewer
+- A `.zip` with everything bundled
+        """
+    )
+
     with gr.Row():
         with gr.Column(scale=2):
-            links_input = gr.Textbox(
-                lines=12, 
-                label="Paste Links Here", 
-                placeholder="https://kemono.cr/patreon/user/.../post/...\n(You can also paste raw <a href=...> tags)"
+            url_input = gr.Textbox(
+                label="URLs (one per line)",
+                placeholder="https://example.com\nhttps://another-site.com/page",
+                lines=12,
+                elem_id="url-box",
             )
-            
-            with gr.Row():
-                delay_input = gr.Slider(minimum=0, maximum=15, value=2.5, step=0.5, label="Delay Between Requests (Seconds)")
-                retry_input = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Max Retries per Link")
-                
-            start_btn = gr.Button("🚀 Start Extraction", variant="primary")
-            
+            delay_slider = gr.Slider(
+                minimum=0,
+                maximum=5,
+                value=1,
+                step=0.5,
+                label="Delay between requests (seconds) — be polite to servers!",
+            )
+            fetch_btn = gr.Button("🚀 Fetch Pages", variant="primary")
+
         with gr.Column(scale=1):
-            status_output = gr.Textbox(label="Status Logging")
-            files_output = gr.File(label="📥 Download Extracted Files")
-            
-    start_btn.click(
-        fn=scrape_kemono,
-        inputs=[links_input, delay_input, retry_input],
-        outputs=[files_output, status_output]
+            log_output = gr.Textbox(
+                label="Progress log",
+                lines=12,
+                interactive=False,
+                elem_id="log-box",
+            )
+
+    with gr.Row():
+        combined_file = gr.File(label="📄 Combined HTML (download)")
+        zip_file      = gr.File(label="📦 All pages ZIP (download)")
+
+    fetch_btn.click(
+        fn=process_urls,
+        inputs=[url_input, delay_slider],
+        outputs=[log_output, combined_file, zip_file],
     )
 
-app.launch()
+if __name__ == "__main__":
+    demo.launch()