remove mcp section, improve readability

app.py

@@ -5,64 +5,24 @@ from markdownify import markdownify as md
 import tempfile
 import zipfile
 import re
-from typing import Tuple, Set, List
+from typing import Tuple
 import os
 import gradio as gr
 from collections import deque
-import time
-
 
 # ===========================================================
-# TEXT CLEANING UTILITIES
+# 🌐 WEBSITE CRAWLER
 # ===========================================================
-def clean_text_content(text: str) -> str:
-    """Clean text: remove extra whitespace, normalize line breaks."""
-    # Remove multiple consecutive newlines
-    text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text)
-    # Remove leading/trailing whitespace from lines
-    lines = [line.strip() for line in text.split('\n')]
-    # Remove empty lines
-    lines = [line for line in lines if line.strip()]
-    # Join and clean up
-    text = '\n'.join(lines)
-    # Fix common HTML entity artifacts
-    text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
-    return text.strip()
-
-
-def extract_main_content(soup: BeautifulSoup) -> str:
-    """Extract main content from HTML, removing boilerplate."""
-    # Remove unwanted elements
-    for tag in soup(["script", "style", "nav", "footer", "header", "aside", "noscript", "meta", "link"]):
-        tag.decompose()
-    
-    # Find main content area
-    main_content = (
-        soup.find("main") or
-        soup.find("article") or
-        soup.find("div", class_=re.compile(r"content|main|post|article|body-text", re.I)) or
-        soup.find("body")
-    )
-    
-    if not main_content:
-        return ""
-    
-    # Convert to markdown
-    markdown_text = md(str(main_content), heading_style="ATX")
-    
-    # Clean up markdown artifacts
-    markdown_text = re.sub(r'\[([^\]]+)\]\(javascript:[^)]*\)', r'\1', markdown_text)  # Remove JS links
-    markdown_text = re.sub(r'\n{3,}', '\n\n', markdown_text)  # Remove excessive newlines
-    markdown_text = re.sub(r'#{7,}', '###', markdown_text)  # Cap heading levels
-    
-    return clean_text_content(markdown_text)
 
+def crawl_site_for_links(start_url: str, max_pages: int = 50, max_depth: int = 2):
+    """
+    Recursively crawl a website and collect:
+    • Internal HTML pages
+    • PDF files
+    
+    We stay inside the same domain for safety.
+    """
 
-# ===========================================================
-# RECURSIVE CRAWLER
-# ===========================================================
-def crawl_site_for_links(start_url: str, max_pages: int = 50, max_depth: int = 2, progress=None):
-    """Crawl the given site (up to max_depth) and return sets of HTML and PDF URLs."""
     visited = set()
     html_links = set()
     pdf_links = set()
@@ -70,33 +30,33 @@ def crawl_site_for_links(start_url: str, max_pages: int = 50, max_depth: int = 2
     parsed_base = urlparse(start_url)
     domain = parsed_base.netloc
 
-    queue = deque([(start_url, 0)])  # (url, depth)
+    queue = deque([(start_url, 0)])
     session = requests.Session()
     session.headers.update({
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        "User-Agent": "Mozilla/5.0"
     })
 
     while queue and len(visited) < max_pages:
         current_url, depth = queue.popleft()
+
         if current_url in visited or depth > max_depth:
             continue
 
         visited.add(current_url)
-        if progress:
-            progress(f"🔍 Crawling: {len(visited)}/{max_pages} pages discovered...")
-        
+
         try:
             response = session.get(current_url, timeout=10)
+
             if "text/html" not in response.headers.get("Content-Type", ""):
                 continue
 
             soup = BeautifulSoup(response.content, "html.parser")
+
             for a in soup.find_all("a", href=True):
                 href = a["href"]
                 full_url = urljoin(current_url, href)
                 parsed = urlparse(full_url)
 
-                # Stay in same domain
                 if parsed.netloc != domain:
                     continue
 
@@ -104,274 +64,166 @@ def crawl_site_for_links(start_url: str, max_pages: int = 50, max_depth: int = 2
                     pdf_links.add(full_url)
                 elif not href.startswith(("#", "javascript:", "mailto:", "tel:")):
                     html_links.add(full_url)
-                    if full_url not in visited and depth + 1 <= max_depth:
+                    if full_url not in visited:
                         queue.append((full_url, depth + 1))
-        except Exception as e:
+
+        except Exception:
             continue
 
     return html_links, pdf_links
 
 
 # ===========================================================
-# MAIN EXTRACTION FUNCTION
+# 📦 EXTRACTION ENGINE
 # ===========================================================
-def extract_all_content_as_zip(url: str, max_links: int = None, max_depth: int = 2, progress=None) -> Tuple[str, str]:
+
+def extract_all_content_as_zip(url: str, max_links: int, max_depth: int) -> Tuple[str, str]:
     """
-    Extract clean text content and PDFs from a website recursively.
-    Content is processed to remove HTML noise and keep only meaningful text.
+    Main function:
+    • Crawls the site
+    • Converts pages to Markdown
+    • Downloads PDFs
+    • Packs everything into a ZIP file
     """
+
     try:
-        if not url.strip():
-            return "❌ Please enter a valid URL", None
-        
         if not url.startswith(("http://", "https://")):
             url = "https://" + url
 
-        if progress:
-            progress("🌐 Starting crawl...")
-        
-        html_links, pdf_links = crawl_site_for_links(
-            url, 
-            max_pages=(max_links or 50), 
-            max_depth=max_depth,
-            progress=progress
-        )
+        html_links, pdf_links = crawl_site_for_links(url, max_links, max_depth)
 
         if not html_links and not pdf_links:
-            return "❌ No internal links or PDFs found to extract.", None
+            return "❌ No internal pages or PDFs found.", None
 
-        # Create ZIP
         with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as temp_zip:
             zip_path = temp_zip.name
 
-        successful_html = 0
-        failed_html = 0
-        successful_pdfs = 0
-        failed_pdfs = 0
-
         session = requests.Session()
-        session.headers.update({
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
-        })
+        session.headers.update({"User-Agent": "Mozilla/5.0"})
+
+        html_ok = 0
+        pdf_ok = 0
 
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_file:
-            # Create index file
-            index_content = f"# Website Extraction Report\n\n"
-            index_content += f"**Source:** {url}\n"
-            index_content += f"**Extracted:** {time.strftime('%Y-%m-%d %H:%M:%S')}\n"
-            index_content += f"**Total Pages:** {len(html_links)}\n"
-            index_content += f"**Total PDFs:** {len(pdf_links)}\n\n"
-            index_content += "## Table of Contents\n\n"
-
-            # Process HTML pages
-            html_list = list(html_links)
-            for i, link_url in enumerate(html_list, 1):
-                if progress:
-                    progress(f"📄 Processing HTML ({i}/{len(html_list)})...")
-                
+
+            # ---- HTML → Markdown ----
+            for i, link_url in enumerate(html_links, 1):
                 try:
                     resp = session.get(link_url, timeout=10)
-                    resp.raise_for_status()
-                    
                     soup = BeautifulSoup(resp.content, "html.parser")
-                    
-                    # Extract title
-                    title_tag = soup.find("title")
-                    title = title_tag.get_text().strip() if title_tag else f"Page {i}"
-                    
-                    # Extract and clean content
-                    content = extract_main_content(soup)
-                    
-                    if not content or len(content) < 20:
-                        failed_html += 1
-                        continue
-                    
-                    # Build markdown file
-                    markdown_text = f"# {title}\n\n"
-                    markdown_text += f"*Source: [{link_url}]({link_url})*\n\n"
-                    markdown_text += content
-
-                    # Safe filename
-                    filename = re.sub(r"[^\w\-_.]", "_", title[:50])
-                    if not filename or filename == "_":
-                        filename = f"page_{i}"
-                    filename = f"{filename}.md"
 
+                    for tag in soup(["script","style","nav","footer","header","aside"]):
+                        tag.decompose()
+
+                    main_content = (
+                        soup.find("main")
+                        or soup.find("article")
+                        or soup.find("body")
+                    )
+
+                    markdown_text = md(str(main_content))
+                    title = soup.find("title")
+                    if title:
+                        markdown_text = f"# {title.text.strip()}\n\n{markdown_text}"
+
+                    filename = f"page_{i}.md"
                     zip_file.writestr(filename, markdown_text)
-                    successful_html += 1
-                    
-                    # Add to index
-                    index_content += f"- [{title}]({filename})\n"
-                    
-                except Exception as e:
-                    failed_html += 1
-                    continue
+                    html_ok += 1
+
+                except Exception:
+                    pass
 
-            # Process PDFs
-            pdf_list = list(pdf_links)
-            for j, pdf_url in enumerate(pdf_list, 1):
-                if progress:
-                    progress(f"📄 Downloading PDFs ({j}/{len(pdf_list)})...")
-                
+            # ---- PDFs ----
+            for j, pdf_url in enumerate(pdf_links, 1):
                 try:
                     resp = session.get(pdf_url, timeout=20)
-                    resp.raise_for_status()
-                    pdf_filename = os.path.basename(urlparse(pdf_url).path)
-                    if not pdf_filename.lower().endswith(".pdf"):
-                        pdf_filename = f"document_{j}.pdf"
-                    zip_file.writestr(f"pdfs/{pdf_filename}", resp.content)
-                    successful_pdfs += 1
-                except Exception as e:
-                    failed_pdfs += 1
-                    continue
+                    zip_file.writestr(f"pdfs/document_{j}.pdf", resp.content)
+                    pdf_ok += 1
+                except Exception:
+                    pass
 
-            # Write index
-            zip_file.writestr("INDEX.md", index_content)
+        message = f"""
+✅ Extraction completed!
 
-        # Build status message
-        status_message = (
-            f"✅ **Extraction Complete!**\n\n"
-            f"📊 **Results:**\n"
-            f"- 📄 HTML Pages: **{successful_html}** extracted"
-        )
-        if failed_html > 0:
-            status_message += f", {failed_html} failed"
-        
-        status_message += f"\n- 📋 PDFs: **{successful_pdfs}** downloaded"
-        if failed_pdfs > 0:
-            status_message += f", {failed_pdfs} failed"
-        
-        status_message += f"\n\n📦 ZIP contains {successful_html} markdown files + {successful_pdfs} PDFs + index"
+• HTML pages saved as Markdown: {html_ok}
+• PDFs downloaded: {pdf_ok}
 
-        return status_message, zip_path
+You can now download the ZIP file below.
+"""
+        return message, zip_path
 
     except Exception as e:
         return f"❌ Error: {str(e)}", None
 
 
 # ===========================================================
-# GRADIO UI WITH CUSTOM THEME
+# 🎨 BEAUTIFUL GRADIO WEB APP
 # ===========================================================
-def gradio_extract(url, max_links, max_depth):
-    message, zip_path = extract_all_content_as_zip(
-        url, 
-        int(max_links) if max_links else None, 
-        int(max_depth),
-        progress=gr.update
-    )
-    return message, zip_path
-
-
-# Custom CSS for better visuals
-custom_css = """
-.gradio-container {
-    max-width: 900px;
-    margin: 0 auto;
-}
-
-.header {
-    text-align: center;
-    padding: 20px;
-    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-    color: white;
-    border-radius: 10px;
-    margin-bottom: 20px;
-}
-
-.header h1 {
-    margin: 0;
-    font-size: 2.5em;
-}
-
-.header p {
-    margin: 10px 0 0 0;
-    font-size: 1.1em;
-    opacity: 0.9;
-}
-"""
 
-with gr.Blocks(theme=gr.themes.Soft(), css=custom_css, title="🌐 Website Content Extractor") as demo:
-    gr.HTML("""
-    <div class="header">
-        <h1>🌐 Website Content Extractor</h1>
-        <p>Extract clean, human-readable content from any website</p>
-    </div>
-    """)
-    
+def run_extraction(url, max_links, depth):
+    return extract_all_content_as_zip(url, int(max_links), int(depth))
+
+
+with gr.Blocks(theme=gr.themes.Soft(), title="Website Content Extractor") as app:
+
     gr.Markdown("""
-    ### How it works:
-    1. Enter a website URL
-    2. Set the maximum number of pages to crawl
-    3. Choose crawl depth (1-3 levels)
-    4. Click **Extract** to start
-    5. Download the ZIP with cleaned markdown files and PDFs
-    
-    The extractor removes HTML noise and keeps only meaningful content.
-    """)
-    
-    with gr.Group():
-        gr.Markdown("### Input Settings")
+# 🌍 Website Content & PDF Extractor
+
+This tool downloads the **text and PDFs from a website** and packages everything into a ZIP file.
+
+Perfect for:
+• Research  
+• Archiving documentation  
+• Creating AI knowledge bases  
+• Offline reading  
+""")
+
+    with gr.Box():
+        gr.Markdown("## 🧭 How to use")
+
+        gr.Markdown("""
+1️⃣ Enter the homepage of a website  
+2️⃣ Choose how deep the crawler should explore  
+3️⃣ Click **Start Extraction**  
+4️⃣ Download your ZIP file  
+
+⚠️ Large websites may take several minutes.
+""")
+
+    with gr.Row():
         url_input = gr.Textbox(
             label="Website URL",
-            placeholder="https://example.com",
-            info="Enter the starting URL"
+            placeholder="https://example.com"
         )
-        
-        with gr.Row():
-            max_links = gr.Number(
-                label="Max Pages to Extract",
-                value=50,
-                minimum=1,
-                maximum=200,
-                step=1,
-                info="Limit the number of pages crawled"
-            )
-            max_depth = gr.Slider(
-                label="Crawl Depth",
-                minimum=1,
-                maximum=3,
-                value=2,
-                step=1,
-                info="How deep to follow links"
-            )
-    
-    extract_btn = gr.Button("🚀 Extract Content", variant="primary", size="lg")
-    
-    gr.Divider()
-    
-    with gr.Group():
-        gr.Markdown("### Results")
-        status_output = gr.Textbox(
-            label="Status",
-            interactive=False,
-            lines=3
+
+    with gr.Row():
+        max_links_input = gr.Slider(
+            10, 200, value=50, step=10,
+            label="Maximum pages to scan",
+            info="Higher = more content but slower"
         )
-        zip_output = gr.File(
-            label="📦 Download ZIP",
-            type="filepath"
+
+        depth_input = gr.Slider(
+            1, 3, value=2, step=1,
+            label="Crawl depth",
+            info="How many clicks away from homepage"
         )
-    
-    # Connect button
-    extract_btn.click(
-        fn=gradio_extract,
-        inputs=[url_input, max_links, max_depth],
-        outputs=[status_output, zip_output]
+
+    run_btn = gr.Button("🚀 Start Extraction", variant="primary")
+
+    status_output = gr.Textbox(label="Status")
+    file_output = gr.File(label="Download ZIP")
+
+    run_btn.click(
+        fn=run_extraction,
+        inputs=[url_input, max_links_input, depth_input],
+        outputs=[status_output, file_output]
     )
-    
-    gr.Markdown("""
-    ---
-    **Made with ❤️ for better web scraping**
-    """)
 
 
 # ===========================================================
-# ENTRY POINT
+# 🚀 ENTRY POINT — WEB APP ONLY
 # ===========================================================
+
 if __name__ == "__main__":
-    port = int(os.environ.get("PORT", 7860))
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=port,
-        ssr_mode=False,
-        show_error=True
-    )
+    app.launch(server_name="0.0.0.0", server_port=7860)