Update app.py

app.py

@@ -1,6 +1,6 @@
 """High-Performance Stealth Web Scraper & Browser Automation MCP Server
 
-All-in-one file designed for 2 vCPU / 16 GB RAM Hugging Face Spaces.
+Resilient, single-file server for Hugging Face Spaces (Port 7860).
 """
 
 import asyncio
@@ -22,25 +22,25 @@ from patchright.async_api import Browser, BrowserContext, Page, async_playwright
 import trafilatura
 
 # ==========================================
-# 1. Logging & Configuration
+# 1. Configuration & Settings
 # ==========================================
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
     handlers=[logging.StreamHandler(sys.stdout)],
 )
-logger = logging.getLogger("mcp_scraper")
+logger = logging.getLogger("mcp_server")
 
 PORT = int(os.getenv("PORT", 7860))
 HOST = "0.0.0.0"
-MAX_CONCURRENT_BROWSERS = 4  # Tuned for 2 vCPU
-MAX_RECYCLE_REQUESTS = 100  # Prevents long-term memory leaks
-MAX_CONTENT_CHARS = 100_000  # Context window safeguard
+MAX_CONCURRENT_BROWSERS = 4
+MAX_RECYCLE_REQUESTS = 100
+MAX_CONTENT_CHARS = 100_000
 HTTP_TIMEOUT_SEC = 12.0
 BROWSER_TIMEOUT_MS = 30000
 
 # ==========================================
-# 2. Patchright Stealth Browser Pool
+# 2. Resilient Browser Pool (Lazy Initialized)
 # ==========================================
 
 
@@ -53,36 +53,38 @@ class StealthBrowserPool:
         self._request_counter = 0
         self._lock = asyncio.Lock()
 
-    async def initialize(self):
-        if not self._playwright:
-            self._playwright = await async_playwright().start()
-        if not self._browser or not self._browser.is_connected():
-            logger.info("Starting Patchright Chromium Stealth Engine...")
-            self._browser = await self._playwright.chromium.launch(
-                headless=True,
-                args=[
-                    "--no-sandbox",
-                    "--disable-setuid-sandbox",
-                    "--disable-dev-shm-usage",
-                    "--disable-gpu",
-                    "--disable-blink-features=AutomationControlled",
-                    "--no-first-run",
-                    "--window-size=1920,1080",
-                ],
-            )
-            self._request_counter = 0
+    async def get_browser(self) -> Browser:
+        async with self._lock:
+            if not self._playwright:
+                self._playwright = await async_playwright().start()
+            if not self._browser or not self._browser.is_connected():
+                logger.info("Initializing Chromium Stealth instance...")
+                self._browser = await self._playwright.chromium.launch(
+                    headless=True,
+                    args=[
+                        "--no-sandbox",
+                        "--disable-setuid-sandbox",
+                        "--disable-dev-shm-usage",
+                        "--disable-gpu",
+                        "--disable-blink-features=AutomationControlled",
+                        "--no-first-run",
+                        "--window-size=1920,1080",
+                    ],
+                )
+                self._request_counter = 0
+            return self._browser
 
     async def _check_recycle(self):
         async with self._lock:
             self._request_counter += 1
             if self._request_counter >= MAX_RECYCLE_REQUESTS:
-                logger.info(
-                    "Recycling browser instance to free memory under 16GB limit..."
-                )
+                logger.info("Recycling browser process to free RAM...")
                 if self._browser:
-                    await self._browser.close()
+                    try:
+                        await self._browser.close()
+                    except Exception:
+                        pass
                 self._browser = None
-                await self.initialize()
 
     @asynccontextmanager
     async def get_page(self) -> AsyncGenerator[Page, None]:
@@ -90,11 +92,8 @@ class StealthBrowserPool:
         context: BrowserContext | None = None
         page: Page | None = None
         try:
-            if not self._browser or not self._browser.is_connected():
-                await self.initialize()
-
-            # High-entropy stealth fingerprint context
-            context = await self._browser.new_context(
+            browser = await self.get_browser()
+            context = await browser.new_context(
                 viewport={"width": 1920, "height": 1080},
                 user_agent=(
                     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -103,10 +102,8 @@ class StealthBrowserPool:
                 ),
                 locale="en-US",
                 timezone_id="America/New_York",
-                device_scale_factor=1,
             )
 
-            # Injected anti-detect overrides
             await context.add_init_script("""
                 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
                 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
@@ -117,7 +114,6 @@ class StealthBrowserPool:
             page = await context.new_page()
             page.set_default_navigation_timeout(BROWSER_TIMEOUT_MS)
             page.set_default_timeout(BROWSER_TIMEOUT_MS)
-
             yield page
         finally:
             if page:
@@ -133,17 +129,11 @@ class StealthBrowserPool:
             self._semaphore.release()
             await self._check_recycle()
 
-    async def shutdown(self):
-        if self._browser:
-            await self._browser.close()
-        if self._playwright:
-            await self._playwright.stop()
-
 
 browser_pool = StealthBrowserPool()
 
 # ==========================================
-# 3. Content Extraction Pipeline
+# 3. HTML & Markdown Processing
 # ==========================================
 
 
@@ -162,7 +152,6 @@ def clean_html(raw_html: str) -> str:
 
 
 def extract_content(html: str, url: str = "") -> dict[str, Any]:
-    """Primary extraction via Trafilatura with automatic Markdownify fallback."""
     extracted = trafilatura.extract(
         html,
         url=url,
@@ -195,7 +184,7 @@ def extract_content(html: str, url: str = "") -> dict[str, Any]:
     if len(extracted) > MAX_CONTENT_CHARS:
         extracted = (
             extracted[:MAX_CONTENT_CHARS]
-            + f"\n\n... [Truncated: Reached {MAX_CONTENT_CHARS} characters limit]"
+            + f"\n\n... [Truncated: reached {MAX_CONTENT_CHARS} limit]"
         )
 
     return {"content": extracted.strip(), "metadata": meta_dict}
@@ -215,14 +204,13 @@ def extract_json_ld(html: str) -> list[dict]:
 
 
 # ==========================================
-# 4. Hybrid Scraping & Automation Core
+# 4. Scrape & Automation Engine
 # ==========================================
 
 FAST_HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
     "Accept-Language": "en-US,en;q=0.9",
-    "Upgrade-Insecure-Requests": "1",
 }
 
 
@@ -231,8 +219,6 @@ async def dismiss_popups(page: Page):
         'button[id*="cookie" i]',
         'button[class*="cookie" i]',
         'button[aria-label*="accept" i]',
-        'button[class*="accept" i]',
-        'button[id*="accept" i]',
         'button:has-text("Accept all")',
         'button:has-text("Accept")',
         'button:has-text("I agree")',
@@ -255,7 +241,6 @@ async def scrape_pipeline(
     auto_scroll: bool = True,
     wait_for_selector: str | None = None,
 ) -> dict:
-    # Tier 1: Fast HTTP
     if not force_browser:
         try:
             async with httpx.AsyncClient(
@@ -267,14 +252,14 @@ async def scrape_pipeline(
                 resp = await client.get(url)
                 if resp.status_code == 200:
                     html = resp.text
-                    signals = [
-                        "cf-challenge",
-                        "challenges.cloudflare.com",
-                        "ray-id",
-                        "enable javascript",
-                        "just a moment...",
-                    ]
-                    if not any(s in html.lower() for s in signals):
+                    if not any(
+                        s in html.lower()
+                        for s in [
+                            "cf-challenge",
+                            "ray-id",
+                            "just a moment...",
+                        ]
+                    ):
                         parsed = extract_content(html, url=url)
                         if len(parsed["content"]) >= 150:
                             return {
@@ -284,15 +269,12 @@ async def scrape_pipeline(
                                 "content": parsed["content"],
                                 "metadata": parsed["metadata"],
                             }
-        except Exception as e:
-            logger.debug(f"HTTP scrape failed for {url}: {e}")
+        except Exception:
+            pass
 
-    # Tier 2: Patchright Stealth Headless Browser
-    logger.info(f"Escalating to Patchright Stealth Engine for: {url}")
     async with browser_pool.get_page() as page:
         await page.goto(url, wait_until="domcontentloaded")
 
-        # Solve Cloudflare Turnstile box if visible
         try:
             cf_frame = await page.query_selector(
                 "iframe[src*='challenges.cloudflare.com']"
@@ -324,7 +306,6 @@ async def scrape_pipeline(
                 await asyncio.sleep(0.3)
 
         await asyncio.sleep(0.5)
-
         html = await page.content()
         final_url = page.url
         title = await page.title()
@@ -343,14 +324,14 @@ async def scrape_pipeline(
 
 
 # ==========================================
-# 5. MCP Server & Tool Definitions
+# 5. MCP Tools Setup
 # ==========================================
 mcp = FastMCP("High-Performance Web Scraper", host=HOST, port=PORT)
 
 
 @mcp.tool(
     name="scrape_url",
-    description="Scrapes cleaned Markdown content and metadata from any URL. Automatically handles Cloudflare, SPA rendering, and popups.",
+    description="Scrapes cleaned Markdown content and metadata from any URL.",
 )
 async def scrape_url(
     url: str,
@@ -358,13 +339,6 @@ async def scrape_url(
     auto_scroll: bool = True,
     wait_for_selector: str | None = None,
 ) -> dict:
-    """Parameters:
-
-    - url: The target URL to scrape.
-    - force_browser: Skip fast HTTP and directly use headless stealth browser.
-    - auto_scroll: Scroll down to trigger lazy loaded elements.
-    - wait_for_selector: CSS selector to wait for before extracting.
-    """
     return await scrape_pipeline(
         url, force_browser, auto_scroll, wait_for_selector
     )
@@ -372,14 +346,9 @@ async def scrape_url(
 
 @mcp.tool(
     name="search_and_scrape",
-    description="Searches DuckDuckGo and concurrently extracts cleaned markdown from the top ranking result pages.",
+    description="Searches DuckDuckGo and concurrently extracts content from top results.",
 )
 async def search_and_scrape(query: str, max_results: int = 3) -> dict:
-    """Parameters:
-
-    - query: Search terms.
-    - max_results: Number of search results to scrape (default: 3, max: 5).
-    """
     max_results = min(max(1, max_results), 5)
     loop = asyncio.get_running_loop()
 
@@ -423,17 +392,11 @@ async def search_and_scrape(query: str, max_results: int = 3) -> dict:
 
 @mcp.tool(
     name="take_screenshot",
-    description="Takes a high-resolution full-page or viewport screenshot of a webpage and returns it as a Base64 PNG.",
+    description="Takes a high-resolution full-page or viewport screenshot of a webpage.",
 )
 async def take_screenshot(
     url: str, full_page: bool = True, wait_seconds: float = 1.0
 ) -> dict:
-    """Parameters:
-
-    - url: The page URL to render and capture.
-    - full_page: Capture entire scrollable document height (True) or standard viewport (False).
-    - wait_seconds: Additional time to wait after loading for animations.
-    """
     async with browser_pool.get_page() as page:
         await page.goto(url, wait_until="networkidle")
         if wait_seconds > 0:
@@ -452,21 +415,11 @@ async def take_screenshot(
 
 @mcp.tool(
     name="interact_page",
-    description="Executes a chain of browser actions: click, fill inputs, press keys, wait, and evaluate JS.",
+    description="Executes a list of browser actions (click, type, press, wait, evaluate).",
 )
 async def interact_page(
     url: str, actions: list[dict[str, Any]], extract_markdown: bool = True
 ) -> dict:
-    """Actions schema list example:
-
-    [
-      {"type": "click", "selector": "button#submit"},
-      {"type": "type", "selector": "input#search", "text": "Hugging Face"},
-      {"type": "press", "key": "Enter"},
-      {"type": "wait", "seconds": 2.0},
-      {"type": "evaluate", "script": "window.scrollTo(0, document.body.scrollHeight);"}
-    ]
-    """
     async with browser_pool.get_page() as page:
         await page.goto(url, wait_until="domcontentloaded")
         logs = []
@@ -492,19 +445,15 @@ async def interact_page(
                     await asyncio.sleep(s)
                     logs.append(f"[{idx}] Waited {s}s")
                 elif act_type == "evaluate":
-                    script = act["script"]
-                    res = await page.evaluate(script)
+                    res = await page.evaluate(act["script"])
                     logs.append(f"[{idx}] Evaluated script -> {res}")
             except Exception as e:
-                logs.append(f"[{idx}] Action failed: {str(e)}")
+                logs.append(f"[{idx}] Failed: {str(e)}")
 
         html = await page.content()
         final_url = page.url
 
-    res = {
-        "final_url": final_url,
-        "logs": logs,
-    }
+    res = {"final_url": final_url, "logs": logs}
     if extract_markdown:
         parsed = extract_content(html, url=final_url)
         res["content"] = parsed["content"]
@@ -515,16 +464,11 @@ async def interact_page(
 
 @mcp.tool(
     name="extract_structured_data",
-    description="Parses structured JSON-LD schemas and arbitrary CSS selector targets from a webpage.",
+    description="Parses structured JSON-LD schemas and arbitrary CSS selector targets.",
 )
 async def extract_structured_data(
     url: str, css_selectors: dict[str, str] | None = None
 ) -> dict:
-    """Parameters:
-
-    - url: The target URL.
-    - css_selectors: Key-value map of names to CSS selectors, e.g. {"prices": ".price-tag", "headers": "h2"}
-    """
     async with browser_pool.get_page() as page:
         await page.goto(url, wait_until="domcontentloaded")
         html = await page.content()
@@ -544,9 +488,10 @@ async def extract_structured_data(
 
 
 # ==========================================
-# 6. Main Entry Point (SSE Transport)
+# 6. Main SSE Runner
 # ==========================================
 if __name__ == "__main__":
-    logger.info(f"Starting Web Scraper MCP Server on http://{HOST}:{PORT}/sse")
-    # Runs the official FastMCP Server on port 7860 via Server-Sent Events (SSE)
+    logger.info(f"Starting MCP Server on http://{HOST}:{PORT}/sse ...")
+    mcp.settings.host = HOST
+    mcp.settings.port = PORT
     mcp.run(transport="sse")