Update app.py

app.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 import os
 import time
+import logging
 import threading
 import asyncio
 from typing import Optional, Dict, Any, Tuple
@@ -13,20 +14,22 @@ from pydantic import BaseModel
 from starlette.responses import JSONResponse
 
 from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+from selenium.common.exceptions import WebDriverException
+from webdriver_manager.chrome import ChromeDriverManager
+from bs4 import BeautifulSoup
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import TimeoutException, WebDriverException
-from webdriver_manager.chrome import ChromeDriverManager
-from bs4 import BeautifulSoup
+from selenium.common.exceptions import TimeoutException
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("fast_fetcher")
 
 # ---------------- BrowserManager ----------------
 class BrowserManager:
-    """
-    Manages a single Chrome webdriver instance.
-    """
     def __init__(
         self,
         headless: bool = True,
@@ -34,7 +37,6 @@ class BrowserManager:
         window_size: str = "1366,768",
         disable_images: bool = True,
         block_resource_urls: Optional[list[str]] = None,
-        page_load_strategy: str = "eager",
     ):
         self.headless = headless
         self.user_agent = user_agent
@@ -46,34 +48,37 @@ class BrowserManager:
             "*.googlesyndication.com/*",
             "*.adservice.google.com/*",
         ]
-        self.page_load_strategy = page_load_strategy
         self._driver_lock = threading.Lock()
         self._driver: Optional[webdriver.Chrome] = None
-        self._start_driver()
+        self._start_driver_with_retries()
 
-    def _start_driver(self):
+    def _build_options(self) -> Options:
         opts = Options()
 
-        # If CHROME_BIN env var is set (we set it in Dockerfile), point to it
+        # set binary location from env, fallback to default
         chrome_bin = os.environ.get("CHROME_BIN", "/usr/bin/google-chrome-stable")
         if os.path.exists(chrome_bin):
             opts.binary_location = chrome_bin
+            logger.debug("Using chrome binary: %s", chrome_bin)
+        else:
+            logger.warning("Chrome binary not found at %s; Selenium Manager may still locate a browser.", chrome_bin)
 
-        # Headless flags
         if self.headless:
-            # keep both forms for compatibility
+            # keep compatibility for versions: add both
             opts.add_argument("--headless=new")
             opts.add_argument("--headless")
 
-        # Docker/container friendly flags
+        # container-friendly and stability flags
         opts.add_argument("--no-sandbox")
         opts.add_argument("--disable-setuid-sandbox")
         opts.add_argument("--disable-dev-shm-usage")
         opts.add_argument("--disable-gpu")
         opts.add_argument("--disable-extensions")
         opts.add_argument("--disable-blink-features=AutomationControlled")
-        opts.add_argument("--single-process")
+        opts.add_argument("--disable-software-rasterizer")
+        opts.add_argument("--disable-accelerated-2d-canvas")
         opts.add_argument(f"--window-size={self.window_size}")
+        opts.add_argument("--remote-debugging-port=0")
 
         if self.user_agent:
             opts.add_argument(f"--user-agent={self.user_agent}")
@@ -88,29 +93,86 @@ class BrowserManager:
 
         opts.add_experimental_option("excludeSwitches", ["enable-logging"])
         opts.add_experimental_option("useAutomationExtension", False)
+        return opts
+
+    def _start_driver_with_retries(self, attempts: int = 3, delay_seconds: float = 1.0):
+        last_exc = None
+        for attempt in range(1, attempts + 1):
+            try:
+                logger.info("Starting Chrome driver (attempt %d/%d)...", attempt, attempts)
+                self._start_driver()
+                logger.info("Chrome driver started successfully.")
+                return
+            except Exception as e:
+                logger.exception("Failed to start driver on attempt %d: %s", attempt, e)
+                last_exc = e
+                time.sleep(delay_seconds)
+        raise RuntimeError(f"Unable to start Chrome driver after {attempts} attempts: {last_exc}") from last_exc
+
+    def _start_driver(self):
+        opts = self._build_options()
 
-        # Attempt to download a matching chromedriver using webdriver_manager
+        # First: try letting Selenium Manager handle driver (selenium >=4.14)
+        try:
+            logger.debug("Trying webdriver.Chrome(options=opts) to allow Selenium Manager to find driver.")
+            self._driver = webdriver.Chrome(options=opts)
+            # quick smoke test: get version via execute_script -> may throw if browser crashed
+            try:
+                self._driver.execute_script("return navigator.userAgent")
+            except Exception:
+                # driver started but browser died; fallthrough to fallback
+                raise RuntimeError("Selenium Manager started a session but the browser crashed immediately.")
+            # if success, configure network blocking best-effort
+            self._post_start_setup()
+            return
+        except Exception as e_primary:
+            logger.warning("Selenium Manager attempt failed: %s", e_primary)
+
+        # Fallback: use webdriver-manager to download compatible chromedriver, ensure executable, and use it
         try:
             driver_path = ChromeDriverManager().install()
+            logger.info("webdriver-manager installed chromedriver: %s", driver_path)
+            # ensure executable bit set
+            try:
+                os.chmod(driver_path, 0o755)
+            except Exception as ex_perm:
+                logger.warning("Could not chmod chromedriver: %s", ex_perm)
             service = Service(driver_path)
             self._driver = webdriver.Chrome(service=service, options=opts)
-        except Exception as e:
-            # If webdriver_manager failed or binary mismatch, try a minimal fallback
+            self._post_start_setup()
+            return
+        except Exception as e_fallback:
+            logger.exception("webdriver-manager fallback failed: %s", e_fallback)
+            # try one more variation: if /usr/bin/chromedriver exists, use it
             try:
-                # attempt to use chromedriver in PATH if available
-                service = Service("/usr/bin/chromedriver")
-                self._driver = webdriver.Chrome(service=service, options=opts)
-            except Exception as ee:
-                raise RuntimeError(f"Failed to start Chrome driver: {e}; fallback: {ee}")
+                system_path = "/usr/bin/chromedriver"
+                if os.path.exists(system_path):
+                    logger.info("Trying system chromedriver at %s", system_path)
+                    os.chmod(system_path, 0o755)
+                    service = Service(system_path)
+                    self._driver = webdriver.Chrome(service=service, options=opts)
+                    self._post_start_setup()
+                    return
+                else:
+                    logger.debug("No system chromedriver at %s", system_path)
+            except Exception as e_sys:
+                logger.exception("System chromedriver attempt failed: %s", e_sys)
+
+            # raise combined error
+            raise RuntimeError(f"Failed to start Chrome driver. primary_error={e_primary}, fallback_error={e_fallback}")
 
-        # Optional quick setup: allow CDP commands to block resources (best-effort)
+    def _post_start_setup(self):
+        # set reasonable timeout and (best-effort) block some urls using CDP
         try:
             self._driver.set_page_load_timeout(60)
             self._driver.execute_cdp_cmd("Network.enable", {})
             if self.block_resource_urls:
-                self._driver.execute_cdp_cmd("Network.setBlockedURLs", {"urls": self.block_resource_urls})
+                try:
+                    self._driver.execute_cdp_cmd("Network.setBlockedURLs", {"urls": self.block_resource_urls})
+                except Exception:
+                    # some Chrome builds may not support this; ignore
+                    pass
         except Exception:
-            # ignore CDP or timeout errors
             pass
 
     def fetch_html(
@@ -118,10 +180,9 @@ class BrowserManager:
         url: str,
         wait_seconds: Optional[float] = 10.0,
         wait_for_selector: Optional[str] = None,
-        force_reload: bool = True,
     ) -> str:
         if self._driver is None:
-            self._start_driver()
+            self._start_driver_with_retries()
 
         with self._driver_lock:
             driver = self._driver
@@ -146,13 +207,13 @@ class BrowserManager:
 
                 return driver.page_source
             except WebDriverException as e:
-                # try to recover by restarting driver
+                logger.exception("WebDriver exception during fetch: %s", e)
+                # restart driver and raise
                 try:
                     self._safe_quit_driver()
                 except Exception:
                     pass
-                # restart and raise to caller
-                self._start_driver()
+                self._start_driver_with_retries()
                 raise RuntimeError(f"WebDriver error during fetch: {e}")
 
     def _safe_quit_driver(self):
@@ -167,7 +228,7 @@ class BrowserManager:
         self._safe_quit_driver()
 
 
-# ---------------- EXTRACT_DATA (kept same as in your original) ----------------
+# ---------------- EXTRACT_DATA ----------------
 def EXTRACT_DATA(html: str) -> Dict[str, Any]:
     soup = BeautifulSoup(html, "html.parser")
     BASE_URL = "https://www.google.com"
@@ -355,7 +416,8 @@ def EXTRACT_DATA(html: str) -> Dict[str, Any]:
 
     return data
 
-# ---------------- BrowserPool and cache ----------------
+
+# ---------------- BrowserPool and API ----------------
 class BrowserPool:
     def __init__(self, pool_size: int = 1, headless: bool = True):
         self.pool_size = max(1, pool_size)
@@ -397,17 +459,13 @@ class SimpleTTLCache:
         with self._lock:
             self._cache[key] = (time.time(), value)
 
-# API models
 class SearchRequest(BaseModel):
     query: Optional[str] = None
     url: Optional[str] = None
     wait_for_selector: Optional[str] = None
     headless: Optional[bool] = True
 
-# FastAPI app
 app = FastAPI(title="fast_fetcher_api", version="0.1")
-
-# Global shared objects (initialized on startup)
 POOL: Optional[BrowserPool] = None
 EXECUTOR: Optional[ThreadPoolExecutor] = None
 CACHE = SimpleTTLCache(ttl_seconds=25)
@@ -415,12 +473,12 @@ CACHE = SimpleTTLCache(ttl_seconds=25)
 @app.on_event("startup")
 async def startup_event():
     global POOL, EXECUTOR
-    # set headless=True for Docker. Tune pool_size as needed.
+    # in Docker use headless True
     POOL = BrowserPool(pool_size=1, headless=True)
     EXECUTOR = ThreadPoolExecutor(max_workers=2)
     app.state.executor = EXECUTOR
     app.state.pool = POOL
-    print("Startup: browser pool created (size=1).")
+    logger.info("Startup: browser pool created (size=%d).", 1)
 
 @app.on_event("shutdown")
 async def shutdown_event():
@@ -429,7 +487,7 @@ async def shutdown_event():
         POOL.close_all()
     if EXECUTOR:
         EXECUTOR.shutdown(wait=True)
-    print("Shutdown: browsers closed and executor stopped.")
+    logger.info("Shutdown: browsers closed and executor stopped.")
 
 def _blocking_fetch_and_extract(manager: BrowserManager, url: str, wait_for_selector: Optional[str], wait_seconds: Optional[float]):
     start = time.time()