Update app.py

app.py

@@ -17,7 +17,7 @@ from starlette.responses import JSONResponse
 # --- bring in your BrowserManager and EXTRACT_DATA functions (adapted) ---
 # For brevity we import them inline. If you already have them in a module,
 # replace with: from your_module import BrowserManager, EXTRACT_DATA
-
+import os
 import threading
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
@@ -60,15 +60,26 @@ class BrowserManager:
 
     def _start_driver(self):
         opts = Options()
+
+        # set binary location from env if provided (in Dockerfile we set CHROME_BIN)
+        chrome_bin = os.environ.get("CHROME_BIN", "/usr/bin/google-chrome-stable")
+        if os.path.exists(chrome_bin):
+            opts.binary_location = chrome_bin
+
+        # headless flags
         if self.headless:
+            # new headless mode for newer chrome versions; keep both to be safe
             opts.add_argument("--headless=new")
             opts.add_argument("--headless")
-        opts.add_argument(f"--window-size={self.window_size}")
+        # essential flags for running chrome in containers
         opts.add_argument("--no-sandbox")
+        opts.add_argument("--disable-setuid-sandbox")
         opts.add_argument("--disable-dev-shm-usage")
         opts.add_argument("--disable-gpu")
         opts.add_argument("--disable-extensions")
         opts.add_argument("--disable-blink-features=AutomationControlled")
+        opts.add_argument("--single-process")
+        opts.add_argument(f"--window-size={self.window_size}")
         if self.user_agent:
             opts.add_argument(f"--user-agent={self.user_agent}")
 
@@ -81,22 +92,22 @@ class BrowserManager:
             opts.add_experimental_option("prefs", prefs)
 
         opts.add_experimental_option("excludeSwitches", ["enable-logging"])
+        # optional: avoid "Chrome is being controlled by automated test software" banner
+        opts.add_experimental_option("useAutomationExtension", False)
 
-        # Some systems may not accept the desired_capabilities arg in this signature,
-        # so we set pageLoadStrategy later via browser options / capabilities if needed.
-        service = Service(ChromeDriverManager().install())
-        self._driver = webdriver.Chrome(service=service, options=opts)
-
-        # quick defaults
+        # create service: webdriver_manager will download a chromedriver binary
+        # that matches the detected Chrome version (usually)
         try:
-            self._driver.set_page_load_timeout(60)
-            # Enable CDP to block some resource URLs (best-effort)
-            self._driver.execute_cdp_cmd("Network.enable", {})
-            if self.block_resource_urls:
-                self._driver.execute_cdp_cmd("Network.setBlockedURLs", {"urls": self.block_resource_urls})
-        except Exception:
-            # ignore CDP failures gracefully
-            pass
+            driver_path = ChromeDriverManager().install()
+            service = Service(driver_path)
+            self._driver = webdriver.Chrome(service=service, options=opts)
+        except Exception as e:
+            # fallback: try system chromedriver if present
+            try:
+                service = Service("/usr/bin/chromedriver")
+                self._driver = webdriver.Chrome(service=service, options=opts)
+            except Exception as ee:
+                raise RuntimeError(f"Failed to start Chrome driver: {e}; fallback: {ee}")
 
     def fetch_html(
         self,