Update scraper.py

scraper.py

@@ -1,4 +1,3 @@
-
 import os
 import io
 import time
@@ -11,14 +10,13 @@ from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.support.expected_conditions import staleness_of
 from selenium.common.exceptions import (
     NoSuchElementException,
     TimeoutException,
     StaleElementReferenceException,
 )
-from selenium.webdriver.firefox.service import Service
-from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
 
 from huggingface_hub import HfApi, HfFolder
 
@@ -56,23 +54,23 @@ def safe_get_text(driver, xpath, retries=1, delay=0.5):
 
 
 def initialize_driver():
-    # point to the geckodriver binary you installed
-    service = Service("/usr/local/bin/geckodriver")
-
-    opts = Options()
-    opts.headless = True
-    # disable sandbox (in many container environments)
-    opts.add_argument("--no-sandbox")
-    opts.add_argument("--disable-gpu")
-    # ensure a full window so responsive pages load properly
-    opts.add_argument("--window-size=1920,1080")
-
-    driver = webdriver.Firefox(service=service, options=opts)
+    # path to the chromedriver you installed in Dockerfile
+    service = Service("/usr/local/bin/chromedriver")
+
+    options = Options()
+    # tell Selenium exactly where Chrome itself lives
+    options.binary_location = "/usr/bin/google-chrome-stable"
+    options.add_argument("--headless=new")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument("--window-size=1920,1080")
+
+    # launch Chrome via the Service
+    driver = webdriver.Chrome(service=service, options=options)
     return driver
 
 
-
-
 def process_batch(start_id, end_id, worker_id):
     print(f"Worker {worker_id} processing IDs {start_id} to {end_id}")
     try: