Update Facebook_Marketplace_Scraper.py

Facebook_Marketplace_Scraper.py

@@ -25,10 +25,18 @@ from datetime import datetime
 from bs4 import BeautifulSoup as soup
 from splinter import Browser
 from webdriver_manager.chrome import ChromeDriverManager
-from webdriver_manager.firefox import GeckoDriverManager
 from selenium.webdriver.chrome.service import Service as ChromeService
 from selenium.webdriver.firefox.service import Service as FirefoxService
 import json
+import geckodriver_autoinstaller
+import platform
+
+# For headless browser in cloud environments
+try:
+    from pyvirtualdisplay import Display
+    VIRTUAL_DISPLAY_AVAILABLE = True
+except ImportError:
+    VIRTUAL_DISPLAY_AVAILABLE = False
 
 class FacebookMarketplaceScraper:
     def __init__(self, headless=True, debug=True):
@@ -43,11 +51,18 @@ class FacebookMarketplaceScraper:
         self.debug = debug
         self.browser = None
         self.listings = []
+        self.display = None
         
         # Create output directory if it doesn't exist
         self.output_dir = "marketplace_output"
         if not os.path.exists(self.output_dir):
             os.makedirs(self.output_dir)
+            
+        # Initialize virtual display for cloud environments if available
+        if VIRTUAL_DISPLAY_AVAILABLE and platform.system() == 'Linux':
+            self.display = Display(visible=0, size=(1920, 1080))
+            self.display.start()
+            print("Virtual display started")
     
     def initialize_browser(self):
         """Initialize browser with Splinter"""
@@ -55,22 +70,50 @@ class FacebookMarketplaceScraper:
             # First try Firefox (better for cloud environments like Hugging Face)
             try:
                 print("Attempting to initialize Firefox browser...")
+                
+                # Auto-install geckodriver
+                geckodriver_path = geckodriver_autoinstaller.install()
+                print(f"Geckodriver installed at: {geckodriver_path}")
+                
+                # Configure Firefox options
+                from selenium import webdriver
+                from selenium.webdriver.firefox.options import Options
+                
+                firefox_options = Options()
+                if self.headless:
+                    firefox_options.add_argument("--headless")
+                
+                firefox_options.add_argument("--no-sandbox")
+                firefox_options.add_argument("--disable-dev-shm-usage")
+                
                 browser_options = {
                     'headless': self.headless,
-                    'service': FirefoxService(GeckoDriverManager().install())
+                    'options': firefox_options
                 }
+                
                 self.browser = Browser('firefox', **browser_options)
                 print("Firefox browser initialized successfully")
                 return True
             except Exception as firefox_error:
                 print(f"Firefox initialization failed: {firefox_error}")
+                import traceback
+                traceback.print_exc()
                 
                 # Fallback to Chrome if Firefox fails
                 print("Falling back to Chrome browser...")
                 service = ChromeService(ChromeDriverManager().install())
+                
+                from selenium.webdriver.chrome.options import Options
+                chrome_options = Options()
+                if self.headless:
+                    chrome_options.add_argument("--headless")
+                chrome_options.add_argument("--no-sandbox")
+                chrome_options.add_argument("--disable-dev-shm-usage")
+                
                 browser_options = {
                     'headless': self.headless,
-                    'service': service
+                    'service': service,
+                    'options': chrome_options
                 }
                 self.browser = Browser('chrome', **browser_options)
                 print("Chrome browser initialized successfully")
@@ -87,9 +130,16 @@ class FacebookMarketplaceScraper:
             try:
                 self.browser.quit()
                 print("Browser closed successfully")
-                self.browser = None
             except Exception as e:
                 print(f"Error closing browser: {e}")
+                
+        # Stop virtual display if it was started
+        if self.display:
+            try:
+                self.display.stop()
+                print("Virtual display stopped")
+            except:
+                pass
     
     def build_search_url(self, location, min_price=None, max_price=None, days_listed=None, 
                          min_mileage=None, max_mileage=None, min_year=None, max_year=None,