Update app.py

app.py

@@ -5,8 +5,6 @@ from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-from webdriver_manager.chrome import ChromeDriverManager
-from webdriver_manager.core.os_manager import ChromeType
 from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
 from sentence_transformers import SentenceTransformer
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -77,12 +75,10 @@ def find_binary(binary_name):
     """Find binary path."""
     try:
         result = subprocess.check_output(
-            ['find', '/', '-name', binary_name, '-type', 'f', '-executable'],
+            ['which', binary_name],
             stderr=subprocess.DEVNULL
         ).decode().strip()
-        paths = result.split('\n') if result else []
-        logging.info(f"Found {binary_name} at: {paths}")
-        return paths[0] if paths else None
+        return result if result else None
     except Exception as e:
         logging.error(f"Error finding {binary_name}: {str(e)}")
         return None
@@ -107,14 +103,28 @@ def setup_driver():
     """Set up Selenium WebDriver."""
     cleanup_chromedriver_processes()  # Clean up before starting
     try:
+        # Set custom cache directory to avoid permission issues
+        os.environ['WDM_CACHE_DIR'] = '/tmp/.wdm'
+        
         chromium_path = os.getenv('CHROMIUM_PATH', '/usr/bin/chromium')
         chromedriver_path = os.getenv('CHROMEDRIVER_PATH', '/usr/bin/chromedriver')
 
-        # Verify binaries
+        # Verify binaries exist
         if not os.path.exists(chromium_path):
             chromium_path = find_binary('chromium') or find_binary('chromium-browser')
+            if not chromium_path:
+                st.error("Chromium not found. Please ensure it's installed.")
+                return None, None
+            
         if not os.path.exists(chromedriver_path):
             chromedriver_path = find_binary('chromedriver')
+            if not chromedriver_path:
+                st.error("ChromeDriver not found. Please ensure it's installed.")
+                return None, None
+
+        # Ensure executables have proper permissions
+        subprocess.run(['chmod', '+x', chromedriver_path], check=True)
+        subprocess.run(['chmod', '+x', chromium_path], check=True)
 
         check_versions(chromium_path, chromedriver_path)
 
@@ -127,21 +137,14 @@ def setup_driver():
         options.add_argument('--disable-extensions')
         options.add_argument('--disable-background-networking')
         options.add_argument('--window-size=1920,1080')
-        options.add_argument('--remote-debugging-port=0')  # Random port to avoid conflicts
+        options.add_argument('--remote-debugging-port=0')
         options.add_argument('--ignore-certificate-errors')
         options.add_argument('--disable-web-security')
-        options.binary_location = chromium_path if chromium_path else '/usr/bin/chromium'
-
-        # Initialize ChromeDriver
-        try:
-            subprocess.run(['chmod', '+x', chromedriver_path], check=True)
-            service = Service(chromedriver_path)
-        except:
-            logging.info("Falling back to webdriver-manager")
-            chromedriver_path = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()
-            subprocess.run(['chmod', '+x', chromedriver_path], check=True)
-            service = Service(chromedriver_path)
+        options.binary_location = chromium_path
 
+        # Initialize ChromeDriver service
+        service = Service(executable_path=chromedriver_path)
+        
         max_attempts = 3
         for attempt in range(max_attempts):
             try:
@@ -149,11 +152,13 @@ def setup_driver():
                     logging.warning(f"Port {service.port} in use, stopping service")
                     service.stop()
                     time.sleep(1)
+                
                 service.start()
                 driver = webdriver.Chrome(service=service, options=options)
                 driver.set_page_load_timeout(60)
                 logging.info(f"ChromeDriver initialized on port {service.port}")
                 return driver, service
+                
             except WebDriverException as e:
                 logging.warning(f"WebDriver init failed (attempt {attempt + 1}/{max_attempts}): {str(e)}")
                 if attempt < max_attempts - 1:
@@ -164,6 +169,7 @@ def setup_driver():
                     time.sleep(2)
                 else:
                     raise
+                    
     except Exception as e:
         logging.error(f"Error setting up WebDriver: {str(e)}")
         logging.error(traceback.format_exc())
@@ -180,57 +186,62 @@ def scrape_website(url):
     """Scrape data from the given URL."""
     driver, service = setup_driver()
     if not driver or not service:
+        st.error("Failed to initialize WebDriver. Please check if Chromium and ChromeDriver are properly installed.")
         return None
-    max_attempts = 3
-    for attempt in range(max_attempts):
-        try:
-            logging.info(f"Attempting to scrape {url} (attempt {attempt + 1}/{max_attempts})")
-            driver.get(url)
-            WebDriverWait(driver, 30).until(
-                EC.presence_of_element_located((By.TAG_NAME, "body"))
-            )
-            title = driver.title
+        
+    try:
+        logging.info(f"Attempting to scrape {url}")
+        driver.get(url)
+        WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located((By.TAG_NAME, "body"))
+        )
+        
+        # Get page title
+        title = driver.title
+        
+        # Try multiple selectors for main content
+        content_selectors = [
+            (By.ID, "content"),
+            (By.CLASS_NAME, "mw-parser-output"),
+            (By.TAG_NAME, "main"),
+            (By.CLASS_NAME, "main-content"),
+            (By.ID, "main"),
+            (By.TAG_NAME, "article")
+        ]
+        
+        main_content = None
+        for by, value in content_selectors:
             try:
-                main_content = WebDriverWait(driver, 10).until(
-                    EC.presence_of_element_located((By.ID, "content"))
-                )
+                main_content = driver.find_element(by, value)
+                break
             except:
-                try:
-                    main_content = driver.find_element(By.CLASS_NAME, "mw-parser-output")
-                except:
-                    main_content = driver.find_element(By.TAG_NAME, "body")
-            text_content = main_content.text
-            cleaned_content = clean_text(text_content)
-            logging.info(f"Scraped {len(cleaned_content)} characters from {url}")
-            return {
-                "title": title,
-                "content": cleaned_content,
-                "url": url
-            }
-        except (WebDriverException, TimeoutException) as e:
-            logging.warning(f"Scraping failed for {url} (attempt {attempt + 1}/{max_attempts}): {str(e)}")
-            if attempt < max_attempts - 1:
-                try:
-                    driver.quit()
-                    service.stop()
-                except:
-                    pass
-                driver, service = setup_driver()
-                if not driver:
-                    return None
-                time.sleep(2)
-            else:
-                logging.error(f"Error scraping {url}: {str(e)}")
-                logging.error(traceback.format_exc())
-                st.error(f"Error scraping {url}: {str(e)}")
-                return None
-        finally:
-            try:
-                driver.quit()
-                service.stop()
-                logging.info("WebDriver and service stopped")
-            except Exception as e:
-                logging.warning(f"Error quitting driver: {str(e)}")
+                continue
+                
+        if not main_content:
+            main_content = driver.find_element(By.TAG_NAME, "body")
+            
+        text_content = main_content.text
+        cleaned_content = clean_text(text_content)
+        logging.info(f"Scraped {len(cleaned_content)} characters from {url}")
+        
+        return {
+            "title": title,
+            "content": cleaned_content,
+            "url": url
+        }
+        
+    except (WebDriverException, TimeoutException) as e:
+        logging.error(f"Error scraping {url}: {str(e)}")
+        st.error(f"Error scraping {url}: {str(e)}")
+        return None
+        
+    finally:
+        try:
+            driver.quit()
+            service.stop()
+            logging.info("WebDriver and service stopped")
+        except Exception as e:
+            logging.warning(f"Error quitting driver: {str(e)}")
 
 @st.cache_resource
 def initialize_qa_model():