Update app.py

app.py

@@ -19,11 +19,10 @@ import traceback
 import os
 import re
 import time
-import numpy as np
-from typing import List
+import psutil
 from selenium.common.exceptions import WebDriverException, TimeoutException
 
-# Set up logging to /tmp/app.log
+# Set up logging
 logging.basicConfig(
     filename='/tmp/app.log',
     level=logging.DEBUG,
@@ -45,7 +44,7 @@ This app combines web scraping with Retrieval-Augmented Generation (RAG) to crea
 It can scrape websites, index the content, and answer your questions about the scraped content.
 """)
 
-# Initialize session state variables
+# Initialize session state
 if 'scraped_content' not in st.session_state:
     st.session_state.scraped_content = ""
 if 'vector_store' not in st.session_state:
@@ -55,8 +54,27 @@ if 'chat_history' not in st.session_state:
 if 'qa_pipeline' not in st.session_state:
     st.session_state.qa_pipeline = None
 
+def cleanup_chromedriver_processes():
+    """Kill any lingering ChromeDriver processes."""
+    try:
+        for proc in psutil.process_iter(['name']):
+            if proc.info['name'] == 'chromedriver':
+                proc.kill()
+                logging.info(f"Killed ChromeDriver process PID {proc.pid}")
+    except Exception as e:
+        logging.warning(f"Error cleaning up ChromeDriver processes: {str(e)}")
+
+def check_port(port):
+    """Check if a port is in use."""
+    try:
+        result = subprocess.run(['netstat', '-tuln'], capture_output=True, text=True)
+        return f':{port}' in result.stdout
+    except Exception as e:
+        logging.warning(f"Error checking port {port}: {str(e)}")
+        return False
+
 def find_binary(binary_name):
-    """Find binary path using 'find' command."""
+    """Find binary path."""
     try:
         result = subprocess.check_output(
             ['find', '/', '-name', binary_name, '-type', 'f', '-executable'],
@@ -70,64 +88,36 @@ def find_binary(binary_name):
         return None
 
 def check_versions(chromium_path, chromedriver_path):
-    """Log Chromium and ChromeDriver versions for debugging."""
+    """Log Chromium and ChromeDriver versions."""
     if chromium_path:
         try:
             chromium_version = subprocess.check_output([chromium_path, '--version']).decode().strip()
             logging.info(f"Chromium path: {chromium_path}, version: {chromium_version}")
         except Exception as e:
             logging.error(f"Error checking Chromium version: {str(e)}")
-    else:
-        logging.error("Chromium binary not found")
     if chromedriver_path:
         try:
             chromedriver_version = subprocess.check_output([chromedriver_path, '--version']).decode().strip()
             logging.info(f"ChromeDriver path: {chromedriver_path}, version: {chromedriver_version}")
         except Exception as e:
             logging.error(f"Error checking ChromeDriver version: {str(e)}")
-    else:
-        logging.error("ChromeDriver binary not found")
 
 @st.cache_resource
 def setup_driver():
-    """Set up Selenium WebDriver with headless Chromium."""
+    """Set up Selenium WebDriver."""
+    cleanup_chromedriver_processes()  # Clean up before starting
     try:
-        # Define possible binary paths
-        possible_chromium_paths = [
-            '/usr/lib/chromium-browser/chromium-browser',
-            '/usr/bin/chromium-browser',
-            '/usr/bin/chromium',
-            '/usr/bin/chrome'
-        ]
-        possible_chromedriver_paths = [
-            '/usr/lib/chromium-browser/chromedriver',
-            '/usr/bin/chromedriver'
-        ]
-
-        # Search for binaries
-        chromium_path = None
-        for path in possible_chromium_paths:
-            if os.path.exists(path):
-                chromium_path = path
-                break
-        if not chromium_path:
-            chromium_path = find_binary('chromium') or find_binary('chromium-browser') or find_binary('chrome')
+        chromium_path = os.getenv('CHROMIUM_PATH', '/usr/bin/chromium')
+        chromedriver_path = os.getenv('CHROMEDRIVER_PATH', '/usr/bin/chromedriver')
 
-        chromedriver_path = None
-        for path in possible_chromedriver_paths:
-            if os.path.exists(path):
-                chromedriver_path = path
-                break
-        if not chromedriver_path:
+        # Verify binaries
+        if not os.path.exists(chromium_path):
+            chromium_path = find_binary('chromium') or find_binary('chromium-browser')
+        if not os.path.exists(chromedriver_path):
             chromedriver_path = find_binary('chromedriver')
 
-        # Log versions for debugging
         check_versions(chromium_path, chromedriver_path)
 
-        if not chromium_path:
-            logging.warning("No Chromium binary found, using default path")
-            chromium_path = '/usr/bin/chromium'
-
         # Configure Chrome options
         options = Options()
         options.add_argument('--headless=new')
@@ -137,36 +127,35 @@ def setup_driver():
         options.add_argument('--disable-extensions')
         options.add_argument('--disable-background-networking')
         options.add_argument('--window-size=1920,1080')
-        options.add_argument('--remote-debugging-port=9222')
-        options.add_argument('--ignore-certificate-errors')  # Handle SSL issues
-        options.add_argument('--disable-web-security')  # Relax CORS for testing
-        options.binary_location = chromium_path
+        options.add_argument('--remote-debugging-port=0')  # Random port to avoid conflicts
+        options.add_argument('--ignore-certificate-errors')
+        options.add_argument('--disable-web-security')
+        options.binary_location = chromium_path if chromium_path else '/usr/bin/chromium'
 
         # Initialize ChromeDriver
-        if chromedriver_path and os.path.exists(chromedriver_path):
-            try:
-                subprocess.run(['chmod', '+x', chromedriver_path], check=True)
-                logging.info(f"Set executable permissions for {chromedriver_path}")
-            except subprocess.CalledProcessError as e:
-                logging.warning(f"Failed to chmod {chromedriver_path}: {str(e)}")
+        try:
+            subprocess.run(['chmod', '+x', chromedriver_path], check=True)
             service = Service(chromedriver_path)
-        else:
-            logging.info("Using webdriver-manager for ChromeDriver")
+        except:
+            logging.info("Falling back to webdriver-manager")
             chromedriver_path = ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install()
             subprocess.run(['chmod', '+x', chromedriver_path], check=True)
             service = Service(chromedriver_path)
 
-        # Start WebDriver with retry
         max_attempts = 3
         for attempt in range(max_attempts):
             try:
+                if check_port(service.port):
+                    logging.warning(f"Port {service.port} in use, stopping service")
+                    service.stop()
+                    time.sleep(1)
                 service.start()
                 driver = webdriver.Chrome(service=service, options=options)
                 driver.set_page_load_timeout(60)
                 logging.info(f"ChromeDriver initialized on port {service.port}")
                 return driver, service
             except WebDriverException as e:
-                logging.warning(f"WebDriver initialization failed (attempt {attempt + 1}/{max_attempts}): {str(e)}")
+                logging.warning(f"WebDriver init failed (attempt {attempt + 1}/{max_attempts}): {str(e)}")
                 if attempt < max_attempts - 1:
                     try:
                         service.stop()
@@ -188,7 +177,7 @@ def clean_text(text):
     return text.strip()
 
 def scrape_website(url):
-    """Scrape data from the given URL with retry logic."""
+    """Scrape data from the given URL."""
     driver, service = setup_driver()
     if not driver or not service:
         return None
@@ -200,9 +189,7 @@ def scrape_website(url):
             WebDriverWait(driver, 30).until(
                 EC.presence_of_element_located((By.TAG_NAME, "body"))
             )
-            # Get page title
             title = driver.title
-            # Get main content (Wikipedia-specific)
             try:
                 main_content = WebDriverWait(driver, 10).until(
                     EC.presence_of_element_located((By.ID, "content"))
@@ -269,7 +256,7 @@ def initialize_qa_model():
 
 @st.cache_resource
 def create_vector_store(text):
-    """Create a FAISS vector store from scraped text."""
+    """Create a FAISS vector store."""
     try:
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=500,