Update app.py

app.py

@@ -1,23 +1,12 @@
 import streamlit as st
 import os
 import re
-import time
-import subprocess
-import shutil
 import logging
-import psutil
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import WebDriverException, TimeoutException
+from playwright.sync_api import sync_playwright
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain.schema import Document
-import chromedriver_autoinstaller
 
 # Try importing transformers with fallback
 try:
@@ -38,21 +27,11 @@ except ImportError as e:
 
 # Set up logging
 logging.basicConfig(
-    filename='/tmp/app.log',
+    filename='/app/cache/app.log',
     level=logging.DEBUG,
     format='%(asctime)s - %(levelname)s - %(message)s'
 )
 
-# Health check
-logging.info("Starting application health check")
-try:
-    logging.info(f"Python version: {subprocess.check_output(['python', '--version']).decode().strip()}")
-    logging.info(f"Pip list: {subprocess.check_output(['pip', 'list']).decode()}")
-    logging.info(f"Environment variables: {os.environ}")
-    logging.info(f"/tmp contents: {subprocess.check_output(['ls', '-la', '/tmp']).decode()}")
-except Exception as e:
-    logging.error(f"Health check failed: {str(e)}")
-
 # Set page configuration
 st.set_page_config(
     page_title="Web Scraping + RAG Chatbot",
@@ -65,7 +44,7 @@ st.set_page_config(
 st.title("🕷️ Web Scraping + RAG Chatbot")
 st.markdown("""
 This app combines web scraping with Retrieval-Augmented Generation (RAG) to create an intelligent chatbot.
-It can scrape websites, index the content, and answer your questions about the scraped content.
+Enter a URL to scrape its content, then ask questions about the scraped data.
 """)
 
 # Initialize session state
@@ -78,156 +57,6 @@ if 'chat_history' not in st.session_state:
 if 'qa_pipeline' not in st.session_state:
     st.session_state.qa_pipeline = None
 
-def cleanup_chromedriver_processes():
-    """Kill any lingering ChromeDriver and Chrome processes."""
-    try:
-        active_processes = []
-        for proc in psutil.process_iter(['name', 'cmdline', 'pid']):
-            try:
-                name = proc.info['name'].lower()
-                cmdline = proc.info.get('cmdline', [])
-                if 'chromedriver' in name or 'chrome' in name or 'chromium' in name:
-                    if any('type=renderer' in arg for arg in cmdline):
-                        continue
-                    active_processes.append(f"{name} (PID {proc.pid})")
-                    proc.terminate()
-                    try:
-                        proc.wait(timeout=5)
-                        logging.info(f"Terminated process {name} PID {proc.pid}")
-                    except psutil.TimeoutExpired:
-                        proc.kill()
-                        logging.info(f"Force killed process {name} PID {proc.pid}")
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                pass
-        if active_processes:
-            logging.info(f"Terminated processes: {', '.join(active_processes)}")
-        else:
-            logging.info("No Chrome-related processes found")
-        # Additional cleanup with killall
-        try:
-            subprocess.run(['killall', '-9', 'chromedriver', 'chromium', 'chrome'], check=False, capture_output=True)
-            logging.info("Ran killall for chromedriver, chromium, and chrome")
-        except Exception as e:
-            logging.warning(f"Error running killall: {str(e)}")
-    except Exception as e:
-        logging.warning(f"Error cleaning up processes: {str(e)}")
-
-def find_binary(binary_name):
-    """Find binary path."""
-    try:
-        result = subprocess.check_output(['which', binary_name], stderr=subprocess.DEVNULL).decode().strip()
-        logging.info(f"Found {binary_name} at: {result}")
-        return result if result else None
-    except Exception as e:
-        logging.error(f"Error finding {binary_name}: {str(e)}")
-        return None
-
-def check_versions(chromium_path, chromedriver_path):
-    """Log Chromium and ChromeDriver versions."""
-    if chromedriver_path:
-        try:
-            chromedriver_version = subprocess.check_output([chromedriver_path, '--version']).decode().strip()
-            logging.info(f"ChromeDriver path: {chromedriver_path}, version: {chromedriver_version}")
-        except Exception as e:
-            logging.error(f"Error checking ChromeDriver version: {str(e)}")
-    if chromium_path:
-        try:
-            chromium_version = subprocess.check_output([chromium_path, '--version']).decode().strip()
-            logging.info(f"Chromium path: {chromium_path}, version: {chromium_version}")
-        except Exception as e:
-            logging.error(f"Error checking Chromium version: {str(e)}")
-
-def check_disk_space():
-    """Log available disk space for /tmp."""
-    try:
-        stat = shutil.disk_usage('/tmp')
-        logging.info(f"Disk space for /tmp: Total={stat.total / (1024**3):.2f}GB, Used={stat.used / (1024**3):.2f}GB, Free={stat.free / (1024**3):.2f}GB")
-    except Exception as e:
-        logging.warning(f"Error checking disk space: {str(e)}")
-
-def setup_driver():
-    """Set up Selenium WebDriver."""
-    logging.info("Setting up Selenium WebDriver")
-    cleanup_chromedriver_processes()
-    check_disk_space()
-    chromium_path = os.getenv('CHROMIUM_PATH', '/usr/bin/chromium')
-
-    # Auto-install ChromeDriver
-    chromedriver_path = None
-    try:
-        chromedriver_path = chromedriver_autoinstaller.install()
-        if chromedriver_path:
-            logging.info(f"Auto-installed ChromeDriver at: {chromedriver_path}")
-        else:
-            logging.warning("chromedriver_autoinstaller.install() returned None")
-    except Exception as e:
-        logging.error(f"Error auto-installing ChromeDriver: {str(e)}")
-
-    # Fallback to environment variable or find_binary
-    if not chromedriver_path:
-        chromedriver_path = os.getenv('CHROMEDRIVER_PATH', '/usr/bin/chromedriver')
-        logging.info(f"Falling back to CHROMEDRIVER_PATH: {chromedriver_path}")
-        if not chromedriver_path or not os.path.exists(chromedriver_path):
-            chromedriver_path = find_binary('chromedriver')
-            if not chromedriver_path:
-                st.error("ChromeDriver not found. Please ensure it's installed.")
-                logging.error("No ChromeDriver found after fallback attempts")
-                return None, None
-
-    # Verify Chromium exists
-    if not os.path.exists(chromium_path):
-        chromium_path = find_binary('chromium') or find_binary('chromium-browser')
-        if not chromium_path:
-            st.error("Chromium not found. Please ensure it's installed.")
-            logging.error("No Chromium found")
-            return None, None
-
-    # Check executable permissions
-    if not os.access(chromedriver_path, os.X_OK):
-        try:
-            subprocess.run(['chmod', '+x', chromedriver_path], check=True)
-            logging.info(f"Set executable permissions on {chromedriver_path}")
-        except subprocess.CalledProcessError:
-            st.warning(f"Could not set executable permissions on {chromedriver_path}")
-    if not os.access(chromium_path, os.X_OK):
-        try:
-            subprocess.run(['chmod', '+x', chromium_path], check=True)
-            logging.info(f"Set executable permissions on {chromium_path}")
-        except subprocess.CalledProcessError:
-            st.warning(f"Could not set executable permissions on {chromium_path}")
-
-    check_versions(chromium_path, chromedriver_path)
-
-    # Configure Chrome options
-    options = Options()
-    options.add_argument('--headless=new')
-    options.add_argument('--no-sandbox')
-    options.add_argument('--disable-dev-shm-usage')
-    options.add_argument('--disable-gpu')
-    options.add_argument('--disable-extensions')
-    options.add_argument('--disable-background-networking')
-    options.add_argument('--window-size=1920,1080')
-    options.add_argument('--ignore-certificate-errors')
-    options.binary_location = chromium_path
-
-    # Initialize ChromeDriver
-    try:
-        service = Service(executable_path=chromedriver_path, port=0)
-        logging.info("Starting ChromeDriver service")
-        service.start()
-        driver = webdriver.Chrome(service=service, options=options)
-        driver.set_page_load_timeout(60)
-        logging.info("ChromeDriver initialized successfully")
-        return driver, service
-    except WebDriverException as e:
-        logging.error(f"Failed to initialize WebDriver: {str(e)}")
-        st.error(f"Failed to initialize WebDriver: {str(e)}")
-        return None, None
-    except Exception as e:
-        logging.error(f"Unexpected error initializing ChromeDriver: {str(e)}")
-        st.error(f"Unexpected error initializing ChromeDriver: {str(e)}")
-        return None, None
-
 def clean_text(text):
     """Clean and normalize scraped text."""
     try:
@@ -239,59 +68,49 @@ def clean_text(text):
         return text
 
 def scrape_website(url):
-    """Scrape data from the given URL."""
+    """Scrape data from the given URL using Playwright."""
     logging.info(f"Starting scrape for URL: {url}")
-    driver, service = setup_driver()
-    if not driver or not service:
-        st.error("Failed to initialize WebDriver. Please check if Chromium and ChromeDriver are installed.")
-        logging.error("WebDriver initialization failed")
-        return None
-    try:
-        logging.info(f"Navigating to {url}")
-        driver.get(url)
-        WebDriverWait(driver, 30).until(
-            EC.presence_of_element_located((By.TAG_NAME, "body"))
-        )
-        title = driver.title
-        content_selectors = [
-            (By.ID, "content"),
-            (By.CLASS_NAME, "mw-parser-output"),
-            (By.TAG_NAME, "main"),
-            (By.CLASS_NAME, "main-content"),
-            (By.ID, "main"),
-            (By.TAG_NAME, "article")
-        ]
-        main_content = None
-        for by, value in content_selectors:
-            try:
-                main_content = driver.find_element(by, value)
-                logging.info(f"Found content with selector: {by}={value}")
-                break
-            except:
-                continue
-        if not main_content:
-            main_content = driver.find_element(By.TAG_NAME, "body")
-            logging.info("Falling back to body tag for content")
-        text_content = main_content.text
-        cleaned_content = clean_text(text_content)
-        logging.info(f"Scraped {len(cleaned_content)} characters from {url}")
-        return {
-            "title": title,
-            "content": cleaned_content,
-            "url": url
-        }
-    except (WebDriverException, TimeoutException) as e:
-        logging.error(f"Error scraping {url}: {str(e)}")
-        st.error(f"Error scraping {url}: {str(e)}")
-        return None
-    finally:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True, args=['--no-sandbox', '--disable-dev-shm-usage'])
+        page = browser.new_page()
         try:
-            driver.quit()
-            service.stop()
-            logging.info("WebDriver and service stopped")
-            cleanup_chromedriver_processes()
+            logging.info(f"Navigating to {url}")
+            page.goto(url, wait_until="domcontentloaded", timeout=30000)
+            title = page.title()
+            content_selectors = [
+                "#content",
+                ".mw-parser-output",
+                "main",
+                ".main-content",
+                "#main",
+                "article"
+            ]
+            main_content = None
+            for selector in content_selectors:
+                try:
+                    main_content = page.query_selector(selector)
+                    if main_content:
+                        logging.info(f"Found content with selector: {selector}")
+                        break
+                except:
+                    continue
+            if not main_content:
+                main_content = page.query_selector("body")
+                logging.info("Falling back to body tag for content")
+            text_content = main_content.inner_text()
+            cleaned_content = clean_text(text_content)
+            logging.info(f"Scraped {len(cleaned_content)} characters from {url}")
+            return {
+                "title": title,
+                "content": cleaned_content,
+                "url": url
+            }
         except Exception as e:
-            logging.warning(f"Error cleaning up WebDriver: {str(e)}")
+            logging.error(f"Error scraping {url}: {str(e)}")
+            st.error(f"Error scraping {url}: {str(e)}")
+            return None
+        finally:
+            browser.close()
 
 @st.cache_resource
 def initialize_qa_model():
@@ -402,7 +221,7 @@ if app_mode == "Web Scraping":
                     with st.expander("View scraped content"):
                         st.text_area("Content", result['content'], height=300)
                 else:
-                    st.error("Failed to scrape the website. Check logs for details.")
+                    st.error("Failed to scrape the website. Check logs at /app/cache/app.log.")
         else:
             st.warning("Please enter a valid URL (e.g., https://example.com).")
 
@@ -427,10 +246,10 @@ elif app_mode == "Chat with Content":
 elif app_mode == "About":
     st.header("ℹ️ About")
     st.markdown("""
-    This app uses Selenium for web scraping, LangChain for vector storage with FAISS,
+    This app uses Playwright for web scraping, LangChain for vector storage with FAISS,
     and Hugging Face models for embeddings and question answering.
-    - **Web Scraping**: Extracts text using headless Chromium.
+    - **Web Scraping**: Extracts text using headless Chromium via Playwright.
     - **RAG**: Indexes content with sentence-transformers and answers questions using FLAN-T5.
-    - **Tech Stack**: Python, Streamlit, Selenium, LangChain, Hugging Face Transformers, FAISS.
+    - **Tech Stack**: Python, Streamlit, Playwright, LangChain, Hugging Face Transformers, FAISS.
     - **Docker**: Runs in a containerized environment.
     """)