Update scrape.py

scrape.py

@@ -2,13 +2,10 @@ from selenium import webdriver  # type: ignore
 from selenium.webdriver.chrome.service import Service  # type: ignore
 from selenium.webdriver.chrome.options import Options  # type: ignore
 from bs4 import BeautifulSoup  # type: ignore
-from dotenv import load_dotenv # type: ignore
-import os
 import time
 
-load_dotenv()
-
-CHROME_DRIVER_PATH = os.getenv("./chrome")
+# Define the ChromeDriver path directly
+CHROME_DRIVER_PATH = "./chrome"  
 
 def scrape_website(website):
     print("Connecting to Chrome Browser...")
@@ -16,6 +13,7 @@ def scrape_website(website):
     # Setup ChromeDriver service and options
     service = Service(CHROME_DRIVER_PATH)
     options = Options()
+    options.add_argument("--headless")  # Run in headless mode for deployment
     driver = webdriver.Chrome(service=service, options=options)
     
     try:
@@ -37,9 +35,7 @@ def scrape_website(website):
 def extract_body_content(html_content):
     soup = BeautifulSoup(html_content, "html.parser")
     body_content = soup.body
-    if body_content:
-        return str(body_content)
-    return ""
+    return str(body_content) if body_content else ""
 
 def clean_body_content(body_content):
     soup = BeautifulSoup(body_content, "html.parser")
@@ -55,6 +51,4 @@ def clean_body_content(body_content):
     return cleaned_content
 
 def split_dom_content(dom_content, max_length=6000):
-    return [
-        dom_content[i : i + max_length] for i in range(0, len(dom_content), max_length)
-    ]
+    return [dom_content[i:i + max_length] for i in range(0, len(dom_content), max_length)]