chat fix, a few other fixes

main.py

@@ -286,7 +286,8 @@ def main():
             new_chat_id = str(datetime.now().timestamp())
             st.session_state.chat_history[new_chat_id] = {
                 "messages": [],
-                "date": datetime.now().strftime("%Y-%m-%d")
+                "date": datetime.now().strftime("%Y-%m-%d"),
+                "name": "🗨️ New Chat"
             }
             st.session_state.current_chat_id = new_chat_id
             st.session_state.web_scraper_chat = None
@@ -305,12 +306,12 @@ def main():
             for chat_id, chat_data in chats:
                 messages = chat_data['messages']
                 if messages:
-                    button_label = f"{messages[0]['content'][:25]}..."
+                    button_label = chat_data.get('name', f"{messages[0]['content'][:25]}...")
                 else:
-                    button_label = "🗨️ Empty Chat"
-                
+                    button_label = chat_data.get('name', "🗨️ Empty Chat")
+
                 col1, col2 = st.columns([0.85, 0.15]) 
-                
+
                 with col1:
                     if st.button(button_label, key=f"history_{chat_id}", use_container_width=True):
                         st.session_state.current_chat_id = chat_id

src/scrapers/playwright_scraper.py

@@ -17,7 +17,7 @@ class ScraperConfig:
                  headless: bool = True,
                  debug: bool = False,
                  timeout: int = 60000,
-                 wait_for: str = 'networkidle'):
+                 wait_for: str = 'domcontentloaded'): # use networkidle instead of domcontentloaded if you want!
         self.use_stealth = use_stealth
         self.simulate_human = simulate_human
         self.use_custom_headers = use_custom_headers

src/web_extractor.py

@@ -21,6 +21,8 @@ import tiktoken
 import csv
 from bs4 import BeautifulSoup, Comment
 from .scrapers.playwright_scraper import PlaywrightScraper, ScraperConfig
+from urllib.parse import urlparse
+import streamlit as st
 
 class WebExtractor:
     def __init__(self, model_name: str = "gpt-4o-mini", model_kwargs: Dict[str, Any] = None, proxy: Optional[str] = None, headless: bool = True, debug: bool = False):
@@ -60,6 +62,13 @@ class WebExtractor:
     def _hash_content(self, content: str) -> str:
         return hashlib.md5(content.encode()).hexdigest()
 
+    def get_website_name(self, url: str) -> str:
+        parsed_url = urlparse(url)
+        domain = parsed_url.netloc
+        if domain.startswith('www.'):
+            domain = domain[4:]
+        return domain.split('.')[0].capitalize()
+
     @lru_cache(maxsize=100)
     async def _cached_api_call(self, content_hash: str, query: str) -> str:
         if isinstance(self.model, OllamaModel):
@@ -114,7 +123,11 @@ class WebExtractor:
             pages = parts[1] if len(parts) > 1 and not parts[1].startswith('-') else None
             url_pattern = parts[2] if len(parts) > 2 and not parts[2].startswith('-') else None
             handle_captcha = '-captcha' in user_input.lower()
-            
+
+            website_name = self.get_website_name(url)
+
+            st.session_state.chat_history[st.session_state.current_chat_id]["name"] = website_name
+
             response = await self._fetch_url(url, pages, url_pattern, handle_captcha)
         elif not self.current_content:
             response = "Please provide a URL first before asking for information."