Fix

api/crawler.py

@@ -1,4 +1,6 @@
 import time
+import re
+import requests
 import trafilatura
 from trafilatura.sitemaps import sitemap_search
 from api.utils import logger, is_valid_url
@@ -33,11 +35,23 @@ def smart_chunk(text: str, chunk_size=1000, overlap=100) -> list[str]:
 def crawl_website(base_url: str, limit: int = 25):
     logger.info(f"Starting crawl for {base_url}")
 
-    urls = sitemap_search(base_url)
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    try:
+        urls = sitemap_search(base_url)
+    except Exception:
+        urls = []
+
     if not urls:
+        logger.warning("No sitemap found (or blocked). Fallback to base URL.")
         urls = [base_url]
 
     valid_urls = [u for u in urls if is_valid_url(u, base_url)]
+    logger.info(f"Found {len(urls)} URLs, {len(valid_urls)} valid.")
 
     count = 0
     for link in valid_urls:
@@ -45,17 +59,50 @@ def crawl_website(base_url: str, limit: int = 25):
             break
 
         try:
-            time.sleep(0.5)
-            downloaded = trafilatura.fetch_url(link)
+            time.sleep(1.0)
+            response = requests.get(link, headers=headers, timeout=10)
 
-            result = trafilatura.bare_extraction(downloaded, include_comments=False)
-
-            if not result or not result.get("text"):  # type: ignore
+            if response.status_code != 200:
+                logger.warning(f"Blocked or missing ({response.status_code}): {link}")
                 continue
 
-            page_title = result.get("title", "Unknown Page")  # type: ignore
-            raw_text = result["text"]  # type: ignore
+            # --- ROBUST EXTRACTION STRATEGY ---
+            page_title = "Unknown Page"
+            raw_text = ""
+
+            # Attempt 1: Bare Extraction (Best Quality)
+            try:
+                result = trafilatura.bare_extraction(
+                    response.text, include_comments=False
+                )
+                if result and isinstance(result, dict) and result.get("text"):
+                    page_title = result.get("title", "Unknown Page")
+                    raw_text = result["text"]
+            except Exception as e:
+                # If trafilatura crashes (AttributeError, etc.), fail silently and try fallback
+                # logger.warning(f"Metadata extraction failed for {link}, using fallback.")
+                pass
+
+            # Attempt 2: Fallback Extraction (If Attempt 1 failed)
+            if not raw_text:
+                raw_text = trafilatura.extract(response.text, include_comments=False)
+
+                # Manual Title Extraction via Regex (since bare_extraction failed)
+                if raw_text:
+                    title_match = re.search(
+                        r"<title>(.*?)</title>",
+                        response.text,
+                        re.IGNORECASE | re.DOTALL,
+                    )
+                    if title_match:
+                        clean_title = title_match.group(1).strip()
+                        if clean_title:
+                            page_title = clean_title
+
+            if not raw_text:
+                continue
 
+            # Chunking and Context Injection
             text_chunks = smart_chunk(raw_text)
 
             contextualized_chunks = [

chrome-extension/sidepanel.css

@@ -348,6 +348,10 @@ input:focus {
   }
 }
 
+#train-status {
+  margin-top: 8px;
+}
+
 .loader {
   border: 2px solid #f3f3f3;
   border-top: 2px solid var(--accent-color);