Improve scraping depth to 20 pages and increase context window to 60k chars

backend/app/services/scrape_pipeline.py

@@ -94,7 +94,7 @@ IMPORTANT_PAGE_KEYWORDS = [
     'connect', 'social', 'links',
 ]
 
-MAX_PAGES_TO_SCRAPE = 10
+MAX_PAGES_TO_SCRAPE = 20
 REQUEST_TIMEOUT = 15
 MAX_RETRIES = 3
 RETRY_DELAY = 1.0  # seconds between retries
@@ -1324,15 +1324,15 @@ async def run_full_research_new(url: str, force_refresh: bool = False, progress=
             system_prompt = f"""You are an AI assistant for {raw_name} ({url}).
 
 RULES:
-1. Answer ONLY from the knowledge base below - never make things up.
-2. Search the knowledge carefully before saying "I don't know".
-3. For bio questions, check the HOMEPAGE section first.
-4. Give partial info if available (e.g., "The site mentions X but not Y...").
-5. Keep answers concise and helpful.
+1. Base your answers primarily on the KNOWLEDGE BASE below.
+2. If the knowledge base doesn't have the answer, you may use your general knowledge to explain standard concepts or fill minor gaps, but MUST clarify that this info comes from general knowledge, not the specific website.
+3. For bio/overview questions, check the HOMEPAGE section first.
+4. Keep answers concise, helpful, and use Markdown (bullet points, bold text) for readability.
+5. If the user asks about something completely unrelated to the site (e.g. "who is batman"), politely bring them back to the topic of {raw_name}.
 
 === KNOWLEDGE BASE ===
 
-{chatbot_context[:10000]}
+{chatbot_context[:60000]}
 
 === END ===
 """
@@ -1470,15 +1470,15 @@ RULES:
     system_prompt = f"""You are an AI assistant for {raw_name} ({url}).
 
 RULES:
-1. Answer ONLY from the knowledge base below - never make things up.
-2. Search the knowledge carefully before saying "I don't know".
-3. For bio questions, check the HOMEPAGE section first.
-4. Give partial info if available (e.g., "The site mentions X but not Y...").
-5. Keep answers concise and helpful.
+1. Base your answers primarily on the KNOWLEDGE BASE below.
+2. If the knowledge base doesn't have the answer, you may use your general knowledge to explain standard concepts or fill minor gaps, but MUST clarify that this info comes from general knowledge, not the specific website.
+3. For bio/overview questions, check the HOMEPAGE section first.
+4. Keep answers concise, helpful, and use Markdown (bullet points, bold text) for readability.
+5. If the user asks about something completely unrelated to the site (e.g. "who is batman"), politely bring them back to the topic of {raw_name}.
 
 === KNOWLEDGE BASE ===
 
-{chatbot_context[:10000]}
+{chatbot_context[:60000]}
 
 === END ===
 """