Update utils.py

utils.py

@@ -496,11 +496,11 @@ def run_research_agent(
     """
     Low-Call approach:
       1) Tavily search (up to 20 URLs).
-      2) Firecrawl scrape => combined text from the URLs.
-      3) (No truncation) Use the full richness of the scraped materials.
-      4) Split the text into chunks (each 4500 tokens) and summarize each chunk individually.
-      5) Use a single final merge call to produce a comprehensive, detailed, and exhaustive research report.
-         The final report must adhere to world-class research report guidelines.
+      2) Firecrawl scrape => combined text
+      3) Truncate to 12k tokens total
+      4) Split into chunks (each 4500 tokens) => Summarize each chunk individually => summaries
+      5) Single final merge => final PDF
+      => 2 or more total LLM calls (but no more than 10) to reduce the chance of rate limit errors.
     """
     print(f"[LOG] Starting LOW-CALL research agent for topic: {topic}")
 
@@ -541,17 +541,16 @@ def run_research_agent(
             print("[LOG] Could not retrieve content from any search results. Exiting.")
             return "Could not retrieve content from any of the search results."
 
-        # Input Sanitization: Remove any chain-of-thought markers from the scraped content.
-        combined_content = re.sub(r"<think>.*?</think>", "", combined_content, flags=re.DOTALL)
-
-        # Note: The previous truncation to 12,000 tokens is removed so the full content is used.
+        # Step 3: Truncate to 12k tokens total
+        print("[LOG] Step 3: Truncating combined text to 12,000 tokens if needed.")
+        combined_content = truncate_text_tokens(combined_content, max_tokens=12000)
 
         # Step 4: Splitting text into chunks (4500 tokens each) and summarizing each chunk.
         print("[LOG] Step 4: Splitting text into chunks (4500 tokens each). Summarizing each chunk.")
         tokenizer = tiktoken.get_encoding("cl100k_base")
         tokens = tokenizer.encode(combined_content)
         chunk_size = 4500          # Reduced chunk size to avoid exceeding the LLM's TPM limit.
-        max_chunks = 10            # Allow up to 10 chunks (and thus up to 10 LLM calls).
+        max_chunks = 10            # Allow up to 10 chunks (and thus 10 LLM calls).
         summaries = []
         start = 0
         chunk_index = 1
@@ -565,7 +564,8 @@ def run_research_agent(
             prompt = f"""
 You are a specialized summarization engine. Summarize the following text 
 for a professional research report. Provide accurate details but do not 
-include any chain-of-thought or internal planning. Keep it concise, yet capture all key points:
+include chain-of-thought or internal reasoning. Keep it concise, but 
+include key data points and context:
 {chunk_text}
 """
             data = {
@@ -580,56 +580,36 @@ include any chain-of-thought or internal planning. Keep it concise, yet capture
             start = end
             chunk_index += 1
 
-        # Step 5: Single final merge call with enhanced instructions.
-        print("[LOG] Step 5: Merging chunk summaries into the final research report.")
+        # Step 5: Single final merge call
+        print("[LOG] Step 5: Doing one final merge of chunk summaries.")
         references_text = "\n".join(f"- {url}" for url in references_list) if references_list else "None"
         truncated_summaries = [truncate_text_for_llm(s, max_tokens=1000) for s in summaries]
         merged_input = "\n\n".join(truncated_summaries)
 
-        # Enhanced final prompt including world-class report guidelines.
         final_prompt = f"""
-IMPORTANT: Do NOT include any chain-of-thought, internal planning, or hidden reasoning in the final output.
-Draft a professional, world-class research report that adheres to the following tenets:
-
-I. Essential Principles and Qualities:
-   - Accuracy: Present accurate facts with no spelling or grammatical errors.
-   - Clarity: Use clear, straightforward language.
-   - Brevity: Be concise yet complete.
-   - Objectivity: Avoid personal bias.
-   - Simplicity: Use simple language, and explain any necessary technical jargon briefly.
-   - Logical Sequence: Arrange points in a logical order with proper planning.
-   - Proper Form and Presentation: Follow required formats with an attractive presentation.
-   - Selectiveness: Include only necessary content.
-   - Comprehensiveness: Provide complete and detailed coverage.
-   - Reliability, Coherence, and Relevance: Ensure a logical flow and relevance to the research questions.
-
-II. Structure the Report as Follows:
-   - Title Page (with a concise descriptive title)
-   - Table of Contents
-   - Executive Summary
-   - Introduction (clearly outlining the research purpose and objectives)
-   - Historical or Contextual Background
-   - Detailed Findings organized into coherent thematic sections
-   - Conclusion (with recommendations and insights)
-   - References/Bibliography (listing the provided URLs)
-
-III. Content and Writing Style:
-   - Use consistent and clear language.
-   - Support arguments with reliable evidence.
-   - Write in active voice with clear headings and a logical flow.
-   - Develop each section in multiple detailed paragraphs.
-
-IV. Steps for Writing the Report:
-   - Write a clear thesis statement.
-   - Prepare an outline and develop content sequentially.
-
-Use the following partial summaries and references as source materials to produce a detailed and exhaustive research report.
+IMPORTANT: Do NOT include chain-of-thought or hidden planning. 
+Produce a long, academic-style research paper with the following structure:
+- Title Page (concise descriptive title)
+- Table of Contents
+- Executive Summary
+- Introduction
+- Historical or Contextual Background
+- Multiple Thematic Sections (with subheadings)
+- Detailed Analysis (multi-paragraph sections)
+- Footnotes or inline citations referencing the URLs
+- Conclusion
+- References / Bibliography (list these URLs at the end)
+Requirements:
+- Minimal bullet points, prefer multi-paragraph
+- Each section at least 2-3 paragraphs
+- Aim for 1500+ words if possible
+- Under 6000 tokens total
+- Professional, academic tone
 Partial Summaries:
 {merged_input}
 References (URLs):
 {references_text}
-
-Now, merge these partial summaries into one thoroughly expanded, detailed, and exhaustive research report:
+Now, merge these partial summaries into one thoroughly expanded research paper:
 """
         final_data = {
             "model": MODEL_COMBINATION,
@@ -640,9 +620,6 @@ Now, merge these partial summaries into one thoroughly expanded, detailed, and e
         final_response = call_llm_with_retry(groq_client, **final_data)
         final_text = final_response.choices[0].message.content.strip()
 
-        # Post-process final_text to remove any lingering chain-of-thought markers.
-        final_text = re.sub(r"<think>.*?</think>", "", final_text, flags=re.DOTALL)
-
         # Step 6: PDF generation
         print("[LOG] Step 6: Generating final PDF from the merged text.")
         final_report = generate_report(final_text)