app.py

# =============================================================================
# By T.Sakai / G.Huet
# =============================================================================
import os, time, io, json, random, re, requests, openai, base64, html
import gradio as gr
import PyPDF2, tempfile, logging, difflib
import markdown, unicodedata, pdfkit
from PIL import Image
from io import BytesIO
from bs4 import BeautifulSoup
from datetime import datetime
from reportlab.lib.pagesizes import A4
from xhtml2pdf import pisa
from bs4.element import Tag
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import WebDriverException

# ============================================================================= Global Settings
MAX_MESSAGE_LENGTH = 1048576
SUMMARIZATION_REQUEST_COUNT = 0
TOTAL_SUMMARIZED_WORDS = 0
# Set up logging basic configuration
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
# ============================================================================= Helper functions

def capture_visual_screenshot(srcdoc: str) -> str:
    """
    Opens a temporary HTML file from the provided srcdoc string,
    loads it in a headless Chrome browser using Selenium,
    waits for the content to render,
    takes a screenshot, crops it, and returns a base64-encoded PNG image.
    """
    options = Options()
    options.add_argument("--headless")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    driver = None
    try:
        driver = webdriver.Chrome(options=options)
        driver.set_window_size(1100, 800)  # Adjust per your expected visual dimensions

        # Write the srcdoc to a temporary HTML file
        with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as tmp_file:
            tmp_file.write(srcdoc.encode("utf-8"))
            tmp_file.flush()
            file_url = "file://" + tmp_file.name

        driver.get(file_url)
        time.sleep(5)  # Allow time for JavaScript (e.g., mermaid) to render

        screenshot_png = driver.get_screenshot_as_png()
        image = Image.open(BytesIO(screenshot_png))
        # Crop the image to remove extra whitespace
        cropped_image = image.crop(image.getbbox())
        buffered = BytesIO()
        cropped_image.save(buffered, format='PNG')
        base64_img = base64.b64encode(buffered.getvalue()).decode("utf-8")
        return base64_img
    except WebDriverException as e:
        logging.error("Error in capture_visual_screenshot: %s", e)
        return ""
    finally:
        if driver:
            driver.quit()

def replace_visual_iframes(soup: BeautifulSoup) -> BeautifulSoup:
    """
    Finds all <iframe class="visual-frame"> tags in the provided BeautifulSoup object,
    uses capture_visual_screenshot() to get a base64 image,
    and replaces each iframe with an <img> tag embedding the screenshot.
    """
    iframes = soup.find_all("iframe", class_="visual-frame")
    for iframe in iframes:
        srcdoc = iframe.get("srcdoc")
        if srcdoc:
            base64_img = capture_visual_screenshot(srcdoc)
            if base64_img:
                new_img = soup.new_tag("img")
                new_img["src"] = "data:image/png;base64," + base64_img
                new_img["style"] = "max-width:100%; display:block; margin:auto; page-break-after:avoid;"
                iframe.replace_with(new_img)
            else:
                logging.error("Failed to capture screenshot for an iframe.")
    return soup

def process_pdf(url: str) -> str:
    try:
        headers = {"User-Agent": get_random_header()}
        r = requests.get(url, headers=headers)
        r.raise_for_status()
        f = io.BytesIO(r.content)
        reader = PyPDF2.PdfReader(f)
        text = ""
        for page in reader.pages:
            page_text = page.extract_text()
            if page_text:
                text += page_text + "\n"
        if text.strip():
            logging.info(f"process_pdf: Successfully extracted text from {url}")
            return text
        else:
            logging.info(f"process_pdf: No text extracted from {url}")
            return "PDF content could not be extracted."
    except Exception as e:
        err = f"Error processing PDF: {e}"
        logging.error(err)
        return err

def compress_text(text: str, target_length: int) -> str:
    global SUMMARIZATION_REQUEST_COUNT, TOTAL_SUMMARIZED_WORDS
    prompt = f"Summarize the following text in a way that preserves all valuable information, and output a compressed version not exceeding {target_length} characters:\n\n{text}"
    summary = llm_call(prompt, model="gpt-4o-mini", max_tokens_param=100000)
    SUMMARIZATION_REQUEST_COUNT += 1
    TOTAL_SUMMARIZED_WORDS += len(summary.split())
    logging.info(f"compress_text: Compressed text length: {len(summary)} -- Requests: {SUMMARIZATION_REQUEST_COUNT}, Total words: {TOTAL_SUMMARIZED_WORDS}")
    return summary

def summarize_large_text(text: str, target_length: int, chunk_size: int = 1000, overlap: int = 200) -> str:
    """
    Summarizes a large text by splitting it into overlapping chunks, summarizing each chunk,
    and then combining the intermediate summaries into a final summary.
    
    The prompt for these intermediate calls explicitly instructs to preserve key details
    and to include any tables or structured data present.
    
    Parameters:
      text         : The input text to summarize.
      target_length: The maximum number of tokens (or an approximate target length) for the final summary.
      chunk_size   : The number of words to include in each chunk.
      overlap      : The number of overlapping words between chunks.
    
    Returns:
      The final combined summary as a string.
    """
    words = text.split()
    if len(words) <= chunk_size:
        # If the text is short, simply return it (or you could call a simple summarization)
        if len(words) < 500:
            return "Not a coherent text or not worth processing - discard."
        else:
            return text
    chunks = []
    i = 0
    while i < len(words):
        # Create a chunk from i to i+chunk_size words.
        chunk = " ".join(words[i:i+chunk_size])
        chunks.append(chunk)
        # Move forward by chunk_size - overlap words.
        i += (chunk_size - overlap)
    
    summary_chunks = []
    for chunk in chunks:
        chunk_prompt = (f"""
Summarize the following text, preserving all key details and ensuring that any tables or structured data are also summarized:
{chunk}
Maintain the original sources.
Keep all mentions of names, people/titles, dates, papers, reports, organisation/institute/NGO/government bodies quotes, products, project names, ... 
"""
        )
        summary_chunk = llm_call(prompt=chunk_prompt, model="gpt-4o-mini", temperature=0, max_tokens_param=500)
        summary_chunk = re.sub(r'\{\[\{(.*?)\}\]\}', r'\1', summary_chunk)
        summary_chunk = re.sub(r'\[\{(.*?)\}\]', r'\1', summary_chunk)
        global SUMMARIZATION_REQUEST_COUNT, TOTAL_SUMMARIZED_WORDS
        SUMMARIZATION_REQUEST_COUNT += 1
        TOTAL_SUMMARIZED_WORDS += len(summary_chunk.split())
        summary_chunks.append(summary_chunk.strip())
    
    combined_summary = "\n".join(summary_chunks)
    # Now, produce one final summary that fuses all the intermediate summaries.
    final_prompt = (f"""
Combine the following summaries into one concise summary that preserves all critical details, including any relevant table or structured data:
{combined_summary}
Maintain the original sources.
Keep all mentions of names, people/titles, dates, papers, reports, organisation/institute/NGO/government bodies quotes, products, project names, ...
"""
    )
    final_summary = llm_call(prompt=final_prompt, model="gpt-4o-mini", temperature=0, max_tokens_param=target_length)
    final_summary = re.sub(r'\{\[\{(.*?)\}\]\}', r'\1', final_summary)
    final_summary = re.sub(r'\[\{(.*?)\}\]', r'\1', final_summary)
    return final_summary.strip()


def analyze_with_gpt4o(query: str, snippet: str, breadth: int, url: str = "url unknown", title: str = "title unknown") -> dict:
    # If snippet is a callable, call it to get the string.
    if callable(snippet):
        snippet = snippet()
    snippet_words = len(snippet.split())
    
    # Define a word threshold after which we start the chunking summarization.
    CHUNK_WORD_THRESHOLD = 1500
    if snippet_words > CHUNK_WORD_THRESHOLD:
        # Adjust the target_length as needed (here using 2000 tokens as an example).
        snippet = summarize_large_text(snippet, target_length=2000, chunk_size=1000, overlap=200)
        snippet_words = len(snippet.split())
    
    client = os.getenv('OPENAI_API_KEY')  # alternatively, pass your API key here if needed.

    prompt = (f"""
Analyze the following content from a query result:

{snippet}

Research topic:
{query}

url:
{url}

title:
{title}

Instructions:
1.  Relevance: Determine if the content is relevant to the research topic. Answer with a single word: "yes" or "no".

2.  Structure: If the content is relevant, provide a comprehensive summary structured into the following sections. Prioritize extreme conciseness and token efficiency while preserving all key information. Aim for the shortest possible summary that retains all essential facts, figures, arguments, and quotes. The total summary should not exceed 1000 words, but shorter is strongly preferred.
    -   Key Facts (at least 10, if present): List the core factual claims using short, declarative sentences or bullet points. Apply lemmatization and standard abbreviations.
    -   Key Figures: Extract numerical data (statistics, dates, percentages) and include any necessary context (units, references, explanations) required to interpret these numbers. Present them concisely (list or table format).
    -   Key Arguments (at least 5, if present): Identify main arguments or claims. Summarize supporting evidence and counter-arguments concisely.
    -   Key Quotes: Include significant quotes (with the author's name in parenthesis) if people verbatim were mentioned in the query result. Attribute quotes correctly. Paraphrase if needed, indicating that it's a paraphrase. Use symbols (e.g., &, +, ->, =) to conserve tokens.
    -   Structured Summary (10 to 50 sentences): Provide a structured summary that includes anecdotes, people, and locations to ensure the report is relatable.

Note: General Optimization Guidelines:
    -   Lemmatize words (e.g., "running" -> "run").
    -   Use common abbreviations.
    -   Remove redundancy and unnecessary words.
    -   Shorten words carefully (e.g., "information" -> "info") without causing ambiguity.
    -   Use symbols where appropriate.

3.  Follow-up Search Queries: Generate at least {breadth} follow-up search queries relevant to the research topic and the summarized content. Use search operators (AND, OR, quotation marks) as needed.

4. Ensure that the summary length and level of detail is proportional to the source length.
Source length: {snippet_words} words. You may produce a more detailed summary if the text is long.

// Special guidance for follow-up search queries
1. Query Progression:
   - Begin with foundational/conceptual queries
   - Progress to methodological/technical terms
   - Culminate in specialized/applied combinations

2. Term Optimization:
   - Use Boolean logic (AND/OR) strategically
   - Include both general terminology AND domain-specific jargon
   - Add temporal filters when relevant (e.g., "since 2018", "2020-2023")
   - Consider geographical/cultural modifiers if applicable

3. Query Structure:
   - Prioritize conceptual combinations over simple keyword matching
   - Use quotation marks for exact phrases and hyphenation for compound terms
   - Include emerging terminology variants (e.g., "LLMs" OR "large language models")

// IMPORTANT: Format your response as a proper JSON object with these fields:
- "relevant": "yes" or "no"
- url: full url
- title: title
- "summary": {{...your structured summary with all parts...}}
- "followups": [array of follow-up queries]
"""
    )

    try:
        response = llm_call(prompt=prompt, model="gpt-4o-mini", temperature=0, max_tokens_param=8000)
        response = re.sub(r'\{\[\{(.*?)\}\]\}', r'\1', response)
        response = re.sub(r'\[\{(.*?)\}\]', r'\1', response)
        if not response:
            logging.error("analyze_with_gpt4o: Empty response received from API.")
            return {"relevant": "no", "summary": "", "followups": []}
        
        # Check if the response already begins with "yes" or "no" (non-JSON format)
        if response.strip().lower().startswith("yes") or response.strip().lower().startswith("no"):
            # Handle non-JSON format
            lines = response.strip().split("\n")
            relevance = "yes" if lines[0].strip().lower() == "yes" else "no"
            
            # Extract the follow-up queries (usually in brackets at the end)
            followups_match = re.search(r'\[(.*?)\]', response, re.DOTALL)
            followups = []
            if followups_match:
                followups_text = followups_match.group(1)
                # Parse the lines within brackets as comma-separated or quoted items
                followups = [q.strip().strip('"\'') for q in re.findall(r'"([^"]*)"', followups_text)]
                if not followups:  # Try without quotes
                    followups = [q.strip() for q in followups_text.split(",")]
            
            # Everything else is the summary
            summary_text = response
            if followups_match:
                summary_text = response[:followups_match.start()].strip()
            if summary_text.startswith("yes") or summary_text.startswith("no"):
                summary_text = "\n".join(lines[1:]).strip()
            
            return {
                "relevant": relevance,
                "summary": summary_text,
                "followups": followups if followups else []
            }
            
        else:
            # Standard JSON parsing
            # Remove Markdown code fences if present
            if response.startswith("```"):
                response = re.sub(r"^```(json)?", "", response)
                response = re.sub(r"```$", "", response).strip()
            response = response.strip()
            # Optionally remove any start/end markers like "json" if present:
            if response.lower().startswith("json"):
                response = response[4:].strip()
            try:
                result = json.loads(response)
                return result
            except json.JSONDecodeError:
                # If JSON parsing fails, try to extract the information using regex
                logging.warning("JSON parsing failed, attempting regex extraction")
                relevance_match = re.search(r'"relevant":\s*"(yes|no)"', response, re.IGNORECASE)
                relevance = relevance_match.group(1) if relevance_match else "no"
                
                # Extract follow-up queries using regex
                followups_match = re.search(r'"followups":\s*\[(.*?)\]', response, re.DOTALL)
                followups = []
                if followups_match:
                    followups_text = followups_match.group(1)
                    followups = [q.strip().strip('"\'') for q in re.findall(r'"([^"]*)"', followups_text)]
                
                # Extract summary (everything else)
                summary = response
                
                return {
                    "relevant": relevance,
                    "summary": summary,
                    "followups": followups
                }
                
    except Exception as e:
        logging.error(f"analyze_with_gpt4o error: {e}")
        return {"relevant": "no", "summary": "", "followups": []}

def get_random_header():
    headers = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36",
        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
        "Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36"
    ]
    return random.choice(headers)

def process_url(url: str, retries: int = 3, timeout: int = 15) -> str:
    """
    Process a URL with multiple user agents and retries to handle 403 errors
    and other common web scraping issues.
    
    Args:
        url: The URL to retrieve content from
        retries: Number of retry attempts (with different user agents)
        timeout: Connection timeout in seconds
        
    Returns:
        The page content as a string, or an error message
    """
    if url.lower().endswith(".pdf"):
        return process_pdf(url)
        
    user_agents = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
        "Mozilla/5.0 (iPad; CPU OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1"
    ]
    
    referrers = [
        "https://www.google.com/",
        "https://www.bing.com/",
        "https://search.yahoo.com/",
        "https://duckduckgo.com/",
        "https://www.baidu.com/"
    ]
    
    for attempt in range(retries):
        try:
            # Choose a different user agent and referrer for each attempt
            headers = {
                "User-Agent": user_agents[attempt % len(user_agents)],
                "Referer": referrers[attempt % len(referrers)],
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.5",
                "Accept-Encoding": "gzip, deflate, br",
                "Connection": "keep-alive",
                "Upgrade-Insecure-Requests": "1",
                "Cache-Control": "max-age=0"
            }
            
            # Add a delay between attempts that increases with each retry
            if attempt > 0:
                delay = random.uniform(2, 5) * attempt
                logging.info(f"Retry {attempt+1}/{retries} for {url} - waiting {delay:.1f} seconds")
                time.sleep(delay)
            
            response = requests.get(url, headers=headers, timeout=timeout)
            
            if response.status_code == 200:
                logging.info(f"Successfully retrieved content from {url} on attempt {attempt+1}")
                return response.text
            elif response.status_code == 403:
                logging.warning(f"403 Forbidden on attempt {attempt+1} for {url}, trying different user agent")
                continue
            elif response.status_code == 404:
                return f"Error: Page not found (404) for {url}"
            else:
                response.raise_for_status()
                
        except requests.exceptions.Timeout:
            logging.warning(f"Timeout on attempt {attempt+1} for {url}")
            if attempt == retries - 1:
                return f"Error: Timeout after {retries} attempts for {url}"
                
        except requests.exceptions.TooManyRedirects:
            return f"Error: Too many redirects for {url}"
            
        except requests.exceptions.ConnectionError:
            logging.warning(f"Connection error on attempt {attempt+1} for {url}")
            if attempt == retries - 1:
                return f"Error: Could not connect to {url} after {retries} attempts"
                
        except Exception as e:
            logging.error(f"Error retrieving content from {url} (attempt {attempt+1}): {e}")
            if attempt == retries - 1:
                return f"Error accessing URL: {str(e)}"
    
    return f"Content could not be retrieved from {url} after {retries} attempts"

def clean_content(raw_content: str) -> str:
    # Parse HTML using BeautifulSoup (if not HTML, it will safely return the text)
    soup = BeautifulSoup(raw_content, "html.parser")
    
    # Remove unwanted elements by tag name
    for unwanted in soup(["script", "style", "header", "footer", "nav", "aside"]):
        unwanted.decompose()

    # Optionally, remove any tags with common advert or boilerplate keywords in the class or id:
    for tag in soup.find_all(attrs={"class": re.compile("ad|banner|popup", re.IGNORECASE)}):
        tag.decompose()
    for tag in soup.find_all(attrs={"id": re.compile("ad|banner|popup", re.IGNORECASE)}):
        tag.decompose()
        
    # Extract text and collapse multiple spaces/newlines.
    text = soup.get_text(separator=" ")
    text = re.sub(r'\s+', ' ', text).strip()
    
    # Additionally, remove any obfuscating strings (example: repeated symbols) if known.
    text = re.sub(r'[\u2022\u2023]+', '', text)  # remove bullet point symbols, adjust as needed

    return text

def display_image():
    return "visual.png"

def parse_link_request(user_message: str):
    # Look for a URL pattern in the message (simple regex for http/https)
    url_regex = r"(https?://\S+)"
    match = re.search(url_regex, user_message)
    if match:
        url = match.group(0)
        # Remove URL from the message to extract the question
        question = user_message.replace(url, "").strip()
        # Optionally, remove trigger words like 'open this link:' if needed
        question = re.sub(r"open\s*this\s*link[:\s]*", "", question, flags=re.I).strip()
        return url, question
    return None, user_message

def handle_link_request(user_message: str):
    # Parse the message to extract URL and question
    url, specific_question = parse_link_request(user_message)
    if not url:
        return "No valid URL was detected. Please include a link."
    
    # Fetch content from the URL
    fetched_content = process_url(url)
    if fetched_content.startswith("Error"):
        return f"Error fetching content from the URL: {fetched_content}"
    
    # Build a prompt to answer the specific question using fetched content
    prompt = f"""Using the details from the following content, answer the question in detail.
Content:\n{fetched_content}
Question: {specific_question}
Answer:"""
    
    answer = llm_call(prompt=prompt, model="o3-mini", temperature=0, max_tokens_param=1000)
    return answer

def clean_llm_response(response: str) -> str:
    """
    Clean the raw LLM response by removing code fences and ensuring
    that the result is valid JSON.
    """
    # Remove any leading/trailing whitespace and code fence markers.
    cleaned = response.strip()
    # Remove code fences (handle cases where there might be a language hint as well).
    cleaned = re.sub(r'^```(?:json)?', '', cleaned)
    cleaned = re.sub(r'```$', '', cleaned)
    
    # Replace newline characters with spaces.
    cleaned = cleaned.replace("\n", " ")
    # Collapse multiple spaces into one.
    cleaned = re.sub(r'\s+', ' ', cleaned)
    
    return cleaned.strip()

# ============================================================================= Improve

def remove_text_from_html(report_html: str, text_to_remove: str) -> str:
    soup = BeautifulSoup(report_html, "html.parser")
    # Iterate over all tags in the document
    for tag in soup.find_all():
        # Get the text contained (stripped)
        tag_text = tag.get_text(strip=True)
        if tag_text == text_to_remove:
            # The only content of this tag is text_to_remove; remove the entire tag.
            tag.decompose()
        else:
            # Otherwise, update all NavigableStrings in that tag.
            # (This covers the case where the text appears within more content.)
            for child in tag.find_all(string=True):
                if text_to_remove in child:
                    new_text = child.replace(text_to_remove, "")
                    child.replace_with(new_text)
    return str(soup)

def fix_visual_after_section(report_html: str, section_title: str, extra_instructions: str) -> str:
    """
    Given a report HTML, a target section name (from inside [[...]]), and extra instructions,
    this function finds the first header (h1–h4) that contains the section name (ignoring common TOC headers),
    then finds the first <iframe> following that header.
    It sends the current iframe 'srcdoc' (which should contain a mermaid diagram)
    to the LLM with extra instructions and expects back a corrected mermaid code (starting with a mermaid diagram keyword).
    It then rebuilds and reassigns the iframe's srcdoc with the improved code.
    The text within the mermaid code should not contain [ ], this would be interpreted otherwise. Use it wisely
    Make sure there's no parenthesis or line break in the mermaid code content.
    """
    soup = BeautifulSoup(report_html, "html.parser")
    header = None
    # Look for header tags (h1–h4) that contain the section title (ignoring "table of contents")
    for tag in soup.find_all(["h1", "h2", "h3", "h4"]):
        header_text = tag.get_text(strip=True)
        if section_title.lower() in header_text.lower() and "table of contents" not in header_text.lower():
            header = tag
            break
    if not header:
        logging.error(f"fix_visual_after_section: Section '{section_title}' not found.")
        return report_html

    # Find the first <iframe> after the header
    iframe = header.find_next("iframe")
    if not iframe or not iframe.has_attr("srcdoc"):
        logging.error(f"fix_visual_after_section: No <iframe> found after section '{section_title}'.")
        return report_html

    current_code = iframe["srcdoc"]
    logging.info(f"fix_visual_after_section: Current iframe srcdoc length: {len(current_code)}")
    
    # Build prompt: ask the LLM to simplify/adjust the mermaid code.
    prompt = f"""Improve the following mermaid visualization code.
Extra instructions: {extra_instructions}
Current mermaid code:
{current_code}

Return only the improved mermaid code (the content that should go inside the <div class="visual-frame">...</div>) with no extra wrapping HTML or commentary.

// Important
- No introduction, conclusions or code fences -> Output the result directly - create only the content for the mermaid
- Do no put any comment of #color coding and classes inside the mermaid code generated, it's supposed to be only focused on the mermaid code required to render it
- Avoid the formatting commands (ex: %%...%% ) and do not use classDef, class or linkStyle for formatting
- Do not skip lines or add line breaks in the content of the mermaid code
- The text within the mermaid code should not contain [ ], this would be interpreted otherwise. Use it wisely
- Do not add items in the mermaid code between parenthesis ex: [xxx(yyy)] , rather use [xxx - yyy] / Do not put it in parenthesis, even after the hyphen - or for units.
- If you need to add a remark in the code, do it like this after an hyphen: "[abc - 95% Context Coverage]". Simply don't use parenthesis except for the below prescribed formatting, mermaid if very format sensitive
Example: do not write:
G --> H[Performance Evaluation (45% Speed Improvement, 35% Risk Profiling, 50% Fraud Detection)]
Instead, use
G --> H[Performance Evaluation - 45% Speed Improvement, 35% Risk Profiling, 50% Fraud Detection]
- Do not add any formatting reference in the content of the mermaid code (ex - green ...), this will lead to errors when submitted to the mermaid api
- Do not use the pipe symbol | in the mermaid code, it would create an error 
- Do not use the gantt chart visual placeholder to generate a graph (ex: barchart), Gantt charts should only be used for timelines / project plans
Note: Visual placeholders are solely supposed to generate diagrams not for graphs.
- Do not use the quotation mark inside the text of the diagram 
- Do not use commas in the content of each object / nodes, they will create rendering issues
- Do not use any [x] reference notation for citations (ex: [1], [2], ...) inside the mermaid code - we will manage sources separately

// Examples
Note: Pay attention for each example to what type of parenthesis / bracket is used and respect it scrupulously

-- flowchart --
Important:
- If the flow is "deeper" than broad (>3 levels), choose TD (Top Down) ==> TD is the default one if you have a doubt
- If the flow is "broader" than deep (>3 branches at the same level), choose LR (Left Right)
- There can be mix of flows that can be elaborate
- Take in consideration that the resulting iframe content will render 1200px width and and 700px high. Make the best of the areas real-estate.

Top Down:

flowchart TD
    A[Christmas] -->|Get money| B(Go shopping)
    B --> C{{Let me think}}
    C -->|One| D[Laptop]
    C -->|Two| E[iPhone]
    C -->|Three| F[fa:fa-car Car]


Left Right:

flowchart LR
    A[Anticipated AI Advancements by 2025]
    
    subgraph Infinite_Context [Infinite Context Framework]
      I1[Token Limits: Unlimited]
      I2[Summarization Accuracy: Significantly Improved]
      I3[Industry Impact: Legal, Academic, IoT]
      I4[Use-case Efficiency: High Impact]
      I5[Integration with Emerging Reasoning Models & Agentic Architecture]
    end

    subgraph Traditional_Context [Traditional Fixed Context Windows]
      T1[Token Limits: 2048-4096]
      T2[Summarization Accuracy: Moderate]
      T3[Industry Applications: Limited Scope]
      T4[Use-case Impact: Constrained]
    end

    subgraph Comparative_Innovations [Comparative Metrics & Emerging Technologies]
      C1[Test Time Compute Enhancements]
      C2[Innovations: VJEPA, Synthetic Data, Deepseek]
      C3[New Paradigms: TTT, Novel Architectural Approaches]
      C4[Disruption Factors: Deployment Speed, Employment Impact, Lower Barriers]
    end

    A --> Infinite_Context
    A --> Traditional_Context
    A --> Comparative_Innovations

    class I1,I2,I3,I4,I5 infinite;
    class T1,T2,T3,T4 trad;
    class C1,C2,C3,C4 comp;

Mixed: For elaborate visuals
// example 1

flowchart LR
    subgraph subgraph1
        direction TB
        top1[top] --> bottom1[bottom]
    end
    subgraph subgraph2
        direction TB
        top2[top] --> bottom2[bottom]
    end
    %% ^ These subgraphs are identical, except for the links to them:

    %% Link *to* subgraph1: subgraph1 direction is maintained
    outside --> subgraph1
    %% Link *within* subgraph2:
    %% subgraph2 inherits the direction of the top-level graph (LR)
    outside ---> top2

// example 2
flowchart LR
subgraph "One"
  a("`The **cat**
  in the hat`") -- "edge label" --> b{{"`The **dog** in the hog`"}}
end
subgraph "`**Two**`"
  c("`The **cat**
  in the hat`") -- "`Bold **edge label**`" --> d("The dog in the hog")
end

-- sequence --
sequenceDiagram
    Alice->>+John: Hello John, how are you?
    Alice->>+John: John, can you hear me?
    John-->>-Alice: Hi Alice, I can hear you!
    John-->>-Alice: I feel great!

-- gantt --
gantt
    title A Gantt Diagram
    dateFormat  YYYY-MM-DD
    section Section
    A task           :a1, 2014-01-01, 30d
    Another task     :after a1  , 20d
    section Another
    Task in sec      :2014-01-12  , 12d
    another task      : 24d

-- pie --
pie title Pets adopted by volunteers
    "Dogs" : 386
    "Cats" : 85
    "Rats" : 15

-- mindmap --
mindmap
  root[mindmap]
    Origins
      Long history
      Popularisation
        British popular psychology author Tony Buzan
    Research
      On effectivness
      On Automatic creation
        Uses
            Creative techniques
            Strategic planning
            Argument mapping
    Tools
      Pen and paper
      Mermaid
"""
    
    improved_code = llm_call(prompt=prompt, model="o3-mini", temperature=0, max_tokens_param=1500)
    # Remove code fence markers and all leading whitespace
    improved_code = improved_code.replace("```", "").lstrip()
    logging.info(f"fix_visual_after_section: Improved code received from LLM:\n{improved_code}")
    
    # Verify that the result starts with a valid mermaid diagram keyword.
    valid_keywords = ["mindmap", "flowchart", "sequence", "gantt", "pie"]
    if not any(improved_code.lower().startswith(keyword) for keyword in valid_keywords):
        logging.error("fix_visual_after_section: Improved code does not start with a valid mermaid keyword.")
        return report_html

    # Rebuild the srcdoc using the improved mermaid code.
    new_srcdoc = f"""<!DOCTYPE html>
<html>
    <head>
        <script src="https://cdn.jsdelivr.net/npm/mermaid@10.4.0/dist/mermaid.min.js"></script>
        <script>
            mermaid.initialize({{ startOnLoad: true, securityLevel: "loose", theme: "default" }});
        </script>
        <style>
            .mermaid {{
                background-color: #f7f2f2;
                color: black;
                padding: 10px;
                border-radius: 5px;
                border: 1px solid #ccc;
                max-width: 1200px;
                margin: 0 auto;
                min-height: 400px;
            }}
        </style>
    </head>
    <body>
        <div class="visual-frame">
{improved_code}
        </div>
    </body>
</html>"""

    # Remove parenthesis in mermaid formatting
    new_srcdoc = re.sub(
        r"<html>\.\.\.\[\.\.\.\((.*?)\)\] </html>",
        r"<html>...[...- \1] </html>",
        new_srcdoc
    )
    iframe["srcdoc"] = new_srcdoc
    logging.info("fix_visual_after_section: Successfully updated the iframe srcdoc with improved mermaid code.")
    return str(soup)

def snippet_in_tag(tag: Tag, snippet: str) -> bool:
    """
    Check if the snippet is found in a tag's visible text.
    If the tag is an iframe with a srcdoc attribute, also check the unparsed srcdoc content.
    """
    # Check tag's normal text.
    if tag.get_text() and snippet in tag.get_text():
        return True
    # For iframes, check the srcdoc attribute.
    if tag.name.lower() == "iframe" and tag.has_attr("srcdoc"):
        srcdoc = tag["srcdoc"]
        # Parse the srcdoc content.
        srcdoc_soup = BeautifulSoup(srcdoc, "html.parser")
        if srcdoc_soup.get_text() and snippet in srcdoc_soup.get_text():
            return True
    return False

def expand_snippet_area(soup: BeautifulSoup, snippet: str) -> Tag:
    allowed_tags = {"div", "table"}

    logging.info("Searching for all elements containing the snippet: '%s'", snippet)
    # Get all tags where the snippet is found either in the text or inside an iframe's srcdoc.
    candidates = soup.find_all(lambda tag: snippet_in_tag(tag, snippet))
    if not candidates:
        logging.info("No element containing the snippet was found. Returning None.")
        return None

    # Choose the candidate with the greatest depth (i.e. the most ancestors).
    candidate = max(candidates, key=lambda tag: len(list(tag.parents)))
    logging.info("Candidate element selected based on depth (<%s>): %s", candidate.name, candidate)

    iframe_candidate = None
    allowed_candidate = None

    # Iterate upward from the candidate's direct parent.
    current = candidate.parent
    while current is not None and current.name.lower() != "body":
        logging.info("Evaluating parent element: <%s>", current.name)
        tag_name = current.name.lower()
        if tag_name == "iframe":
            iframe_candidate = current
            logging.info("Found an <iframe> container; updating iframe_candidate.")
        elif tag_name in allowed_tags and allowed_candidate is None:
            allowed_candidate = current
            logging.info("Found allowed container <%s>; setting allowed_candidate.", tag_name)
        current = current.parent

    if iframe_candidate is not None:
        logging.info("Returning the iframe container.")
        return iframe_candidate
    elif allowed_candidate is not None:
        logging.info("No iframe found; returning the first allowed container (div/table).")
        return allowed_candidate
    else:
        logging.info("No iframe, div, or table container found; returning candidate element.")
        return candidate

def fine_tune_report(adjustment_request: str, openai_api_key: str, serpapi_api_key: str, report_html: str, 
                     initial_request: str, qa: str, target_style: str, knowledge_crumbs: str, 
                     complementary_guidance: str) -> (str, str):
    # unescape report
    report_html = html.unescape(report_html)
    
    # Quick adjustments
    if "remove the following text:" in adjustment_request.lower():
        # Extract the target text and process removal directly.
        text_to_remove_match = re.search(r"remove the following text:\s*\[\[([^\]]+)\]\]", adjustment_request, re.I)
        if text_to_remove_match:
            text_to_remove = text_to_remove_match.group(1).strip()
            # Return the report after locally removing the text.
            return remove_text_from_html(report_html, text_to_remove), qa
    
    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    logging.info("fine_tune_report: Starting fine-tuning process based on the adjustment request.")

    # Step 1: (LLM call to get unique strings)
    
    prompt_identify = (f"""
You are a meticulous technical editor.

Below is the full report HTML and a user adjustment request.
Extract one or more unique plain-text string(s) (without any HTML tags or formatting) that uniquely appear in the area(s) targeted by the adjustment request.

// Examples

    if the user request to "Add xyz in the conclusion", the unique string to identify should be specific to the conclusion
    if the user request to "correct the graph after section 1.2", the unique string should be one of the string that appear specifically in the graph after section 1.2 (ex: the title)
    if the user request is "Remove any mention about the car industry", the unique string(s) should be a sentence that would be in a paragraph of the report that would talk about car industry
    --> The unique string is what would allow to identify precisely through a search the section targeted by the user request, it has to be concise and unique.

Output them in a JSON object with the key "identified_unique_strings" mapped to a list of strings.
Ensure these strings exactly match the content in the report.

Important:
- If the unique string appears within a code snippet (like a javascript graph or a mermaid diagram), do not use the code as part of the snippet. For example, instead of 
 "A[Fundamental AI Research - Emerging Theories and Paradigms] --> B[Algorithm Innovation - Novel ML and NLP Models]"
simply use "Fundamental AI Research - Emerging Theories and Paradigms".
- The unique string identified should not contain a line break (ex: <br> or \n), take only a snippet that originally didn't contain a line break

Full Report HTML:
{report_html}

User Adjustment Request:
{adjustment_request}

Only output valid JSON."""
    )
    response_identify = llm_call(prompt=prompt_identify, model="o3-mini", temperature=0, max_tokens_param=10000)
    logging.info("fine_tune_report: Raw unique string identification response: %s", response_identify)

    try:
        response_identify = clean_llm_response(response_identify.strip().strip("```"))
        id_data = json.loads(response_identify)
        unique_strings = id_data.get("identified_unique_strings", [])
    except Exception as e:
        logging.error("fine_tune_report: Error parsing unique strings JSON: %s", e)
        unique_strings = []
    
    if not unique_strings:
        logging.warning("fine_tune_report: No unique strings were identified for adjustment. Returning original report.")
        return report_html, qa
    
    # Step 2: Parse the report HTML once.
    soup = BeautifulSoup(report_html, "html.parser")
    corrections_summary = []
    
    for uniq_str in unique_strings:
        uniq_str = uniq_str.strip()
        logging.info("fine_tune_report: Processing unique string: '%s'", uniq_str)
        # Use expand_snippet_area to get the container Tag directly.
        container_tag = expand_snippet_area(soup, uniq_str)
        if container_tag is None:
            logging.warning("fine_tune_report: Could not locate a container for unique string: '%s'", uniq_str)
            continue

        if "remove" in adjustment_request.lower():
            # Attempt to extract target phrase from the removal instruction.
            m = re.search(r'remove\s+(?:the\s+duplicate\s+)?mention\s+of\s+the\s+source:\s*(.+)', adjustment_request.lower())
            if m:
                target = m.group(1).strip()
                text_lower = container_tag.get_text().lower()
                if target in text_lower:
                    # Remove this container entirely.
                    container_tag.replace_with(BeautifulSoup("", "html.parser"))
                    logging.info("fine_tune_report: Removed section containing target '%s'", target)
                    corrections_summary.append(f"Section containing {target} removed as per request.")
                    continue  # Skip further processing of this snippet.
        
        original_container_html = str(container_tag)
        logging.info("fine_tune_report: Found container for unique string adjustment:\n\n%s\n", original_container_html)

        # Step 3: Call the LLM to adjust this container.
        prompt_adjust = (f"""
You are a technical editor.
Given the following HTML container (with its outer tags) extracted from a larger report and based on the user adjustment request,
produce a corrected version by making only the necessary changes. Preserve inline citations, formatting, and context.
The updated version will be put back in the exact same location and must have the same outer tags.
If the original container is a mermaid code, make sure the new version does not include parenthesis or line break in the content of the mermaid pseudo-code.

- Overall Report HTML:
{report_html}
- Knowledge Crumbs:
{knowledge_crumbs}

- Original Container to Adjust:
{original_container_html}

- User Adjustment Request:
{adjustment_request}

Additional Guidance:
- Target Style: {target_style}
- Complementary Guidance:
{complementary_guidance}

Ensure the updated content remains consistent with the overall report style.
Output a JSON object with exactly two keys:
- "improved" (the corrected container's full HTML) and
- "summary" (a brief explanation of the changes)

Only output valid JSON with no comments or code fences."""
        )
        
        response_adjust = llm_call(prompt=prompt_adjust, model="o3-mini", temperature=0, max_tokens_param=10000)
        logging.info("fine_tune_report: Raw container adjustment response: %s", response_adjust)
        try:
            response_adjust = clean_llm_response(response_adjust.strip().strip("```"))
            logging.info("Cleaned container adjustment response: %s", response_adjust)
            adjust_data = json.loads(response_adjust)
            corrected_container = adjust_data.get("improved", "").strip()
            container_summary = adjust_data.get("summary", "").strip()
        except Exception as e:
            logging.error("fine_tune_report: Error parsing container adjustment JSON: %s", e)
            continue

        if not corrected_container:
            logging.warning("fine_tune_report: No improved container was generated; skipping correction for this container.")
            continue

        corrections_summary.append(f"Container corrected: {container_summary}")

        # Step 4: Replace the original container with the updated one in our soup.
        container_tag.replace_with(BeautifulSoup(corrected_container, "html.parser"))
        logging.info("fine_tune_report: Updated container re-injected.")

    updated_report_html = str(soup)
    
    # Step 5 (and 6): Update the reference table if needed.
    prompt_refs = (
        f"\nYou are a technical editor.\n\n"
        "Review the following updated report HTML. If any new inline citations (e.g., [x]) have been added that are not in the original reference table,\n"
        "generate an updated References Summary Table as valid HTML. Output only the updated table without any additional comments.\n\n"
        f"Updated Report HTML:\n{updated_report_html}"
    )
    # Increase token limit to 1500 for this call
    updated_refs = llm_call(prompt=prompt_refs, model="o3-mini", temperature=0, max_tokens_param=1500)
    updated_refs = updated_refs.strip().strip("```").strip()
    
    if updated_refs and not updated_refs.lower().startswith("error: empty response"):
        soup_updated = BeautifulSoup(updated_report_html, "html.parser")
        ref_heading = soup_updated.find(lambda tag: tag.name in ["h1", "h2", "h3", "h4"] and "references summary table" in tag.get_text(strip=True).lower())
        if ref_heading:
            next_sibling = ref_heading.find_next_sibling()
            if next_sibling:
                try:
                    new_ref_html = BeautifulSoup(updated_refs, "html.parser")
                    next_sibling.replace_with(new_ref_html)
                    logging.info("fine_tune_report: Reference table updated successfully.")
                except Exception as e:
                    logging.error("fine_tune_report: Error updating reference table: %s", e)
            else:
                logging.info("fine_tune_report: No sibling after reference heading; skipping reference update.")
            updated_report_html = str(soup_updated)
        else:
            logging.info("fine_tune_report: No reference table heading found; reference update skipped.")
    else:
        logging.info("fine_tune_report: No valid updated reference table returned; leaving unchanged.")

    global_summary = "Corrections Applied Based on User Request:\n" + "\n".join(corrections_summary)
    updated_qa = qa.strip() + "\n----------\n" + global_summary

    logging.info("fine_tune_report: Fine-tuning complete.")
    return updated_report_html, updated_qa

def suggest_improvements(report_html: str, openai_api_key: str, serpapi_api_key: str, initial_query: str, clarification_text: str, crumbs_box: str, additional_clarifications: str) -> str:
    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    prompt = (f"""
You are a technical editor.
Based on the following full HTML report, generate improvement suggestions - at least 3."
Format each proposal as a numbered list item in the following style:\n"
Examples:
1) in the section xyz, adjust ...
2) after the paragraph abc, detail the graph further ...
3) in the focus placeholder xxx, add a mention about ...
4) make a reference to ... in the section 3.2
...
n) final improvement suggestion...

// Pay attention to:
- The initial user request:
{initial_query}
- The Q&A:
{clarification_text}
- the knowledge crumbs (from search):
{crumbs_box}
- additional clarifications form the user:
{additional_clarifications}

Note: Discard impractical recommendation (in pdf or gr.html)
Only output the suggestions exactly as a numbered list (text)

Full Report HTML:
{report_html}

Now provide your suggestions."""
    )
    suggestions = llm_call(prompt=prompt, model="o3-mini", temperature=0, max_tokens_param=1000)
    return suggestions.strip().strip("```").strip()

def improve_report_from_chat(user_message: str, chat_history: list, report_text: str, crumbs_text: str, style: str, initial_request: str, qa: str, complementary_guidance: str):
    adjustment_request = user_message.replace("@improve", "").strip()

    report_text = html.unescape(report_text)
    
    # --- CASE 1: Removal request ---
    removal_match = re.search(r"remove the following text:\s*\[\[([^\]]+)\]\]", adjustment_request, re.I)
    if removal_match:
        text_to_remove = removal_match.group(1).strip()
        updated_report = remove_text_from_html(report_text, text_to_remove)
        answer = f"Removed text: '{text_to_remove}' from the report."
        chat_history.append([user_message, answer])
        return chat_history, "", updated_report

    # --- CASE 2: Visual fix request ---
    visual_fix_match = re.search(r"fix visual after section\s+\[\[([^\]]+)\]\](?::\s*(.*))?", adjustment_request, re.I)
    if visual_fix_match:
        section_name = visual_fix_match.group(1).strip()
        extra_instructions = visual_fix_match.group(2).strip() if visual_fix_match.group(2) else ""
        updated_report = fix_visual_after_section(report_text, section_name, extra_instructions)
        answer = f"Fixed visual after section '{section_name}'."
        chat_history.append([user_message, answer])
        return chat_history, "", updated_report

    # --- DEFAULT: Proceed with existing LLM-based improvement (fine_tune_report) ---
    updated_report, _ = fine_tune_report(
        adjustment_request,
        os.getenv("OPENAI_API_KEY"),
        os.getenv("SERPAPI_API_KEY"),
        report_text,
        initial_request=initial_request,
        qa=qa,
        target_style=style,
        knowledge_crumbs=crumbs_text,
        complementary_guidance=complementary_guidance
    )
    answer = f"Report updated with your improvement: {adjustment_request}"
    chat_history.append([user_message, answer])
    return chat_history, "", updated_report

# ============================================================================= Expand

def get_max_reference(report_html: str) -> int:
    """
    Searches the provided report HTML for the References Summary Table and returns
    the maximum reference number currently used.
    """
    soup_ = BeautifulSoup(report_html, "html.parser")
    max_ref = 0
    # Locate a heading that includes "references summary table" (case insensitive)
    ref_heading = soup_.find(lambda tag: tag.name in ["h1", "h2", "h3", "h4"] and "references summary table" in tag.get_text(strip=True).lower())
    if ref_heading:
        next_sibling = ref_heading.find_next_sibling()
        if next_sibling:
            text = next_sibling.get_text(separator="\n")
            for line in text.splitlines():
                m = re.match(r'\s*(\d+)\s*\|', line)
                if m:
                    num = int(m.group(1))
                    max_ref = max(max_ref, num)
    return max_ref

def expand_report(expansion_request: str, openai_api_key: str, serpapi_api_key: str, report_html: str, 
                  initial_request: str, qa: str, target_style: str, knowledge_crumbs: str, 
                  complementary_guidance: str) -> (str, str):
    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    # unescape report
    report_html = html.unescape(report_html)
    
    logging.info("expansion_report: Starting fine-tuning process based on the adjustment request.")

    # Step 1: Identify unique strings in the targeted sections.
    prompt_identify = (f"""
You are a meticulous technical editor.

Below is the full report HTML and a user expansion request.
Extract one or more unique plain-text string(s) (without any HTML tags or formatting) that uniquely appear in the area(s) targeted by the expansion request.

// Examples:
    - If the user requests "Expand xyz in the conclusion", the unique string should be specific to the conclusion.
    - If the user requests "Elaborate on the graph after section 1.2", the unique string should be one of the strings that appear specifically in that graph (e.g., its title).
    - If the user requests "Add any mention about the car industry", the unique string should be a sentence that appears in a paragraph discussing that industry.
--> The unique string is what allows you to identify precisely the targeted section; it must be concise and unique.

Output them in a JSON object with the key "identified_unique_strings" mapped to a list of strings.
Ensure these strings exactly match the content in the report.

Important:
- If the unique string appears within a code snippet (like a javascript graph or a mermaid diagram), do not use the code as part of the snippet. For example, instead of 
 "A[Fundamental AI Research - Emerging Theories and Paradigms] --> B[Algorithm Innovation - Novel ML and NLP Models]"
simply use "Fundamental AI Research - Emerging Theories and Paradigms".
- The unique string identified should not contain a line break (ex: <br> or \n), take only a snippet that originally didn't contain a line break


Full Report HTML:
{report_html}

User Expansion Request:
{expansion_request}

Only output valid JSON.
""")
    response_identify = llm_call(prompt=prompt_identify, model="o3-mini", temperature=0, max_tokens_param=5000)
    logging.info("expansion_report: Raw unique string identification response: %s", response_identify)

    try:
        response_identify = clean_llm_response(response_identify.strip().strip("```"))
        id_data = json.loads(response_identify)
        unique_strings = id_data.get("identified_unique_strings", [])
    except Exception as e:
        logging.error("expansion_report: Error parsing unique strings JSON: %s", e)
        unique_strings = []

    if not unique_strings:
        logging.warning("expansion_report: No unique strings were identified for adjustment. Returning original report.")
        return report_html, qa

    # Determine current maximum reference number.
    current_max_ref = get_max_reference(report_html)
    logging.info(f"expansion_report: Current max reference number is {current_max_ref}")

    new_references_list = []
    soup = BeautifulSoup(report_html, "html.parser")
    corrections_summary = []

    for uniq_str in unique_strings:
        uniq_str = uniq_str.strip()
        logging.info("expansion_report: Processing unique string: '%s'", uniq_str)
        container_tag = expand_snippet_area(soup, uniq_str)
        if container_tag is None:
            logging.warning("expansion_report: Could not locate a container for unique string: '%s'", uniq_str)
            continue

        original_container_html = str(container_tag)
        logging.info("expansion_report: Found container for unique string adjustment (length %d): %s", 
                     len(original_container_html), original_container_html[:200])
        
        # Check if it is a removal request.
        if "remove" in expansion_request.lower():
            m = re.search(r'remove\s+(?:the\s+duplicate\s+)?mention\s+of\s+the\s+source:\s*(.+)', expansion_request.lower())
            if m:
                target = m.group(1).strip()
                text_lower = container_tag.get_text().lower()
                if target in text_lower:
                    container_tag.replace_with(BeautifulSoup("", "html.parser"))
                    logging.info("expansion_report: Removed section containing target '%s'", target)
                    corrections_summary.append(f"Section containing {target} removed as per request.")
                    continue
        
        # --- Modified prompt: focus only on the container / snippet.
        prompt_adjust = (f"""
You are a technical editor. You are tasked to read a report and expand a very specific snippet in that report.
Read the report first:

{report_html}
-------------
Having the report in memory and using only the following container snippet (including its outer HTML tags),
produce an expanded version of the container by adding between one and three paragraphs containing additional relevant details.

// IMPORTANT
- DO NOT output the full report; output only the updated snippet with no comments
- DO NOT output the full report; output only the updated snippet with no comments
- DO NOT output the full report; output only the updated snippet with no comments
- No title or subtitles, only the snippet expanded with complementary paragraphs aligned with the user request
- No title or subtitles, only the snippet expanded with complementary paragraphs aligned with the user request
- No title or subtitles, only the snippet expanded with complementary paragraphs aligned with the user request
- Do not refer or duplicate other concepts already addressed in other parts of the report.
- Do not refer or duplicate other concepts already addressed in other parts of the report.
- Do not refer or duplicate other concepts already addressed in other parts of the report.

// Inputs
- Original Container to Adjust:
{original_container_html}

- Full User Expansion Request:
{expansion_request}
Note: the user request is addressing several containers, just apply the guidance that are relevant for this specific container

// Additional Guidance:
- Target Style: {target_style}
- Complementary Guidance: {complementary_guidance}
- Preserve inline citations (formatted as [x]); if new citations are added, list them separately in "newref".
- The citations ref should be aligned with those in the References summary table if the reference used already exists.

// Output
Output a JSON object with exactly three keys:
  "result": the expanded container's HTML (only the snippet),
  "newref": a string with new reference lines (each line formatted as "new reference number | name | author | url"; empty if none),
  "summary": a brief explanation of the changes made.
""")
        response_adjust = llm_call(prompt=prompt_adjust, model="o3-mini", temperature=0, max_tokens_param=10000)
        logging.info("expansion_report: Raw container adjustment response: %s", response_adjust)
        try:
            response_adjust = response_adjust.strip().strip("json").strip("```").strip()
            logging.info("expansion_report: Cleaned container adjustment response: %s", response_adjust)
            adjust_data = json.loads(response_adjust)
            corrected_container = adjust_data.get("result", "").strip()
            new_refs_str = adjust_data.get("newref", "").strip()
            container_summary = adjust_data.get("summary", "").strip()
        except Exception as e:
            logging.error("expansion_report: Error parsing container adjustment JSON: %s", e)
            continue

        if new_refs_str:
            for line in new_refs_str.splitlines():
                line = line.strip()
                if line:
                    ref_parts = line.split("|")
                    if len(ref_parts) >= 4:
                        current_max_ref += 1
                        ref_name = ref_parts[1].strip()
                        ref_author = ref_parts[2].strip()
                        ref_url = ref_parts[3].strip()
                        new_ref_line = f"{current_max_ref} | {ref_name} | {ref_author} | {ref_url}"
                        new_references_list.append(new_ref_line)
                        logging.info("expansion_report: Added new reference: %s", new_ref_line)
        else:
            logging.info("expansion_report: No new references found for this container.")

        corrections_summary.append(f"Container expanded: {container_summary}")
        # Reintegrate: replace the original container with only the expanded snippet.
        new_content_soup = BeautifulSoup(corrected_container, "html.parser")
        if new_content_soup.contents:
            container_tag.replace_with(new_content_soup.contents[0])
        else:
            container_tag.replace_with(new_content_soup)
        logging.info("expansion_report: Updated container re-injected.")

    updated_report_html = str(soup)

    new_refs_text = "\n".join(new_references_list) if new_references_list else ""
    prompt_refs = (
        "\nYou are a technical editor.\n\n"
        "Review the following updated report HTML along with the list of new references added during the expansion process. "
        "Every inline citation found in the updated report must have a corresponding entry in the References Summary Table. "
        "Merge any new references with the existing ones without duplication.\n\n" +
        (f"New References Added:\n{new_refs_text}\n\n" if new_refs_text else "New References Added: None\n\n") +
        f"Updated Report HTML:\n{updated_report_html}\n\n"
        "Output only the updated References Summary Table as valid HTML with no additional commentary."
    )
    updated_refs = llm_call(prompt=prompt_refs, model="o3-mini", temperature=0, max_tokens_param=10000)
    updated_refs = updated_refs.strip().strip("```").strip()

    if updated_refs and not updated_refs.lower().startswith("error: empty response"):
        soup_updated = BeautifulSoup(updated_report_html, "html.parser")
        ref_heading = soup_updated.find(lambda tag: tag.name in ["h1", "h2", "h3", "h4"] and "references summary table" in tag.get_text(strip=True).lower())
        if ref_heading:
            next_sibling = ref_heading.find_next_sibling()
            if next_sibling:
                try:
                    new_ref_html = BeautifulSoup(updated_refs, "html.parser")
                    next_sibling.replace_with(new_ref_html)
                    logging.info("expansion_report: Reference table updated successfully.")
                except Exception as e:
                    logging.error("expansion_report: Error updating reference table: %s", e)
            else:
                logging.info("expansion_report: No sibling after reference heading; skipping reference update.")
            updated_report_html = str(soup_updated)
        else:
            logging.info("expansion_report: No reference table heading found; reference update skipped.")
    else:
        logging.info("expansion_report: No valid updated reference table returned; leaving unchanged.")

    global_summary = "Corrections Applied Based on User Request:\n" + "\n".join(corrections_summary)
    updated_qa = qa.strip() + "\n----------\n" + global_summary
    logging.info("expansion_report: Expansion complete. Summary of changes:\n%s", global_summary)
    return updated_report_html, updated_qa

def expand_report_from_chat(user_message: str, chat_history: list, report_text: str, crumbs_text: str, style: str, initial_request: str, qa: str, complementary_guidance: str):
    """
    If the user enters a message with '@expand', first generate expansion suggestions from the current report.
    Then, use those suggestions (if any) as the expansion instruction to update the report HTML.
    Returns the updated chat history and the updated report.
    """

    # unescape report
    report_text = html.unescape(report_text)
    
    expand_request = user_message.replace("@expand", "").strip()
    
    suggestions = suggest_expansions(report_text, os.getenv("OPENAI_API_KEY"), os.getenv("SERPAPI_API_KEY"),
                                     initial_request, qa, crumbs_text, complementary_guidance)
    
    if suggestions.strip():
        expansion_instruction = suggestions
    else:
        expansion_instruction = expand_request
    
    updated_report, _ = expand_report(
        expansion_instruction,
        os.getenv("OPENAI_API_KEY"),
        os.getenv("SERPAPI_API_KEY"),
        report_text,
        initial_request=initial_request,
        qa=qa,
        target_style=style,
        knowledge_crumbs=crumbs_text,
        complementary_guidance=complementary_guidance
    )
    
    answer = f"Report updated with expansion: {expansion_instruction}"
    chat_history.append([user_message, answer])
    return chat_history, "", updated_report

def suggest_expansions(report_html: str, openai_api_key: str, serpapi_api_key: str, initial_query: str, clarification_text: str, crumbs_box: str, additional_clarifications: str) -> str:
    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    prompt = (f"""
You are a technical editor.
Based on the following full HTML report, generate expansion suggestions - at least 3."
Format each proposal as a numbered list item in the following style:\n"
Examples:
1) in the section xyz, elaborate on this topic ...
2) after the paragraph abc, develop the graph further to integrate abc...
3) in the focus placeholder xxx, add a paragraph about ...
4) add several pragraphs on ... in the section 3.2
...
n) final expansion suggestion...

// Pay attention to:
- The initial user request:
{initial_query}
- The Q&A:
{clarification_text}
- the knowledge crumbs (from search):
{crumbs_box}
- additional clarifications form the user:
{additional_clarifications}

Note: Discard impractical recommendation (in pdf or gr.html)
Only output the suggestions exactly as a numbered list (text)

Full Report HTML:
{report_html}

// Important
- Your suggestions should not conflict with the content already present in other sections of the report
- This an "expansion", not an "addition", which means you should not ask to add a new section or sub-section but just add a paragraph or two at most
- You're not supposed to add any new title or brand new section
- Focus on the paragraph (within p tag) or sections (between 2 h1 titles - excluding visuals) that are not of the proper dimension for the topic addressed.

Your suggestions will help to make the report more comprehensive and ensure all the relevant knowledge available is used. 
Now provide your suggestions to expand the report."""
    )
    suggestions = llm_call(prompt=prompt, model="o3-mini", temperature=0, max_tokens_param=5000)
    return suggestions.strip().strip("```").strip()

# ============================================================================= Chat

def send_chat_message(user_message, openai_api_key, serpapi_api_key, chat_history, report_text, crumbs_text, style, initial_request, qa, complementary_guidance):
    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    report_text = html.unescape(report_text)
    
    if "@improve" in user_message.lower():
        return improve_report_from_chat(user_message, chat_history, report_text, crumbs_text, style, initial_request, qa, complementary_guidance)
    elif "@expand" in user_message.lower():
        return expand_report_from_chat(user_message, chat_history, report_text, crumbs_text, style, initial_request, qa, complementary_guidance)
    if "http://" in user_message or "https://" in user_message:
        answer = handle_link_request(user_message)
    else:
        system_prompt = f"""
You are a knowledgeable research assistant.

Based on the following
- Report:
{report_text}

- Source Crumbs:
{crumbs_text}

- User Question:
{user_message}

Provide a response in the desired style: {style}

Your Answer:"""
        answer = llm_call(prompt=system_prompt, model="o3-mini", temperature=0, max_tokens_param=10000)
    updated_history = chat_history + [[user_message, answer]]
    return updated_history, "", report_text

# ============================================================================= Graph placeholder

def generate_graph_snippet(placeholder_text: str, context: str, initial_query: str, crumbs: str) -> str:    
    graph_examples = """
--- Example 1 ---

<html>
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Global Comparative Analysis of Ethical Framework Adoption</title>
    <style>
      body {
        font-family: Arial, sans-serif;
        margin: 0;
        padding: 20px;
      }
      h2 {
        text-align: left;
        margin-bottom: 0;
      }
      #chart {
        max-width: 100%;
        max-height: 700px;
      }
      .axis-label {
        font-size: 14px;
        fill: #333;
      }
      .tooltip {
        position: absolute;
        background: #f9f9f9;
        padding: 5px 10px;
        border: 1px solid #d3d3d3;
        border-radius: 4px;
        pointer-events: none;
        font-size: 12px;
      }
    </style>
  </head>
  <body>
    <h2>Global Comparative Analysis of Ethical Framework Adoption</h2>
    <div id="chart"></div>
    <script src="https://d3js.org/d3.v7.min.js"></script>
    <script>
      const data = [
        { region: "North America", adoption: 80, initiatives: 5, trust: 75 },
        { region: "Europe",         adoption: 85, initiatives: 6, trust: 80 },
        { region: "Asia",           adoption: 70, initiatives: 4, trust: 65 },
        { region: "South America",  adoption: 60, initiatives: 3, trust: 70 },
        { region: "Africa",         adoption: 50, initiatives: 2, trust: 55 },
        { region: "Oceania",        adoption: 75, initiatives: 4, trust: 78 }
      ];

      const margin = { top: 50, right: 30, bottom: 50, left: 60 };
      const containerWidth = 1100px;
      const width = containerWidth - margin.left - margin.right;
      const height = 700 - margin.top - margin.bottom;
      
      const svg = d3.select("#chart")
                    .append("svg")
                    .attr("width", width + margin.left + margin.right)
                    .attr("height", height + margin.top + margin.bottom)
                    .append("g")
                    .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
      
      const x = d3.scaleLinear()
                  .domain([40, 90])
                  .range([0, width]);
      
      const y = d3.scaleLinear()
                  .domain([50, 85])
                  .range([height, 0]);
      
      const r = d3.scaleSqrt()
                  .domain([2, 6])
                  .range([10, 30]);
      
      svg.append("g")
         .attr("transform", "translate(0," + height + ")")
         .call(d3.axisBottom(x));
      
      svg.append("g")
         .call(d3.axisLeft(y));
      
      svg.append("text")
         .attr("class", "axis-label")
         .attr("x", width)
         .attr("y", height + 40)
         .attr("text-anchor", "end")
         .text("Ethical Guideline Adoption (%)");
      
      svg.append("text")
         .attr("class", "axis-label")
         .attr("transform", "rotate(-90)")
         .attr("y", -50)
         .attr("x", 0)
         .attr("text-anchor", "end")
         .text("Public Trust Index");
      
      svg.selectAll("circle")
         .data(data)
         .enter()
         .append("circle")
           .attr("cx", d => x(d.adoption))
           .attr("cy", d => y(d.trust))
           .attr("r", d => r(d.initiatives))
           .attr("fill", "#69b3a2")
           .attr("opacity", 0.7);
      
      const tooltip = d3.select("body")
                        .append("div")
                        .attr("class", "tooltip")
                        .style("opacity", 0);
      
      svg.selectAll("circle")
         .on("mouseover", function(event, d) {
           tooltip.transition()
                  .duration(200)
                  .style("opacity", 0.9);
           tooltip.html("Region: " + d.region + "<br/>Adoption: " + d.adoption + "%<br/>Initiatives: " + d.initiatives + "<br/>Trust: " + d.trust)
                  .style("left", (event.pageX + 10) + "px")
                  .style("top", (event.pageY - 28) + "px");
         })
         .on("mouseout", function() {
           tooltip.transition()
                  .duration(500)
                  .style("opacity", 0);
         });
    </script>
  </body>
</html>


--- Example 2 ---

<html>
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Hybrid Architectures vs Traditional Deep Learning: Speed and Accuracy Comparison</title>
  <style>
    body {
      margin: 0;
      font-family: Arial, sans-serif;
    }
    h1 {
      text-align: left;
      margin: 20px 0;
    }
    #chart {
      max-width: 100%;
      max-height: 700px;
      margin: auto;
      background-color: #fff;
      border: 1px solid #ddd;
      box-shadow: 0px 0px 8px rgba(0,0,0,0.1);
    }
  </style>
</head>
<body>
  <h1>Hybrid Architectures vs Traditional Deep Learning: Speed and Accuracy</h1>
  <div id="chart"></div>
  <script src="https://d3js.org/d3.v6.min.js"></script>
  <script>
    // Set margins and dimensions
    const margin = { top: 40, right: 40, bottom: 60, left: 60 },
          width = 1100px - margin.left - margin.right,
          height = 600 - margin.top - margin.bottom;

    // Sample benchmarking data
    const data = [
      { arch: "Hybrid Architectures", processing: 110, accuracy: 94, source: "Maryland Graduate" },
      { arch: "Hybrid Architectures", processing: 130, accuracy: 90, source: "Study [2]" },
      { arch: "Hybrid Architectures", processing: 150, accuracy: 88, source: "Study [3]" },
      { arch: "Traditional Deep Learning", processing: 180, accuracy: 85, source: "Study [2]" },
      { arch: "Traditional Deep Learning", processing: 220, accuracy: 80, source: "Study [3]" },
      { arch: "Traditional Deep Learning", processing: 240, accuracy: 83, source: "Maryland Graduate" }
    ];

    // Create SVG container
    const svg = d3.select("#chart")
                  .append("svg")
                  .attr("width", width + margin.left + margin.right)
                  .attr("height", height + margin.top + margin.bottom)
                  .append("g")
                  .attr("transform", `translate(${margin.left},${margin.top})`);

    // Scales
    const x = d3.scaleLinear()
                .domain([d3.min(data, d => d.processing) - 10, d3.max(data, d => d.processing) + 10])
                .range([0, width]);

    const y = d3.scaleLinear()
                .domain([d3.min(data, d => d.accuracy) - 5, d3.max(data, d => d.accuracy) + 5])
                .range([height, 0]);

    // Color scale based on architecture type
    const color = d3.scaleOrdinal()
                    .domain(["Hybrid Architectures", "Traditional Deep Learning"])
                    .range(["#1f77b4", "#ff7f0e"]);

    // Add X Axis
    svg.append("g")
       .attr("class", "axis")
       .attr("transform", `translate(0, ${height})`)
       .call(d3.axisBottom(x));

    // Add Y Axis
    svg.append("g")
       .attr("class", "axis")
       .call(d3.axisLeft(y));

    // X Axis label
    svg.append("text")
       .attr("x", width / 2)
       .attr("y", height + margin.bottom - 10)
       .attr("text-anchor", "middle")
       .text("Processing Time (ms)");

    // Y Axis label
    svg.append("text")
       .attr("transform", "rotate(-90)")
       .attr("x", -height / 2)
       .attr("y", -margin.left + 15)
       .attr("text-anchor", "middle")
       .text("Accuracy (%)");

    // Tooltip
    const tooltip = d3.select("body")
                      .append("div")
                      .style("position", "absolute")
                      .style("padding", "8px")
                      .style("background", "rgba(0,0,0,0.6)")
                      .style("color", "#fff")
                      .style("border-radius", "4px")
                      .style("visibility", "hidden")
                      .style("font-size", "12px");

    // Draw data points
    svg.selectAll("circle")
       .data(data)
       .enter()
       .append("circle")
       .attr("cx", d => x(d.processing))
       .attr("cy", d => y(d.accuracy))
       .attr("r", 7)
       .attr("fill", d => color(d.arch))
       .attr("stroke", "#333")
       .attr("stroke-width", 1)
       .on("mouseover", (event, d) => {
          tooltip.html(`<strong>${d.arch}</strong><br/>Processing: ${d.processing} ms<br/>Accuracy: ${d.accuracy}%<br/>Source: ${d.source}`)
                 .style("visibility", "visible");
       })
       .on("mousemove", (event) => {
          tooltip.style("top", (event.pageY - 10) + "px")
                 .style("left", (event.pageX + 10) + "px");
       })
       .on("mouseout", () => {
          tooltip.style("visibility", "hidden");
       });

    // Legend
    const legendData = color.domain();
    const legend = svg.selectAll(".legend")
                      .data(legendData)
                      .enter()
                      .append("g")
                      .attr("class", "legend")
                      .attr("transform", (d, i) => `translate(0, ${i * 20})`);

    legend.append("rect")
          .attr("x", width - 18)
          .attr("width", 18)
          .attr("height", 18)
          .style("fill", color);

    legend.append("text")
          .attr("x", width - 24)
          .attr("y", 9)
          .attr("dy", ".35em")
          .style("text-anchor", "end")
          .text(d => d);
  </script>
</head>
<body>
</body>
</html>


--- Example 3 ---

<html>
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Performance Improvements in AI Applications: Healthcare, Education, and Finance</title>
  <style>
    body {
      font-family: Arial, sans-serif;
      margin: 0;
      padding: 10px;
      background-color: #f5f5f5;
    }
    h1 {
      text-align: left;
      margin-bottom: 10px;
      color: #333;
    }
    #chart {
      max-width: 100%;
      max-height: 700px;
      background-color: #fff;
      border: 1px solid #ddd;
      box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.1);
      margin: auto;
    }
    .axis text {
      fill: #555;
    }
    .axis path,
    .axis line {
      stroke: #ccc;
    }
  </style>
</head>
<body>
  <h1>Performance Improvements in AI Applications</h1>
  <div id="chart"></div>
  <script src="https://d3js.org/d3.v7.min.js"></script>
  <script>
    const data = [
      { sector: "Healthcare", metric: "Accuracy", value: 87 },
      { sector: "Healthcare", metric: "Speed", value: 72 },
      { sector: "Healthcare", metric: "Cost Efficiency", value: 95 },
      { sector: "Education", metric: "Accuracy", value: 80 },
      { sector: "Education", metric: "Speed", value: 68 },
      { sector: "Education", metric: "Cost Efficiency", value: 75 },
      { sector: "Finance", metric: "Accuracy", value: 92 },
      { sector: "Finance", metric: "Speed", value: 85 },
      { sector: "Finance", metric: "Cost Efficiency", value: 88 }
    ];

    const sectors = ["Healthcare", "Education", "Finance"];
    const metrics = ["Accuracy", "Speed", "Cost Efficiency"];

    const margin = { top: 60, right: 30, bottom: 50, left: 60 };
    const containerWidth = 1100px;
    const width = containerWidth - margin.left - margin.right;
    const height = 700 - margin.top - margin.bottom;

    const svg = d3.select("#chart")
      .append("svg")
      .attr("width", width + margin.left + margin.right)
      .attr("height", height + margin.top + margin.bottom)
      .style("display", "block")
      .append("g")
      .attr("transform", `translate(${margin.left},${margin.top})`);

    const x0 = d3.scaleBand()
      .domain(sectors)
      .range([0, width])
      .paddingInner(0.2);

    const x1 = d3.scaleBand()
      .domain(metrics)
      .range([0, x0.bandwidth()])
      .padding(0.1);

    const y = d3.scaleLinear()
      .domain([0, 100])
      .range([height, 0])
      .nice();

    const color = d3.scaleOrdinal()
      .domain(metrics)
      .range(["#4682b4", "#6b8e23", "#d2691e"]);

    svg.append("g")
      .attr("class", "x-axis")
      .attr("transform", `translate(0, ${height})`)
      .call(d3.axisBottom(x0))
      .selectAll("text")
      .style("font-size", "12px");

    svg.append("g")
      .attr("class", "y-axis")
      .call(d3.axisLeft(y).ticks(10).tickFormat(d => d + "%"))
      .selectAll("text")
      .style("font-size", "12px");

    const sectorGroups = svg.selectAll(".sector")
      .data(sectors)
      .enter()
      .append("g")
      .attr("class", "sector")
      .attr("transform", d => `translate(${x0(d)},0)`);

    sectorGroups.selectAll("rect")
      .data(sector => data.filter(d => d.sector === sector))
      .enter()
      .append("rect")
      .attr("x", d => x1(d.metric))
      .attr("y", d => y(d.value))
      .attr("width", x1.bandwidth())
      .attr("height", d => height - y(d.value))
      .attr("fill", d => color(d.metric));

    sectorGroups.selectAll("text.label")
      .data(sector => data.filter(d => d.sector === sector))
      .enter()
      .append("text")
      .attr("class", "label")
      .attr("x", d => x1(d.metric) + x1.bandwidth() / 2)
      .attr("y", d => y(d.value) - 5)
      .attr("text-anchor", "middle")
      .style("font-size", "11px")
      .style("fill", "#333")
      .text(d => d.value + "%");

    svg.append("text")
      .attr("x", width / 2)
      .attr("y", -30)
      .attr("text-anchor", "middle")
      .style("font-size", "18px")
      .style("fill", "#333")
      .text("Improvement in Key Performance Metrics for AI Applications");

    const legend = svg.selectAll(".legend")
      .data(metrics)
      .enter()
      .append("g")
      .attr("class", "legend")
      .attr("transform", (d, i) => `translate(0,${i * 20})`);

    legend.append("rect")
      .attr("x", width - 18)
      .attr("width", 18)
      .attr("height", 18)
      .attr("fill", d => color(d));

    legend.append("text")
      .attr("x", width - 24)
      .attr("y", 9)
      .attr("dy", ".35em")
      .style("text-anchor", "end")
      .style("fill", "#333")
      .style("font-size", "12px")
      .text(d => d);
  </script>
</html>"""
    
    prompt = f"""
Generate a full HTML code (including CSS and JavaScript) that displays a simple, elegant graph using D3.js.
The output MUST include only:
 • A descriptive title for the graph
 • The graph code (using D3.js) that uses data relevant to:
{placeholder_text}

Ensure that no extraneous content (such as discussion, concluding remarks, or additional text) is included.
Take in consideration that the resulting  content will render 1200px width and and 700px high.
Output the result:
- starting with <html> and
- ending with </html>
- with no additional comments or code fences
- use proper indentation in the output
// Sources:
Keep in mind the:
- context:
{context}

- the knowledge inputs:
{crumbs}

// Requirements
- Use elaborate but effective visual through call to the D3.js library.
- No introduction, conclusions or code fences -> Output the result directly
- It should start with <html> and end with </html>
- Use a responsive design. Do not fix the width of the main container in the generated HTML. Instead, set the meta viewport to "width=device-width, initial-scale=1" and use CSS (such as viewBox attributes in SVG) to ensure the chart scales properly.
- Make the visuals content rich, there's no point having a visual if its content has no real value.
- Make sure all the items are visible and don't require hovering the mouse to be displayed - this report is meant to be printed.
- For any element where text is displayed, if you choose a dark background, the fond has to be white / if you choose a clear background, the font has to be black
- Use your judgement to decide between box plots, bubble charts, calendar view, chord diagrams, histograms, ...
- Your response should start with <html> and end with </html> - no intro before, no comments after.

// Purpose and grounding
- It has to convey some relevant insights grounded from the context and knowledge input
- Make sure that the message conveyed really adds value once integrated in a report or posted on social media
- It CANNOT be hallucinated or hypothetical, it has to be factual and based on relevant info
- You will be penalised if you lie or mix things that are not related to each other
- Do not "wing it" with clumsy output, have a deep thinking to it

// Examples
{graph_examples}

Now provide the result directly.
"""
    result = llm_call(prompt, model="o3-mini", temperature=0, max_tokens_param=10000)
    result = result.strip().strip("```").strip()
    
   
    # Create a responsive container with proper width constraints - fixed width and height for consistency
    htmloutput = f"""
    <div class="graph-container" style="max-width:1100px; width:100%; margin:0 auto; overflow:hidden;">
        <iframe 
            class="visual-frame" 
            style="border:none; width:100%; height:700px;" 
            srcdoc="{result}">
        </iframe>
    </div>
    """
    
    logging.info(f"The code produced for this graph placeholder:\n{placeholder_text}")
    return htmloutput

def replace_graph_placeholders(report_html: str, context: str, initial_query: str, crumbs: str) -> str:
    pattern = r"\[\[Graph Placeholder (\d+):(.*?)\]\]"
    
    def placeholder_replacer(match):
        placeholder_num = match.group(1)
        instructions = match.group(2).strip()
        logging.info(f"Generating graph {placeholder_num}")
        
        try:
            visual_html = generate_graph_snippet(instructions, context, initial_query, crumbs)
            prompt_improve = f"Based on the instructions:\n{instructions}\n\nThe context:\n{context}\n\nThe knowledge crumbs:\n{crumbs}\n\nProvide a refined version to this graph code:\n{visual_html}\n\nThe improved result should be EXACTLY in the same html format (no introduction, no conclusion, no code fences)."
            visual_html = llm_call(prompt_improve, model="o3-mini", temperature=0, max_tokens_param=10000)
            visual_html = visual_html.strip().strip("```").strip()

            return f'<!-- Graph {placeholder_num} Start -->\n<div class="graph-container" style="margin:20px 0;">\n{visual_html}\n</div>\n<!-- Graph {placeholder_num} End -->'
        except Exception as e:
            logging.error(f"Graph {placeholder_num} failed: {str(e)}")
            error_msg = f'<div class="error-message" style="color:red; border:1px solid red; padding:10px; margin:10px 0;">Error generating Graph {placeholder_num}: {str(e)}</div>'
            return error_msg

    return re.sub(pattern, placeholder_replacer, report_html, flags=re.DOTALL)

# ============================================================================= Visual placeholder

def generate_visual_snippet(placeholder_text: str, context: str, initial_query: str, crumbs: str) -> str:
    # remove special lines
    def remove_special_lines(input_string):
        lines = input_string.splitlines()
        filtered_lines = [
            line for line in lines
            if not (
                (line.strip().startswith("%%") and line.strip().endswith("%%")) or
                line.strip().startswith("class")
            )
        ]
        cleanedlines = "\n".join(filtered_lines)
        return cleanedlines

    def process_multiline_string(input_string):
        def process_parentheses(match):
            content = match.group(2)  # Extract text inside parentheses
            return f"{match.group(1)} - {content}"  # Replace with a dash and the content
        pattern = r"(\[.*)\(([^)]+)\)\]"  # Captures lines with content followed by parentheses
        processed_string = re.sub(pattern, lambda m: process_parentheses(m), input_string, flags=re.MULTILINE)
    
    prompt = f"""
Generate a mermaid code displaying a simple but effective and elegant visualization based on the following requirements:
{placeholder_text}

It will be integrated in a broader report about:
{initial_query}

// Sources:
Keep in mind the:
- context:
{context}
- the knowledge inputs
{crumbs}

// Requirements
- Use standard mermaid visual rendering (ex: flowchart, sequence, gantt, pie, mindmap)
- Ieep it simple, no fancy characters, just simple text in the pseudo-code (not event parenthesis)
- Make the visuals content rich, there's no point having a visual if its content has no real value.
- It has to convey some relevant insights grounded in data provided in the search results, do not make things up - particularly on the relationship between items
- Use only this type of quotation marks: ", others would lead to failure to render
- Take in consideration that the resulting iframe content will render 1200px width and and 700px high.
- It has to convey some relevant insights grounded from the context and knowledge input
- It CANNOT be hallucinated or hypothetical, it has to be factual and based on relevant info
- You will be penalised if you lie or mix things that are not related to each other
- Do not "wing it" with clumsy output, have a deep thinking to it

// Important
- No introduction, conclusions or code fences -> Output the result directly - create only the content for the mermaid
- Do no put any comment of #color coding and classes inside the mermaid code generated, it's supposed to be only focused on the mermaid code required to render it
- Avoid the formatting commands (ex: %%...%% ) and do not use classDef, class or linkStyle for formatting
- Do not add items in the mermaid code between parenthesis ex: [xxx(yyy)] , rather use [xxx - yyy] / Do not put it in parenthesis, even after the hyphen - or for units.
- If you need to add a remark in the code, do it like this after an hyphen: "[abc - 95% Context Coverage]". Simply don't use parenthesis except for the below prescribed formatting, mermaid if very format sensitive
Example: do not write:
G --> H[Performance Evaluation (45% Speed Improvement, 35% Risk Profiling, 50% Fraud Detection)]
Instead, use
G --> H[Performance Evaluation - 45% Speed Improvement, 35% Risk Profiling, 50% Fraud Detection]
- Do not add any formatting reference in the content of the mermaid code (ex - green ...), this will lead to errors when submitted to the mermaid api
- Do not use the pipe symbol | in the mermaid code, it would create an error 
- Do not use the gantt chart visual placeholder to generate a graph (ex: barchart), Gantt charts should only be used for timelines / project plans
Note: Visual placeholders are solely supposed to generate diagrams not for graphs.
- Do not use the quotation mark inside the text of the diagram 
- Do not use commas in the content of each object / nodes, they will create rendering issues
- Do not use any [x] reference notation for citations (ex: [1], [2], ...) inside the mermaid code - we will manage sources separately

// Examples
Note: Pay attention for each example to what type of parenthesis / bracket is used and respect it scrupulously

-- flowchart --
Important:
- If the flow is "deeper" than broad (>3 levels), choose TD (Top Down) ==> TD is the default one if you have a doubt
- If the flow is "broader" than deep (>3 branches at the same level), choose LR (Left Right)
- There can be mix of flows that can be elaborate
- Take in consideration that the resulting iframe content will render 1200px width and and 700px high. Make the best of the areas real-estate.

Top Down:

flowchart TD
    A[Christmas] -->|Get money| B(Go shopping)
    B --> C{{Let me think}}
    C -->|One| D[Laptop]
    C -->|Two| E[iPhone]
    C -->|Three| F[fa:fa-car Car]


Left Right:

flowchart LR
    A[Anticipated AI Advancements by 2025]
    
    subgraph Infinite_Context [Infinite Context Framework]
      I1[Token Limits: Unlimited]
      I2[Summarization Accuracy: Significantly Improved]
      I3[Industry Impact: Legal, Academic, IoT]
      I4[Use-case Efficiency: High Impact]
      I5[Integration with Emerging Reasoning Models & Agentic Architecture]
    end

    subgraph Traditional_Context [Traditional Fixed Context Windows]
      T1[Token Limits: 2048-4096]
      T2[Summarization Accuracy: Moderate]
      T3[Industry Applications: Limited Scope]
      T4[Use-case Impact: Constrained]
    end

    subgraph Comparative_Innovations [Comparative Metrics & Emerging Technologies]
      C1[Test Time Compute Enhancements]
      C2[Innovations: VJEPA, Synthetic Data, Deepseek]
      C3[New Paradigms: TTT, Novel Architectural Approaches]
      C4[Disruption Factors: Deployment Speed, Employment Impact, Lower Barriers]
    end

    A --> Infinite_Context
    A --> Traditional_Context
    A --> Comparative_Innovations

    class I1,I2,I3,I4,I5 infinite;
    class T1,T2,T3,T4 trad;
    class C1,C2,C3,C4 comp;

Mixed: For elaborate visuals
// example 1

flowchart LR
    subgraph subgraph1
        direction TB
        top1[top] --> bottom1[bottom]
    end
    subgraph subgraph2
        direction TB
        top2[top] --> bottom2[bottom]
    end
    %% ^ These subgraphs are identical, except for the links to them:

    %% Link *to* subgraph1: subgraph1 direction is maintained
    outside --> subgraph1
    %% Link *within* subgraph2:
    %% subgraph2 inherits the direction of the top-level graph (LR)
    outside ---> top2

// example 2
flowchart LR
subgraph "One"
  a("`The **cat**
  in the hat`") -- "edge label" --> b{{"`The **dog** in the hog`"}}
end
subgraph "`**Two**`"
  c("`The **cat**
  in the hat`") -- "`Bold **edge label**`" --> d("The dog in the hog")
end

-- sequence --
sequenceDiagram
    Alice->>+John: Hello John, how are you?
    Alice->>+John: John, can you hear me?
    John-->>-Alice: Hi Alice, I can hear you!
    John-->>-Alice: I feel great!

-- gantt --
gantt
    title A Gantt Diagram
    dateFormat  YYYY-MM-DD
    section Section
    A task           :a1, 2014-01-01, 30d
    Another task     :after a1  , 20d
    section Another
    Task in sec      :2014-01-12  , 12d
    another task      : 24d

-- pie --
pie title Pets adopted by volunteers
    "Dogs" : 386
    "Cats" : 85
    "Rats" : 15

-- mindmap --
mindmap
  root[mindmap]
    Origins
      Long history
      Popularisation
        British popular psychology author Tony Buzan
    Research
      On effectivness
      On Automatic creation
        Uses
            Creative techniques
            Strategic planning
            Argument mapping
    Tools
      Pen and paper
      Mermaid

"""
    
    result = llm_call(prompt, model="o3-mini", temperature=0, max_tokens_param=10000)
    result = result.strip().strip("```").strip()
    #result = remove_special_lines(result)
    #result = process_multiline_string(result)

    htmloutput = f"""<iframe class="visual-frame" srcdoc='
        <!DOCTYPE html>
        <html>
            <head>
                <script src="https://cdn.jsdelivr.net/npm/mermaid@10.4.0/dist/mermaid.min.js"></script>
                <script>
                    mermaid.initialize({{ startOnLoad: true, securityLevel: "loose", theme: "default" }});
                </script>
                <style>
                    .mermaid {{
                        background-color: #f7f2f2;
                        color: black;
                        padding: 10px;
                        border-radius: 5px;
                        border: 1px solid #ccc;
                        max-width: 1200px;
                        margin: 0 auto;
                        min-height: 400px;
                    }}
                </style>
            </head>
            <body>
                <div class="mermaid">
    {result}
                </div>
            </body>
        </html>' width="1200px" height="700px" style="border:none;"></iframe>
        """
    
    logging.info(f"The code produced for this visual placeholder:\n{placeholder_text}\n\n {htmloutput}\n\n")

    # Remove parenthesis in mermaid formatting
    htmloutput = re.sub(
        r"<iframe>\.\.\.\[\.\.\.\((.*?)\)\] </iframe>",
        r"<iframe>...[...- \1] </iframe>",
        htmloutput
    )
    return htmloutput

def replace_visual_placeholders(report_html: str, context: str, initial_query: str, crumbs: str) -> str:
    pattern = r"\[\[Visual Placeholder (\d+):(.*?)\]\]"  # Capture placeholder number
    replacements = []

    def placeholder_replacer(match):
        placeholder_num = match.group(1)
        instructions = match.group(2).strip()
        logging.info(f"Generating visual {placeholder_num}")
        
        try:
            visual_html = generate_visual_snippet(instructions, context, initial_query, crumbs)
            prompt_improve = f"Based on the instructions:\n{instructions}\n\nThe context:\n{context}\n\nThe knowledge crumbs:\n{crumbs}\n\nProvide a refined version to this visual code:\n{visual_html}\n\nThe improved result should be EXACTLY in the same html format (no introduction, no conclusion, no code fences)."
            visual_html = llm_call(prompt_improve, model="o3-mini", temperature=0, max_tokens_param=10000)
            visual_html = visual_html.strip().strip("```").strip()
            # Add error boundary and logging
            return f'<!-- Visual {placeholder_num} Start -->\n{visual_html}\n<!-- Visual {placeholder_num} End -->'
        except Exception as e:
            logging.error(f"Visual {placeholder_num} failed: {str(e)}")
            return f'<!-- ERROR GENERATING VISUAL {placeholder_num} -->'

    return re.sub(pattern, placeholder_replacer, report_html, flags=re.DOTALL)

# ============================================================================= Focus placeholder

def replace_focus_placeholders(report_html: str, context: str, initial_query: str, crumbs: str) -> str:
    pattern = r"\[\[Focus Placeholder (\d+):(.*?)\]\]"  # Capture placeholder number
    def placeholder_replacer(match):
        placeholder_num = match.group(1)
        instructions = match.group(2).strip()
        logging.info(f"Generating focus box {placeholder_num}")
        try:
            focus_html = generate_focus_snippet(instructions, context, initial_query, crumbs)
            # Remove any outer HTML or body tags from the generated snippet:
            focus_html = re.sub(r"<\/?(html|head|body)[^>]*>", "", focus_html, flags=re.DOTALL|re.IGNORECASE).strip()
            # Wrap the entire focus content in one div with a dedicated CSS class
            return (
                f'<!-- Focus {placeholder_num} Start -->'
                f'<div class="focus-placeholder">{focus_html}</div>'
                f'<!-- Focus {placeholder_num} End -->'
            )
        except Exception as e:
            logging.error(f"Focus {placeholder_num} failed: {str(e)}")
            return f'<!-- ERROR GENERATING FOCUS {placeholder_num} -->'
    return re.sub(pattern, placeholder_replacer, report_html, flags=re.DOTALL)

def generate_focus_snippet(placeholder_text: str, context: str, initial_query: str, crumbs: str) -> str:
    prompt = f"""
Generate a complete, self-contained inner-HTML code (only the content inside <div> tags, no outer <html> tags) with inline CSS.
The code should provide an elaborate technical drill‐down on the topic:
{placeholder_text}

The output must be between 1000 and 2000 words.
- Use clear sub‑titles (bold them or use <h3> tags) and paragraphs to break the content.

It will be integrated in a broader report (in html, within the <body>, no need to add it) about:
{initial_query}

// Sources:
Keep in mind the:
- context:
{context}
- the knowledge inputs
{crumbs}

// Critical Requirements
- It MUST include inline citations using the format [1], [2], etc., that exactly match the reference numbering used in the overall report.
Note: no need to add a reference table at the end of the focus box, all the references / citations have already been listed at the end of the report (just refer to the [x] provided in the placeholder instructions)
Note2: citations sources in-line need to be in this format: blablabla - Source [x] / "pdf" is not a source, provide the title or author
- Do not add inline citations for every sentence, use is ontly when people, numbers or organisation names or projects are mentioned
- Use html with css to render it, keep it sober and focused on the content
- The output has to be boxed in a border of 1px Black
- Font size minimum 12px for readability in PDF
- Skip lines and add intermediate titles for key paragraphs
- You have freedom on the structure, but it has to cover all potential aspects on the topic in between 500 and 1000 words

// Mentioning sources, organisations and individuals
- We will perform a post-processing on the output
- For this reasons use this format for any specific name, organisation or project: {{[{{name}}]}}
ex1: {{[{{Google}}]}} CEO, {{[{{Sundar Pichai}}]}} ...
ex2: in a report from the {{[{{university of Berkeley}}]}} titled "{{[{{The great acceleration}}]}}"...
ex3: the CEO of {{[{{Softbank}}]}} , {{[{{Masayoshi Son}}]}}, said that "the best way to..."
ex4: the project {{[{{Stargate}}]}}, anounced by {{[{{OpenAI}}]}} in collaboration with {{[{{Salesforce}}]}}
ex5: Mr. {{[{{Michael Parrot}}]}}, Marketing director in {{[{{Panasonic}}]}}, mentioned that ...
Note: the output will be processed through regex and the identifiers removed, but this way we can keep track of all sources and citations without disclosing them.
- This should apply to names, people/titles, dates, papers, reports, organisation/institute/NGO/government bodies quotes, products, project names, ... 
- You should have approximately 10 mention of organisations, people, projects or people, use the prescribed format
- DO NOT MENTION this formmatting requirement, just apply it. The user doesn't have to know about this technicality.
- Make sure not to bundle together several sources, use one source at a time when mentioning it, do the same for the authors. They can be put together only if both were involved and mentioned in the research results.
Note: LinkedIn is not a source - if you want to use a source related to LinkedIn, you should check the author of the page visited, this is the real source, mention the name of the author as "'authorName' from LinkedIn Pulse"

// Purpose and grounding
- It has to convey some relevant insights grounded from the context and knowledge input
- It CANNOT be hallucinated or hypothetical, it has to be factual and based on relevant info
- You will be penalised if you lie or mix things that are not related to each other
- Do not "wing it" with clumsy output, have a deep thinking to it

// Important
- Make it real, with anecdotes from the content
- Ground the content on the inputs shared above (context and knowledge inputs) with facts and numbers
- Take a deep breath, think step by step and think it well.
- Title should be h2 level (inside the focus box) then the content
- No introduction, conclusions, code fences -> Output the result directly
""" 
    result = llm_call(prompt, model="o3-mini", temperature=0, max_tokens_param=10000)
    result = result.strip().strip("```").strip()
    result = re.sub(r'\{\[\{(.*?)\}\]\}', r'\1', result)
    result = re.sub(r'\[\{(.*?)\}\]', r'\1', result)
    logging.info(f"The code produced for this focus placeholder:\n{placeholder_text}\n\n {result}\n\n")
    return result
    
# ============================================================================= LLMCall

def llm_call(prompt: str, messages: list = None, model: str = "o3-mini", temperature: float = 0.0, max_tokens_param: int = 10000) -> str:
    if messages is None or len(messages) == 0:
        messages = [{"role": "user", "content": prompt}]
    if len(messages[0]['content']) > MAX_MESSAGE_LENGTH:
        messages[0]['content'] = messages[0]['content'][:MAX_MESSAGE_LENGTH]
    try:
        client = openai.OpenAI(api_key=os.getenv('OPENAI_API_KEY'))
        params = {
            "model": model,
            "messages": messages
        }
        if model == "o3-mini":
            params["max_completion_tokens"] = max_tokens_param
        else:
            params["max_tokens"] = max_tokens_param
            params["temperature"]= temperature
        response = client.chat.completions.create(**params)
        result = response.choices[0].message.content
        result = result.strip().strip("json").strip("```").strip()
        
        if not result:
            logging.error("Empty response from LLM for prompt: " + prompt)
            return "Error: empty response"
        
        logging.info(f"llm_call completed with model {model}. Response preview: {result}\n\n\nThe original prompt sent was:\n{prompt[:5000]}")
        return result
    except Exception as e:
        err_msg = f"Error calling OpenAI API: {e}"
        logging.error(err_msg)
        return err_msg

# ============================================================================= Search Engine

def filter_search_results(results: list, visited_urls: set, query: str, clarifications: str) -> list:
    # Filter out already seen results
    new_results = []
    for idx, res in enumerate(results):
        url = res.get("link", "")
        if url and url not in visited_urls:
            new_results.append(res)
    if not new_results:
        return []
    # Build text describing the new results
    results_text = ""
    for idx, res in enumerate(new_results):
        title = res.get("title", "No Title")
        snippet = res.get("snippet", "No Snippet")
        url = res.get("link", "")
        results_text += f"\nResult {idx}:\nTitle: {title}\nSnippet: {snippet}\nURL: {url}\n"
    prompt = (f"""
The following search results were obtained for the query
'{query}'

with clarifications:
{clarifications}

// Requirements
- For each result, decide whether it might be of interest for deeper research.
- Even if not completely certain, lean towards including more potential references.
- Return your decision as a JSON object where each key is the result index (as an integer) and the value is either 'yes' or 'no'.

// EXAMPLE
{{"0": "yes", "1":"no", "2": "yes", ...}}

Consider the title, snippet, and URL in your decision.

// Results to process
{results_text}

// IMPORTANT
Output only the JSON object, no code fences."""
    )
    llm_response = llm_call(prompt, model="gpt-4o-mini", max_tokens_param=500, temperature=0)
    # Remove any Markdown code fences if present
    llm_response = llm_response.strip().strip("json").strip("```").strip()
    try:
        decision_map = json.loads(llm_response)
    except Exception as e:
        logging.error(f"filter_search_results: JSON decode error: {e}; Full response: {llm_response}")
        decision_map = {}
    filtered = []
    for idx, res in enumerate(new_results):
        url = res.get("link", "")
        # Add URL to visited regardless
        visited_urls.add(url)
        decision = decision_map.get(str(idx), "no").strip().lower()
        if decision == "yes":
            filtered.append(res)
    # If no results were judged "yes", fall back to all new results so that we still process them.
    if not filtered:
        logging.info("filter_search_results: No results selected after filtering, defaulting to all new results.")
        return new_results
    logging.info(f"filter_search_results: {len(filtered)} results selected out of {len(new_results)} candidates")
    return filtered

def make_multilingual_query(query: str, context: str, languagesdetected: str) -> str:
    finalquery = f"({query})"  # original query is wrapped in parentheses
    languages_detected_list = languagesdetected.split(",")
    for lang in languages_detected_list:
        prompt2 = f"""The research query is: "{query}".
Based on this query and context: "{context}", and with the detected language {lang}, provide the translated version of the query in that language.
The translation must be less than 20 words and preserve search operators like AND, OR, parenthesis, quotation marks, and exclusion hyphens.
Output only the translated query."""
        translatedquery = llm_call(prompt2, model="gpt-4o-mini", max_tokens_param=50, temperature=0)
        finalquery += f" OR ({translatedquery})"
    logging.info(f"make_multilingual_query: Transformed query: {finalquery}")
    return finalquery

def generate_query_tree(context: str, breadth: int, depth: int, selected_engines: list) -> list:
    # If selected_engines is None, provide a fallback string
    list_engines = "all relevant search engines" if selected_engines is None else ','.join(map(str, selected_engines))
    
    prompt = f"""
Generate {breadth} search queries for academic research exploring: "{context}"

// Research Strategy Requirements
1. Query Progression:
   - Begin with foundational/conceptual queries
   - Progress to methodological/technical terms
   - Culminate in specialized/applied combinations

2. Term Optimization:
   - Use Boolean logic (AND/OR) strategically
   - Include both general terminology AND domain-specific jargon
   - Add temporal filters when relevant (e.g., "since 2018", "2020-2023")
   - Consider geographical/cultural modifiers if applicable

3. Query Structure:
   - Prioritize conceptual combinations over simple keyword matching
   - Use quotation marks for exact phrases and hyphenation for compound terms
   - Include emerging terminology variants (e.g., "LLMs" OR "large language models")

// Output Requirements
- Strictly valid JSON format: {{"queries": ["query1", "query2"]}}
- No Markdown, code fences, or supplementary text
- Clean string formatting without special characters

// Example (breadth=4):
{{
  "queries": [
    "Fundamental theories AND (Artificial Intelligence OR machine learning)",
    "(Computational mathematics OR statistical modeling) AND research paradigms",
    "\"Deep learning architectures\" AND (optimization techniques OR neural networks)",
    "\"Generative AI\" AND industrial applications AND (2020-2024 OR recent developments)"
  ]
}}

Generate queries that systematically explore the research landscape from multiple conceptual angles."""

    messages = []
    llm_response = llm_call(prompt=prompt, messages=messages, model="o3-mini", temperature=0, max_tokens_param=1500)
    logging.info(f"Generated query tree: {llm_response}")

    cleaned_response = llm_response.strip().strip("`").strip()
    try:
        queries = json.loads(cleaned_response)['queries']
        logging.info(f"Queries generated in queries: {queries}")
        final_queries = queries[:min(len(queries), breadth)]
        logging.info(f"Queries generated in final_queries: {final_queries}")
    except (json.JSONDecodeError, KeyError, TypeError) as e:
        logging.error(f"Error parsing LLM response in generate_query_tree: {e}")
        final_queries = []  # or use a fallback default query list, e.g.
        # final_queries = [ "default search query" ]
    return final_queries

def generate_serp_queries(context: str, breadth: int, depth: int, initial_query: str, selected_engines=None, results_per_query: int = 10) -> list:
    queries = generate_query_tree(context, breadth, depth, selected_engines)
    logging.info(f"Queries generated from generate_query_tree:{queries}")
    prompt = f"""The research topic is: "{initial_query}".
Based on this query and the overall context: "{context}", suggest one or several languages (other than English) that might be relevant.

Output either:
- "No local attributes detected"
- One language (e.g., "Spanish")
- Multiple languages comma separated (e.g., "Italian,Putonghua,Cantonese")
Output only the result.
"""
    languages_detected = llm_call(prompt, model="gpt-4o-mini", temperature=0, max_tokens_param=50)
    logging.info(f"languages detected: {languages_detected}")
    if languages_detected != "No local attributes detected":
        queries = [make_multilingual_query(q, context, languages_detected) for q in queries]
    if not selected_engines or len(selected_engines) == 0:
        prompt_engines = f"""
Examine these queries:
{queries}
and considering the research context:
{context}
Identify among these search engines:
google,google_scholar,google_trends,google_news,google_ai_overview,bing,bing_news,baidu,baidu_news,yandex,youtube_video,linkedin,linkedin_profile,duckduckgo_news,yelp_reviews
Which are most relevant? Output a comma separated list (e.g., "google,baidu").
If none are found, output "google".
"""
        identified_engines = llm_call(prompt_engines, model="o3-mini", temperature=0, max_tokens_param=50)
        logging.info(f"Identified engines are:{identified_engines}")

        # Split and strip engines; if result is empty (or all empty strings), default to ["google"]
        selected_engines = [e.strip() for e in identified_engines.split(",") if e.strip()]
        if not selected_engines:
            selected_engines = ["google"]
    else:
        selected_engines = selected_engines
    final_queries = []
    for q in queries:
        for engine in selected_engines:
            final_queries.append((q, engine))
    logging.info(f"generate_serp_queries: Final query-engine pairs: {final_queries}")
    return final_queries

def perform_serpapi_search(query: str, engine: str, num_results: int = 10) -> list:
    api_key = os.getenv("SERPAPI_API_KEY")
    params = {
        "q": query,
        "api_key": api_key,
        "engine": engine,
        "num": str(num_results)
    }
    attempt = 0
    results = []
    while attempt < 3:
        try:
            headers = {"User-Agent": get_random_header()}
            response = requests.get("https://serpapi.com/search", params=params, headers=headers)
            if response.status_code == 200:
                try:
                    data = response.json()
                    # Check if data is a string; if so, convert it
                    if isinstance(data, str):
                        data = json.loads(data)
                except Exception as je:
                    logging.error("perform_serpapi_search: Failed to parse JSON response. Raw response: " + response.text)
                    data = {}
                # Double-check the data type and try to convert otherwise if necessary
                if not isinstance(data, dict):
                    try:
                        data = json.loads(data)
                    except Exception as e:
                        logging.error("perform_serpapi_search: Could not convert response to a dictionary. Error: " + str(e))
                        data = {}
                results = data.get("organic_results", [])
                if len(results) > 0:
                    logging.info(f"perform_serpapi_search: Found {len(results)} results for engine {engine}")
                    break
            time.sleep(random.uniform(1, 3))
        except Exception as e:
            logging.error(f"perform_serpapi_search attempt {attempt} error: {e}")
            time.sleep(random.uniform(1, 3))
        attempt += 1
        time.sleep(2)
    if not results:
        logging.info(f"perform_serpapi_search: No results found for query: {query}, engine: {engine}")
    return results

# ============================================================================= Final report

def generate_final_report(initial_query: str, context: str, reportstyle: str, learnings: list, visited_urls: list,
                          aggregated_crumbs: str, references: list, pages: int = 8) -> str:
    fallback_text = ""
    if not learnings:
        fallback_text = "No external summaries were directly extracted. It is not possible to analyze relevance."
    combined_learnings = "\n".join(learnings) if learnings else fallback_text
    word_count = pages * 500
    prompt = (f"""

--------------- Objectives -----------
Produce a comprehensive report in html format.
The report should be detailed, grounded in facts and sources provided
It should be  {pages} long () or {word_count} words ) (excluding html formatting).
It should include tables, focus boxes as well as visuals and graphs (see below).
Exclusing Abstract, Table of contents, Abstracts, Conclusion and all placeholders, there should be approximately {round(pages/2,0)}
The report must follow this writing style: {reportstyle}.

--------------- Formatting -----------
- Add on top of the report the report title (with the <h1> tag) - this is the only part that should be centered (in-line style)
- All text alignment has to be on the left
- Titles for sections and sub-sections should systematically use the tags:
  <h1> for sections (ex: <h1>3. Examination of State-of-the-Art of AI</h1>)
  <h2> for sub-sections (ex: <h2>3.2 AI Performance in Mathematics</h2>)
  <h3> for sub-sub-sections (ex: <h3>3.2.1 Illustration with math conjecture demonstration</h3>)
  <h4> for bulletpoint title (ex: <h4>item to detail:</h4> description of the item to detail ...)
  Note:
  * Put "Table of contents" and "Abstract" title in h1 format.
- No more than 10 sentences per div blocks, skip lines and add line breaks when changing topic.
- For the numbering of titles or numbered lists, use numbers (ex: 1.) and sub-units (1.1, 1.2... 1.1.1...,1.1.2...).
  Note:
    * Exclude the use of html numbered lists format, they don't get correctly implemented. Use plain text format for numbering of sections and sub-sections
    * Do not put a numbered list (ex: 1.1, ...) for every sentences! It should be used parcimoniously for real sub-sections.
- Put paragraphs, sentences that are part of the same section in a div tag, this will be used for formatting.
- Avoid Chinese characters in the output (use the Pinyin version) since they won't display correcly in the pdf (it displays as black boxes)
- For the Table of contents: do not mention the pages, but make each item on separate line
- The Table of contents should not mention the abstract and table of contents, the numbering should start from the introduction and end with References Summary Table
- Exceptionally - for sections requiring specific improvements - put it between <div class="improvable-chunk">...</div> (but don't mention it in the report, this will be managed through post-processing)

--------------- Tables -----------
- The report must include between {round(pages/10,0)} and {round(pages/5,0)} tables from the sources used (add citations if necessary) and use facts and figures extensively to ground the analysis.
- Use inline formatting for the tables with homogeneous border and colors

--------------- Citations -----------
- The report must include inline citations (e.g., [1], [2], etc.) from real sources provided in the search results below - be selective, don't put it at every sentence or every paragraph.
   Note: citations sources in-line need to be in this format: blablabla - Source [x] / "pdf" is not a source, provide the title or author
- Do not add inline citations for every sentence, use is ontly when people, numbers or organisation names or projects are mentioned
- The name of the reference table should be: "References Summary Table"
- The reference table at the end containing the citations details should have 4 columns: the ref number, the title of the document, the author(s, the URL - with hyperlink)
- The report MUST include a References Summary Table with between 10 (for a 8 page report) and 30 references (for a 40 pages report). All inline citations (e.g., [1], [2], …) present in the report and in any focus placeholders MUST have a corresponding entry in this table with its full URL.
- For the reference citations, add systematically the urls from the Learnings (no need to put them in numbered list format since we alredy have the [x] that serves as number list)
- Do not add any inline citations reference in the visual and graph placeholders descriptions belo, you can add them in focus though.
- Do not make false references / citations. It has to be grounded from the sources in the rsearch results / crumbs below (no example.com/... type references!)
- The references / citations should be only coming from the most reputable sources amongst all the Learnings and Results from searches below
- The table generated should have in-line styling to have word-wrap and 100% width

// Instructions:
1. Integrate numbers from the sources but always mention the source
2. Whenever you mention a figure or quote, add an inline reference [x] matching its source from the references.
3. Again, Specifically name relevant organizations, tools, project names, and people encountered in the crumbs or learnings.
   Note: This is for academic purposes, so thorough citations and referencing are essential.
4. Focus on reputable sources that will not be disputed (generally social media posts cannot be an opposable sources, but some of them may mention reputable sources)
   Note: put the full reference url (no generic domain address), down to the html page or the pdf

// Format when mentioning sources, organisations and individuals
- We will perform a post-processing on the output
- For this reasons use this format for any specific name, organisation or project: {{[{{name}}]}}
example 1: {{[{{Organisation}}]}}'s CEO, {{[{{CEO name}}]}} ...
example 2: in a report from the {{[{{University name}}]}} titled "{{[{{report title}}]}}"...
example 3: the CEO of {{[{{Company name}}]}} , {{[{{Name}}]}}, said that "the best way to..."
eexample 4: the project {{[{{project name}}]}}, anounced by {{[{{...}}]}} in collaboration with {{[{{...}}]}}
example 5: Mr. {{[{{person}}]}}, Marketing director in {{[{{company}}]}}, mentioned that ...
Note: the output will be processed through regex and the identifiers removed, but this way we can keep track of all sources and citations without disclosing them.
- This should apply to names, people/titles, dates, papers, reports, organisation/institute/NGO/government bodies quotes, products, project names, ... 
- You should have approximately {2 * pages} mention of organisations, people, projects or people, use the prescribed format
- The same item cannot be mentioned more than 3 times, don't over do it
- Do not mix sources that are not directly related in the search results, don't put together organisations or people that have nothing to do with each other
- DO NOT MENTION this formmatting requirement, just apply it. The user doesn't have to know about this technicality.
Note: LinkedIn is not a relevant source - if you want to use a source related to LinkedIn, you should check the author of the page visited, this is the real source, mention the name of the author as "'authorName' from LinkedIn Pulse"

--------------- Source grounding -----------
// Use the following learnings and merged reference details from a deep research process on this topic:
'{initial_query}'

// Taking also into consideration the context:
{context}

// Key learnings to use as source:
{json.dumps(learnings, indent=2)}

// Results from searches you can mention in details:
{aggregated_crumbs}

--------------- Placeholders -----------
In order to enrich the content, within the core sections (between introduction and conclusion), you can inject some placeholders that will be developped later on.
There are 3 types: visual, graphs, focus - each with their own purpose

// Visual placeholders //
- Create special visual placeholders that will be rendered in mermaid afterwards.
- The Visual placeholders should follow this format:

Source:source_name [y]
[[Visual Placeholder n:
- Purpose of this visual is:...
- Relevant content to generate it:
o ex: arguments
o ex: lists of elements
o ex: data points
o ...
- Message to convey: ...
]]

with:
- n as the reference number,
- source_name as the full name of the main source used and
- y as the number ref of the source reference in the reference table.
Important note for visual placeholders:
- on the line before [[...]] mention the source with the reference number [x] in the form: ""Source: abc [n]" - only one source should be mentioned
- after [[ put "Visual Placeholder n:" explicitly (with n as the ref number of the placeholder box created). This will be used in a regex
- the only types of mermaid diagram that can be generated are: flowchart, sequence, gantt, pie, mindmap (no charts) // Take this into consideration when providing the instructions for the diagram
- do not make mention in the report to "visual placeholders" just mention the visual and the number..
- in the placeholder, no need to add the references to the source or its ref number, but make sure ALL of the data points required has a source from the learning and reference material hereafter 
- these placeholders text should contain:
    o the purpose of the future visual 
    o the relevant data to generate it
- there should be between {round(pages/10,0)} and {round(pages/5,0)} of these visuals placeholders within the report (all between introduction and conclusion)
- 2 visual placeholders cannot be in the same section
Note: the placeholders will then be processed separately by a llm to generate the specific code to display each of them so the instruction need to be clear enough.

// Graph placeholders //
- Create special graph placeholders that will be rendered in d3.js afterwards based on your guidance:

Source:source_name [y]
[[Graph Placeholder n:
- Purpose of this graph is:...
- Relevant numbers to generate it:
table format
- Message to convey: ...
]]

with:
- n as the reference number,
- source_name as the  full name of the main source used and
- y as the source reference in the reference table.
- the table containing all the required data has to include data points FROM the learnings / results from the search below
Important note for graph placeholders:
- on the line before [[...]] mention the source with the reference number [x] in the form: ""Source: abc [n]" - only one source should be mentioned
- use p tag for the source and source reference number
- after [[ put "Graph Placeholder n:" explicitly (with n as the ref number of the graph created). This will be used in a regex
- Do not make things up - every data points have to be from a real source
- All types of graphs (using d3.js library) can be generated // Take this into consideration when providing the instructions for the graph data
- do not make mention in the report to "graph placeholders" just mention graph.
- in the placeholder, no need to add the references to the source or its ref number, but make sure ALL of the data points required has a source from the learning and reference material hereafter 
- these placeholders text should contain:
    o the purpose of the future graph 
    o the relevant data to generate it
- there should be between {round(pages/10,0)} and {round(pages/5,0)} of these graphs placeholders within the report (all between introduction and conclusion)
- 2 graph placeholders cannot be in the same section 
Note: the placeholders will then be processed separately by a llm to generate the specific code to display each of them so the instruction need to be clear enough.

// Focus placeholders //
- To drill down on specific topic that would be deserve to be developped extensively separately, create special focus placeholders in [[...]] double backets
Note: outside of the placeholder, do not make reference in the report to "focus placeholders" just mention the "Focus box n".
- in the Focus placeholder, make a mention to the prescribed sources used (no need to add the source before or after the placeholder)
- do not make the placeholder on the exact same topic as the section or the sub-section where it is positioned, it has to be either:
    o a special case that deserves attention
    o a recent development / innovation
    o a theoretical drill-down
    o a contrarian point of view / objection
- these placeholders text should contain:
    o the purpose of the focus box 
    o the relevant data to generate it
    o the guidance in terms of style and message to convey
    Note: Be specific if you want some particular point developped, keep it consistent across the report.
- there should be between {round(pages/20,0)} and {round(pages/10,0)} of these focus placeholders within the report (all between introduction and conclusion)
- 2 focus placeholders cannot be in the same section and should be a few pages apart in the report
- Mention all the sources that should be used to generate this focus placeholder and list also the references that will be mentioned in the References section later (ex: [1], [2])
Note: the Focus placeholders will then be processed separately by a llm to generate the specific code to display each of them so the instruction need to be clear enough.

// Format:
[[Focus Placeholder n:
- Topic of this focus:...
- Relevant info to generate it:...
- Specific angle of the focus placeholder:...
- Key elements to mention:
o ...
o ...
...
]]

with:
- n as the reference number,

Important note for focus placeholders:
- after [[ put "Focus Placeholder n:" explicitly (with n as the ref number of the focus box created). This will be used in a regex
- Do not add a title for the Focus placeholder just before the [[...]], the content that will replace the focus placeholder - generated later on - will already include a title
- placeholders (visual, graph or focus) can only appear in the sections or sub-sections not in introduction, the conclusion, the references or after the references

--------------- Report structure -----------
Use the following report structure with consistency:

{{Do not add anything before - no introductory meta comment or content}}

- Abstract
- Table of contents
- Introduction
- [Sections and sub-sections, depending on the size and relevant topic - including visual, graph and focus placeholders]
- Conclusion
- References Summary Table
- Report ending formatting (as mentioned before)

{{Do not add anything after - no conclusive meta comment or content}}

--------------- Best practices  -----------
You generated the below best practices based on millions of reports reviewed and analyzed, follow them:

I. Overall Structure and Organization:

    Standard Structure: Follow the clear and organized structure as prescribed above
    Logical Flow: Ensure a logical flow of ideas, connecting various parts to form a unified whole. Use narrative links between sentences and paragraphs.
    Headings and Subheadings: Use clear, informative, and short headings and subheadings to divide the report into sections and facilitate browsing and scanning. Number them for longer reports.
    Consistent Layout: Maintain a consistent page layout and design features throughout the report. Use the same formatting for titles, headings, bulleted lists, and labels.
    White Space: Use plenty of margins and white space to avoid a "wall of words" and give the reader's eyes a chance to rest.

II. Content and Writing Style:

    Reader-Centric Approach: Design the report with the reader's needs and expectations in mind.
    Clear and Concise Language: Use clear, direct, and precise language. Avoid meaningless expressions. Explain any technical terms.
    Objective Voice: Strive to be as objective as possible when expressing opinions. Record opinions in the context of supporting facts.
    Active Voice: Use active voice to make sentences more direct and engaging.
    Conciseness: Avoid repetition and redundant phrases.
    Accurate and Factual: Be factual, consistent, and accurate.
    Evidence-Based: Support claims with credible evidence and properly cited sources.
    Paragraph Structure: Write strong paragraphs that present a topic, discuss it, and conclude it. Avoid one-sentence paragraphs.
    Avoid Bias: Represent the work of other researchers and your own point of view fairly and accurately.

III. Abstract/Executive Summary:

    Concise Overview: Provide a quick overview of the report's purpose, methods, findings, and conclusions.
    Key Points: Express the thesis or central idea and key points.
    Keywords: Include important keywords related to method and content for searchability.
    Standalone: The abstract should be a self-contained text, understandable on its own.

IV. Introduction:

    Engaging Opening: Capture the reader's attention with a hook, such as a compelling story, quotation, question, or statistic.
    Background Information: Provide necessary context and background information on the topic.
    Purpose and Scope: Clearly state the purpose and scope of the report.
    Thesis Statement: Include a clear and focused thesis statement that expresses the aim or focus of the paper.
    Roadmap: (Optional) Provide a brief overview of the report's structure.

V. Body (Main Sections):

    Organize by Theme/Topic: Structure each section to focus on a specific aspect of the report.
    Methodology: Describe the research methods and materials used in detail.
    Results: Present the research results clearly and objectively.
    Discussion: Interpret the results and explain their significance.

VI. Conclusion:

    Summarize Main Points: Briefly restate the primary findings and analysis.
    Restate Thesis: Reiterate the thesis statement in a fresh way.
    Offer Recommendations: (If applicable) Suggest actions based on the findings.
    Highlight Implications: Explain the significance of the findings and their potential impact.
    Concluding Statement: End with a powerful statement that leaves a lasting impression.
    Avoid New Information: Do not introduce new ideas or evidence in the conclusion.

VII. Visuals (Tables, Graphs, Images):

    Purposeful Use: Incorporate visuals to illustrate complex data and make findings easier to understand.
    Appropriate Chart Types: Choose chart types that effectively convey the message and suit the nature of the data (e.g., bar charts for comparison, line charts for trends).
    Simplicity: Opt for simplicity in design, avoiding unnecessary embellishments.
    Clear Labeling: Label axes, legends, and data points clearly.
    Consistent Formatting: Use a consistent color scheme and style.
    Data Accuracy: Verify the accuracy of data (it has to be related to the source provided) before creating visualizations.
    Titles: Tables and charts should have sequential numbered titles above them (e.g., "Table 1 – Table Title").

VIII. References and Citations:

    Accurate Citations: Cite all sources properly and consistently, following a recognized citation style (e.g., MLA, APA).
    Reputable Sources: Use reputable and industry-vetted sources.
    Current Sources: Use current sources (published within the last 5 years, if possible).
    Complete Reference List: Include a complete list of references in alphabetical order.

--------------- Ending -----------
End the report with the following sequence:

    <iframe class="visual-frame" srcdoc='
    <!DOCTYPE html>
    <html>
    </head>
        <body>
            <div>
-end-
            </div>
        </body>
    </html>' width="100px" height="15px" style="border:none;"></iframe>

Then close the html code from the broader report
</body>
</html>

--------------- Your turn -----------
Take a deep breath, do your best.
Now, produce the report please.
"""
    )
    tokentarget = word_count * 5  # rough multiplier for token target
    report = llm_call(prompt, model="o3-mini", max_tokens_param=tokentarget)
    # Post-processing
    report = re.sub(r'\{\[\{(.*?)\}\]\}', r'\1', report)
    report = re.sub(r'\[\{(.*?)\}\]', r'\1', report)
    
    # If the report is too long, compress it.
    if len(report) > MAX_MESSAGE_LENGTH:
        report = compress_text(report, MAX_MESSAGE_LENGTH)
    if report.startswith("Error calling OpenAI API"):
        logging.error(f"generate_final_report error: {report}")
        return f"Error generating report: {report}"
    logging.info("generate_final_report: Report generated successfully.")
    return report

# ============================================================================= Final report



def generate_tailored_questions(openai_api_key: str, query: str, existing_qa: str, existing_report: str, 
                                existing_log: str, crumbs: str) -> str:
    os.environ["OPENAI_API_KEY"] = openai_api_key
    query = query or ""
    existing_qa = existing_qa or ""
    existing_report = existing_report or ""
    existing_log = existing_log or ""
    crumbs = crumbs or ""
    prompt = (
        f"Context:\nSearch Topic: {query}\nExisting Clarification Q&A: {existing_qa}\nExisting Report: {existing_report}\n"
        f"Existing Process Log: {existing_log}\nCrumbs: {crumbs}\n\n"
        "Based on the above, generate FIVE new clarification questions, preferably open questions like 'What objectives would you like to reach with this search?' or 'Are there specific aspects you would like to search in particular such as xxx, yyy, ...?' "
        "Each question should be followed by 'Response:' then 2 empty lines."
    )
    new_questions = llm_call(prompt, model="gpt-4o", temperature=0, max_tokens_param=1000)
    logging.info(f"generate_tailored_questions: Generated questions: {new_questions}")
    if not new_questions.strip():
        new_questions = "No questions asked."
    return new_questions + "\n\n" + existing_qa

def backup_fields(research_query: str, 
                  include_domains: str, exclude_keywords: str, additional_clarifications: str, 
                  selected_engines, results_per_query, breadth, depth, clarification_text: str, 
                  existing_report: str, existing_log: str, crumbs_box: str, final_report: str, existing_queries_box: str) -> str:
    data = {
        "openai_api_key": "",
        "serpapi_api_key": "",
        "research_query": research_query,
        "include_domains": include_domains,
        "exclude_keywords": exclude_keywords,
        "additional_clarifications": additional_clarifications,
        "selected_engines": selected_engines,
        "results_per_query": results_per_query,
        "breadth": breadth,
        "depth": depth,
        "clarification_text": clarification_text,
        "existing_report": existing_report,
        "existing_log": existing_log,
        "crumbs_box": crumbs_box,
        "final_report": final_report,
        "existing_queries": existing_queries_box
    }
    backup_json = json.dumps(data, indent=2)
    logging.info(f"backup_fields: Data backed up: {backup_json}")
    return backup_json

def load_fields(backup_json: str):
    try:
        data = json.loads(backup_json)
        logging.info(f"load_fields: Loaded data: {data}")
        return (data.get("openai_api_key", ""),
                data.get("serpapi_api_key", ""),
                data.get("research_query", ""),
                data.get("include_domains", ""),
                data.get("exclude_keywords", ""),
                data.get("additional_clarifications", ""),
                data.get("selected_engines", []),
                data.get("results_per_query", 10),
                data.get("breadth", 4),
                data.get("depth", 2),
                data.get("clarification_text", ""),
                data.get("existing_report", ""),
                data.get("existing_log", ""),
                data.get("crumbs_box", ""),
                data.get("final_report", ""),
                data.get("existing_queries",""))
    except Exception as e:
        logging.error(f"load_fields error: {e}")
        return ("", "", "", "", "", "", [], 10, 4, 2, "", "", "", "", "", "")

def refine_query(query: str, openai_api_key: str, reportstyle: str) -> str:
    os.environ["OPENAI_API_KEY"] = openai_api_key
    if not query:
        logging.info("refine_query: Empty query provided.")
        return ""
    prompt = f"""Reformulate the following research query: '{query}'
    Use this following style: {reportstyle}
    The result should be a string of text, without any quotation mark before and after - Only the updated and refined version of the query, nothing else."""
    refined = llm_call(prompt, model="gpt-4o-mini", temperature=0, max_tokens_param=100)
    logging.info(f"refine_query: Refined query: {refined}")
    return refined

def validate_visual_html(html: str) -> bool:
    """Basic sanity check for generated visuals"""
    checks = [
        ("<svg" in html) or ("plotly" in html.lower()),
        "background" not in html.lower() or "#fff" in html.lower(),
        not re.search(r"color\s*:\s*#000000", html, re.I)
    ]
    return all(checks)

class ReportGenerator:
    def __init__(self, render_with_selenium: bool = False):
        # Flag to determine if we are rendering the final PDF using Selenium
        self.render_with_selenium = render_with_selenium

    def generate_report_html(self, solution_content: str) -> str:
        # Normalize text and fix dash characters.
        solution_content = unicodedata.normalize('NFKC', solution_content)
        solution_content = re.sub(r'[\u2010\u2011\u2012\u2013\u2014\u2015]', "-", solution_content)
        solution_content = re.sub(r'\[(.*?)\]\(.*?\)', r'\1', solution_content)
        html_content = markdown.markdown(solution_content, extensions=['extra', 'tables'])
        # Insert page break divs before key sections
        html_content = html_content.replace("<h2>Table of Contents</h2>", "<div class='page-break'></div><h2>Table of Contents</h2>")
        html_content = html_content.replace("<h2>Introduction</h2>", "<div class='page-break'></div><h2>Introduction</h2>")
        html_content = html_content.replace("<h2>Conclusion</h2>", "<div class='page-break'></div><h2>Conclusion</h2>")
        html_content = html_content.replace("<h2>References</h2>", "<div class='page-break'></div><h2>References</h2>")
        html_content = html_content.replace("<h2>Surprise-Me Extension Report</h2>", "<div class='page-break'></div><h2>Surprise-Me Extension Report</h2>")
        
        logging.info(f"ReportGenerator: HTML report generated successfully:\n{html_content}")
        return html_content

    def generate_report_pdf(self, solution_content: str, metadata: dict = None) -> bytes:
        # Generate the full HTML report (including text, placeholders, and mermaid visuals as iframes)
        html_report = self.generate_report_html(solution_content)

        # Add header if provided in metadata
        date_str = datetime.now().strftime("%Y-%m-%d")
        header = ""
        if metadata:
            header = (f"<p>Search Query: {metadata.get('Query name', 'N/A')}<br>"
                      f"Author: {metadata.get('User name', 'N/A')} | Date: {metadata.get('Date', date_str)}</p>")
        soup = BeautifulSoup(html_report, "html.parser")
        if soup.body:
            soup.body.insert(0, BeautifulSoup(header, "html.parser"))
        logging.info("ReportGenerator: Soup report generated:\n%s", soup)

        # Replace all iframes (class 'visual-frame') with images
        soup = replace_visual_iframes(soup)
        
        # Extract only the body content if <body> exists, otherwise use full HTML
        body_tag = soup.find("body")
        body_content = body_tag.decode_contents() if body_tag else str(soup)

        # Reassemble a clean HTML document with inline CSS styles for PDF conversion.
        final_html = f"""<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <style>
        body {{ 
            font-family: Helvetica, sans-serif; 
            margin: 40px; 
            background: white; 
        }}
        h1 {{ 
            font-size: 20pt; 
            margin-bottom: 12px; 
            text-align: left; 
            font-weight: bold;
        }}
        h2 {{ 
            font-size: 16pt; 
            margin-bottom: 10px; 
            text-align: left; 
            font-weight: bold;
        }}
        h3 {{ 
            font-size: 14pt; 
            margin-bottom: 8px; 
            text-align: left; 
            font-weight: bold;
        }}
        h4 {{ 
            font-size: 12pt; 
            text-align: left; 
            font-weight: bold;
        }}
        p {{ 
            font-size: 11pt; 
            line-height: 1.5; 
            margin-bottom: 10px; 
            white-space: pre-wrap;
        }}
        pre, div {{ 
            white-space: pre-wrap;
        }}
        ol, ul {{ 
            font-size: 11pt; 
            margin-left: 20px; 
            line-height: 1.5;
        }}
        hr {{ 
            border: 1px solid #ccc; 
            margin: 20px 0;
        }}
        /* ---------------------- TABLE STYLES ---------------------- */
        table {{ 
             border-collapse: collapse; 
             width: 100%; 
             margin-bottom: 10px; 
             table-layout: fixed;  /* Added to force a fixed layout for wrapping */
        }}
        th, td {{ 
             border: 1px solid #ccc; 
             padding: 8px; 
             text-align: left; 
             word-wrap: break-word;      /* Ensure long words are wrapped */
             overflow-wrap: break-word;  /* Modern equivalent */
        }}
        th {{ 
             background-color: #f2f2f2; 
        }}
        /* ------------------ FOCUS PLACEHOLDER STYLES ------------------ */
        .focus-placeholder {{
             border: 1px solid black;  /* This creates the outer border */
             padding: 10px;
             font-size: 12pt;
             line-height: 1.6;
             margin-bottom: 10px;
             word-wrap: break-word;
        }}
        /* Ensure that child elements inside the focus placeholder do not get extra borders */
        .focus-placeholder > * {{
             border: none;
        }}
        .page-break {{ 
             page-break-before: always; 
        }}
    </style>
  </head>
  <body>
    {body_content}
  </body>
</html>
"""
        logging.info("ReportGenerator: Final HTML for PDF conversion generated.")

        # Generate the PDF using xhtml2pdf (pisa)
        pdf_buffer = io.BytesIO()
        pisa_status = pisa.CreatePDF(final_html, dest=pdf_buffer, 
                                     link_callback=lambda uri, rel: uri)
        if pisa_status.err:
            logging.error("Error generating PDF with xhtml2pdf.")
            return None
        logging.info("ReportGenerator: PDF generated successfully.")
        return pdf_buffer.getvalue()

def handle_generate_report(query_name: str, user_name: str, final_report: str):
    try:
        report_generator = ReportGenerator(render_with_selenium=False)
        metadata = {
            "Query name": query_name,
            "User name": user_name,
            "Date": datetime.now().strftime("%Y-%m-%d"),
            "Time": datetime.now().strftime("%H:%M:%S"),
        }

        pdf_bytes = report_generator.generate_report_pdf(solution_content=final_report, metadata=metadata)
        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as tmp_file:
            tmp_file.write(pdf_bytes)
            tmp_path = tmp_file.name
        logging.info(f"handle_generate_report: PDF report generated at {tmp_path}")
        return "Report generated successfully.", gr.update(value=tmp_path, visible=True)
    except Exception as e:
        logging.error(f"handle_generate_report error: {e}", exc_info=True)
        return f"Error generating report: {str(e)}", None

def extract_summary_from_crumbs(crumbs_list: list) -> str:
    """
    Given a list of crumb records (each with 'url', 'summary', and 'full_content'),
    extract and aggregate only the summary parts.
    """
    aggregated = "\n".join([f"URL: {c['url']}\nSummary: {c['summary']}" for c in crumbs_list])
    logging.info("extract_summary_from_crumbs: Aggregated crumb summary created.")
    return aggregated

def generate_surprise_report(previous_report: str, crumbs_list: list, initial_query: str, reportstyle: str,
                             breadth: int, depth: int, followup_clarifications: str,
                             include_domains: str, exclude_keywords: str, additional_clarifications: str,
                             results_per_query: int, selected_engines, existing_queries: str) -> str:
    crumb_summary = "\n\n".join([f"Title: {c.get('title', 'No Title')}\nURL: {c['url']}\nSummary: {c['summary']}" for c in crumbs_list])
    new_prompt = ("Based on the previous research report:\n"
                  f"{previous_report}\n\n"
                  "And the following summaries of the scraped sources:\n"
                  f"{crumb_summary}\n\n"
                  f"And the initial query: '{initial_query}',\n\n"
                  "please propose a disruptive, experimental, and ambitious new research hypothesis that goes beyond conventional boundaries (Overton-window) related to the topic. "
                  "Formulate this as a new research query that could lead to innovative insights.")
    disruptive_query = llm_call(new_prompt, model="o3-mini", temperature=0, max_tokens_param=2000)
    logging.info(f"generate_surprise_report: Disruptive new query generated: {disruptive_query}")

    # Generate tailored clarification questions for the disruptive query
    clarifications_for_new = generate_tailored_questions(
        os.getenv("OPENAI_API_KEY"),
        disruptive_query + "\n\n IMPORTANT NOTE: in this specific iteration, generate also the responses for the questions asked (simulated)",
        "", "", "", ""
    )
    logging.info(f"generate_surprise_report: Clarification questions for new query: {clarifications_for_new}")

    # Run iterative deep research for the disruptive query
    generator = iterative_deep_research_gen(
        disruptive_query, reportstyle, breadth, depth, followup_clarifications,
        include_domains, exclude_keywords, additional_clarifications,
        extra_context="", selected_engines=selected_engines, results_per_query=results_per_query, existing_queries=existing_queries, go_deeper=1
    )
    extension_report = ""
    for progress, rep, proc_log, new_crumbs, existing_queries in generator:
        if rep is not None:
            extension_report = rep
            break
    logging.info("generate_surprise_report: Extension report generated successfully.")
    appended_report = previous_report + "\n\n<div style='page-break-before: always;'></div>\n<h2>Surprise-Me Extension Report</h2>\n\n" + clarifications_for_new + "\n\n" + extension_report
    return appended_report

def extract_structured_insights(html_text: str) -> str:
    """
    Extract only facts, figures, arguments, and quotes in a concise manner.
    Use BeautifulSoup to parse and remove anything not relevant to these categories.
    This function returns a short text suitable for summarization by the LLM.
    """
    soup = BeautifulSoup(html_text, "html.parser")

    # We can decide to keep paragraphs that contain digits (numbers),
    # or words like "claim", "argument", "quote", etc. This is just an example heuristic.
    paragraphs = soup.find_all('p')
    curated_excerpts = []
    for p in paragraphs:
        text = p.get_text().strip()
        # If it has digits or certain keywords, we keep it
        if re.search(r'\d+', text) or re.search(r'\bargument\b|\bfact\b|\bfigure\b|\bstudy\b|\bquote\b', text, re.IGNORECASE):
            curated_excerpts.append(text)

    # Combine them into a shorter snippet
    snippet = "\n".join(curated_excerpts)
    # If snippet is too short, fallback to the entire cleaned text
    if len(snippet.split()) < 30:
        snippet = clean_content(html_text)[:2000]  # or some fallback length

    return snippet

def generate_reference_table(references_list: list) -> str:
    # Build a simple HTML table with reference number, title, and URL.
    if not references_list:
        return "<p>No references available.</p>"
    table_html = "<table style='width:100%; border-collapse: collapse;'>"
    table_html += "<tr><th style='border: 1px solid #ccc; padding: 8px;'>Ref No.</th><th style='border: 1px solid #ccc; padding: 8px;'>Title/Description</th><th style='border: 1px solid #ccc; padding: 8px;'>URL</th></tr>"
    for ref_num, url in references_list:
        # In a more complete implementation, you might also retrieve a proper title from your scraped data.
        table_html += f"<tr><td style='border: 1px solid #ccc; padding: 8px;'>{ref_num}</td><td style='border: 1px solid #ccc; padding: 8px;'>{url}</td><td style='border: 1px solid #ccc; padding: 8px;'><a href='{url}'>{url}</a></td></tr>"
    table_html += "</table>"
    return table_html
    
def iterative_deep_research_gen(initial_query: str, reportstyle: str, breadth: int, depth: int,
                                followup_clarifications: str,
                                include_domains: str,
                                exclude_keywords: str,
                                additional_clarifications: str,
                                extra_context: str = "",
                                selected_engines=None,
                                results_per_query: int = 10,
                                existing_queries: str = "",
                                go_deeper: int = 8):
    overall_context = extra_context + f"Initial Query: {initial_query}\n"
    if followup_clarifications.strip():
        overall_context += "User Clarifications: " + followup_clarifications.strip() + "\n"
    process_log = "Starting research with context:\n" + overall_context + "\n"
    overall_learnings = []
    visited_urls = set()
    # Parse previously processed queries from existing_queries if provided
    processed_queries = set()
    for q_line in existing_queries.splitlines():
        q_line = q_line.strip()
        if q_line:
            processed_queries.add(q_line)
    crumbs_list = []
    ref_counter = 1
    references_list = []
    followup_suggestions = []
    logging.info("iterative_deep_research_gen: Research started.")
    for iteration in range(1, depth + 1):
        process_log += f"\n--- Iteration {iteration} ---\n"
        logging.info(f"iterative_deep_research_gen: Starting iteration {iteration}.")
        combined_context = overall_context
        if followup_suggestions:
            # Deduplicate follow-up suggestions before adding them to context.
            unique_suggestions = list(set(followup_suggestions))
            combined_context += "\n\nFollow-up suggestions: " + ", ".join(unique_suggestions)
        queries = generate_serp_queries(combined_context, breadth, depth, initial_query, selected_engines, results_per_query)

        # ===================================================================
        # Skip queries already in processed_queries
        filtered_query_tuples = []
        for q_tuple in queries:
            q_text, eng = q_tuple
            if q_text not in processed_queries:
                filtered_query_tuples.append(q_tuple)
                processed_queries.add(q_text)  # remember we've processed it
        # ===================================================================
        process_log += f"\nWill run {len(filtered_query_tuples)} new queries this iteration instead of {len(queries)} total.\n"
        iteration_learnings = []
        followup_suggestions = []  # reset for current iteration
        for query_tuple in filtered_query_tuples:
            query_str, engine = query_tuple
            mod_query = query_str
            if include_domains.strip():
                domains = [d.strip() for d in include_domains.split(",") if d.strip()]
                domain_str = " OR ".join([f"site:{d}" for d in domains])
                mod_query += f" ({domain_str})"
            if exclude_keywords.strip():
                for ex in [ex.strip() for ex in exclude_keywords.split(",") if ex.strip()]:
                    mod_query += f" -{ex}"
            process_log += f"\nPerforming SERPAPI search with query: {mod_query} using engine: {engine}\n"
            results = perform_serpapi_search(mod_query, engine, results_per_query)
            
            # Filter out visited results
            filtered_results = filter_search_results(results, visited_urls, initial_query, followup_clarifications)
            process_log += f"After filtering, {len(filtered_results)} results remain for processing.\n"
            for res in filtered_results:
                url = res.get("link", "")
                title = res.get("title", "No Title")
                if not url:
                    continue
                    
                raw_content = process_url(url)
                if raw_content.startswith("Error"):
                    process_log += f"{raw_content}\n"
                    continue
                    
                # Skip processing if raw_content is empty or too short (< 1000 characters)
                if not raw_content or len(raw_content) < 1000 or "could not be extracted" in raw_content.lower():
                    process_log += f"Content from {url} is too short (<500 characters), skipping.\n"
                    continue
                
                process_log += f"Successfully extracted content from {url}\n"
                
                # 1) Clean and do minimal parse
                cleaned_html = clean_content(raw_content)
                # 2) Extract structured data
                semantically_rich_snippet = extract_structured_insights(cleaned_html)
                # 3) Summarize with LLM
                analysis = analyze_with_gpt4o(initial_query, semantically_rich_snippet, breadth, url, title)
                
                # Analyze the cleaned content with GPT-4o-mini
                cleaned_text = clean_content(raw_content)  # Call the function to get a string.
                analysis = analyze_with_gpt4o(initial_query, cleaned_text, breadth, url, title)
                analysis_summary = analysis.get("summary", "")
                if isinstance(analysis_summary, (dict, list)):
                    analysis_summary = json.dumps(analysis_summary)
                analysis_summary = analysis_summary.strip()
                                
                process_log += f"Summary: {analysis.get('summary')}, Follow-ups: {analysis.get('followups')}\n"
                if not analysis_summary:
                    analysis_summary = (raw_content[:200] + "..." if len(raw_content) > 200 else raw_content)
                
                # Append crumb (even if not relevant)  
                crumbs_list.append({
                    "url": url,
                    "title": title,
                    "summary": analysis_summary,
                    "clean_content": clean_content,
                    "raw_excerpt": raw_content[:1000]
                })
                
                if analysis.get("relevant", "no").lower() == "yes":
                    if url.startswith("http://") or url.startswith("https://"):
                        link_str = f" <a href='{url}'>[{ref_counter}]</a>"
                    else:
                        link_str = f" [{ref_counter}]"
                    summary_with_ref = analysis_summary + link_str
                    iteration_learnings.append(summary_with_ref)
                    references_list.append((ref_counter, url))
                    logging.info(f"Appended crumb: URL: {url}, Title: {title}, Summary snippet: {analysis_summary[:100]}")
                    ref_counter += 1
                    if isinstance(analysis.get("followups"), list):
                        if iteration < depth:
                            followup_suggestions.extend(analysis.get("followups"))
        process_log += f"Iteration {iteration} extracted {len(iteration_learnings)} learnings.\n"
        logging.info(f"iterative_deep_research_gen: Iteration {iteration} extracted {len(iteration_learnings)} learnings.")
        if len(iteration_learnings) == 0:
            process_log += f"Iteration {iteration} extracted no learnings; proceeding to next iteration instead of aborting.\n"
            # Optionally, you could continue here rather than break.
            continue
        overall_learnings.extend(iteration_learnings)
        overall_context += f"\nIteration {iteration} learnings:\n" + "\n".join(iteration_learnings) + "\n"
        if additional_clarifications.strip():
            overall_context += "\nAdditional Clarifications from user: " + additional_clarifications.strip() + "\n"
            process_log += "Appended additional clarifications to the context.\n"
        progress_pct = int((iteration / depth) * 100)
        yield (f"Progress: {progress_pct}%", None, None, None, None)

    if breadth > 3 and depth > 2:
        filtered_crumbs_list = filter_crumbs_in_batches(crumbs_list, initial_query, followup_clarifications)
    else:
        filtered_crumbs_list = crumbs_list

    # Now build aggregated crumb text from filtered_crumbs_list only
    aggregated_crumbs = "\n\n".join([
        f"Title: {c.get('title','No Title')}\nURL: {c['url']}\nSummary: {c['summary']}"
        for c in filtered_crumbs_list
    ])
    final_report = generate_final_report(initial_query, combined_context, reportstyle, overall_learnings, list(visited_urls), aggregated_crumbs, references_list, pages=go_deeper)
    
    # --- NEW STEP: Post-process final_report to replace visual and focus placeholders ---
    final_report = replace_visual_placeholders(final_report, combined_context, initial_query, aggregated_crumbs)
    final_report = replace_focus_placeholders(final_report, combined_context, initial_query, aggregated_crumbs)
    final_report = replace_graph_placeholders(final_report, combined_context, initial_query, aggregated_crumbs)
    
    alignment_assessment = assess_report_alignment(final_report, initial_query, followup_clarifications)
    final_report = final_report.replace(
        "<p>---end---</p>    </div>  </body></html>",
        f"<p>---------</p><p><b>Report alignment assessment:</b> {alignment_assessment}</p>    </div>  </body></html>"
    )
    logging.info("iterative_deep_research_gen: Final report generated.")
    # We convert processed_queries to a string suitable for storing
    all_processed_queries_str = "\n".join(sorted(processed_queries))

    yield ("", final_report, process_log, aggregated_crumbs, all_processed_queries_str)

def filter_crumbs_in_batches(crumbs_list: list, initial_query: str, clarifications: str) -> list:
    accepted = []
    batch_size = 20
    for i in range(0, len(crumbs_list), batch_size):
        batch = crumbs_list[i:i+batch_size]
        # Build a prompt describing each crumb
        prompt = f"""
We have a set of crumbs. For each crumb, decide if it adds new facts, figures, references, or quotes to the topic being investigated.
Don't be too selective, we still need to keep between 10 and 15 in each batch out of {batch_size}.
Mark 'yes' if it is valuable for the final report, otherwise 'no'. Output JSON.
Ensure consistency with the initial query:
{initial_query}

And the provided clarifications:
{clarifications}

Previously selected batch:

Now provide the JSON
"""
        listing = []
        for idx, c in enumerate(batch):
            snippet_for_prompt = c["summary"][:1000]  # short snippet
            listing.append(f"Crumb {idx}: {snippet_for_prompt}")
        prompt += "\n".join(listing)

        prompt += """
Return a JSON object with structure:
{
 "0": "yes" or "no",
 "1": "yes" or "no",
 ...
}

No code fences, no 'json' mentioned before the result, only the json result formatted.
"""
        decision_str = llm_call(prompt, model="o3-mini", temperature=0, max_tokens_param=1500)
        # parse JSON
        try:
            decisions = json.loads(decision_str)
        except:
            decisions = {}
        for idx, c in enumerate(batch):
            d = decisions.get(str(idx), "no").lower()
            if d == "yes":
                accepted.append(c)
    return accepted
    
def assess_report_alignment(report: str, initial_query: str, clarifications: str) -> str:
    prompt = f"""
Please assess the following research report in terms of its alignment with the initial user request and the clarification Q&A provided.
Ensure that the report covers key points of the topic.

Initial Query:
{initial_query}

Clarifications:
{clarifications}

Research Report:
{report}

Provide a short assessment in one paragraph (between 2 html div tags) on how well the report aligns with these requirements, you can put in html tags for bold or italic.
The output will be integrated in a html page for rendering.
"""
    assessment = llm_call(prompt, model="o3-mini", temperature=0, max_tokens_param=400)
    logging.info(f"assess_report_alignment: Assessment result: {assessment}")
    return assessment

def run_deep_research(openai_api_key: str, serpapi_api_key: str, initial_query: str, reportstyle: str, breadth: int, depth: int,
                      followup_clarifications: str, include_domains: str,
                      exclude_keywords: str, additional_clarifications: str,
                      results_per_query: int, selected_engines, existing_crumbs: str, existing_report: str, existing_log: str,
                      existing_queries: str, pages: str, surprise_me: bool):
    if not openai_api_key or not serpapi_api_key:
        logging.error("run_deep_research: Invalid API keys provided.")
        return "Please input valid API keys", "", "", "", ""

    os.environ["OPENAI_API_KEY"] = openai_api_key
    os.environ["SERPAPI_API_KEY"] = serpapi_api_key

    extra_context = ""
    if existing_report:
        extra_context += f"Existing Report:\n{existing_report}\n"
    if existing_log:
        extra_context += f"Existing Log:\n{existing_log}\n"
    if existing_crumbs:
        extra_context += f"Existing Crumbs:\n{existing_crumbs}\n"

    final_progress = ""
    final_report = ""
    final_process_log = ""
    final_crumbs = ""
    logging.info("run_deep_research: Starting deep research process.")
    for progress, rep, proc_log, crumbs, all_processed_queries_str in iterative_deep_research_gen(
            initial_query, reportstyle, breadth, depth, followup_clarifications,
            include_domains, exclude_keywords, additional_clarifications,
            extra_context, selected_engines, results_per_query, existing_queries, go_deeper=int(pages)):
        if rep is None:
            final_progress = progress
            yield final_progress, None, None, None, None, all_processed_queries_str
        else:
            final_report = rep
            final_process_log = proc_log
            final_crumbs = crumbs
            break
    if surprise_me:
        extended_report = generate_surprise_report(
            final_report, final_crumbs, initial_query, reportstyle, breadth, depth,
            followup_clarifications, include_domains, exclude_keywords, additional_clarifications,
            results_per_query, selected_engines, existing_queries
        )
        final_report = extended_report
        final_progress = "Progress: 100% (\"Surprise Me\" extension complete)"
    logging.info("run_deep_research: Deep research process completed.")
    yield (final_progress, final_report, final_report, final_process_log, final_crumbs, all_processed_queries_str)

def load_example(example_choice: str) -> str:
    filename = ""
    if example_choice == "Implications of the release of advanced Deep Research solutions":
        filename = "example1.txt"
    elif example_choice == "AI regulation in finance":
        filename = "example2.txt"
    elif example_choice == "AI top voices":
        filename = "example3.txt"
    try:
        with open(filename, "r", encoding="utf-8") as f:
            content = f.read()
        logging.info(f"load_example: Loaded content from {filename}")
        return content
    except Exception as e:
        logging.error(f"load_example:  Error loading {filename}: {e}")
        return ""

def main():
    custom_css = """
    /* Overall container customization */
    .gradio-container {
        max-width: 2000px;
        margin: auto;
        padding: 15px;
        background: linear-gradient(135deg, #f8f9fa, #e9ecef);
    }
    body {
        background: linear-gradient(135deg, #fefefe, #f4f4f8);
    }
    h1, h2, h3, label {
        color: #343a40;
        font-weight: bold;
    }
    .gr-button {
        background-color: #007bff;
        color: #fff;
        font-weight: bold;
        border-radius: 5px;
        border: none;
        padding: 8px 16px;
    }
    #final-report {
        background-color: #fafaf5;
        box-shadow: 0px 2px 5px rgba(0,0,0,0.1);
        max-width: 1400px;
        margin: auto;
        padding: 20px;
    }
    /* Progress display style */
    #progress-display {
        text-align: center;
        font-size: 1.2em;
        color: #007bff;
        margin-bottom: 10px;
    }
    """

    with gr.Blocks(css=custom_css, title="DeepSearch It – MyWay") as demo:
        with gr.Row():
            with gr.Column():
                gr.Markdown("# DeepSearch It  – MyWay")
                gr.Markdown("### === by T.Sakai / G.Huet ===")
                gr.Markdown("Input your API Keys, choose your search parameters (engine, depth, breadth, results, exclusions...), answer the clarifications questions and let the DeepSearch start.")
            gr.Image(value=display_image, height=300, width=300, interactive=False, show_label=False, show_download_button=False, show_share_button=False, show_fullscreen_button=False, elem_id="image")

        with gr.Accordion("1] API Keys", open=False):
            openai_api_key_input = gr.Textbox(label="OpenAI API Key", placeholder="Enter your OpenAI API Key here...", type="password")
            serpapi_api_key_input = gr.Textbox(label="SERPAPI API Key", placeholder="Enter your SERPAPI API Key here...", type="password")
            gr.Markdown("[Create OpenAI API Key](https://platform.openai.com/account/api-keys) | [Create SERPAPI API Key](https://serpapi.com/manage-api-key)")
            gr.Markdown("You can check the open-source code - None of the user API keys are stored or logged.")

        with gr.Accordion ("2] Research topic", open=False):
            with gr.Row():
                research_query = gr.Textbox(label="Research Query", placeholder="Enter your research query here...", lines=2, elem_id="research-query", scale=4)
                with gr.Column(scale=1):
                    reportstyle = gr.Textbox(label="Report style", placeholder="The report style", lines=1, value="Academic style")
                    refine_query_button = gr.Button("Refine my Query")

        with gr.Accordion("3] Q&A", open=False):
            with gr.Row():
                gen_followups = gr.Button("Generate Tailored Clarification Questions", scale=1)
                clarification_text = gr.Textbox(label="Clarification / Follow-Up Questions", placeholder="Tailored clarifying suggestions will appear here...", lines=6, scale = 4)
                
        with gr.Accordion("4] Search Parameters", open=False):
            with gr.Column():
                with gr.Row():
                    with gr.Column():
                        include_domains = gr.Textbox(label="Include Domains", placeholder="e.g., example.com, anotherdomain.org", lines=1)
                        exclude_keywords = gr.Textbox(label="Exclude Keywords/Mentions", placeholder="e.g., politics, rumor", lines=1)
                    with gr.Column():
                        pages_dropdown = gr.Dropdown(label="Report Length (pages)", choices=[str(i) for i in range(4, 41)], value="8")
                        surprise_me_checkbox = gr.Checkbox(label="Surprise me", value=False)
                with gr.Column():
                    additional_clarifications = gr.Textbox(label="Additional Clarifications", placeholder="Clarifications to be applied between iterations", lines=2, scale=4)
            selected_engines = gr.CheckboxGroup(label="Specific engines to Use (AI to choose by default)", choices=[
                "google",
                "google_jobs_listing",
                "google_trends",
                "google_news",
                "google_scholar",
                "google_ai_overview",
                "bing",
                "bing_news",
                "baidu",
                "baidu_news",
                "yandex",
                "youtube_video",
                "linkedin",
                "linkedin_profile",
                "duckduckgo_news",
                "yelp_reviews"
            ])
            with gr.Row():
                results_per_query = gr.Slider(label="Results per Query", minimum=5, maximum=20, step=1, value=10)
                breadth = gr.Slider(label="Research Breadth (queries per iteration)", minimum=0, maximum=6, step=1, value=2)
                depth = gr.Slider(label="Research Depth (iterations)", minimum=1, maximum=2, step=1, value=2)

        with gr.Accordion("5] Report", open=False, elem_classes="folder"):
            progress_display = gr.Markdown("", elem_id="progress-display")
            run_btn = gr.Button("Generate report")
            with gr.Accordion("Improvement query", open=False, elem_classes="folder"):
                with gr.Row():
                    adjustmentguidelines = gr.Textbox(label="Improvement instructions", placeholder="Input here your directions.\n\nFor example:\n- 'remove the following text:[[ABC]]'\n- 'fix visual after section [[XYZ]]'", lines=5)
                    with gr.Column():
                        suggest_improvements_button = gr.Button("Suggest Improvements")
                        fine_tune_button = gr.Button("Improve the Report")
                    with gr.Column():
                        suggest_expansions_button = gr.Button("Suggest Expansion")
                        expand_button = gr.Button("Expand the Report")
            with gr.Accordion("PDF generation", open=False, elem_classes="folder"):
                with gr.Column():
                    query_name = gr.Textbox(label="Query name", placeholder="Enter query name...", lines=1)
                    user_name = gr.Textbox(label="User name", placeholder="Enter your name...", lines=1)
                    report_status = gr.Textbox(label="Report Status", interactive=False, lines=2, value="Click 'Generate Report' to create your PDF report.")
                    report_file = gr.File(label="Download Report", visible=False, interactive=False, file_types=[".pdf"])
                    generate_button = gr.Button("Create pdf report")
            final_report = gr.HTML(label="Final Report", max_height = 800, min_height = 200, elem_id="final-report")

        
        with gr.Accordion("6] Extra Context (Crumbs, Existing Report & Log, Processed Queries)", open=False):
            existing_report = gr.Textbox(label="Existing Report (if any)", placeholder="Paste previously generated report here...", lines=4)
            existing_log = gr.Textbox(label="Existing Process Log (if any)", placeholder="Paste previously generated log here...", lines=4)
            crumbs_box = gr.Textbox(label="Existing Crumbs (All scraped sources, JSON)", placeholder="Paste existing crumbs JSON here...", lines=4)
            existing_queries_box = gr.Textbox(label="Existing Queries (processed queries)", placeholder="Paste processed queries here...", lines=4)

        with gr.Accordion("7] Chat with search", open=False):
            chatbot = gr.Chatbot(label="Chat with Search")
            chat_input = gr.Textbox(
                label="Your Message", 
                placeholder="Ask a question about the research report or its sources..."
            )
            send_button = gr.Button("Send")
        
        with gr.Accordion("8] Backup / Restore Fields", open=False):
            with gr.Row():
                backup_button = gr.Button("Backup Fields")
                load_button = gr.Button("Load Fields")
            backup_text = gr.Textbox(label="Backup JSON", placeholder="Backup output will appear here. You can also paste JSON here to load fields.", lines=6, interactive=True)

        with gr.Accordion("-- EXAMPLES --", open=False):
            example_dropdown = gr.Dropdown(label="Select an Example", choices=["Implications of the release of advanced Deep Research solutions", "AI regulation in finance", "AI top voices"])
            load_example_button = gr.Button("Load Example")

        # Button actions
        refine_query_button.click(
            fn=refine_query,
            inputs=[research_query, openai_api_key_input, reportstyle],
            outputs=[research_query]
        )
        gen_followups.click(
            fn=generate_tailored_questions,
            inputs=[openai_api_key_input, research_query, clarification_text, existing_report, existing_log, crumbs_box],
            outputs=clarification_text
        )
        run_btn.click(
            fn=run_deep_research,
            inputs=[openai_api_key_input, serpapi_api_key_input, research_query, reportstyle, breadth, depth, clarification_text, include_domains, exclude_keywords,
                    additional_clarifications, results_per_query, selected_engines, existing_report, existing_log, existing_queries_box, crumbs_box,
                    pages_dropdown, surprise_me_checkbox],
            outputs=[progress_display, final_report, existing_report, existing_log, crumbs_box, existing_queries_box],
            show_progress=True,
            api_name="deep_research"
        )
        backup_button.click(
            fn=backup_fields,
            inputs=[research_query, include_domains, exclude_keywords,
                    additional_clarifications, selected_engines, results_per_query, breadth, depth, clarification_text, existing_report, existing_log, crumbs_box, existing_report, existing_queries_box],
            outputs=[backup_text]
        )
        load_button.click(
            fn=load_fields,
            inputs=[backup_text],
            outputs=[openai_api_key_input, serpapi_api_key_input, research_query, include_domains, exclude_keywords,
                     additional_clarifications, selected_engines, results_per_query, breadth, depth, clarification_text, existing_report, existing_log, crumbs_box, final_report, existing_queries_box]
        )
        load_example_button.click(
            fn=load_example,
            inputs=[example_dropdown],
            outputs=[backup_text]
        )
        generate_button.click(
            fn=handle_generate_report,
            inputs=[query_name, user_name, final_report],
            outputs=[report_status, report_file]
        ) 

        fine_tune_button.click(
            fn=fine_tune_report,
            inputs=[adjustmentguidelines, openai_api_key_input, serpapi_api_key_input, final_report, research_query, clarification_text, reportstyle, crumbs_box, additional_clarifications],
            outputs=[final_report, clarification_text]
        )

        expand_button.click(
            fn=expand_report,
            inputs=[adjustmentguidelines, openai_api_key_input, serpapi_api_key_input, final_report, research_query, clarification_text, reportstyle, crumbs_box, additional_clarifications],
            outputs=[final_report, clarification_text]
        )
        
        suggest_expansions_button.click(
            fn=suggest_expansions,
            inputs=[final_report, openai_api_key_input, serpapi_api_key_input, research_query, clarification_text, crumbs_box, additional_clarifications],
            outputs=[adjustmentguidelines]
        )
        
        suggest_improvements_button.click(
            fn=suggest_improvements,
            inputs=[final_report, openai_api_key_input, serpapi_api_key_input, research_query, clarification_text, crumbs_box, additional_clarifications],
            outputs=[adjustmentguidelines]
        )
        
        send_button.click(
            fn=send_chat_message,
            inputs=[chat_input, openai_api_key_input, serpapi_api_key_input, chatbot, final_report, crumbs_box, reportstyle, research_query, clarification_text, additional_clarifications],
            outputs=[chatbot, chat_input, final_report]
        )
        
        demo.launch()

if __name__ == "__main__":
            main()