app/DocumentGeneration/markdown_renderer.py

# Markdown Renderer Module
# Contains all markdown rendering functions for both PDF and Word

import re
import base64
import os
import sys
import tempfile
from typing import List, Dict, Any, Optional

# Import from the same directory (Document Generation)
from grouped import parse_content_into_groups, should_break_page_for_group
from chart_generator import create_chart_from_data

def clean_markdown_text(text: str) -> str:
    """Clean markdown formatting for PDF generation - PRESERVE HEADINGS"""
    if not text:
        return ""
    
    # Remove markdown formatting BUT PRESERVE HEADING STRUCTURE
    cleaned = text
    
    # PRESERVE HEADINGS - don't strip the # markers, keep them for rendering
    # We'll handle styling at render time instead of stripping them
    
    # Remove bold, italic formatting - strip ** or * symbols
    cleaned = re.sub(r'\*\*(.*?)\*\*', r'\1', cleaned)  # Remove bold
    cleaned = re.sub(r'\*(.*?)\*', r'\1', cleaned)  # Remove italic
    cleaned = re.sub(r'`(.*?)`', r'\1', cleaned)  # Remove code
    cleaned = re.sub(r'~~(.*?)~~', r'\1', cleaned)  # Remove strikethrough
    
    # Remove links (keep text)
    cleaned = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', cleaned)  # Remove links, keep text
    cleaned = re.sub(r'!\[([^\]]*)\]\([^)]+\)', '', cleaned)  # Remove images
    
    # Remove list markers (more aggressive)
    cleaned = re.sub(r'^\s*[-*+]\s*', '', cleaned, flags=re.MULTILINE)  # Remove list markers
    cleaned = re.sub(r'^\s*\d+\.\s*', '', cleaned, flags=re.MULTILINE)  # Remove numbered list markers
    cleaned = re.sub(r'^\s*[-*+]\s*', '', cleaned, flags=re.MULTILINE)  # Remove any remaining list markers
    
    # Remove blockquotes
    cleaned = re.sub(r'^\s*>\s*', '', cleaned, flags=re.MULTILINE)  # Remove blockquotes
    
    # Remove horizontal rules
    cleaned = re.sub(r'^\s*[-*_]{3,}\s*$', '', cleaned, flags=re.MULTILINE)  # Remove horizontal rules
    
    # Remove code blocks
    cleaned = re.sub(r'```[\s\S]*?```', '', cleaned)  # Remove code blocks
    cleaned = re.sub(r'`.*?`', '', cleaned)  # Remove any remaining inline code
    
    # Remove emphasis markers - strip ** or * symbols
    cleaned = re.sub(r'_{1,2}(.*?)_{1,2}', r'\1', cleaned)  # Remove underscores
    cleaned = re.sub(r'\*{1,2}(.*?)\*{1,2}', r'\1', cleaned)  # Remove asterisks
    
    # PRESERVE CHART DATA MARKERS for inline processing
    # DO NOT REMOVE chart markers - they will be processed inline
    # cleaned = re.sub(r'<!--CHARTDATASTART-->.*?<!--CHARTDATAEND-->', '', cleaned, flags=re.DOTALL)
    # cleaned = re.sub(r'<!--CHARTDATASTART-->.*$', '', cleaned, flags=re.DOTALL)
    # cleaned = re.sub(r'<!--CHARTDATAEND-->', '', cleaned)
    
    # REMOVE CHART DATA MARKERS for PDF rendering since they're not being processed
    cleaned = re.sub(r'<!--CHART_DATA_START-->.*?<!--CHART_DATA_END-->', '', cleaned, flags=re.DOTALL)
    cleaned = re.sub(r'<!--CHARTDATASTART-->.*?<!--CHARTDATAEND-->', '', cleaned, flags=re.DOTALL)
    
    # Remove any remaining HTML-like tags
    cleaned = re.sub(r'<[^>]+>', '', cleaned)
    
    # Clean up extra whitespace and formatting
    cleaned = re.sub(r'\n\s*\n\s*\n+', '\n\n', cleaned)  # Remove excessive line breaks
    cleaned = re.sub(r'^\s+', '', cleaned, flags=re.MULTILINE)  # Remove leading whitespace
    cleaned = re.sub(r'\s+$', '', cleaned, flags=re.MULTILINE)  # Remove trailing whitespace
    cleaned = re.sub(r' +', ' ', cleaned)  # Replace multiple spaces with single space
    
    # Final cleanup
    cleaned = cleaned.strip()
    
    return cleaned

def clean_text_for_pdf(text: str) -> str:
    """Clean text specifically for PDF generation, handling Unicode issues"""
    if not text:
        return ""
    
    # First clean markdown
    cleaned = clean_markdown_text(text)
    
    # Replace problematic Unicode characters with ASCII equivalents
    unicode_replacements = {
        '\u2019': "'",  # Right single quotation mark
        '\u2018': "'",  # Left single quotation mark
        '\u201C': '"',  # Left double quotation mark
        '\u201D': '"',  # Right double quotation mark
        '\u2013': '-',  # En dash
        '\u2014': '--', # Em dash
        '\u2022': '•',  # Bullet
        '\u2026': '...', # Horizontal ellipsis
        '\u00A0': ' ',  # Non-breaking space
        '\u00B0': '°',  # Degree sign
        '\u00AE': '(R)', # Registered trademark
        '\u2122': '(TM)', # Trademark
        '\u00A9': '(C)', # Copyright
    }
    
    for unicode_char, replacement in unicode_replacements.items():
        cleaned = cleaned.replace(unicode_char, replacement)

    # Additional space normalization after Unicode replacements to prevent double spaces
    cleaned = re.sub(r' +', ' ', cleaned)  # Replace multiple spaces with single space
    
    return cleaned

def process_content_with_inline_charts(doc, content: str, business_idea: str):
    """Process content and render charts inline where they appear in the text"""
    try:
        from docx.shared import Inches
    except ImportError:
        raise Exception("python-docx is not installed. Please install it with: pip install python-docx")
    
    # Split content by chart markers - handle both formats
    chart_pattern = r'<!--CHART_DATA_START-->(.*?)<!--CHART_DATA_END-->'
    parts = re.split(chart_pattern, content, flags=re.DOTALL)
    
    # If no charts found, try alternative format
    if len(parts) == 1:
        chart_pattern = r'<!--CHARTDATASTART-->(.*?)<!--CHARTDATAEND-->'
        parts = re.split(chart_pattern, content, flags=re.DOTALL)
    
    for i, part in enumerate(parts):
        if i % 2 == 0:  # Regular content
            if part.strip():
                render_markdown_to_docx_grouped(doc, part.strip())
        else:  # Chart data
            try:
                chart_array = eval(part.strip())
                for chart_item in chart_array:
                    if len(chart_item) >= 5:
                        chart_type = chart_item[1]
                        chart_title = chart_item[2]
                        data = chart_item[4]
                        
                        # Add chart title
                        doc.add_heading(f"{chart_title}", level=2)
                        
                        # Create and add chart
                        chart_info = {
                            "type": chart_type,
                            "title": chart_title,
                            "data": data
                        }
                        
                        chart_bytes = create_chart_from_data(chart_info, business_idea)
                        
                        if chart_bytes:
                            with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp_img:
                                tmp_img.write(chart_bytes)
                                temp_img_path = tmp_img.name
                            
                            # Add image to Word document
                            doc.add_picture(temp_img_path, width=Inches(6.0))  # 6 inches width
                            
                            # Clean up temp file
                            os.remove(temp_img_path)
            except Exception as e:
                # If chart processing fails, just render the raw content
                if part.strip():
                    render_markdown_to_docx_grouped(doc, part.strip())


def render_markdown_to_pdf(pdf, content: str, primary_color, accent_color, text_color):
    """Render markdown content to PDF with proper heading styling and page break logic"""
    if not content:
        return
    
    lines = content.split('\n')
    current_y = pdf.get_y()
    
    for line in lines:
        line = line.strip()
        if not line:
            # Check if we need a page break for empty line
            if current_y + 10 > 280:  # 280mm is roughly where we want to break
                pdf.add_page()
                current_y = 25
            else:
                pdf.ln(3)  # Consistent space for empty lines
                current_y = pdf.get_y()
            continue
        
        # Calculate estimated height needed for this line
        estimated_height = 12  # Base height for text
        
        # Handle different heading levels
        if line.startswith("# "):    # H1 - Main heading (like "Company Profile")
            estimated_height = 25  # Heading + line + spacing
            if current_y + estimated_height > 280:
                pdf.add_page()
                current_y = 25
            
            text = line[2:].strip()
            pdf.set_font("Arial", "B", 20)  # Bold, size 20
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 10, text)
            
            # Calculate text width and make line autofit
            text_width = pdf.get_string_width(text)
            line_width = min(text_width + 12, 160)  # Add 12mm padding, max 160mm
            
            # Add subtle line under main heading - consistent with section titles
            pdf.set_draw_color(221, 221, 221)  # Lighter gray line (#DDDDDD)
            pdf.set_line_width(0.3)  # Thin line
            pdf.line(25, current_y + 10, 25 + line_width, current_y + 10)
            
            current_y = pdf.get_y()
            pdf.ln(8)  # Consistent space after main heading
            current_y = pdf.get_y()
            
        elif line.startswith("## ") or re.match(r'^\d+\.\d+\s+', line):   # H2 - Subheading or numbered subheading
            estimated_height = 20  # Subheading + line + spacing
            if current_y + estimated_height > 280:
                pdf.add_page()
                current_y = 25
            
            text = line[3:].strip() if line.startswith("## ") else line.strip()
            pdf.set_font("Arial", "B", 16)  # Bold, size 16
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 8, text)
            
            # Calculate text width and make line autofit
            text_width = pdf.get_string_width(text)
            line_width = min(text_width + 10, 160)  # Add 10mm padding, max 160mm
            
            # Add subtle line under subheading - consistent styling
            pdf.set_draw_color(221, 221, 221)  # Lighter gray line (#DDDDDD)
            pdf.set_line_width(0.3)  # Thin line
            pdf.line(25, current_y + 8, 25 + line_width, current_y + 8)
            
            current_y = pdf.get_y()
            pdf.ln(6)  # Consistent space after subheading
            current_y = pdf.get_y()
            
        elif re.match(r'^\d+\.\d+\s+[A-Z]', line):  # Numbered subheadings like "3.6 SWOT Analysis"
            estimated_height = 20  # Subheading + line + spacing
            if current_y + estimated_height > 280:
                pdf.add_page()
                current_y = 25
            
            text = line.strip()
            pdf.set_font("Arial", "B", 16)  # Bold, size 16
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 8, text)

            
            current_y = pdf.get_y()
            pdf.ln(6)  # Consistent space after subheading
            current_y = pdf.get_y()
            
        elif line.startswith("### ") or line.startswith("#### "):  # H3/H4 - Smaller subheadings
            estimated_height = 18  # Subheading + spacing
            if current_y + estimated_height > 280:
                pdf.add_page()
                current_y = 25
            
            text = line[4:].strip() if line.startswith("### ") else line[5:].strip()
            pdf.set_font("Arial", "B", 14)  # Bold, size 14
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 8, text)
            
            current_y = pdf.get_y()
            pdf.ln(4)  # Consistent space after subheading
            current_y = pdf.get_y()
            
        elif line.startswith("**") and line.endswith("**"):  # Bold text that might be a heading
            text = line.strip()  # Keep ** markers
            # Check if this looks like a main heading (no numbers, not too long)
            if not re.match(r'^\d+\.', text) and len(text) < 50:
                estimated_height = 22  # Heading + line + spacing
                if current_y + estimated_height > 280:
                    pdf.add_page()
                    current_y = 25
                
                pdf.set_font("Arial", "B", 18)  # Bold, size 18
                pdf.set_text_color(*primary_color)  # Black
                pdf.set_xy(25, current_y)
                pdf.multi_cell(160, 9, text)
                
                current_y = pdf.get_y()
                pdf.ln(6)  # Consistent space after heading
                current_y = pdf.get_y()
            else:
                # Regular bold text
                estimated_height = 12
                if current_y + estimated_height > 280:
                    pdf.add_page()
                    current_y = 25
                
                pdf.set_font("Arial", "B", 12)  # Bold, size 12
                pdf.set_text_color(*text_color)  # Dark gray
                pdf.set_xy(25, current_y)
                pdf.multi_cell(160, 8, text)
                current_y = pdf.get_y()
            
        else:  # Normal paragraph text
            # For long paragraphs, estimate height based on text length
            estimated_height = max(12, len(line) // 80 * 12)  # Rough estimate
            if current_y + estimated_height > 280:
                pdf.add_page()
                current_y = 25
            
            pdf.set_font("Arial", "", 12)  # Regular font, size 12
            pdf.set_text_color(*text_color)  # Dark gray
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 8, line, align='L')  # Left align to prevent text justification
            current_y = pdf.get_y()


def render_markdown_to_pdf_grouped(pdf, content: str, primary_color, accent_color, text_color):
    """Render markdown content to PDF with subheading-content grouping and smart page breaks"""
    if not content:
        return
    
    # print(f"DEBUG: render_markdown_to_pdf_grouped called with content length: {len(content)}")
    # print(f"DEBUG: FULL CONTENT:")
    # print("=" * 80)
    # print(content)
    # print("=" * 80)
    
    # Parse content into groups
    groups = parse_content_into_groups(content)
    
    for group in groups:
        subheading = group['subheading']
        group_content = group['content']
        subheading_type = group['subheading_type']
        
        # Get current Y position from PDF
        current_y = pdf.get_y()
        page_height = 280  # Approximate usable page height
        
        # Use smart page break logic
        if should_break_page_for_group(pdf, group, current_y, page_height):
            pdf.add_page()
            current_y = 25
            pdf.set_y(current_y)  # Set the PDF's Y position
        
        # Render subheading based on type - using original styling
        # Use the pre-cleaned text from the group
        text = group['subheading_clean']
        
        if subheading_type == 'h1':
            pdf.set_font("Arial", "B", 20)  # Bold, size 20 (original size)
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 10, text)
            
            # Calculate text width and make line autofit
            text_width = pdf.get_string_width(text)
            line_width = min(text_width + 12, 160)  # Add 12mm padding, max 160mm
            
            # Add subtle line under main heading - consistent with section titles
            pdf.set_draw_color(221, 221, 221)  # Lighter gray line (#DDDDDD)
            pdf.set_line_width(0.3)  # Thin line
            pdf.line(25, current_y + 10, 25 + line_width, current_y + 10)
            
            current_y = pdf.get_y()
            pdf.ln(8)  # Consistent space after main heading
            current_y = pdf.get_y()
            
        elif subheading_type == 'h2':
            pdf.set_font("Arial", "B", 18)  # Bold, size 18 (original size)
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 9, text)
            
            # Add underline (original styling)
            text_width = pdf.get_string_width(text)
            line_width = min(text_width + 12, 160)
            pdf.set_draw_color(221, 221, 221)  # Light gray line
            pdf.set_line_width(0.3)
            pdf.line(25, current_y + 9, 25 + line_width, current_y + 9)
            
            current_y = pdf.get_y()
            pdf.ln(6)  # Space after subheading
            current_y = pdf.get_y()
            
        elif subheading_type == 'h3':
            pdf.set_font("Arial", "B", 14)  # Bold, size 14 (original size)
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 8, text)
            
            current_y = pdf.get_y()
            pdf.ln(4)  # Space after subheading
            current_y = pdf.get_y()
            
        elif subheading_type == 'h4':
            pdf.set_font("Arial", "B", 12)  # Bold, size 12 (original size)
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 7, text)
            
            current_y = pdf.get_y()
            pdf.ln(3)  # Space after subheading
            current_y = pdf.get_y()
            
        elif subheading_type == 'bold':
            line_height_heading = 9 # For size 18 heading
            line_height_text = 8 # For size 12 text
            
            if not re.match(r'^\d+\.', text) and len(text) < 80:
                # Main heading style
                pdf.set_font("Arial", "B", 12) # Bold, size 12
                pdf.set_text_color(*primary_color) # Black
                pdf.set_xy(25, current_y)
                pdf.multi_cell(160, 8, text) # Multi-cell for text
                current_y = pdf.get_y() # Get updated Y position
                pdf.ln(6)
                current_y = pdf.get_y()
            else:
                # Regular bold text
                pdf.set_font("Arial", "B", 12) # Bold, size 12
                pdf.set_text_color(*text_color) # Dark gray
                pdf.set_xy(25, current_y)
                pdf.multi_cell(160, 8, text) # Multi-cell for text
                current_y = pdf.get_y() # Get updated Y position
            
        elif subheading_type == 'numbered':
            pdf.set_font("Arial", "B", 18)  # Bold, size 18 (original size)
            pdf.set_text_color(*primary_color)  # Black
            pdf.set_xy(25, current_y)
            pdf.multi_cell(160, 9, text)
            
            # Add underline (original styling)
            text_width = pdf.get_string_width(text)
            line_width = min(text_width + 12, 160)
            pdf.set_draw_color(221, 221, 221)  # Light gray line
            pdf.set_line_width(0.3)
            pdf.line(25, current_y + 9, 25 + line_width, current_y + 9)
            
            current_y = pdf.get_y()
            pdf.ln(6)  # Space after subheading
            current_y = pdf.get_y()
        
        # Render grouped content with simple styling
        if group_content:
            content_lines = group_content.split('\n')
            for line in content_lines:
                if line.strip():
                    # Check if line contains text ending with ":" that should be bold
                    if ':' in line:
                        # Split the line by ":" to separate the label from the content
                        parts = line.split(':', 1)  # Split only on first ":"
                        if len(parts) == 2:
                            label = parts[0].strip()
                            content = parts[1].strip()
                            
                            # Render label in bold
                            pdf.set_font("Arial", "B", 12)  # Bold
                            pdf.set_text_color(*text_color)
                            pdf.set_xy(25, current_y)
                            pdf.multi_cell(160, 8, label + ":")
                            current_y = pdf.get_y()
                            
                            # Render content in regular font
                            if content:
                                pdf.set_font("Arial", "", 12)  # Regular
                                pdf.set_text_color(*text_color)
                                pdf.set_xy(25, current_y)
                                pdf.multi_cell(160, 8, content, align='L')  # Left align to prevent text justification
                                current_y = pdf.get_y()
                        else:
                            # No content after ":", just render the whole line
                            pdf.set_font("Arial", "B", 12)  # Bold for labels ending with ":"
                            pdf.set_text_color(*text_color)
                            pdf.set_xy(25, current_y)
                            pdf.multi_cell(160, 8, line, align='L')  # Left align to prevent text justification
                            current_y = pdf.get_y()
                    elif ' - ' in line:
                        # Split the line by " - " to separate the label from the content
                        parts = line.split(' - ', 1)  # Split only on first " - "
                        if len(parts) == 2:
                            label = parts[0].strip()
                            content = parts[1].strip()
                            
                            # Render label in bold
                            pdf.set_font("Arial", "B", 12)  # Bold
                            pdf.set_text_color(*text_color)
                            pdf.set_xy(25, current_y)
                            pdf.multi_cell(160, 8, label + " -")
                            current_y = pdf.get_y()
                            
                            # Render content in regular font
                            if content:
                                pdf.set_font("Arial", "", 12)  # Regular
                                pdf.set_text_color(*text_color)
                                pdf.set_xy(25, current_y)
                                pdf.multi_cell(160, 8, content, align='L')  # Left align to prevent text justification
                                current_y = pdf.get_y()
                        else:
                            # No content after " - ", just render the whole line
                            pdf.set_font("Arial", "B", 12)  # Bold for labels ending with " -"
                            pdf.set_text_color(*text_color)
                            pdf.set_xy(25, current_y)
                            pdf.multi_cell(160, 8, line, align='L')  # Left align to prevent text justification
                            current_y = pdf.get_y()
                    else:
                        # Regular paragraph rendering without mixed text processing
                        pdf.set_font("Arial", "", 12)
                        pdf.set_text_color(*text_color)
                        pdf.set_xy(25, current_y)
                        pdf.multi_cell(160, 8, line, align='L')  # Left align to prevent text justification
                        current_y = pdf.get_y()  # Update current_y after each line
                else:
                    # Empty line spacing
                    pdf.ln(3)
                    current_y = pdf.get_y()  # Update current_y after spacing


def render_markdown_to_docx(doc, content: str):
    """Render markdown content to Word document with proper heading styling"""
    if not content:
        return
    
    lines = content.split('\n')
    
    for line in lines:
        line = line.strip()
        if not line:
            doc.add_paragraph()  # Empty paragraph for spacing
            continue
        
        # Handle different heading levels
        if line.startswith("# "):    # H1 - Main heading (like "Company Profile")
            text = line[2:].strip()
            doc.add_heading(text, level=1)
        elif line.startswith("## ") or re.match(r'^\d+\.\d+\s+', line):   # H2 - Subheading or numbered subheading
            text = line[3:].strip() if line.startswith("## ") else line.strip()
            doc.add_heading(text, level=2)
        elif re.match(r'^\d+\.\d+\s+[A-Z]', line):  # Numbered subheadings like "3.6 SWOT Analysis"
            text = line.strip()
            doc.add_heading(text, level=2)
        elif line.startswith("### ") or line.startswith("#### "):  # H3/H4 - Smaller subheadings
            text = line[4:].strip() if line.startswith("### ") else line[5:].strip()
            doc.add_heading(text, level=3)
        elif line.startswith("**") and line.endswith("**"):  # Bold text that might be a heading
            text = line.strip()  # Keep ** markers
            # Check if this looks like a main heading (no numbers, not too long)
            if not re.match(r'^\d+\.', text) and len(text) < 50:
                doc.add_heading(text, level=1)  # Treat as main heading
            else:
                # Regular bold text
                doc.add_paragraph(text)
        else:  # Normal paragraph text
            # Check if line contains text ending with ":" that should be bold
            if ':' in line:
                # Split the line by ":" to separate the label from the content
                parts = line.split(':', 1)  # Split only on first ":"
                if len(parts) == 2:
                    label = parts[0].strip()
                    content = parts[1].strip()
                    
                    # Create paragraph with mixed formatting
                    paragraph = doc.add_paragraph()
                    # Add label in bold
                    run = paragraph.add_run(label + ":")
                    run.bold = True
                    # Add content in regular font
                    if content:
                        run = paragraph.add_run(" " + content)
                        run.bold = False
                else:
                    # No content after ":", just render the whole line in bold
                    paragraph = doc.add_paragraph(line)
                    paragraph.runs[0].bold = True
            elif ' - ' in line:
                # Split the line by " - " to separate the label from the content
                parts = line.split(' - ', 1)  # Split only on first " - "
                if len(parts) == 2:
                    label = parts[0].strip()
                    content = parts[1].strip()
                    
                    # Create paragraph with mixed formatting
                    paragraph = doc.add_paragraph()
                    # Add label in bold
                    run = paragraph.add_run(label + " -")
                    run.bold = True
                    # Add content in regular font
                    if content:
                        run = paragraph.add_run(" " + content)
                        run.bold = False
                else:
                    # No content after " - ", just render the whole line in bold
                    paragraph = doc.add_paragraph(line)
                    paragraph.runs[0].bold = True
            else:
                doc.add_paragraph(line)


def render_markdown_to_docx_grouped(doc, content: str):
    """Render markdown content to Word document with subheading-content grouping and smart page breaks"""
    if not content:
        return
    
    # Parse content into groups
    groups = parse_content_into_groups(content)
    
    for i, group in enumerate(groups):
        subheading = group['subheading']
        group_content = group['content']
        subheading_type = group['subheading_type']
        
        # Add page break before group if it's not the first group and the previous group was large
        if i > 0:
            # Check if we should add a page break to keep groups together
            # This is a simple heuristic - in Word, we rely more on the natural flow
            # but we can add manual page breaks for very large groups
            if group['content_length'] > 1000:  # Large group threshold
                doc.add_page_break()
        
        # Render subheading based on type - using original styling
        # Use the pre-cleaned text from the group
        text = group['subheading_clean']
        
        if subheading_type == 'h1':
            doc.add_heading(text, level=1)
        elif subheading_type == 'h2':
            doc.add_heading(text, level=2)
        elif subheading_type == 'h3':
            doc.add_heading(text, level=3)
        elif subheading_type == 'h4':
            doc.add_heading(text, level=4)
        elif subheading_type == 'bold':
            if not re.match(r'^\d+\.', text) and len(text) < 80:
                # Simple paragraph with bold text
                paragraph = doc.add_paragraph(text)
                paragraph.style = doc.styles['Heading 1'] # Apply heading style
            else:
                # Regular paragraph
                doc.add_paragraph(text)
        elif subheading_type == 'numbered':
            doc.add_heading(text, level=2)
        
        # Render grouped content with simple styling
        if group_content:
            content_lines = group_content.split('\n')
            for line in content_lines:
                if line.strip():
                    # Check if line contains text ending with ":" that should be bold
                    if ':' in line:
                        # Split the line by ":" to separate the label from the content
                        parts = line.split(':', 1)  # Split only on first ":"
                        if len(parts) == 2:
                            label = parts[0].strip()
                            content = parts[1].strip()
                            
                            # Create paragraph with mixed formatting
                            paragraph = doc.add_paragraph()
                            # Add label in bold
                            run = paragraph.add_run(label + ":")
                            run.bold = True
                            # Add content in regular font
                            if content:
                                run = paragraph.add_run(" " + content)
                                run.bold = False
                        else:
                            # No content after ":", just render the whole line in bold
                            paragraph = doc.add_paragraph(line)
                            paragraph.runs[0].bold = True
                    elif ' - ' in line:
                        # Split the line by " - " to separate the label from the content
                        parts = line.split(' - ', 1)  # Split only on first " - "
                        if len(parts) == 2:
                            label = parts[0].strip()
                            content = parts[1].strip()
                            
                            # Create paragraph with mixed formatting
                            paragraph = doc.add_paragraph()
                            # Add label in bold
                            run = paragraph.add_run(label + " -")
                            run.bold = True
                            # Add content in regular font
                            if content:
                                run = paragraph.add_run(" " + content)
                                run.bold = False
                        else:
                            # No content after " - ", just render the whole line in bold
                            paragraph = doc.add_paragraph(line)
                            paragraph.runs[0].bold = True
                    else:
                        # Simple paragraph addition without mixed text processing
                        doc.add_paragraph(line)
                else:
                    doc.add_paragraph()  # Empty paragraph for spacing



def clean_text_for_docx(text: str) -> str:
    # Replace problematic Unicode characters with ASCII equivalents
    unicode_replacements = {
        '\u2019': "'",  # Right single quotation mark
        '\u2018': "'",  # Left single quotation mark
        '\u201C': '"',  # Left double quotation mark
        '\u201D': '"',  # Right double quotation mark
        '\u2013': '-',  # En dash
        '\u2014': '--', # Em dash
        '\u2022': '•',  # Bullet
        '\u2026': '...', # Horizontal ellipsis
        '\u00A0': ' ',  # Non-breaking space
        '\u00B0': '°',  # Degree sign
        '\u00AE': '(R)', # Registered trademark
        '\u2122': '(TM)', # Trademark
        '\u00A9': '(C)', # Copyright
    }
    
    for unicode_char, replacement in unicode_replacements.items():
        text = text.replace(unicode_char, replacement)
    
    # Simple text cleaning - keep basic punctuation and common symbols
    text = ''.join(char for char in text if ord(char) < 128 or char in '•°$€£¥₹₩₽₪₺₴₼₸₾֏₲₡₣₦₵₨₱₫₭៛৳؋﷼%')
    
    return text









def clean_unicode_for_pdf(text: str) -> str:
    # Replace problematic Unicode characters with ASCII equivalents
    unicode_replacements = {
        '\u2019': "'",  # Right single quotation mark
        '\u2018': "'",  # Left single quotation mark
        '\u201C': '"',  # Left double quotation mark
        '\u201D': '"',  # Right double quotation mark
        '\u2013': '-',  # En dash
        '\u2014': '--', # Em dash
        '\u2022': '•',  # Bullet
        '\u2026': '...', # Horizontal ellipsis
        '\u00A0': ' ',  # Non-breaking space
        '\u00B0': '°',  # Degree sign
        '\u00AE': '(R)', # Registered trademark
        '\u2122': '(TM)', # Trademark
        '\u00A9': '(C)', # Copyright
    }
    
    for unicode_char, replacement in unicode_replacements.items():
        text = text.replace(unicode_char, replacement)
    
    # Simple text cleaning - keep basic punctuation and common symbols
    text = ''.join(char for char in text if ord(char) < 128 or char in '•°$€£¥₹₩₽₪₺₴₼₸₾֏₲₡₣₦₵₨₱₫₭៛৳؋﷼%')
    
    return text