# Markdown Renderer Module # Contains all markdown rendering functions for both PDF and Word import re import base64 import os import sys import tempfile from typing import List, Dict, Any, Optional # Import from the same directory (Document Generation) from grouped import parse_content_into_groups, should_break_page_for_group from chart_generator import create_chart_from_data def clean_markdown_text(text: str) -> str: """Clean markdown formatting for PDF generation - PRESERVE HEADINGS""" if not text: return "" # Remove markdown formatting BUT PRESERVE HEADING STRUCTURE cleaned = text # PRESERVE HEADINGS - don't strip the # markers, keep them for rendering # We'll handle styling at render time instead of stripping them # Remove bold, italic formatting - strip ** or * symbols cleaned = re.sub(r'\*\*(.*?)\*\*', r'\1', cleaned) # Remove bold cleaned = re.sub(r'\*(.*?)\*', r'\1', cleaned) # Remove italic cleaned = re.sub(r'`(.*?)`', r'\1', cleaned) # Remove code cleaned = re.sub(r'~~(.*?)~~', r'\1', cleaned) # Remove strikethrough # Remove links (keep text) cleaned = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', cleaned) # Remove links, keep text cleaned = re.sub(r'!\[([^\]]*)\]\([^)]+\)', '', cleaned) # Remove images # Remove list markers (more aggressive) cleaned = re.sub(r'^\s*[-*+]\s*', '', cleaned, flags=re.MULTILINE) # Remove list markers cleaned = re.sub(r'^\s*\d+\.\s*', '', cleaned, flags=re.MULTILINE) # Remove numbered list markers cleaned = re.sub(r'^\s*[-*+]\s*', '', cleaned, flags=re.MULTILINE) # Remove any remaining list markers # Remove blockquotes cleaned = re.sub(r'^\s*>\s*', '', cleaned, flags=re.MULTILINE) # Remove blockquotes # Remove horizontal rules cleaned = re.sub(r'^\s*[-*_]{3,}\s*$', '', cleaned, flags=re.MULTILINE) # Remove horizontal rules # Remove code blocks cleaned = re.sub(r'```[\s\S]*?```', '', cleaned) # Remove code blocks cleaned = re.sub(r'`.*?`', '', cleaned) # Remove any remaining inline code # Remove emphasis markers - strip ** or * symbols cleaned = re.sub(r'_{1,2}(.*?)_{1,2}', r'\1', cleaned) # Remove underscores cleaned = re.sub(r'\*{1,2}(.*?)\*{1,2}', r'\1', cleaned) # Remove asterisks # PRESERVE CHART DATA MARKERS for inline processing # DO NOT REMOVE chart markers - they will be processed inline # cleaned = re.sub(r'.*?', '', cleaned, flags=re.DOTALL) # cleaned = re.sub(r'.*$', '', cleaned, flags=re.DOTALL) # cleaned = re.sub(r'', '', cleaned) # REMOVE CHART DATA MARKERS for PDF rendering since they're not being processed cleaned = re.sub(r'.*?', '', cleaned, flags=re.DOTALL) cleaned = re.sub(r'.*?', '', cleaned, flags=re.DOTALL) # Remove any remaining HTML-like tags cleaned = re.sub(r'<[^>]+>', '', cleaned) # Clean up extra whitespace and formatting cleaned = re.sub(r'\n\s*\n\s*\n+', '\n\n', cleaned) # Remove excessive line breaks cleaned = re.sub(r'^\s+', '', cleaned, flags=re.MULTILINE) # Remove leading whitespace cleaned = re.sub(r'\s+$', '', cleaned, flags=re.MULTILINE) # Remove trailing whitespace cleaned = re.sub(r' +', ' ', cleaned) # Replace multiple spaces with single space # Final cleanup cleaned = cleaned.strip() return cleaned def clean_text_for_pdf(text: str) -> str: """Clean text specifically for PDF generation, handling Unicode issues""" if not text: return "" # First clean markdown cleaned = clean_markdown_text(text) # Replace problematic Unicode characters with ASCII equivalents unicode_replacements = { '\u2019': "'", # Right single quotation mark '\u2018': "'", # Left single quotation mark '\u201C': '"', # Left double quotation mark '\u201D': '"', # Right double quotation mark '\u2013': '-', # En dash '\u2014': '--', # Em dash '\u2022': '•', # Bullet '\u2026': '...', # Horizontal ellipsis '\u00A0': ' ', # Non-breaking space '\u00B0': '°', # Degree sign '\u00AE': '(R)', # Registered trademark '\u2122': '(TM)', # Trademark '\u00A9': '(C)', # Copyright } for unicode_char, replacement in unicode_replacements.items(): cleaned = cleaned.replace(unicode_char, replacement) # Additional space normalization after Unicode replacements to prevent double spaces cleaned = re.sub(r' +', ' ', cleaned) # Replace multiple spaces with single space return cleaned def process_content_with_inline_charts(doc, content: str, business_idea: str): """Process content and render charts inline where they appear in the text""" try: from docx.shared import Inches except ImportError: raise Exception("python-docx is not installed. Please install it with: pip install python-docx") # Split content by chart markers - handle both formats chart_pattern = r'(.*?)' parts = re.split(chart_pattern, content, flags=re.DOTALL) # If no charts found, try alternative format if len(parts) == 1: chart_pattern = r'(.*?)' parts = re.split(chart_pattern, content, flags=re.DOTALL) for i, part in enumerate(parts): if i % 2 == 0: # Regular content if part.strip(): render_markdown_to_docx_grouped(doc, part.strip()) else: # Chart data try: chart_array = eval(part.strip()) for chart_item in chart_array: if len(chart_item) >= 5: chart_type = chart_item[1] chart_title = chart_item[2] data = chart_item[4] # Add chart title doc.add_heading(f"{chart_title}", level=2) # Create and add chart chart_info = { "type": chart_type, "title": chart_title, "data": data } chart_bytes = create_chart_from_data(chart_info, business_idea) if chart_bytes: with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp_img: tmp_img.write(chart_bytes) temp_img_path = tmp_img.name # Add image to Word document doc.add_picture(temp_img_path, width=Inches(6.0)) # 6 inches width # Clean up temp file os.remove(temp_img_path) except Exception as e: # If chart processing fails, just render the raw content if part.strip(): render_markdown_to_docx_grouped(doc, part.strip()) def render_markdown_to_pdf(pdf, content: str, primary_color, accent_color, text_color): """Render markdown content to PDF with proper heading styling and page break logic""" if not content: return lines = content.split('\n') current_y = pdf.get_y() for line in lines: line = line.strip() if not line: # Check if we need a page break for empty line if current_y + 10 > 280: # 280mm is roughly where we want to break pdf.add_page() current_y = 25 else: pdf.ln(3) # Consistent space for empty lines current_y = pdf.get_y() continue # Calculate estimated height needed for this line estimated_height = 12 # Base height for text # Handle different heading levels if line.startswith("# "): # H1 - Main heading (like "Company Profile") estimated_height = 25 # Heading + line + spacing if current_y + estimated_height > 280: pdf.add_page() current_y = 25 text = line[2:].strip() pdf.set_font("Arial", "B", 20) # Bold, size 20 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 10, text) # Calculate text width and make line autofit text_width = pdf.get_string_width(text) line_width = min(text_width + 12, 160) # Add 12mm padding, max 160mm # Add subtle line under main heading - consistent with section titles pdf.set_draw_color(221, 221, 221) # Lighter gray line (#DDDDDD) pdf.set_line_width(0.3) # Thin line pdf.line(25, current_y + 10, 25 + line_width, current_y + 10) current_y = pdf.get_y() pdf.ln(8) # Consistent space after main heading current_y = pdf.get_y() elif line.startswith("## ") or re.match(r'^\d+\.\d+\s+', line): # H2 - Subheading or numbered subheading estimated_height = 20 # Subheading + line + spacing if current_y + estimated_height > 280: pdf.add_page() current_y = 25 text = line[3:].strip() if line.startswith("## ") else line.strip() pdf.set_font("Arial", "B", 16) # Bold, size 16 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) # Calculate text width and make line autofit text_width = pdf.get_string_width(text) line_width = min(text_width + 10, 160) # Add 10mm padding, max 160mm # Add subtle line under subheading - consistent styling pdf.set_draw_color(221, 221, 221) # Lighter gray line (#DDDDDD) pdf.set_line_width(0.3) # Thin line pdf.line(25, current_y + 8, 25 + line_width, current_y + 8) current_y = pdf.get_y() pdf.ln(6) # Consistent space after subheading current_y = pdf.get_y() elif re.match(r'^\d+\.\d+\s+[A-Z]', line): # Numbered subheadings like "3.6 SWOT Analysis" estimated_height = 20 # Subheading + line + spacing if current_y + estimated_height > 280: pdf.add_page() current_y = 25 text = line.strip() pdf.set_font("Arial", "B", 16) # Bold, size 16 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) current_y = pdf.get_y() pdf.ln(6) # Consistent space after subheading current_y = pdf.get_y() elif line.startswith("### ") or line.startswith("#### "): # H3/H4 - Smaller subheadings estimated_height = 18 # Subheading + spacing if current_y + estimated_height > 280: pdf.add_page() current_y = 25 text = line[4:].strip() if line.startswith("### ") else line[5:].strip() pdf.set_font("Arial", "B", 14) # Bold, size 14 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) current_y = pdf.get_y() pdf.ln(4) # Consistent space after subheading current_y = pdf.get_y() elif line.startswith("**") and line.endswith("**"): # Bold text that might be a heading text = line.strip() # Keep ** markers # Check if this looks like a main heading (no numbers, not too long) if not re.match(r'^\d+\.', text) and len(text) < 50: estimated_height = 22 # Heading + line + spacing if current_y + estimated_height > 280: pdf.add_page() current_y = 25 pdf.set_font("Arial", "B", 18) # Bold, size 18 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 9, text) current_y = pdf.get_y() pdf.ln(6) # Consistent space after heading current_y = pdf.get_y() else: # Regular bold text estimated_height = 12 if current_y + estimated_height > 280: pdf.add_page() current_y = 25 pdf.set_font("Arial", "B", 12) # Bold, size 12 pdf.set_text_color(*text_color) # Dark gray pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) current_y = pdf.get_y() else: # Normal paragraph text # For long paragraphs, estimate height based on text length estimated_height = max(12, len(line) // 80 * 12) # Rough estimate if current_y + estimated_height > 280: pdf.add_page() current_y = 25 pdf.set_font("Arial", "", 12) # Regular font, size 12 pdf.set_text_color(*text_color) # Dark gray pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, line, align='L') # Left align to prevent text justification current_y = pdf.get_y() def render_markdown_to_pdf_grouped(pdf, content: str, primary_color, accent_color, text_color): """Render markdown content to PDF with subheading-content grouping and smart page breaks""" if not content: return # print(f"DEBUG: render_markdown_to_pdf_grouped called with content length: {len(content)}") # print(f"DEBUG: FULL CONTENT:") # print("=" * 80) # print(content) # print("=" * 80) # Parse content into groups groups = parse_content_into_groups(content) for group in groups: subheading = group['subheading'] group_content = group['content'] subheading_type = group['subheading_type'] # Get current Y position from PDF current_y = pdf.get_y() page_height = 280 # Approximate usable page height # Use smart page break logic if should_break_page_for_group(pdf, group, current_y, page_height): pdf.add_page() current_y = 25 pdf.set_y(current_y) # Set the PDF's Y position # Render subheading based on type - using original styling # Use the pre-cleaned text from the group text = group['subheading_clean'] if subheading_type == 'h1': pdf.set_font("Arial", "B", 20) # Bold, size 20 (original size) pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 10, text) # Calculate text width and make line autofit text_width = pdf.get_string_width(text) line_width = min(text_width + 12, 160) # Add 12mm padding, max 160mm # Add subtle line under main heading - consistent with section titles pdf.set_draw_color(221, 221, 221) # Lighter gray line (#DDDDDD) pdf.set_line_width(0.3) # Thin line pdf.line(25, current_y + 10, 25 + line_width, current_y + 10) current_y = pdf.get_y() pdf.ln(8) # Consistent space after main heading current_y = pdf.get_y() elif subheading_type == 'h2': pdf.set_font("Arial", "B", 18) # Bold, size 18 (original size) pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 9, text) # Add underline (original styling) text_width = pdf.get_string_width(text) line_width = min(text_width + 12, 160) pdf.set_draw_color(221, 221, 221) # Light gray line pdf.set_line_width(0.3) pdf.line(25, current_y + 9, 25 + line_width, current_y + 9) current_y = pdf.get_y() pdf.ln(6) # Space after subheading current_y = pdf.get_y() elif subheading_type == 'h3': pdf.set_font("Arial", "B", 14) # Bold, size 14 (original size) pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) current_y = pdf.get_y() pdf.ln(4) # Space after subheading current_y = pdf.get_y() elif subheading_type == 'h4': pdf.set_font("Arial", "B", 12) # Bold, size 12 (original size) pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 7, text) current_y = pdf.get_y() pdf.ln(3) # Space after subheading current_y = pdf.get_y() elif subheading_type == 'bold': line_height_heading = 9 # For size 18 heading line_height_text = 8 # For size 12 text if not re.match(r'^\d+\.', text) and len(text) < 80: # Main heading style pdf.set_font("Arial", "B", 12) # Bold, size 12 pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) # Multi-cell for text current_y = pdf.get_y() # Get updated Y position pdf.ln(6) current_y = pdf.get_y() else: # Regular bold text pdf.set_font("Arial", "B", 12) # Bold, size 12 pdf.set_text_color(*text_color) # Dark gray pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, text) # Multi-cell for text current_y = pdf.get_y() # Get updated Y position elif subheading_type == 'numbered': pdf.set_font("Arial", "B", 18) # Bold, size 18 (original size) pdf.set_text_color(*primary_color) # Black pdf.set_xy(25, current_y) pdf.multi_cell(160, 9, text) # Add underline (original styling) text_width = pdf.get_string_width(text) line_width = min(text_width + 12, 160) pdf.set_draw_color(221, 221, 221) # Light gray line pdf.set_line_width(0.3) pdf.line(25, current_y + 9, 25 + line_width, current_y + 9) current_y = pdf.get_y() pdf.ln(6) # Space after subheading current_y = pdf.get_y() # Render grouped content with simple styling if group_content: content_lines = group_content.split('\n') for line in content_lines: if line.strip(): # Check if line contains text ending with ":" that should be bold if ':' in line: # Split the line by ":" to separate the label from the content parts = line.split(':', 1) # Split only on first ":" if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Render label in bold pdf.set_font("Arial", "B", 12) # Bold pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, label + ":") current_y = pdf.get_y() # Render content in regular font if content: pdf.set_font("Arial", "", 12) # Regular pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, content, align='L') # Left align to prevent text justification current_y = pdf.get_y() else: # No content after ":", just render the whole line pdf.set_font("Arial", "B", 12) # Bold for labels ending with ":" pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, line, align='L') # Left align to prevent text justification current_y = pdf.get_y() elif ' - ' in line: # Split the line by " - " to separate the label from the content parts = line.split(' - ', 1) # Split only on first " - " if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Render label in bold pdf.set_font("Arial", "B", 12) # Bold pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, label + " -") current_y = pdf.get_y() # Render content in regular font if content: pdf.set_font("Arial", "", 12) # Regular pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, content, align='L') # Left align to prevent text justification current_y = pdf.get_y() else: # No content after " - ", just render the whole line pdf.set_font("Arial", "B", 12) # Bold for labels ending with " -" pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, line, align='L') # Left align to prevent text justification current_y = pdf.get_y() else: # Regular paragraph rendering without mixed text processing pdf.set_font("Arial", "", 12) pdf.set_text_color(*text_color) pdf.set_xy(25, current_y) pdf.multi_cell(160, 8, line, align='L') # Left align to prevent text justification current_y = pdf.get_y() # Update current_y after each line else: # Empty line spacing pdf.ln(3) current_y = pdf.get_y() # Update current_y after spacing def render_markdown_to_docx(doc, content: str): """Render markdown content to Word document with proper heading styling""" if not content: return lines = content.split('\n') for line in lines: line = line.strip() if not line: doc.add_paragraph() # Empty paragraph for spacing continue # Handle different heading levels if line.startswith("# "): # H1 - Main heading (like "Company Profile") text = line[2:].strip() doc.add_heading(text, level=1) elif line.startswith("## ") or re.match(r'^\d+\.\d+\s+', line): # H2 - Subheading or numbered subheading text = line[3:].strip() if line.startswith("## ") else line.strip() doc.add_heading(text, level=2) elif re.match(r'^\d+\.\d+\s+[A-Z]', line): # Numbered subheadings like "3.6 SWOT Analysis" text = line.strip() doc.add_heading(text, level=2) elif line.startswith("### ") or line.startswith("#### "): # H3/H4 - Smaller subheadings text = line[4:].strip() if line.startswith("### ") else line[5:].strip() doc.add_heading(text, level=3) elif line.startswith("**") and line.endswith("**"): # Bold text that might be a heading text = line.strip() # Keep ** markers # Check if this looks like a main heading (no numbers, not too long) if not re.match(r'^\d+\.', text) and len(text) < 50: doc.add_heading(text, level=1) # Treat as main heading else: # Regular bold text doc.add_paragraph(text) else: # Normal paragraph text # Check if line contains text ending with ":" that should be bold if ':' in line: # Split the line by ":" to separate the label from the content parts = line.split(':', 1) # Split only on first ":" if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Create paragraph with mixed formatting paragraph = doc.add_paragraph() # Add label in bold run = paragraph.add_run(label + ":") run.bold = True # Add content in regular font if content: run = paragraph.add_run(" " + content) run.bold = False else: # No content after ":", just render the whole line in bold paragraph = doc.add_paragraph(line) paragraph.runs[0].bold = True elif ' - ' in line: # Split the line by " - " to separate the label from the content parts = line.split(' - ', 1) # Split only on first " - " if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Create paragraph with mixed formatting paragraph = doc.add_paragraph() # Add label in bold run = paragraph.add_run(label + " -") run.bold = True # Add content in regular font if content: run = paragraph.add_run(" " + content) run.bold = False else: # No content after " - ", just render the whole line in bold paragraph = doc.add_paragraph(line) paragraph.runs[0].bold = True else: doc.add_paragraph(line) def render_markdown_to_docx_grouped(doc, content: str): """Render markdown content to Word document with subheading-content grouping and smart page breaks""" if not content: return # Parse content into groups groups = parse_content_into_groups(content) for i, group in enumerate(groups): subheading = group['subheading'] group_content = group['content'] subheading_type = group['subheading_type'] # Add page break before group if it's not the first group and the previous group was large if i > 0: # Check if we should add a page break to keep groups together # This is a simple heuristic - in Word, we rely more on the natural flow # but we can add manual page breaks for very large groups if group['content_length'] > 1000: # Large group threshold doc.add_page_break() # Render subheading based on type - using original styling # Use the pre-cleaned text from the group text = group['subheading_clean'] if subheading_type == 'h1': doc.add_heading(text, level=1) elif subheading_type == 'h2': doc.add_heading(text, level=2) elif subheading_type == 'h3': doc.add_heading(text, level=3) elif subheading_type == 'h4': doc.add_heading(text, level=4) elif subheading_type == 'bold': if not re.match(r'^\d+\.', text) and len(text) < 80: # Simple paragraph with bold text paragraph = doc.add_paragraph(text) paragraph.style = doc.styles['Heading 1'] # Apply heading style else: # Regular paragraph doc.add_paragraph(text) elif subheading_type == 'numbered': doc.add_heading(text, level=2) # Render grouped content with simple styling if group_content: content_lines = group_content.split('\n') for line in content_lines: if line.strip(): # Check if line contains text ending with ":" that should be bold if ':' in line: # Split the line by ":" to separate the label from the content parts = line.split(':', 1) # Split only on first ":" if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Create paragraph with mixed formatting paragraph = doc.add_paragraph() # Add label in bold run = paragraph.add_run(label + ":") run.bold = True # Add content in regular font if content: run = paragraph.add_run(" " + content) run.bold = False else: # No content after ":", just render the whole line in bold paragraph = doc.add_paragraph(line) paragraph.runs[0].bold = True elif ' - ' in line: # Split the line by " - " to separate the label from the content parts = line.split(' - ', 1) # Split only on first " - " if len(parts) == 2: label = parts[0].strip() content = parts[1].strip() # Create paragraph with mixed formatting paragraph = doc.add_paragraph() # Add label in bold run = paragraph.add_run(label + " -") run.bold = True # Add content in regular font if content: run = paragraph.add_run(" " + content) run.bold = False else: # No content after " - ", just render the whole line in bold paragraph = doc.add_paragraph(line) paragraph.runs[0].bold = True else: # Simple paragraph addition without mixed text processing doc.add_paragraph(line) else: doc.add_paragraph() # Empty paragraph for spacing def clean_text_for_docx(text: str) -> str: # Replace problematic Unicode characters with ASCII equivalents unicode_replacements = { '\u2019': "'", # Right single quotation mark '\u2018': "'", # Left single quotation mark '\u201C': '"', # Left double quotation mark '\u201D': '"', # Right double quotation mark '\u2013': '-', # En dash '\u2014': '--', # Em dash '\u2022': '•', # Bullet '\u2026': '...', # Horizontal ellipsis '\u00A0': ' ', # Non-breaking space '\u00B0': '°', # Degree sign '\u00AE': '(R)', # Registered trademark '\u2122': '(TM)', # Trademark '\u00A9': '(C)', # Copyright } for unicode_char, replacement in unicode_replacements.items(): text = text.replace(unicode_char, replacement) # Simple text cleaning - keep basic punctuation and common symbols text = ''.join(char for char in text if ord(char) < 128 or char in '•°$€£¥₹₩₽₪₺₴₼₸₾֏₲₡₣₦₵₨₱₫₭៛৳؋﷼%') return text def clean_unicode_for_pdf(text: str) -> str: # Replace problematic Unicode characters with ASCII equivalents unicode_replacements = { '\u2019': "'", # Right single quotation mark '\u2018': "'", # Left single quotation mark '\u201C': '"', # Left double quotation mark '\u201D': '"', # Right double quotation mark '\u2013': '-', # En dash '\u2014': '--', # Em dash '\u2022': '•', # Bullet '\u2026': '...', # Horizontal ellipsis '\u00A0': ' ', # Non-breaking space '\u00B0': '°', # Degree sign '\u00AE': '(R)', # Registered trademark '\u2122': '(TM)', # Trademark '\u00A9': '(C)', # Copyright } for unicode_char, replacement in unicode_replacements.items(): text = text.replace(unicode_char, replacement) # Simple text cleaning - keep basic punctuation and common symbols text = ''.join(char for char in text if ord(char) < 128 or char in '•°$€£¥₹₩₽₪₺₴₼₸₾֏₲₡₣₦₵₨₱₫₭៛৳؋﷼%') return text