import re from typing import List, Dict, Any def calculate_group_height(pdf, group: Dict[str, Any]) -> int: """Calculate the estimated height needed for a complete group""" subheading_type = group['subheading_type'] content_lines = group['content'].split('\n') if group['content'] else [] # Base height for subheading if subheading_type == 'h1': subheading_height = 25 # 20pt font + spacing elif subheading_type == 'h2': subheading_height = 20 # 16pt font + underline + spacing elif subheading_type == 'h3': subheading_height = 16 # 14pt font + spacing elif subheading_type == 'h4': subheading_height = 14 # 12pt font + spacing elif subheading_type == 'bold': subheading_height = 20 # 16pt font + spacing elif subheading_type == 'numbered': subheading_height = 20 # 16pt font + spacing else: subheading_height = 16 # Calculate content height more accurately content_height = 0 for line in content_lines: if line.strip(): # More accurate line height calculation words = line.split() if len(words) > 0: # Estimate characters per line (roughly 80-90 chars per line) chars_per_line = 85 estimated_lines = max(1, len(line) // chars_per_line + 1) line_height = estimated_lines * 8 # 8pt per line content_height += line_height else: content_height += 8 # Empty line else: content_height += 3 # Empty line spacing # Add spacing between groups group_spacing = 8 total_height = subheading_height + content_height + group_spacing # print(f"DEBUG: Group '{group.get('subheading_clean', 'Unknown')[:30]}...' height calculation:") # print(f" Subheading height: {subheading_height}") # print(f" Content height: {content_height}") # print(f" Group spacing: {group_spacing}") # print(f" Total height: {total_height}") return total_height def should_break_page_for_group(pdf, group: Dict[str, Any], current_y: int, page_height: int = 280) -> bool: """Determine if we should break to a new page for this group""" estimated_height = calculate_group_height(pdf, group) # print(f"DEBUG: should_break_page_for_group called") # print(f"DEBUG: Subheading: '{group.get('subheading_clean', 'Unknown')[:30]}...'") # print(f"DEBUG: Current Y: {current_y}") # print(f"DEBUG: Estimated height: {estimated_height}") # print(f"DEBUG: Page height: {page_height}") # print(f"DEBUG: Available space: {page_height - current_y}") # print(f"DEBUG: Will fit? {current_y + estimated_height <= page_height}") # CRITICAL FIX: Only break if the group absolutely won't fit # This prevents splitting subheadings from their content # If the group won't fit on current page, break if current_y + estimated_height > page_height: # print(f"DEBUG: -> BREAKING: Group won't fit ({current_y} + {estimated_height} > {page_height})") return True # If we're very close to the end of the page (less than 50mm remaining) and group is significant, break if current_y > page_height - 50 and estimated_height > 30: # print(f"DEBUG: -> BREAKING: Very close to end of page ({current_y} > {page_height - 50}) and group significant ({estimated_height} > 30)") return True # print(f"DEBUG: -> NO BREAK: Group will fit on current page") return False def clean_markdown_from_line(line: str) -> str: """Clean markdown formatting from a single line""" if not line: return "" cleaned = line # Remove bold, italic formatting cleaned = re.sub(r'\*\*(.*?)\*\*', r'\1', cleaned) # Remove bold cleaned = re.sub(r'\*(.*?)\*', r'\1', cleaned) # Remove italic cleaned = re.sub(r'`(.*?)`', r'\1', cleaned) # Remove code cleaned = re.sub(r'~~(.*?)~~', r'\1', cleaned) # Remove strikethrough # Remove emphasis markers cleaned = re.sub(r'_{1,2}(.*?)_{1,2}', r'\1', cleaned) # Remove underscores cleaned = re.sub(r'\*{1,2}(.*?)\*{1,2}', r'\1', cleaned) # Remove asterisks # Remove links (keep text) cleaned = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', cleaned) # Remove links, keep text # Remove any remaining HTML-like tags cleaned = re.sub(r'<[^>]+>', '', cleaned) # CRITICAL FIX: Replace multiple spaces with single space to prevent double spaces cleaned = re.sub(r' +', ' ', cleaned) # Replace multiple spaces with single space return cleaned.strip() def parse_content_into_groups(content: str) -> List[Dict[str, Any]]: """Parse markdown content into subheading-content groups""" if not content: return [] # print(f"DEBUG: parse_content_into_groups called with content length: {len(content)}") lines = content.split('\n') groups = [] current_group = None for i, line in enumerate(lines): line = line.strip() if not line: if current_group: current_group['content_lines'].append('') # Preserve empty lines continue # Check if this is a heading (H1, H2, H3, H4, or numbered) is_heading = ( line.startswith('# ') or # H1 (line.startswith('## ') and not line.startswith('### ')) or # H2 (line.startswith('### ') and not line.startswith('#### ')) or # H3 (line.startswith('#### ') and not line.startswith('##### ')) or # H4 re.match(r'^\d+\.\d+\s+', line) # Numbered subheadings like "3.6 SWOT Analysis" ) # Check if this is bold text is_bold_text = ( (line.startswith('**') and line.endswith('**') and len(line) > 4) or # Bold headings (re.match(r'^\*\*.*?\*\*:', line) and len(line) > 4) # Bold text before colon as heading ) # Check if this is a bold list item (e.g., "- **Item**") is_bold_list_item = re.match(r'^[-*]\s*\*\*(.*?)\*\*$', line) # Special handling for bold text following numbered subheadings should_create_new_group = False if is_heading: # Always create new group for regular headings should_create_new_group = True # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Regular heading, creating new group") elif is_bold_text: # Check if we should treat this bold text as content for a numbered subheading if groups and groups[-1].get('subheading_type') == 'h3': # Check if the previous group's subheading looks like a numbered subheading previous_subheading = groups[-1].get('subheading', '') if re.match(r'^\d+\.\d+\s+', previous_subheading): # This is content for a numbered subheading, add to previous group groups[-1]['content_lines'].append(line) # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Adding to previous numbered subheading '{previous_subheading[:30]}...'") continue # Skip creating new group else: # Not a numbered subheading, treat as new heading should_create_new_group = True # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (not numbered subheading)") else: # No previous group or not a numbered subheading, treat as new heading should_create_new_group = True # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (no numbered subheading)") elif is_bold_list_item: # If it's a bold list item, treat it as a new heading should_create_new_group = True # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Bold list item, creating new group") if should_create_new_group: # Save previous group if it exists if current_group: current_group['content'] = '\n'.join(current_group['content_lines']) groups.append(current_group) # print(f"DEBUG: Saved group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines") # Clean markdown symbols BEFORE determining type and storing clean_text = line heading_type = 'text' if line.startswith('# '): clean_text = line[2:].strip() # Remove "# " heading_type = 'h1' elif line.startswith('## '): clean_text = line[3:].strip() # Remove "## " heading_type = 'h2' elif line.startswith('### '): clean_text = line[4:].strip() # Remove "### " heading_type = 'h3' elif line.startswith('#### '): clean_text = line[5:].strip() # Remove "#### " heading_type = 'h4' elif re.match(r'^\d+\.\d+\s+', line): clean_text = line.strip() # Keep numbered headings as-is heading_type = 'numbered' elif is_bold_text: # Remove ** markers from bold text clean_text = clean_markdown_from_line(line.strip()) heading_type = 'bold' elif is_bold_list_item: # Extract the bold text from list item and treat as H3 subheading # Remove the list marker and ** markers clean_text = re.sub(r'^[-*]\s*\*\*(.*?)\*\*', r'\1', line.strip()) heading_type = 'h3' # Treat as H3 subheading # print(f"DEBUG: Creating new group '{clean_text[:30]}...' (type: {heading_type})") # Start new group with cleaned text current_group = { 'subheading': line, # Keep original for reference 'content_lines': [], 'content': '', 'subheading_clean': clean_text, # Pre-cleaned text 'subheading_type': heading_type } else: # Add to current group's content - clean the line if current_group: cleaned_line = clean_markdown_from_line(line) current_group['content_lines'].append(cleaned_line) if i < 5: # Log first few content lines # print(f"DEBUG: Added content line to group '{current_group.get('subheading_clean', 'Unknown')[:30]}...': '{cleaned_line[:50]}...'") pass else: # Content before any subheading - create a default group if not groups or groups[-1].get('subheading') != '## Introduction': cleaned_line = clean_markdown_from_line(line) current_group = { 'subheading': '## Introduction', 'content_lines': [cleaned_line], 'content': '', 'subheading_clean': 'Introduction', 'subheading_type': 'h2' } # Don't forget the last group if current_group: current_group['content'] = '\n'.join(current_group['content_lines']) groups.append(current_group) # print(f"DEBUG: Saved final group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines") # Add metadata to each group for i, group in enumerate(groups): group['group_id'] = f"group_{i}" group['content_length'] = len(group['content']) group['word_count'] = len(group['content'].split()) if group['content'] else 0 group['has_charts'] = 'CHART' in group['content'].upper() if group['content'] else False group['line_count'] = len([line for line in group['content'].split('\n') if line.strip()]) # print(f"DEBUG: Created {len(groups)} groups total") return groups def extract_key_phrases(content: str) -> List[tuple]: """Extract key phrases from content for grouping metadata""" if not content: return [] # Simple key phrase extraction words = re.findall(r'\b\w+\b', content.lower()) word_freq = {} for word in words: if len(word) > 3: # Only words longer than 3 characters word_freq[word] = word_freq.get(word, 0) + 1 # Return top 5 most frequent words return sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:5] def get_content_groups_info(content: str) -> Dict[str, Any]: """Get detailed information about content groups for analysis and debugging""" if not content: return {"groups": [], "summary": {"total_groups": 0, "total_content_length": 0}} groups = parse_content_into_groups(content) # Add key phrases to each group for group in groups: group['key_phrases'] = extract_key_phrases(group['content']) # Calculate summary statistics total_content_length = sum(group['content_length'] for group in groups) total_word_count = sum(group['word_count'] for group in groups) groups_with_charts = sum(1 for group in groups if group['has_charts']) summary = { "total_groups": len(groups), "total_content_length": total_content_length, "total_word_count": total_word_count, "groups_with_charts": groups_with_charts, "average_group_size": total_content_length / len(groups) if groups else 0, "largest_group": max(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None, "smallest_group": min(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None } return { "groups": groups, "summary": summary }