app/DocumentGeneration/grouped.py

import re
from typing import List, Dict, Any

def calculate_group_height(pdf, group: Dict[str, Any]) -> int:
    """Calculate the estimated height needed for a complete group"""
    subheading_type = group['subheading_type']
    content_lines = group['content'].split('\n') if group['content'] else []
    
    # Base height for subheading
    if subheading_type == 'h1':
        subheading_height = 25  # 20pt font + spacing
    elif subheading_type == 'h2':
        subheading_height = 20  # 16pt font + underline + spacing
    elif subheading_type == 'h3':
        subheading_height = 16  # 14pt font + spacing
    elif subheading_type == 'h4':
        subheading_height = 14  # 12pt font + spacing
    elif subheading_type == 'bold':
        subheading_height = 20  # 16pt font + spacing
    elif subheading_type == 'numbered':
        subheading_height = 20  # 16pt font + spacing
    else:
        subheading_height = 16
    
    # Calculate content height more accurately
    content_height = 0
    for line in content_lines:
        if line.strip():
            # More accurate line height calculation
            words = line.split()
            if len(words) > 0:
                # Estimate characters per line (roughly 80-90 chars per line)
                chars_per_line = 85
                estimated_lines = max(1, len(line) // chars_per_line + 1)
                line_height = estimated_lines * 8  # 8pt per line
                content_height += line_height
            else:
                content_height += 8  # Empty line
        else:
            content_height += 3  # Empty line spacing
    
    # Add spacing between groups
    group_spacing = 8
    
    total_height = subheading_height + content_height + group_spacing
    
    # print(f"DEBUG: Group '{group.get('subheading_clean', 'Unknown')[:30]}...' height calculation:")
    # print(f"  Subheading height: {subheading_height}")
    # print(f"  Content height: {content_height}")
    # print(f"  Group spacing: {group_spacing}")
    # print(f"  Total height: {total_height}")
    
    return total_height

def should_break_page_for_group(pdf, group: Dict[str, Any], current_y: int, page_height: int = 280) -> bool:
    """Determine if we should break to a new page for this group"""
    estimated_height = calculate_group_height(pdf, group)
    
    # print(f"DEBUG: should_break_page_for_group called")
    # print(f"DEBUG:   Subheading: '{group.get('subheading_clean', 'Unknown')[:30]}...'")
    # print(f"DEBUG:   Current Y: {current_y}")
    # print(f"DEBUG:   Estimated height: {estimated_height}")
    # print(f"DEBUG:   Page height: {page_height}")
    # print(f"DEBUG:   Available space: {page_height - current_y}")
    # print(f"DEBUG:   Will fit? {current_y + estimated_height <= page_height}")
    
    # CRITICAL FIX: Only break if the group absolutely won't fit
    # This prevents splitting subheadings from their content
    
    # If the group won't fit on current page, break
    if current_y + estimated_height > page_height:
        # print(f"DEBUG:   -> BREAKING: Group won't fit ({current_y} + {estimated_height} > {page_height})")
        return True
    
    # If we're very close to the end of the page (less than 50mm remaining) and group is significant, break
    if current_y > page_height - 50 and estimated_height > 30:
        # print(f"DEBUG:   -> BREAKING: Very close to end of page ({current_y} > {page_height - 50}) and group significant ({estimated_height} > 30)")
        return True
    
    # print(f"DEBUG:   -> NO BREAK: Group will fit on current page")
    return False

def clean_markdown_from_line(line: str) -> str:
    """Clean markdown formatting from a single line"""
    if not line:
        return ""
    
    cleaned = line
    
    # Remove bold, italic formatting
    cleaned = re.sub(r'\*\*(.*?)\*\*', r'\1', cleaned)  # Remove bold
    cleaned = re.sub(r'\*(.*?)\*', r'\1', cleaned)  # Remove italic
    cleaned = re.sub(r'`(.*?)`', r'\1', cleaned)  # Remove code
    cleaned = re.sub(r'~~(.*?)~~', r'\1', cleaned)  # Remove strikethrough
    
    # Remove emphasis markers
    cleaned = re.sub(r'_{1,2}(.*?)_{1,2}', r'\1', cleaned)  # Remove underscores
    cleaned = re.sub(r'\*{1,2}(.*?)\*{1,2}', r'\1', cleaned)  # Remove asterisks
    
    # Remove links (keep text)
    cleaned = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', cleaned)  # Remove links, keep text
    
    # Remove any remaining HTML-like tags
    cleaned = re.sub(r'<[^>]+>', '', cleaned)
    
    # CRITICAL FIX: Replace multiple spaces with single space to prevent double spaces
    cleaned = re.sub(r' +', ' ', cleaned)  # Replace multiple spaces with single space
    
    return cleaned.strip()

def parse_content_into_groups(content: str) -> List[Dict[str, Any]]:
    """Parse markdown content into subheading-content groups"""
    if not content:
        return []
    
    # print(f"DEBUG: parse_content_into_groups called with content length: {len(content)}")
    
    lines = content.split('\n')
    groups = []
    current_group = None
    
    for i, line in enumerate(lines):
        line = line.strip()
        if not line:
            if current_group:
                current_group['content_lines'].append('')  # Preserve empty lines
            continue
        
        # Check if this is a heading (H1, H2, H3, H4, or numbered)
        is_heading = (
            line.startswith('# ') or  # H1
            (line.startswith('## ') and not line.startswith('### ')) or  # H2
            (line.startswith('### ') and not line.startswith('#### ')) or  # H3
            (line.startswith('#### ') and not line.startswith('##### ')) or  # H4
            re.match(r'^\d+\.\d+\s+', line)  # Numbered subheadings like "3.6 SWOT Analysis"
        )
        
        # Check if this is bold text
        is_bold_text = (
            (line.startswith('**') and line.endswith('**') and len(line) > 4) or  # Bold headings
            (re.match(r'^\*\*.*?\*\*:', line) and len(line) > 4)  # Bold text before colon as heading
        )

        # Check if this is a bold list item (e.g., "- **Item**")
        is_bold_list_item = re.match(r'^[-*]\s*\*\*(.*?)\*\*$', line)
        
        # Special handling for bold text following numbered subheadings
        should_create_new_group = False
        
        if is_heading:
            # Always create new group for regular headings
            should_create_new_group = True
            # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Regular heading, creating new group")
        elif is_bold_text:
            # Check if we should treat this bold text as content for a numbered subheading
            if groups and groups[-1].get('subheading_type') == 'h3':
                # Check if the previous group's subheading looks like a numbered subheading
                previous_subheading = groups[-1].get('subheading', '')
                if re.match(r'^\d+\.\d+\s+', previous_subheading):
                    # This is content for a numbered subheading, add to previous group
                    groups[-1]['content_lines'].append(line)
                    # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Adding to previous numbered subheading '{previous_subheading[:30]}...'")
                    continue  # Skip creating new group
                else:
                    # Not a numbered subheading, treat as new heading
                    should_create_new_group = True
                    # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (not numbered subheading)")
            else:
                # No previous group or not a numbered subheading, treat as new heading
                should_create_new_group = True
                # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (no numbered subheading)")
        elif is_bold_list_item:
            # If it's a bold list item, treat it as a new heading
            should_create_new_group = True
            # print(f"DEBUG: Line {i+1} '{line[:50]}...' - Bold list item, creating new group")
        
        if should_create_new_group:
            # Save previous group if it exists
            if current_group:
                current_group['content'] = '\n'.join(current_group['content_lines'])
                groups.append(current_group)
                # print(f"DEBUG: Saved group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines")
            
            # Clean markdown symbols BEFORE determining type and storing
            clean_text = line
            heading_type = 'text'
            
            if line.startswith('# '):
                clean_text = line[2:].strip()  # Remove "# "
                heading_type = 'h1'
            elif line.startswith('## '):
                clean_text = line[3:].strip()  # Remove "## "
                heading_type = 'h2'
            elif line.startswith('### '):
                clean_text = line[4:].strip()  # Remove "### "
                heading_type = 'h3'
            elif line.startswith('#### '):
                clean_text = line[5:].strip()  # Remove "#### "
                heading_type = 'h4'
            elif re.match(r'^\d+\.\d+\s+', line):
                clean_text = line.strip()  # Keep numbered headings as-is
                heading_type = 'numbered'
            elif is_bold_text:
                # Remove ** markers from bold text
                clean_text = clean_markdown_from_line(line.strip())
                heading_type = 'bold'
            elif is_bold_list_item:
                # Extract the bold text from list item and treat as H3 subheading
                # Remove the list marker and ** markers
                clean_text = re.sub(r'^[-*]\s*\*\*(.*?)\*\*', r'\1', line.strip())
                heading_type = 'h3'  # Treat as H3 subheading
            
            # print(f"DEBUG: Creating new group '{clean_text[:30]}...' (type: {heading_type})")
            
            # Start new group with cleaned text
            current_group = {
                'subheading': line,  # Keep original for reference
                'content_lines': [],
                'content': '',
                'subheading_clean': clean_text,  # Pre-cleaned text
                'subheading_type': heading_type
            }
        else:
            # Add to current group's content - clean the line
            if current_group:
                cleaned_line = clean_markdown_from_line(line)
                current_group['content_lines'].append(cleaned_line)
                if i < 5:  # Log first few content lines
                    # print(f"DEBUG: Added content line to group '{current_group.get('subheading_clean', 'Unknown')[:30]}...': '{cleaned_line[:50]}...'")
                    pass
            else:
                # Content before any subheading - create a default group
                if not groups or groups[-1].get('subheading') != '## Introduction':
                    cleaned_line = clean_markdown_from_line(line)
                    current_group = {
                        'subheading': '## Introduction',
                        'content_lines': [cleaned_line],
                        'content': '',
                        'subheading_clean': 'Introduction',
                        'subheading_type': 'h2'
                    }
    
    # Don't forget the last group
    if current_group:
        current_group['content'] = '\n'.join(current_group['content_lines'])
        groups.append(current_group)
        # print(f"DEBUG: Saved final group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines")
    
    # Add metadata to each group
    for i, group in enumerate(groups):
        group['group_id'] = f"group_{i}"
        group['content_length'] = len(group['content'])
        group['word_count'] = len(group['content'].split()) if group['content'] else 0
        group['has_charts'] = 'CHART' in group['content'].upper() if group['content'] else False
        group['line_count'] = len([line for line in group['content'].split('\n') if line.strip()])
    
    # print(f"DEBUG: Created {len(groups)} groups total")
    return groups

def extract_key_phrases(content: str) -> List[tuple]:
    """Extract key phrases from content for grouping metadata"""
    if not content:
        return []
    
    # Simple key phrase extraction
    words = re.findall(r'\b\w+\b', content.lower())
    word_freq = {}
    
    for word in words:
        if len(word) > 3:  # Only words longer than 3 characters
            word_freq[word] = word_freq.get(word, 0) + 1
    
    # Return top 5 most frequent words
    return sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:5]

def get_content_groups_info(content: str) -> Dict[str, Any]:
    """Get detailed information about content groups for analysis and debugging"""
    if not content:
        return {"groups": [], "summary": {"total_groups": 0, "total_content_length": 0}}
    
    groups = parse_content_into_groups(content)
    
    # Add key phrases to each group
    for group in groups:
        group['key_phrases'] = extract_key_phrases(group['content'])
    
    # Calculate summary statistics
    total_content_length = sum(group['content_length'] for group in groups)
    total_word_count = sum(group['word_count'] for group in groups)
    groups_with_charts = sum(1 for group in groups if group['has_charts'])
    
    summary = {
        "total_groups": len(groups),
        "total_content_length": total_content_length,
        "total_word_count": total_word_count,
        "groups_with_charts": groups_with_charts,
        "average_group_size": total_content_length / len(groups) if groups else 0,
        "largest_group": max(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None,
        "smallest_group": min(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None
    }
    
    return {
        "groups": groups,
        "summary": summary
    }