Delete app/DocumentGeneration/grouped.py

app/DocumentGeneration/grouped.py

@@ -1,304 +0,0 @@
-import re
-from typing import List, Dict, Any
-
-def calculate_group_height(pdf, group: Dict[str, Any]) -> int:
-    """Calculate the estimated height needed for a complete group"""
-    subheading_type = group['subheading_type']
-    content_lines = group['content'].split('\n') if group['content'] else []
-    
-    # Base height for subheading
-    if subheading_type == 'h1':
-        subheading_height = 25  # 20pt font + spacing
-    elif subheading_type == 'h2':
-        subheading_height = 20  # 16pt font + underline + spacing
-    elif subheading_type == 'h3':
-        subheading_height = 16  # 14pt font + spacing
-    elif subheading_type == 'h4':
-        subheading_height = 14  # 12pt font + spacing
-    elif subheading_type == 'bold':
-        subheading_height = 20  # 16pt font + spacing
-    elif subheading_type == 'numbered':
-        subheading_height = 20  # 16pt font + spacing
-    else:
-        subheading_height = 16
-    
-    # Calculate content height more accurately
-    content_height = 0
-    for line in content_lines:
-        if line.strip():
-            # More accurate line height calculation
-            words = line.split()
-            if len(words) > 0:
-                # Estimate characters per line (roughly 80-90 chars per line)
-                chars_per_line = 85
-                estimated_lines = max(1, len(line) // chars_per_line + 1)
-                line_height = estimated_lines * 8  # 8pt per line
-                content_height += line_height
-            else:
-                content_height += 8  # Empty line
-        else:
-            content_height += 3  # Empty line spacing
-    
-    # Add spacing between groups
-    group_spacing = 8
-    
-    total_height = subheading_height + content_height + group_spacing
-    
-    print(f"DEBUG: Group '{group.get('subheading_clean', 'Unknown')[:30]}...' height calculation:")
-    print(f"  Subheading height: {subheading_height}")
-    print(f"  Content height: {content_height}")
-    print(f"  Group spacing: {group_spacing}")
-    print(f"  Total height: {total_height}")
-    
-    return total_height
-
-def should_break_page_for_group(pdf, group: Dict[str, Any], current_y: int, page_height: int = 280) -> bool:
-    """Determine if we should break to a new page for this group"""
-    estimated_height = calculate_group_height(pdf, group)
-    
-    print(f"DEBUG: should_break_page_for_group called")
-    print(f"DEBUG:   Subheading: '{group.get('subheading_clean', 'Unknown')[:30]}...'")
-    print(f"DEBUG:   Current Y: {current_y}")
-    print(f"DEBUG:   Estimated height: {estimated_height}")
-    print(f"DEBUG:   Page height: {page_height}")
-    print(f"DEBUG:   Available space: {page_height - current_y}")
-    print(f"DEBUG:   Will fit? {current_y + estimated_height <= page_height}")
-    
-    # CRITICAL FIX: Only break if the group absolutely won't fit
-    # This prevents splitting subheadings from their content
-    
-    # If the group won't fit on current page, break
-    if current_y + estimated_height > page_height:
-        print(f"DEBUG:   -> BREAKING: Group won't fit ({current_y} + {estimated_height} > {page_height})")
-        return True
-    
-    # If we're very close to the end of the page (less than 50mm remaining) and group is significant, break
-    if current_y > page_height - 50 and estimated_height > 30:
-        print(f"DEBUG:   -> BREAKING: Very close to end of page ({current_y} > {page_height - 50}) and group significant ({estimated_height} > 30)")
-        return True
-    
-    print(f"DEBUG:   -> NO BREAK: Group will fit on current page")
-    return False
-
-def clean_markdown_from_line(line: str) -> str:
-    """Clean markdown formatting from a single line"""
-    if not line:
-        return ""
-    
-    cleaned = line
-    
-    # Remove bold, italic formatting
-    cleaned = re.sub(r'\*\*(.*?)\*\*', r'\1', cleaned)  # Remove bold
-    cleaned = re.sub(r'\*(.*?)\*', r'\1', cleaned)  # Remove italic
-    cleaned = re.sub(r'`(.*?)`', r'\1', cleaned)  # Remove code
-    cleaned = re.sub(r'~~(.*?)~~', r'\1', cleaned)  # Remove strikethrough
-    
-    # Remove emphasis markers
-    cleaned = re.sub(r'_{1,2}(.*?)_{1,2}', r'\1', cleaned)  # Remove underscores
-    cleaned = re.sub(r'\*{1,2}(.*?)\*{1,2}', r'\1', cleaned)  # Remove asterisks
-    
-    # Remove links (keep text)
-    cleaned = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', cleaned)  # Remove links, keep text
-    
-    # Remove any remaining HTML-like tags
-    cleaned = re.sub(r'<[^>]+>', '', cleaned)
-    
-    # CRITICAL FIX: Replace multiple spaces with single space to prevent double spaces
-    cleaned = re.sub(r' +', ' ', cleaned)  # Replace multiple spaces with single space
-    
-    return cleaned.strip()
-
-def parse_content_into_groups(content: str) -> List[Dict[str, Any]]:
-    """Parse markdown content into subheading-content groups"""
-    if not content:
-        return []
-    
-    print(f"DEBUG: parse_content_into_groups called with content length: {len(content)}")
-    
-    lines = content.split('\n')
-    groups = []
-    current_group = None
-    
-    for i, line in enumerate(lines):
-        line = line.strip()
-        if not line:
-            if current_group:
-                current_group['content_lines'].append('')  # Preserve empty lines
-            continue
-        
-        # Check if this is a heading (H1, H2, H3, H4, or numbered)
-        is_heading = (
-            line.startswith('# ') or  # H1
-            (line.startswith('## ') and not line.startswith('### ')) or  # H2
-            (line.startswith('### ') and not line.startswith('#### ')) or  # H3
-            (line.startswith('#### ') and not line.startswith('##### ')) or  # H4
-            re.match(r'^\d+\.\d+\s+', line)  # Numbered subheadings like "3.6 SWOT Analysis"
-        )
-        
-        # Check if this is bold text
-        is_bold_text = (
-            (line.startswith('**') and line.endswith('**') and len(line) > 4) or  # Bold headings
-            (re.match(r'^\*\*.*?\*\*:', line) and len(line) > 4)  # Bold text before colon as heading
-        )
-
-        # Check if this is a bold list item (e.g., "- **Item**")
-        is_bold_list_item = re.match(r'^[-*]\s*\*\*(.*?)\*\*$', line)
-        
-        # Special handling for bold text following numbered subheadings
-        should_create_new_group = False
-        
-        if is_heading:
-            # Always create new group for regular headings
-            should_create_new_group = True
-            print(f"DEBUG: Line {i+1} '{line[:50]}...' - Regular heading, creating new group")
-        elif is_bold_text:
-            # Check if we should treat this bold text as content for a numbered subheading
-            if groups and groups[-1].get('subheading_type') == 'h3':
-                # Check if the previous group's subheading looks like a numbered subheading
-                previous_subheading = groups[-1].get('subheading', '')
-                if re.match(r'^\d+\.\d+\s+', previous_subheading):
-                    # This is content for a numbered subheading, add to previous group
-                    groups[-1]['content_lines'].append(line)
-                    print(f"DEBUG: Line {i+1} '{line[:50]}...' - Adding to previous numbered subheading '{previous_subheading[:30]}...'")
-                    continue  # Skip creating new group
-                else:
-                    # Not a numbered subheading, treat as new heading
-                    should_create_new_group = True
-                    print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (not numbered subheading)")
-            else:
-                # No previous group or not a numbered subheading, treat as new heading
-                should_create_new_group = True
-                print(f"DEBUG: Line {i+1} '{line[:50]}...' - Treating as NEW HEADING (no numbered subheading)")
-        elif is_bold_list_item:
-            # If it's a bold list item, treat it as a new heading
-            should_create_new_group = True
-            print(f"DEBUG: Line {i+1} '{line[:50]}...' - Bold list item, creating new group")
-        
-        if should_create_new_group:
-            # Save previous group if it exists
-            if current_group:
-                current_group['content'] = '\n'.join(current_group['content_lines'])
-                groups.append(current_group)
-                print(f"DEBUG: Saved group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines")
-            
-            # Clean markdown symbols BEFORE determining type and storing
-            clean_text = line
-            heading_type = 'text'
-            
-            if line.startswith('# '):
-                clean_text = line[2:].strip()  # Remove "# "
-                heading_type = 'h1'
-            elif line.startswith('## '):
-                clean_text = line[3:].strip()  # Remove "## "
-                heading_type = 'h2'
-            elif line.startswith('### '):
-                clean_text = line[4:].strip()  # Remove "### "
-                heading_type = 'h3'
-            elif line.startswith('#### '):
-                clean_text = line[5:].strip()  # Remove "#### "
-                heading_type = 'h4'
-            elif re.match(r'^\d+\.\d+\s+', line):
-                clean_text = line.strip()  # Keep numbered headings as-is
-                heading_type = 'numbered'
-            elif is_bold_text:
-                # Remove ** markers from bold text
-                clean_text = clean_markdown_from_line(line.strip())
-                heading_type = 'bold'
-            elif is_bold_list_item:
-                # Extract the bold text from list item and treat as H3 subheading
-                # Remove the list marker and ** markers
-                clean_text = re.sub(r'^[-*]\s*\*\*(.*?)\*\*', r'\1', line.strip())
-                heading_type = 'h3'  # Treat as H3 subheading
-            
-            print(f"DEBUG: Creating new group '{clean_text[:30]}...' (type: {heading_type})")
-            
-            # Start new group with cleaned text
-            current_group = {
-                'subheading': line,  # Keep original for reference
-                'content_lines': [],
-                'content': '',
-                'subheading_clean': clean_text,  # Pre-cleaned text
-                'subheading_type': heading_type
-            }
-        else:
-            # Add to current group's content - clean the line
-            if current_group:
-                cleaned_line = clean_markdown_from_line(line)
-                current_group['content_lines'].append(cleaned_line)
-                if i < 5:  # Log first few content lines
-                    print(f"DEBUG: Added content line to group '{current_group.get('subheading_clean', 'Unknown')[:30]}...': '{cleaned_line[:50]}...'")
-            else:
-                # Content before any subheading - create a default group
-                if not groups or groups[-1].get('subheading') != '## Introduction':
-                    cleaned_line = clean_markdown_from_line(line)
-                    current_group = {
-                        'subheading': '## Introduction',
-                        'content_lines': [cleaned_line],
-                        'content': '',
-                        'subheading_clean': 'Introduction',
-                        'subheading_type': 'h2'
-                    }
-    
-    # Don't forget the last group
-    if current_group:
-        current_group['content'] = '\n'.join(current_group['content_lines'])
-        groups.append(current_group)
-        print(f"DEBUG: Saved final group '{current_group.get('subheading_clean', 'Unknown')[:30]}...' with {len(current_group['content_lines'])} content lines")
-    
-    # Add metadata to each group
-    for i, group in enumerate(groups):
-        group['group_id'] = f"group_{i}"
-        group['content_length'] = len(group['content'])
-        group['word_count'] = len(group['content'].split()) if group['content'] else 0
-        group['has_charts'] = 'CHART' in group['content'].upper() if group['content'] else False
-        group['line_count'] = len([line for line in group['content'].split('\n') if line.strip()])
-    
-    print(f"DEBUG: Created {len(groups)} groups total")
-    return groups
-
-def extract_key_phrases(content: str) -> List[tuple]:
-    """Extract key phrases from content for grouping metadata"""
-    if not content:
-        return []
-    
-    # Simple key phrase extraction
-    words = re.findall(r'\b\w+\b', content.lower())
-    word_freq = {}
-    
-    for word in words:
-        if len(word) > 3:  # Only words longer than 3 characters
-            word_freq[word] = word_freq.get(word, 0) + 1
-    
-    # Return top 5 most frequent words
-    return sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:5]
-
-def get_content_groups_info(content: str) -> Dict[str, Any]:
-    """Get detailed information about content groups for analysis and debugging"""
-    if not content:
-        return {"groups": [], "summary": {"total_groups": 0, "total_content_length": 0}}
-    
-    groups = parse_content_into_groups(content)
-    
-    # Add key phrases to each group
-    for group in groups:
-        group['key_phrases'] = extract_key_phrases(group['content'])
-    
-    # Calculate summary statistics
-    total_content_length = sum(group['content_length'] for group in groups)
-    total_word_count = sum(group['word_count'] for group in groups)
-    groups_with_charts = sum(1 for group in groups if group['has_charts'])
-    
-    summary = {
-        "total_groups": len(groups),
-        "total_content_length": total_content_length,
-        "total_word_count": total_word_count,
-        "groups_with_charts": groups_with_charts,
-        "average_group_size": total_content_length / len(groups) if groups else 0,
-        "largest_group": max(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None,
-        "smallest_group": min(groups, key=lambda g: g['content_length'])['subheading_clean'] if groups else None
-    }
-    
-    return {
-        "groups": groups,
-        "summary": summary
-    }