backend/finance/export_views.py

import io
from collections import defaultdict
from django.http import HttpResponse
from rest_framework.views import APIView
from rest_framework import permissions
from rest_framework.response import Response
from rest_framework import status
from expense_tracker.utils import MongoDBClient
from datetime import datetime, timedelta
from reportlab.lib import colors
from reportlab.lib.pagesizes import letter
from reportlab.platypus import SimpleDocTemplate, Table, TableStyle, Paragraph, Spacer, Image, PageBreak
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.units import inch
from reportlab.lib.enums import TA_CENTER, TA_RIGHT, TA_LEFT
from bson.binary import Binary
from bson.objectid import ObjectId
import os
import re
import gc
import traceback
from xml.sax.saxutils import escape as xml_escape
import concurrent.futures
import time
import matplotlib
matplotlib.use('Agg') # Set non-interactive backend at start
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg
import matplotlib.cm as cm
from .email_service import send_financial_report_email 
from analytics.forecast import get_forecast
from .utils_mongo import get_user_db_id
import threading
import uuid

# --- GLOBAL TASK REGISTRY FOR BACKGROUND PDF GENERATION ---
REPORT_TASKS = {} # Stores {task_id: {status: 'pending'|'completed'|'failed', data: bytes, created_at: timestamp}}

class ExportDataView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    def generate_ai_insight(self, total_income, total_expense, net_savings, sorted_cats, budget_health, monthly_trends=None, max_txn_desc=None, provider=None):
        """
        Generates an extensive CFO-style full report using sequential Chaining for maximum depth.
        Returns (email_summary, full_report_text).
        """
        from finance.ai_helper import generate_text_with_llm
        
        # Shared Context - Enriched for CEO Analysis
        # Use top 15 categories for more breadth
        top_expenses_str = ", ".join([f"{k} (${v:,.0f})" for k,v in sorted_cats[:15]])
        
        # Detailed Budget Deltas
        budget_details = []
        if budget_health:
            for b in budget_health:
                delta = b['limit'] - b['spent']
                status = "OVER" if delta < 0 else "UNDER"
                budget_details.append(f"{b['category']}: Spent ${b['spent']:,.0f} / Limit ${b['limit']:,.0f} ({status} by ${abs(delta):,.0f})")
        budget_health_str = "\n".join(budget_details) if budget_details else "No budget data available."

        # Historical Context
        trends_str = "No historical trend data available."
        if monthly_trends:
            trend_lines = []
            for month, vals in sorted(monthly_trends.items(), reverse=True)[:6]: # Last 6 months for context
                trend_lines.append(f"{month}: Inc ${vals['income']:,.0f} / Exp ${vals['expense']:,.0f}")
            trends_str = "\n".join(trend_lines)

        common_data = f"""
        **FINANCIAL INTELLIGENCE CORE**:
        - Current Month Income: ${total_income:,.2f}
        - Current Month Expenses: ${total_expense:,.2f}
        - Current Net Savings: ${net_savings:,.2f}
        - Savings Rate: {(net_savings/total_income*100 if total_income > 0 else 0):,.1f}%
        - Top 15 Categories: {top_expenses_str}
        - Largest Outlier Transaction: {max_txn_desc or "N/A"}
        
        **BUDGET COMPLIANCE (DETAILED)**:
        {budget_health_str}
        
        **HISTORICAL VELOCITY (LAST 6 MONTHS)**:
        {trends_str}
        """
 
        sys_msg = """You are a Tier-1 CFO and Wealth Strategist. 
        Your goal is to produce a board-level financial report. Use Markdown (##, ###).
        TONE: Authoritative, verbose, analytical. 
        RULES: ALWAYS finish sentences. NO table markers."""
 
        ai_session_state = {"failed_models": set()}

        def get_ai_part(prompt, tokens=8192): # Increased from 4000
            try:
                print(f"DEBUG: Sequential AI Request (Part {getattr(get_ai_part, 'step', 1)}): {len(prompt)} chars")
                res = generate_text_with_llm(
                    prompt=prompt,
                    system_instruction=sys_msg,
                    max_output_tokens=tokens,
                    session_state=ai_session_state,
                    provider_override=provider
                )
                get_ai_part.step = getattr(get_ai_part, 'step', 1) + 1
                if res and res.get('text'):
                    return res['text'] + f"\n\n*Generated by {res.get('source', 'AI')}*\n"
            except Exception as e:
                print(f"ERROR: Sequential AI Part failed: {e}")
            return ""
        
        get_ai_part.step = 1
 
        # STEP 1: Clinical Executive Financial Intelligence
        prompt_1 = f"""
        {common_data}
        **TASK: Part 1 - Clinical Executive Financial Intelligence**
        Sections to cover: 
        1. **Executive Intelligence Assessment**: A sophisticated deep-dive into liquidity, savings-to-income benchmarks, and historical trajectory. Analyze the 50/30/20 rule application specifically for these numbers.
        2. **Strategic Cash Flow Audit**: CFA-level analysis of income stability vs. expense volatility. What are the long-term implications of the current net savings?
        TONE: Clinical, authoritative, board-level executive. Be extremely verbose.
        """
        print("DEBUG: Starting AI Step 1 (Executive Intelligence)...")
        content_1 = get_ai_part(prompt_1)
        
        # STEP 2: Behavioral Spending & Operational Efficiency
        prompt_2 = f"""
        {common_data}
        **CONTEXT FROM PART 1**: {content_1[:500]}...
        **TASK: Part 2 - Behavioral Behavioral Analytics & Efficiency**
        Sections to cover:
        1. **Category Concentration Risk**: Identify if spending is dangerously weighted in specific categories from {top_expenses_str}.
        2. **Operational Expense Audit**: analyze the ratio of Discretionary vs. Mandatory spending. Identify "Category Creep" and behavioral psychological patterns affecting the budget.
        Provide specific mandates for *each* major category mentioned.
        """
        print("DEBUG: Starting AI Step 2 (Behavioral Analytics)...")
        content_2 = get_ai_part(prompt_2)
 
        # STEP 3: Wealth Engineering & Asset Optimization
        prompt_3 = f"""
        {common_data}
        **TASK: Part 3 - Wealth Engineering & Asset Allocation Strategy**
        Sections to cover:
        1. **ROI & Wealth Building Model**: Based on current savings, provide specific ROI-driven allocation strategies (ETFs, Emergency Fund scaling, Debt Arbitrage).
        2. **Strategic Asset Benchmarks**: Specific engineering steps to transition from "Savings" to "Wealth Accumulation" mode based on the current savings rate.
        Focus on compound interest impacts and immediate cash flow optimization.
        """
        content_3 = get_ai_part(prompt_3)
 
        # STEP 4: Risk Analytics & Stress Testing
        prompt_4 = f"""
        {common_data}
        **TASK: Part 4 - Risk Analytics & Stress Testing**
        Sections to cover:
        1. **Functional Stress Test**: How does this current budget handle a 30% income shock or unplanned liability? Identify specific vulnerabilities.
        2. **Concentration & Anomaly Resilience**: Analyze budget overruns and concentration risks with professional rigor.
        """
        content_4 = get_ai_part(prompt_4)
 
        # STEP 5: Master Strategic Mandate & Projections
        prompt_5 = f"""
        {common_data}
        **TASK: Part 5 - Master Strategic Mandate & 24-Month Projections**
        Sections to cover:
        1. **Projected Financial Trajectory**: 12-24 month mathematical projection of net worth if current behavior persists.
        2. **Master High-Impact Mandates**: Provide 3-5 "Mandates" (Top-priority strategic shifts) to accelerate financial freedom.
        Conclude with a powerful, personalized executive directive.
        """
        content_5 = get_ai_part(prompt_5)
        
        # STEP 6: ULTIMATE CONCISE EXECUTIVE BRIEFING (FOR EMAIL BODY)
        prompt_6 = f"""
        {common_data}
        **TASK: Part 6 - ULTIMATE PROFESSIONAL BRIEFING (EMAIL BODY)**
        Sections:
        1. **Professional Snapshot**: High-impact snapshot of the current month.
        2. **Financial Health Pulse**: Brief analysis of current liquidity and savings health.
        3. **Critical Executive Analysis**: The single most important takeaway for this month.
        4. **Key Recommendations**: Provide 6 personalized, sharp bullet points of immediate actions.
        5. **Strategic Advises**: Final high-level strategic counsel.
        TONE: Premium, concise, high-impact.
        """
        email_briefing = get_ai_part(prompt_6, tokens=8192) # Ultra-boosted for depth

# Post-processing: Replace square symbols with professional bullets at line starts, strip otherwise
        def clean_symbols(text):
            if not text: return ""
            # Professional cleaning of square/box symbols that break PDF fonts
            for sym in ['■', '▪', '●', '◆', '◈', '□', '◦', '○', '∙', '⊙', '⊚']:
                text = text.replace(sym, '-')
            
            # Corrected: Splits by '**' and wraps every second element in <b></b> tags
            return "".join([f"<b>{part}</b>" if i % 2 else part for i, part in enumerate(text.replace('***', '').split('**'))])

        full_report = "\n\n".join([
            "Board-Level Comprehensive Financial Analysis (CFO Report)",
            content_1, content_2, content_3, content_4, content_5
        ])
        
        full_report = clean_symbols(full_report)
        full_report = re.sub(r'\|\s*[:\- ]+\s*\|', '|', full_report)
        
        email_summary = clean_symbols(email_briefing)
        
        # FINAL FALLBACK
        if not full_report or len(full_report) < 100:
            print("WARNING: Sequential AI chain produced minimal content. Using fallback.")
            fallback = self.generate_fallback_insight(total_income, total_expense, net_savings, sorted_cats, budget_health)
            return {
                "summary": fallback,
                "full_text": fallback
            }
 
        return {
            "summary": email_summary,
            "full_text": full_report
        }

    def format_text_for_pdf(self, text):
        """
        Converts simple Markdown to ReportLab XML tags and purges non-standard symbols.
        """
        import re
        if not text: return []
        
        # --- GLOBAL SYMBOL CLEANING ---
        # Purge all box/square symbols that break PDF fonts and cause '■' appearances
        for sym in ['■', '▪', '●', '◆', '◈', '□', '◦', '○', '∙', '⊙', '⊚', '☐', '☑', '☒', '★', '☆']:
            text = text.replace(sym, '-')
            
        # Replace smart quotes and dashes with standard ASCII
        replacements = {
            '–': '-',  # En Dash
            '—': '-',  # Em Dash
            '−': '-',  # Minus Sign
            '‐': '-',  # Hyphen
            '‑': '-',  # Non-breaking Hyphen
            '“': '"',  # Smart Quote Open
            '”': '"',  # Smart Quote Close
            '‘': "'",  # Smart Single Quote Open
            '’': "'",  # Smart Single Quote Close
            '…': '...', # Ellipsis
            '<br>': '<br/>', # Fix ReportLab tag parsing
            '<br >': '<br/>',
            '</br>': '', # invalid
        }
        for k, v in replacements.items():
            text = text.replace(k, v)
        
        # Remove triple and double stars (handling bolding residually)
        text = text.replace('***', '')
        # Handle bolding before splitting into paragraphs/lines
        # Convert **text** to <b>text</b> globally
        text = re.sub(r'\*\*\s*(.+?)\s*\*\*', r'<b>\1</b>', text)
        # Final safety check for unmatched ** symbols
        text = text.replace('**', '')

        # 1. Split lines
        lines = text.split('\n')
        formatted_paragraphs = []
        
        for line in lines:
            line = line.strip()
            if not line: continue
            
            # 2. Header Handling (### or ##)
            is_header = False
            header_match = re.match(r'^(#{2,6})\s*(.+)$', line)
            if header_match:
                content = header_match.group(2).strip()
                content = content.replace('<b>', '').replace('</b>', '').strip() # Strip tags from headers
                h_level = len(header_match.group(1))
                size = 14 if h_level == 2 else 12 if h_level == 3 else 11
                line = f"<b><font size='{size}' color='#334155'>{content}</font></b>"
                is_header = True
            
            # 3. Bullet Handling
            if line.startswith(('-', '*', '•')):
                line = line[1:].strip().lstrip('-').lstrip('*').strip()
                line = f"&bullet; {line}"
            
            # 4. Italic Handling (*text*)
            line = re.sub(r'(?<!\*)\*([^\*]+?)\*(?!\*)', r'<i>\1</i>', line)
            
            formatted_paragraphs.append({
                'text': line,
                'is_header': is_header
            })
            
        return formatted_paragraphs

    def enrich_data_for_pdf(self, data):
        import math
        # Convert list of dicts to something easier to analyze if needed
        # ensuring amounts are floats
        for r in data:
            # Clean Amount
            val = r.get('Amount', 0)
            if isinstance(val, str):
                try:
                    val = float(val.replace('$','').replace(',',''))
                except:
                    val = 0.0
            
            # Handle potential None or NaN from DB/Pandas interactions
            if val is None:
                val = 0.0
            try:
                if math.isnan(val):
                    val = 0.0
            except:
                pass # val is not a number, keep as 0.0 from initialization if needed

            r['Amount'] = val
            
            # Clean Category
            if not r.get('Category'):
                 r['Category'] = 'Uncategorized'
            
        return data

    def generate_analytical_encyclopedia(self, cats, inc_cats):
        """
        Generates a massive categorical breakdown with detailed analysis for EVERY category.
        """
        text = ["# Analytical Categorical Encyclopedia\n"]
        text.append("This section provides a clinical deep-dive into every identified categorical node in your financial ecosystem. Each assessment is based on real-time transactional velocity and historical allocation patterns.")
        
        text.append("\n## A. Revenue Node Analysis (Incomes)")
        for cat, amt in inc_cats:
            text.append(f"### Category: {xml_escape(str(cat))}")
            text.append(f"- **Total Inflow**: ${amt:,.2f}")
            text.append(f"- **Strategic Assessment**: This revenue node represents a {'primary' if amt > 5000 else 'secondary'} capital influx point. Maintaining the health of this channel is critical for baseline liquidity.")
            text.append("- **Audit Observations**: Historical data indicates consistent influx. We recommend monitoring the volatility of this category to ensure it aligns with operational requirements.")
            text.append("- **Clinical Directive**: Optimize the timing of these receipts to maximize the compounding potential of your sitting capital.")
        
        text.append("\n## B. Operational Outflow Audit (Expenses)")
        for i, (cat, amt) in enumerate(cats):
            priority = "HIGH-IMPACT" if i < 3 else "SIGNIFICANT" if i < 7 else "OPERATIONAL-VAR"
            text.append(f"### Category Audit: {xml_escape(str(cat))}")
            text.append(f"- **Total Burn**: ${amt:,.2f} [Status: {priority}]")
            text.append(f"- **Detailed Assessment**: Allocation to this cost center is currently {priority} relative to total monthly outflow. A clinical review of all line items within this node is suggested to identify latent capital leakage.")
            text.append(f"- **Heuristic Analysis**: Categorical spending in {xml_escape(str(cat))} often correlates with discretionary behavioral triggers. Restructuring the allocation strategy here could yield a {amt * 0.1:,.2f} increase in monthly surplus.")
            text.append("- **Micro-Bullet Audit Points**:")
            text.append(f"  - Evaluate individual transaction frequency within {xml_escape(str(cat))}.")
            text.append("  - Cross-reference with historical benchmarks for similar fiscal profiles.")
            text.append(f"  - Restore liquidity by applying a 10% reduction mandate to this specific node.")
            
        return "\n".join(text)

    def generate_behavioral_risk_audit(self, data, rate, savings):
        """
        Analyzes spending patterns and assigns archetypes.
        """
        text = ["# Behavioral Strategic Risk Audit\n"]
        text.append("Financial health is not just about numbers; it is about the behavioral algorithms governing capital movement.")
        
        # Determine archetype
        if rate > 30: archetype = "ELITE CAPITAL ARCHITECT"
        elif rate > 20: archetype = "DISCIPLINED GROWTH GUARDIAN"
        elif rate > 10: archetype = "STABLE OPERATIONALIST"
        else: archetype = "LIQUIDITY RISK SURVIVOR"
        
        text.append(f"## Assigned Financial Archetype: **{archetype}**")
        text.append(f"Based on your savings rate of **{rate:.1f}%**, you have been classified as a {archetype}. This profile highlights a {'highly efficient' if rate > 20 else 'developing'} capital retention strategy.")
        
        text.append("\n## Behavioral Node Analysis")
        text.append("### 1. Temporal Spending Velocity")
        text.append("Our audit of your transaction timestamps reveals the following temporal risk profiles:")
        text.append("- **Weekday Momentum**: Standard operational burn. Influx tends to occur in predictable windows.")
        text.append("- **Weekend Surge Liability**: Variable spending typically increases by 15-20% during rest intervals. This represents an area for potential optimization.")
        
        text.append("\n### 2. Discretionary Frequency Rating")
        if rate < 15:
            text.append("- **Status: ELEVATED RISK.** The frequency of small-value discretionary transactions is creating 'death by a thousand cuts' scenario.")
        else:
            text.append("- **Status: OPTIMIZED.** High-value capital is prioritized for mandatory nodes, with discretionary flow kept under tight governance.")
            
        return "\n".join(text)

    def generate_liquidity_stress_model(self, income, expense, savings):
        """
        Deterministic shock simulations.
        """
        text = ["# Liquidity & Solvency Stress Test Models\n"]
        text.append("To ensure long-term resilience, we subject your financial profile to three high-impact stress shocks.")
        
        reserves = savings * 6 # Assume 6 months as baseline for simulation
        
        text.append("## Scenario A: Gross Revenue Cessation (100% Impact)")
        text.append(f"In the event of total income stoppage, your current burn rate of **${expense:,.2f}** per period would exhaust estimated baseline reserves in **{(reserves / expense if expense > 0 else 0):.1f} months**.")
        text.append("### Resilience Rating: " + ("STABLE" if reserves/expense > 6 else "CAUTION" if reserves/expense > 3 else "CRITICAL"))

        text.append("\n## Scenario B: Post-Inflationary Burn Spike (20% Expense Increase)")
        new_exp = expense * 1.2
        new_savings = income - new_exp
        text.append(f"Under a 20% inflationary shock, your expenses would rise to **${new_exp:,.2f}**. This would compress your savings rate to **{(new_savings/income*100 if income > 0 else 0):.1f}%**.")
        
        text.append("\n## Scenario C: Black Swan Event (50% Income Drop + 10% Expense Rise)")
        new_inc = income * 0.5
        new_exp_c = expense * 1.1
        deficit = new_inc - new_exp_c
        text.append(f"Under a dual-shock event, your monthly delta would shift to **${deficit:,.2f}**. This would create a mandatory liquidity draw of ${abs(deficit):,.2f} per month.")
        
        return "\n".join(text)


    def generate_strategic_benchmarking_audit(self, income, expense, fixed_sum):
        """
        Deep metabolic analysis using the 50/30/20 rule.
        """
        text = ["# Strategic Financial Benchmarking Audit (50/30/20 Model)\n"]
        text.append("We have subjected your financial profile to the gold-standard 50/30/20 capital allocation model. This provides a baseline for elite financial management.")
        
        needs_pct = (fixed_sum / income * 100) if income > 0 else 0
        wants_pct = ((expense - fixed_sum) / income * 100) if income > 0 else 0
        savings_pct = ((income - expense) / income * 100) if income > 0 else 0
        
        text.append(f"## Current Allocation Metabolism")
        text.append(f"- **Needs (Mandatory Outflow)**: {needs_pct:.1f}% (Benchmark: 50%)")
        text.append(f"- **Wants (Discretionary Velocity)**: {wants_pct:.1f}% (Benchmark: 30%)")
        text.append(f"- **Savings (Capital Retention)**: {savings_pct:.1f}% (Benchmark: 20%)")
        
        text.append("\n## Optimization Mandates")
        if needs_pct > 50:
            text.append(f"### MANDATE-1: STABILIZE FIXED OBLIGATIONS")
            text.append(f"Your mandatory outflows ({needs_pct:.1f}%) exceed the 50% efficiency threshold. You are currently in an 'Obligation Trap'. Focus on reducing fixed categorical nodes to restore structural agility.")
        else:
            text.append(f"### MANDATE-1: MAINTAIN LEVERAGE")
            text.append(f"Your fixed obligations are well-governed at {needs_pct:.1f}%. This provides a robust foundation for aggressive investment or liability retirement.")

        if wants_pct > 30:
            text.append(f"### MANDATE-2: CURB BEHAVIORAL LEAKAGE")
            text.append(f"Discretionary spending at {wants_pct:.1f}% is creating 'Capital Erosion'. Every percent above 30 is a direct withdrawal from your future wealth compounding pool.")
            
        return "\n".join(text)

    def generate_wealth_efficiency_projections(self, savings):
        """
        Linear and geometric net-worth modeling (Pseudo-AI).
        """
        text = ["# Wealth Intelligence & Capital Projections\n"]
        text.append("This section models your wealth trajectory assuming consistent adherence to current operational efficiency.")
        
        # 12, 60 month linear
        ann_surplus = savings * 12
        five_yr = savings * 60
        
        # 10yr compounding (Pseudo-AI ROI)
        # Formula: A = P(1+r)^t
        roi_8 = savings * ((1+0.08/12)**(120)-1) / (0.08/12) # 10 years at 8%
        
        text.append("## Linear Trajectory (Static Burn)")
        text.append(f"- **12-Month Projected Growth**: ${ann_surplus:,.2f}")
        text.append(f"- **60-Month Strategic Reserve**: ${five_yr:,.2f}")
        
        text.append("\n## Exponential Asset Optimization (8% ROI Model)")
        text.append(f"If your current monthly surplus of **${savings:,.2f}** were successfully diverted into diversified index-tracking vehicles with an 8% annualized yield, your 10-year capital node would reach approximately **${roi_8:,.2f}**.")
        text.append("This highlights the massive 'Compounding Delta' between static savings and active capital deployment.")
        
        return "\n".join(text)

    def generate_capital_allocation_mandates(self, rate):
        """
        Final high-impact clinical advice.
        """
        text = ["# Master Financial Strategy & Capital Mandates\n"]
        text.append("Based on the complete audit, we issue the following elite mandates for total financial optimization.")
        
        mandates = [
            ("LIQUIDITY MANDATE", "Maintain a minimum cash reserve of 6 months operational burn to ensure black-swan resilience."),
            ("ASSET TRANSITION", "Once the savings rate exceeds 25%, move all surplus capital above the reserve threshold into yielding assets within 48 hours of influx."),
            ("TAX ARBITRAGE", "Review categorical nodes for tax-deductible operational expenses to optimize net realization."),
            ("SUBSCRIPTION PURGE", "Execute a clinical audit of all recurring digital outflows; eliminate any node with a utilization rate below 40%."),
            ("REVENUE DIVERSIFICATION", "Identify a secondary income channel that correlates negatively with your primary influx node to minimize systemic risk.")
        ]
        
        for title, desc in mandates:
            text.append(f"### {title}")
            text.append(desc)
            
        return "\n".join(text)

    def generate_master_strategic_report(self, income, expense, savings, cats, budgets, inc_cats):
        """
        Extremely verbose, high-detail clinical report utilizing all real-time data.
        Designed to exceed expectations for deterministic financial analysis.
        """
        rate = (savings / income * 100) if income > 0 else 0
        
        # --- SECTION 1: EXECUTIVE INTELLIGENCE & LIQUIDITY ---
        if rate > 25:
            exec_status = "ALPHA-OPTIMIZED"
            exec_desc = "Your capital velocity is currently performing at an elite level. The structural efficiency of your income-to-asset conversion process is robust."
        elif rate > 15:
            exec_status = "STABLE-CORE"
            exec_desc = "Your financial base is resilient. You are successfully maintaining a surplus that supports both operational safety and moderate growth."
        else:
            exec_status = "LIQUIDITY-VULNERABLE"
            exec_desc = "The current audit reveals a high burn rate relative to gross receipts. A strategic pivot toward cost-containment is mandatory."

        # --- SECTION 2: REVENUE ARCHITECTURE AUDIT (All Income) ---
        revenue_lines = []
        for cat, amt in inc_cats:
            rev_pct = (amt / income * 100) if income > 0 else 0
            revenue_lines.append(f"- **{xml_escape(str(cat))}**: ${amt:,.2f} ({rev_pct:.1f}% share)")
        
        revenue_analysis = f"Your revenue portfolio is currently distributed across **{len(inc_cats)}** distinct channels. "
        if len(inc_cats) == 1:
            revenue_analysis += "Warning: High dependency on a single income stream represents a structural risk. Diversification of revenue channels is recommended."
        else:
            revenue_analysis += "Revenue diversification is healthy, distributing risk across multiple categorical nodes."

        # --- SECTION 2.5: REVENUE STREAM RESILIENCE MATRIX ---
        resilience_lines = []
        for cat, amt in inc_cats[:5]:
            score = "HIGH" if amt > income * 0.5 else "MEDIUM" if amt > income * 0.2 else "STABLE"
            resilience_lines.append(f"- **{xml_escape(str(cat))} Resilience Profile**: Status {score}. Current influx momentum is sustained.")

        # --- SECTION 3: SPENDING COMPOSITION AUDIT (Total Categorical Breakdown) ---
        # Fixed vs Discretionary
        fixed_cats = ['Rent', 'Mortgage', 'Utilities', 'Insurance', 'Taxes', 'Bills', 'Loan', 'EMI', 'Subscription']
        fixed_sum = sum(amt for cat, amt in cats if any(f.lower() in cat.lower() for f in fixed_cats))
        discretionary_sum = expense - fixed_sum
        over_budget_cats = [b for b in budgets if b['health'] > 100]

        # Categorical Resilience Audit (Top 10 Deep Dive)
        cat_deep_dive = []
        for i, (cat, amt) in enumerate(cats[:10]):
            pct = (amt / expense * 100) if expense > 0 else 0
            impact = "CRITICAL" if pct > 20 else "SIGNIFICANT" if pct > 10 else "MODERATE"
            cat_deep_dive.append(f"#### {i+1}. {xml_escape(str(cat))} Audit\nCategorical impact is **{impact}** at **{pct:.1f}%** of total period outflow (${amt:,.2f}). This segment represents a {'primary' if i == 0 else 'major' if i < 3 else 'secondary'} cost center. Managing the variance in this categorical node is essential for maintaining liquidity targets.")

        # Detailed Category Audit (ALL)
        expense_breakdown = []
        for i, (cat, amt) in enumerate(cats):
            pct = (amt / expense * 100) if expense > 0 else 0
            priority = "HIGH-IMPACT" if i < 3 else "SECONDARY" if i < 7 else "OPERATIONAL-VAR"
            expense_breakdown.append(f"- **{xml_escape(str(cat))}** [{priority}]: ${amt:,.2f} ({pct:.1f}%)")

        # --- SECTION 4: BENCHMARK VARIANCE & EFFICIENCY (50/30/20) ---
        needs_pct = (fixed_sum / income * 100) if income > 0 else 0
        wants_pct = (discretionary_sum / income * 100) if income > 0 else 0
        
        efficiency_status = "OPTIMIZED" if rate >= 20 and needs_pct <= 50 else "REBALANCING-REQUIRED"
        
        # --- SECTION 5: PROJECTED CAPITAL TRAJECTORY ---
        monthly_growth = savings
        annual_projection = monthly_growth * 12
        five_year_projection = monthly_growth * 60 # Simple linear for pseudo-ai
        
        # --- SECTION 6: STRATEGIC MASTER DIRECTIVES (Mandates) ---
        mandates = []
        if rate < 20:
            mandates.append(f"**LIQUIDITY MANDATE**: Execute a clinical reduction of 15% in **{xml_escape(cats[0][0]) if cats else 'Top Category'}** to restore liquidity.")
        if over_budget_cats:
            mandates.append(f"**BUDGET MANDATE**: Immediate Spending Freeze on **{xml_escape(over_budget_cats[0]['category'])}**; current variance is ${over_budget_cats[0]['spent'] - over_budget_cats[0]['limit']:,.2f} over limit.")
        mandates.append("**RESERVE MANDATE**: Reallocate 10% of gross receipts toward an emergency liquidity reserve to enhance financial resilience.")
        mandates.append("**OPERATIONAL MANDATE**: Perform a subscription audit to eliminate latent capital leakage in operational expenses.")
        mandates.append("**DIVERSIFICATION MANDATE**: Identify secondary revenue streams to reduce single-source income dependency.")
        mandates.append("**ASSET MANDATE**: Once savings rate exceeds 30%, initiate transition from cash holdings to non-inflationary assets.")
        mandates.append("**RISK MANDATE**: Review all 'Elevated' status days in the Daily Operational Journal to identify behavioral spending triggers.")

        full_report = f"""
### 1. Executive Intelligence Assessment
**Status: {exec_status}**
{exec_desc} 
Your current savings rate of **{rate:.1f}%** indicates an annual surplus growth potential of **${annual_projection:,.2f}**. This creates a foundation for advanced investment strategies or liability retirement.

### 2. Revenue Portfolio Diversification & Resilience
{revenue_analysis}
**Income Stream Breakdown:**
{chr(10).join(revenue_lines)}

**Resilience Portfolio:**
{chr(10).join(resilience_lines)}

### 3. Micro-Categorical Spending Audit & Deep Dive
A complete clinical audit of your spending composition across all **{len(cats)}** identified cost centers.

{chr(10).join(cat_deep_dive)}

**Complete Categorical Roster:**
{chr(10).join(expense_breakdown)}

### 4. Spending Architecture: Fixed vs. Discretionary
Your spending architecture consists of **${fixed_sum:,.2f}** in Fixed Obligations and **${discretionary_sum:,.2f}** in Discretionary flow.
- **Fixed/Needs Ratio**: {needs_pct:.1f}% (Benchmark: 50.0%)
- **Discretionary/Wants Ratio**: {wants_pct:.1f}% (Benchmark: 30.0%)
- **Efficiency Status**: {efficiency_status}

### 5. Multi-Period Capital Trajectory Modeling
*Linear Projection based on current real-time velocity (Zero Market Variance Model):*
- **12-Month Projected Growth**: ${annual_projection:,.2f}
- **60-Month Projected Growth**: ${five_year_projection:,.2f}
This modeling assumes no change in operational burn or revenue influx. Consistency in this trajectory is key to long-term solvency and wealth accumulation.

### 6. Strategic Master Directives (Economic Mandates)
{chr(10).join(f"- {m}" for m in mandates)}

---
*Note: This Deterministic Strategic Intelligence Report is compiled through clinical analysis of real-time transactional nodes and established financial heuristics.*
"""
        return full_report

    def generate_pseudo_ai_insights(self, income, expense, savings, cats, budgets):
        """
        Generates sophisticated, CFO-style insights using real-time data and rule-based triggers.
        Mimics AI reasoning without LLM calls.
        """
        rate = (savings / income * 100) if income > 0 else 0
        cat_map = dict(cats)
        
        # 1. Executive Strategic Assessment
        if rate > 25:
            assessment = "Your financial velocity is currently in the 'High-Alpha' zone. You are converting gross income into net wealth at an elite rate. This allows for aggressive capital allocation toward long-term assets."
        elif rate > 15:
            assessment = "The current trajectory is 'Stable-Positive'. You are maintaining a healthy margin between operational costs and gross receipts. Focus should now shift toward optimizing recurring fixed costs to expand your savings delta."
        elif rate > 5:
            assessment = "Your financial delta is currently 'Vulnerable'. While not in a deficit, the margin for error is thin. A clinical audit of discretionary spending is required to ensure long-term liability coverage."
        else:
            assessment = "The current status is 'Critical Deficit'. Operational expenses are exceeding or nearly matching income. This is an unsustainable burn rate that requires immediate structural intervention."

        # 2. Operational Expense Audit (Fixed vs Discretionary)
        fixed_cats = ['Rent', 'Mortgage', 'Utilities', 'Insurance', 'Taxes', 'Bills', 'Loan', 'EMI', 'Subscription']
        fixed_sum = sum(amt for cat, amt in cats if any(f.lower() in cat.lower() for f in fixed_cats))
        discretionary_sum = expense - fixed_sum
        disc_ratio = (discretionary_sum / expense * 100) if expense > 0 else 0
        
        audit_text = f"Analyzed operational burn: **${fixed_sum:,.2f}** in Fixed/Required costs vs. **${discretionary_sum:,.2f}** in Discretionary flow. "
        if disc_ratio > 40:
            audit_text += "Your Discretionary Ratio is elevated at **{:.1f}%**. This indicates 'Behavioral Leakage'—capital that could be redirected to wealth building is being consumed by variable lifestyle choices.".format(disc_ratio)
        else:
            audit_text += "Your Discretionary Ratio of **{:.1f}%** indicates strong fiscal discipline. You are effectively limiting variable costs to preserve capital.".format(disc_ratio)

        # 3. Category Concentration Risk
        top_cat_name, top_cat_amt = cats[0] if cats else ("N/A", 0)
        top_cat_pct = (top_cat_amt / expense * 100) if expense > 0 else 0
        
        risk_text = f"Primary concentration risk identified in **{xml_escape(str(top_cat_name))}**, which commands **{top_cat_pct:.1f}%** of total period outflow. "
        if top_cat_pct > 35:
            risk_text += "This 'Category Overweighting' creates a structural vulnerability. If costs in this specific sector rise, your entire savings rate could collapse. Diversification of spending is recommended."
        else:
            risk_text += "Spending is reasonably diversified across the portfolio, indicating balanced lifestyle management."

        # 4. Projected 12-Month Trajectory
        projected_savings = savings * 12
        if projected_savings > 0:
            trajectory_text = f"At the current velocity, your annual net capital accumulation is projected at **${projected_savings:,.2f}**. "
            trajectory_text += "This trajectory supports a transition toward asset acquisition. Consistency is the primary variable for success."
        else:
            trajectory_text = f"At the current velocity, you face an annual capital erosion of **${abs(projected_savings):,.2f}**. "
            trajectory_text += "This trajectory leads to liquidity depletion. Structural cost reduction is mandatory to reverse this trend."

        # 5. Benchmark Variance (50/30/20 Rule)
        # Assuming Fixed = Needs (50), Discretionary = Wants (30), Savings = 20
        needs_pct = (fixed_sum / income * 100) if income > 0 else 0
        wants_pct = (discretionary_sum / income * 100) if income > 0 else 0
        
        benchmark_text = f"Benchmark Variance Audit: **Needs: {needs_pct:.1f}%** (Target: 50%) | **Wants: {wants_pct:.1f}%** (Target: 30%) | **Savings: {rate:.1f}%** (Target: 20%). "
        if rate < 20:
            benchmark_text += "You are currently 'Under-Beta' in savings. Rebalancing from the 'Wants' segment is the most efficient path to optimization."
        else:
            benchmark_text += "You are 'Alpha-Positive' against standard benchmarks. This surplus should be leveraged into high-yield instruments."

        # 6. Master Financial Directives (Dynamic Mandates)
        mandates = []
        if rate < 15:
            mandates.append(f"**MANDATE 1**: Execute a 15% reduction in **{xml_escape(str(top_cat_name))}** over the next 30 days to widen the net delta.")
        else:
            mandates.append("**MANDATE 1**: Accelerate wealth allocation. Redirect 10% of existing cash flow toward a diverse ETF or retirement fund portfolio.")
            
        over_budgets = [b for b in budgets if b['health'] > 100]
        if over_budgets:
            top_over = sorted(over_budgets, key=lambda x: x['spent'], reverse=True)[0]
            mandates.append(f"**MANDATE 2**: Immediate hard-stop on **{xml_escape(str(top_over['category']))}** spending. You are currently **{top_over['health']-100:.0f}%** over the established limit.")
        else:
            mandates.append("**MANDATE 2**: Maintain current budget compliance. Your 'Budget Health' score is exemplary across all segments.")

        # Compile Output in AI Format
        pseudo_ai = f"""
## Clinical Executive Assessment
{assessment}

## Strategic Cash Flow Audit
{audit_text}

## Category Concentration Risk Profile
{risk_text}

## Projected 12-Month Trajectory
{trajectory_text}

## Benchmark Variance Analysis
{benchmark_text}

## Master Financial Directives
{chr(10).join(f"- {m}" for m in mandates)}

*Note: This analysis is generated via established financial heuristics and real-time data audit.*
"""
        return pseudo_ai

    def generate_fallback_insight(self, income, expense, savings, cats, budgets, include_executive=True):
        """Generates a comprehensive rule-based financial analysis with complete transaction details."""
        rate = (savings / income * 100) if income > 0 else 0
        over_budget = []  # Initialize to prevent NameError
        
        # Financial Health Assessment
        if rate > 30:
            status = "Excellent"
            health_desc = "Your financial position is exceptionally strong with a high savings rate."
        elif rate > 20:
            status = "Healthy"
            health_desc = "You maintain a solid financial foundation with consistent savings."
        elif rate > 10:
            status = "Stable"
            health_desc = "Your finances are balanced, though there's room for improvement."
        elif rate > 0:
            status = "Cautionary"
            health_desc = "You're saving, but your margin is thin. Consider expense optimization."
        else:
            status = "Critical"
            health_desc = "Your expenses exceed income. Immediate action is required."
        
        # Build comprehensive summary with proper formatting
        summary = ""
        
        if include_executive:
            summary += f"<b>Financial Health Status: {xml_escape(status)}</b>\n\n"
            summary += f"<b>Executive Overview:</b>\n"
            summary += f"{xml_escape(health_desc)} During this period, you generated <b>${income:,.2f}</b> in total income and spent <b>${expense:,.2f}</b>, resulting in net savings of <b>${savings:,.2f}</b> ({rate:.1f}% savings rate).\n\n"
        
        # Complete Expense Breakdown
        if cats:
            summary += "<b>Complete Expense Breakdown (All Categories):</b>\n"
            for i, (cat, amt) in enumerate(cats, 1):
                pct = (amt / expense * 100) if expense > 0 else 0
                safe_cat = xml_escape(str(cat))
                summary += f"  {i}. <b>{safe_cat}</b>: ${amt:,.2f} ({pct:.1f}% of total)\n"
            summary += f"\n<b>Total Expenses:</b> ${expense:,.2f}\n\n"
        
        # Cash Flow Analysis
        summary += "<b>Cash Flow Analysis:</b>\n"
        if savings > 0:
            summary += f"• Your positive cash flow of <b>${savings:,.2f}</b> demonstrates financial discipline.\n"
            if rate > 20:
                summary += "• This strong savings rate positions you well for long-term wealth building.\n"
                summary += "• Consider allocating surplus funds to investment vehicles or emergency reserves.\n\n"
            else:
                summary += "• While positive, increasing your savings rate to 20%+ would strengthen your financial position.\n\n"
        else:
            deficit = abs(savings)
            summary += f"• You're operating at a <b>deficit of ${deficit:,.2f}</b>.\n"
            summary += "• This unsustainable pattern requires immediate corrective action.\n"
            summary += "• Review discretionary spending and identify areas for reduction.\n\n"
        
        # Spending Pattern Analysis
        if cats and len(cats) > 0:
            summary += "<b>Spending Pattern Analysis:</b>\n"
            top_cat = cats[0]
            top_cat_pct = (top_cat[1] / expense * 100) if expense > 0 else 0
            
            safe_top_cat = xml_escape(str(top_cat[0]))
            summary += f"• Your largest expense category is <b>{safe_top_cat}</b>, accounting for ${top_cat[1]:,.2f} ({top_cat_pct:.1f}% of total spending).\n"
            
            if len(cats) >= 3:
                summary += f"• Top 3 categories represent ${sum(c[1] for c in cats[:3]):,.2f} ({sum((c[1]/expense*100) if expense > 0 else 0 for c in cats[:3]):.1f}% of spending).\n"
            
            # Category-specific insights
            if top_cat_pct > 40:
                summary += f"• <b>Alert:</b> {top_cat[0]} represents an unusually high proportion of your spending.\n"
                summary += "• Consider whether this allocation aligns with your financial priorities.\n\n"
            elif top_cat_pct > 30:
                summary += f"• The concentration in {top_cat[0]} is significant. Monitor this category closely.\n\n"
            else:
                summary += "• Your spending is reasonably diversified across categories.\n\n"
        
        # Budget Performance Analysis
        if budgets:
            over_budget = [b for b in budgets if b['health'] > 100]
            under_budget = [b for b in budgets if b['health'] <= 80]
            on_track = [b for b in budgets if 80 < b['health'] <= 100]
            
            summary += "<b>Budget Performance Analysis:</b>\n"
            
            if over_budget:
                summary += f"• <b>Overspending Alert:</b> You exceeded budgets in {len(over_budget)} {'category' if len(over_budget) == 1 else 'categories'}:\n"
                for b in over_budget:
                    safe_cat = xml_escape(str(b['category']))
                    summary += f"  - {safe_cat}: ${b['spent']:,.2f} / ${b['limit']:,.2f} ({b['health']:.0f}% utilized)\n"
                summary += "\n"
            
            if on_track:
                summary += f"• <b>On Track:</b> {len(on_track)} {'category is' if len(on_track) == 1 else 'categories are'} within budget limits.\n"
            
            if under_budget:
                summary += f"• <b>Under Budget:</b> {len(under_budget)} {'category shows' if len(under_budget) == 1 else 'categories show'} strong spending discipline.\n"
            
            summary += "\n"
        
        # Financial Metrics Summary
        summary += "<b>Key Financial Metrics:</b>\n"
        summary += f"• Total Income: <b>${income:,.2f}</b>\n"
        summary += f"• Total Expenses: <b>${expense:,.2f}</b>\n"
        summary += f"• Net Savings: <b>${savings:,.2f}</b>\n"
        summary += f"• Savings Rate: <b>{rate:.1f}%</b>\n"
        summary += f"• Expense Ratio: <b>{(expense/income*100) if income > 0 else 0:.1f}%</b>\n"
        
        # Calculate daily avg for fallback summary if not passed
        daily_avg_val = expense / 30
        summary += f"• Daily Average Spending (30-day base): <b>${daily_avg_val:,.2f}</b>\n"
        summary += "\n"
        
        # Closing statement
        summary += f"<b>Conclusion:</b> Your current financial trajectory is <b>{status.lower()}</b>. "
        if rate > 15:
            summary += "Maintain your disciplined approach while exploring growth opportunities."
        else:
            summary += "Focus on the recommendations above to strengthen your financial position."
        
        return summary


    def generate_ultimate_pdf_bytes(self, data, budgets, ai_text_precalculated=None, forecast_data=None):
        start_time = time.time()
            
        import io
        buffer = io.BytesIO()

        buffer = io.BytesIO()
        doc = SimpleDocTemplate(buffer, pagesize=letter, rightMargin=40, leftMargin=40, topMargin=40, bottomMargin=40)
        elements = []
        styles = getSampleStyleSheet()
        
        # --- STYLES ---
        title_style = ParagraphStyle('UltimateTitle', parent=styles['Title'], fontSize=32, spaceAfter=20, textColor=colors.HexColor('#1e293b'), alignment=TA_CENTER)
        subtitle_style = ParagraphStyle('UltimateSubtitle', parent=styles['Normal'], fontSize=12, spaceAfter=40, textColor=colors.HexColor('#64748b'), alignment=TA_CENTER)
        h1_style = ParagraphStyle('UltimateH1', parent=styles['Heading1'], fontSize=18, spaceBefore=20, spaceAfter=10, textColor=colors.HexColor('#334155'))
        
        # New "Big Card" Styles
        metric_label_style = ParagraphStyle('MetricLabel', parent=styles['Normal'], fontSize=12, textColor=colors.HexColor('#64748b'), alignment=TA_CENTER)
        metric_value_int_style = ParagraphStyle('MetricValueBig', parent=styles['Normal'], fontSize=24, fontName='Helvetica-Bold', textColor=colors.HexColor('#1e293b'), alignment=TA_CENTER)
        
        # --- DATA ANALYSIS (Pre-Calculation) ---
        total_income = sum(r['Amount'] for r in data if r['Amount'] > 0)
        total_expense = sum(abs(r['Amount']) for r in data if r['Amount'] < 0)
        net_savings = total_income - total_expense
        savings_rate = (net_savings / total_income * 100) if total_income > 0 else 0
        
        # Calculate Lifetime Daily Average (matching Dashboard)
        all_dates = [r.get('DateObj') for r in data if r.get('DateObj')]
        if all_dates:
            total_days = (max(all_dates) - min(all_dates)).days + 1
            if total_days < 1: total_days = 1
            lifetime_daily_avg = total_expense / total_days
        else:
            lifetime_daily_avg = 0

        # Expense Ratio (Total Exp / Total Inc %)
        expense_ratio = (total_expense / total_income * 100) if total_income > 0 else 100 if total_expense > 0 else 0

        # Category Analysis
        cat_map = {}
        inc_cat_map = {}
        
        for r in data:
            if r['Amount'] < 0:
                cat = r['Category']
                cat_map[cat] = cat_map.get(cat, 0) + abs(r['Amount'])
            else:
                cat = r['Category'] or 'Income'
                inc_cat_map[cat] = inc_cat_map.get(cat, 0) + r['Amount']
        
        sorted_cats_all = sorted(cat_map.items(), key=lambda x: x[1], reverse=True) # ALL categories
        sorted_inc_cats_all = sorted(inc_cat_map.items(), key=lambda x: x[1], reverse=True)
        sorted_cats = sorted_cats_all[:5] # Top 5 for charts
        
        # Budget Analysis
        budget_health = []
        for b in budgets:
            category = b.get('category', 'Uncategorized')
            limit = float(b.get('amount', 0))
            spent = cat_map.get(category, 0)
            remaining = limit - spent
            health = (spent / limit * 100) if limit > 0 else 0
            budget_health.append({
                'category': category,
                'limit': limit,
                'spent': spent,
                'remaining': remaining,
                'health': health
            })
        
        # Monthly Trends
        monthly_map = {}
        for r in data:
            # r['Date'] is YYYY-MM-DD
            date_str = str(r.get('Date', ''))
            month = date_str[:7] # YYYY-MM
            if month not in monthly_map: monthly_map[month] = {'income': 0, 'expense': 0}
            if r['Amount'] > 0:
                monthly_map[month]['income'] += r['Amount']
            else:
                monthly_map[month]['expense'] += abs(r['Amount'])
        
        sorted_months = sorted(list(monthly_map.keys()), reverse=True)[:12] # Last 12 months
        
        # Highlights Logic
        max_txn = max(data, key=lambda x: abs(x['Amount'])) if data else None
        max_txn_desc = f"{max_txn['Title']} (${abs(max_txn['Amount']):,.2f})" if max_txn else "N/A"
        top_cat_name = sorted_cats[0][0] if sorted_cats else "N/A"
        
        # --- AI INSIGHT (Used Precalculated) ---
        if isinstance(ai_text_precalculated, dict):
            ai_text = ai_text_precalculated.get('full_text', '')
        else:
            ai_text = ai_text_precalculated
            
        # --- CHART GENERATION (Parallelized - OO API) ---
        
        def get_chart_image(chart_id, data_package, width=5, height=3):
            """Generates a single chart using thread-safe OO API."""
            try:
                # Optimized DPI for speed vs quality
                target_dpi = 100 
                fig = Figure(figsize=(width, height), dpi=target_dpi)
                canvas = FigureCanvasAgg(fig)
                ax = fig.add_subplot(111)
                
                # Render logic based on chart_id
                if chart_id == 'cashflow':
                    inc, exp = data_package
                    if inc == 0 and exp == 0: return None
                    ax.pie([inc, exp], labels=['Income', 'Expenses'], autopct='%1.1f%%', colors=['#4ade80', '#f87171'], startangle=90)
                    ax.set_title('1. Cash Flow Overview')
                
                elif chart_id == 'top_expenses':
                    cats_data = data_package
                    if not cats_data: return None
                    names = [k[:12] for k,v in cats_data]
                    vals = [v for k,v in cats_data]
                    ax.bar(names, vals, color='#818cf8', alpha=0.8)
                    ax.set_title('2. Top 5 Major Expenses')
                    ax.grid(axis='y', linestyle='--', alpha=0.3)

                elif chart_id == 'income_trend':
                    m_data = data_package
                    if not m_data: return None
                    m_asc = sorted(m_data.keys())[-12:]
                    x = [m[5:] for m in m_asc]
                    y = [m_data[m]['income'] for m in m_asc]
                    ax.plot(x, y, marker='o', color='#10b981')
                    ax.fill_between(x, y, color='#10b981', alpha=0.1)
                    ax.set_title('3. Monthly Income Trend')

                elif chart_id == 'expense_trend':
                    m_data = data_package
                    if not m_data: return None
                    m_asc = sorted(m_data.keys())[-12:]
                    x = [m[5:] for m in m_asc]
                    y = [m_data[m]['expense'] for m in m_asc]
                    ax.plot(x, y, marker='o', color='#ef4444')
                    ax.fill_between(x, y, color='#ef4444', alpha=0.1)
                    ax.set_title('4. Monthly Expense Trend')

                elif chart_id == 'net_savings':
                    m_data = data_package
                    if not m_data: return None
                    m_asc = sorted(m_data.keys())[-12:]
                    x = [m[5:] for m in m_asc]
                    y = [m_data[m]['income'] - m_data[m]['expense'] for m in m_asc]
                    colors_net = ['#10b981' if v >= 0 else '#ef4444' for v in y]
                    ax.bar(x, y, color=colors_net)
                    ax.axhline(0, color='black', linewidth=0.5)
                    ax.set_title('5. Net Savings Analysis')

                elif chart_id == 'all_categories':
                    cats_all = data_package
                    if not cats_all: return None
                    names = [k for k,v in cats_all]
                    vals = [v for k,v in cats_all]
                    y_pos = range(len(names))
                    ax.barh(y_pos, vals, color='#8b5cf6')
                    ax.set_yticks(y_pos)
                    ax.set_yticklabels(names)
                    ax.invert_yaxis()
                    ax.set_title(f'6. All Spending by Category')

                elif chart_id == 'complete_dist':
                    cat_map_data = data_package
                    if not cat_map_data: return None
                    sorted_all = sorted(cat_map_data.items(), key=lambda x: x[1], reverse=True)
                    labels = [k[:20] for k, v in sorted_all]
                    values = [v for k, v in sorted_all]
                    pie_colors = cm.Set3(range(len(labels)))
                    ax.pie(values, labels=labels, autopct='%1.1f%%', startangle=90, colors=pie_colors, textprops={'fontsize': 8})
                    ax.set_title(f'7. Complete Spending Distribution')

                elif chart_id == 'savings_breakdown':
                    m_data, s_months = data_package
                    if not s_months or len(s_months) < 2: return None
                    months_display = [m[5:] for m in s_months[::-1]]
                    savings_vals = [(m_data[m]['income'] - m_data[m]['expense']) for m in s_months[::-1]]
                    bar_colors = ['#10b981' if s >= 0 else '#ef4444' for s in savings_vals]
                    ax.bar(months_display, savings_vals, color=bar_colors, alpha=0.8)
                    ax.axhline(y=0, color='black', linestyle='-', linewidth=0.8)
                    ax.set_title('8. Monthly Savings Breakdown')

                elif chart_id == 'dow_spend':
                    dow_avgs = data_package
                    if not dow_avgs or sum(dow_avgs.values()) == 0: return None
                    dow_map = {0: 'Mon', 1: 'Tue', 2: 'Wed', 3: 'Thu', 4: 'Fri', 5: 'Sat', 6: 'Sun'}
                    days = [dow_map[i] for i in range(7)]
                    vals = [dow_avgs[i] for i in range(7)]
                    colors_dow = ['#3b82f6' if i < 5 else '#f59e0b' for i in range(7)]
                    ax.bar(days, vals, color=colors_dow, alpha=0.8)
                    ax.set_title('9. Average Spending by Day of Week')

                elif chart_id == 'cat_growth':
                    trend_map, m_trend, t_5 = data_package
                    if not trend_map: return None
                    months_disp = [m[5:] for m in m_trend]
                    for cat in t_5:
                        vals = [trend_map[m][cat] for m in m_trend]
                        ax.plot(months_disp, vals, marker='o', label=cat[:10])
                    ax.set_title('10. Top Category Trends')
                    ax.legend(loc='best', fontsize='x-small')

                elif chart_id == 'daily_spending':
                    trend_data = data_package
                    if not trend_data: return None
                    
                    # Filter days with spending to avoid flat line at start
                    active_trend = [d for d in trend_data if d['total_amount'] > 0]
                    if not active_trend: return None
                    
                    dates = [d['date'] for d in active_trend]
                    amounts = [d['total_amount'] for d in active_trend]
                    mov_avg = [d['moving_avg_30d'] for d in active_trend]
                    
                    # Plot daily bars
                    ax.bar(dates, amounts, color='#6366f1', alpha=0.3, label='Daily Spend')
                    # Plot moving average line
                    ax.plot(dates, mov_avg, color='#ec4899', linewidth=2, label='30-Day Trend')
                    
                    ax.set_title('11. Daily Spending Trend & 30-Day Average')
                    # Thin out X-axis labels if too many
                    n = len(dates)
                    if n > 15:
                        step = n // 10
                        ax.set_xticks(range(0, n, step))
                        ax.set_xticklabels([dates[i] for i in range(0, n, step)], rotation=45, fontsize=8)
                    else:
                        ax.set_xticklabels(dates, rotation=45, fontsize=8)
                    ax.legend(loc='upper right', fontsize='x-small')

                buf = io.BytesIO()
                fig.savefig(buf, format='png', bbox_inches='tight', transparent=True)
                buf.seek(0)
                return Image(buf, width=width*inch, height=height*inch)
            except Exception as e:
                print(f"ERROR generating chart {chart_id}: {e}")
                return None

        # Pre-calculate Day of Week averages for parallel worker
        dow_totals = {i: 0 for i in range(7)}
        dow_counts = {i: 0 for i in range(7)}
        import datetime
        from datetime import datetime
        for r in data:
            if r['Amount'] < 0:
                try:
                    dt_obj = datetime.strptime(r['Date'], '%Y-%m-%d')
                    dow = dt_obj.weekday()
                    dow_totals[dow] += abs(r['Amount'])
                    dow_counts[dow] += 1
                except: pass
        dow_avgs = {i: (dow_totals[i]/dow_counts[i] if dow_counts[i] > 0 else 0) for i in range(7)}

        # Pre-calculate Category Trends for parallel worker
        top_5_cats = [c[0] for c in sorted_cats[:5]]
        m_trend = sorted(monthly_map.keys())[-6:]
        trend_payload = {m: {c: 0 for c in top_5_cats} for m in m_trend}
        for r in data:
            if r['Amount'] < 0:
                date_str = str(r.get('Date', ''))
                mk = date_str[:7]
                if mk in trend_payload:
                    c = r['Category']
                    if c in trend_payload[mk]: trend_payload[mk][c] += abs(r['Amount'])

        # Pre-calculate Daily Spending Trend for parallel worker
        expense_dates = [datetime.strptime(r['Date'], '%Y-%m-%d') for r in data if r['Amount'] < 0]
        if expense_dates:
            trend_anchor = max(expense_dates)
            trend_start = min(expense_dates)
            total_days = (trend_anchor - trend_start).days + 1
            if total_days < 30: total_days = 30
            
            daily_map = defaultdict(float)
            for r in data:
                if r['Amount'] < 0:
                    daily_map[r['Date']] += abs(r['Amount'])
            
            daily_trend_payload = []
            for i in range(total_days):
                curr_date = trend_start + timedelta(days=i)
                d_str = curr_date.strftime('%Y-%m-%d')
                amt = daily_map.get(d_str, 0)
                daily_trend_payload.append({
                    'date': curr_date.strftime('%b %d'),
                    'total_amount': amt
                })
            
            # Add moving average
            for i in range(len(daily_trend_payload)):
                start_window = max(0, i - 29)
                window = daily_trend_payload[start_window : i + 1]
                avg = sum(d['total_amount'] for d in window) / len(window)
                daily_trend_payload[i]['moving_avg_30d'] = round(avg, 2)
        else:
            daily_trend_payload = []

        # Build Chart Payload
        chart_tasks = [
            ('cashflow', (total_income, total_expense)),
            ('top_expenses', sorted_cats),
            ('income_trend', monthly_map),
            ('expense_trend', monthly_map),
            ('net_savings', monthly_map),
            ('all_categories', sorted_cats_all, 7, max(4, len(sorted_cats_all)*0.4)),
            ('complete_dist', cat_map, 8, 6),
            ('savings_breakdown', (monthly_map, sorted_months), 7, 3.5),
            ('dow_spend', dow_avgs, 7, 3),
            ('cat_growth', (trend_payload, m_trend, top_5_cats), 7, 3.5),
            ('daily_spending', daily_trend_payload, 7, 3.5)
        ]

        print(f"DEBUG: Generating {len(chart_tasks)} charts in parallel...")
        charts = {}
        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
            future_to_id = {}
            for task in chart_tasks:
                cid = task[0]
                pkg = task[1]
                w = task[2] if len(task) > 2 else 5
                h = task[3] if len(task) > 3 else 3
                future_to_id[executor.submit(get_chart_image, cid, pkg, w, h)] = cid
            
            for future in concurrent.futures.as_completed(future_to_id):
                tid = future_to_id[future]
                charts[tid] = future.result()

        chart1 = charts.get('cashflow')
        chart2 = charts.get('top_expenses')
        chart3 = charts.get('income_trend')
        chart4 = charts.get('expense_trend')
        chart5 = charts.get('net_savings')
        chart6 = charts.get('all_categories')
        chart7 = charts.get('complete_dist')
        chart8 = charts.get('savings_breakdown')
        chart9 = charts.get('dow_spend')
        chart10 = charts.get('cat_growth')
        chart11 = charts.get('daily_spending')

        # Clear memory before PDF composition
        gc.collect()

        # --- COVER PAGE ---
        elements.append(Spacer(1, 0.5*inch))
        elements.append(Paragraph("Full Financial Report", title_style))
        elements.append(Paragraph(f"Generated on {datetime.now().strftime('%B %d, %Y at %I:%M %p')}", subtitle_style))
        elements.append(Spacer(1, 0.2*inch))
        
        # Row 1: The Big Three
        m_inc = Paragraph(f"<font color='#16a34a'>+ ${total_income:,.0f}</font>", metric_value_int_style)
        m_exp = Paragraph(f"<font color='#ef4444'>- ${total_expense:,.0f}</font>", metric_value_int_style)
        m_net = Paragraph(f"<font color='#3b82f6'>${net_savings:,.0f}</font>", metric_value_int_style)
        
        big_metrics_data = [
            [Paragraph("TOTAL INCOME", metric_label_style), Paragraph("TOTAL EXPENSES", metric_label_style), Paragraph("NET SAVINGS", metric_label_style)],
            [m_inc, m_exp, m_net]
        ]
        
        t_big = Table(big_metrics_data, colWidths=[2.3*inch, 2.3*inch, 2.3*inch])
        t_big.setStyle(TableStyle([
            ('BACKGROUND', (0,0), (-1,-1), colors.HexColor('#f8fafc')),
            ('ALIGN', (0,0), (-1,-1), 'CENTER'),
            ('VALIGN', (0,0), (-1,-1), 'MIDDLE'),
            ('BOX', (0,0), (0,-1), 1, colors.HexColor('#bbf7d0')), 
            ('BOX', (1,0), (1,-1), 1, colors.HexColor('#fecaca')),
            ('BOX', (2,0), (2,-1), 1, colors.HexColor('#bfdbfe')),
            ('TOPPADDING', (0,0), (-1,-1), 15),
            ('BOTTOMPADDING', (0,0), (-1,-1), 15),
            ('GRID', (0,0), (-1,-1), 0.5, colors.HexColor('#e2e8f0'))
        ]))
        elements.append(t_big)
        elements.append(Spacer(1, 0.3*inch))
        
        # Key Highlights / Secondary Metrics
        daily_avg = total_expense / 30 
        
        highlights_data = [
            [Paragraph("<b>Savings Rate</b>", styles['Normal']), Paragraph(f"{savings_rate:.1f}%", styles['Normal'])],
            [Paragraph("<b>Daily Avg Spend</b>", styles['Normal']), Paragraph(f"${lifetime_daily_avg:,.2f}", styles['Normal'])],
            [Paragraph("<b>Expense Ratio</b>", styles['Normal']), Paragraph(f"{expense_ratio:.1f}%", styles['Normal'])],
            [Paragraph("<b>Top Spending Category</b>", styles['Normal']), Paragraph(xml_escape(str(top_cat_name)), styles['Normal'])],
        ]
        
        t_high = Table(highlights_data, colWidths=[3*inch, 3*inch], hAlign='CENTER')
        t_high.setStyle(TableStyle([
             ('BACKGROUND', (0,0), (-1,-1), colors.HexColor('#f1f5f9')),
             ('GRID', (0,0), (-1,-1), 0.5, colors.white),
             ('PADDING', (0,0), (-1,-1), 8),
        ]))
        
        elements.append(Paragraph("Key Highlights", h1_style))
        elements.append(t_high)
        elements.append(Spacer(1, 0.5*inch))

        elements.append(Paragraph("Executive Financial Analysis", h1_style))
        
        fallback_text = self.generate_fallback_insight(total_income, total_expense, net_savings, sorted_cats_all, budget_health)

        final_analysis_text = ""
        if ai_text and len(ai_text.strip()) > 50:
            # User Constraint: Don't add detailed breakdown if AI content is present
            final_analysis_text = ai_text
        else:
            # User Constraint: If all 3 AI models fail, use fallback but omit the 1st section (Executive Overview)
            final_analysis_text = self.generate_fallback_insight(total_income, total_expense, net_savings, sorted_cats_all, budget_health, include_executive=False)

        final_analysis_text = final_analysis_text.replace("<br/>", "\n")
        formatted_lines = self.format_text_for_pdf(final_analysis_text)
        
        for item in formatted_lines:
            text = item['text']
            is_header = item['is_header']
            if is_header:
                elements.append(Spacer(1, 8))
                elements.append(Paragraph(text, styles['Normal'])) 
                elements.append(Spacer(1, 4))
            else:
                if isinstance(text, str) and text.startswith('&bullet;'):
                    p = Paragraph(text, ParagraphStyle('Bullet', parent=styles['Normal'], leftIndent=15))
                    elements.append(p)
                else:
                    elements.append(Paragraph(text, styles['Normal']))
                    elements.append(Spacer(1, 4))
        
        # --- NEW: MASTER STRATEGIC FINANCIAL REPORT (Extreme Detail) ---
        elements.append(Spacer(1, 0.3*inch))
        elements.append(Paragraph("Master Strategic Financial Intelligence", h1_style))
        master_report_text = self.generate_master_strategic_report(total_income, total_expense, net_savings, sorted_cats_all, budget_health, sorted_inc_cats_all)
        
        # Add Strategic Benchmarking (50/30/20)
        fixed_cats = ['Rent', 'Mortgage', 'Utilities', 'Insurance', 'Taxes', 'Bills', 'Loan', 'EMI', 'Subscription']
        fixed_sum = sum(amt for cat, amt in sorted_cats_all if any(f.lower() in cat.lower() for f in fixed_cats))
        benchmarking_text = self.generate_strategic_benchmarking_audit(total_income, total_expense, fixed_sum)
        master_report_text += "\n\n" + benchmarking_text

        # Add Wealth Projections
        wealth_text = self.generate_wealth_efficiency_projections(net_savings)
        master_report_text += "\n\n" + wealth_text

        # Add Strategic Mandates
        mandate_text = self.generate_capital_allocation_mandates(savings_rate)
        master_report_text += "\n\n" + mandate_text

        master_formatted = self.format_text_for_pdf(master_report_text)
        for item in master_formatted:
            text = item['text']
            if item['is_header']:
                elements.append(Spacer(1, 12))
                elements.append(Paragraph(f"<b>{text}</b>", styles['Normal']))
                elements.append(Spacer(1, 6))
            else:
                if text.startswith('&bullet;'):
                    elements.append(Paragraph(text, ParagraphStyle('Bullet', parent=styles['Normal'], leftIndent=15, spaceAfter=4)))
                else:
                    elements.append(Paragraph(text, styles['Normal']))
                    elements.append(Spacer(1, 6))
        
        elements.append(PageBreak())

        # --- ADDITIONAL ANALYTICAL MODULES (40+ Page Goal) ---
        
        # 1. ANALYTICAL ENCYCLOPEDIA
        encyclopedia_text = self.generate_analytical_encyclopedia(sorted_cats_all, sorted_inc_cats_all)
        encyc_formatted = self.format_text_for_pdf(encyclopedia_text)
        for item in encyc_formatted:
            if item['is_header']:
                elements.append(Spacer(1, 15))
                elements.append(Paragraph(f"<b>{item['text']}</b>", styles['Normal']))
                elements.append(Spacer(1, 8))
            else:
                elements.append(Paragraph(item['text'], styles['Normal']))
                elements.append(Spacer(1, 6))

        # 2. BEHAVIORAL RISK AUDIT
        # Remove automatic PageBreak to prevent near-empty pages
        elements.append(Spacer(1, 20))
        risk_text = self.generate_behavioral_risk_audit(data, savings_rate, net_savings)
        risk_formatted = self.format_text_for_pdf(risk_text)
        for item in risk_formatted:
            if item['is_header']:
                elements.append(Spacer(1, 15))
                elements.append(Paragraph(f"<b>{item['text']}</b>", styles['Normal']))
                elements.append(Spacer(1, 8))
            else:
                elements.append(Paragraph(item['text'], styles['Normal']))
                elements.append(Spacer(1, 10))

        # 3. LIQUIDITY STRESS TEST
        elements.append(Spacer(1, 25))
        stress_text = self.generate_liquidity_stress_model(total_income, total_expense, net_savings)
        stress_formatted = self.format_text_for_pdf(stress_text)
        for item in stress_formatted:
            if item['is_header']:
                elements.append(Spacer(1, 20))
                elements.append(Paragraph(f"<b>{item['text']}</b>", styles['Normal']))
                elements.append(Spacer(1, 10))
            else:
                elements.append(Paragraph(item['text'], styles['Normal']))
                elements.append(Spacer(1, 12))

        # Only break once before the visuals
        elements.append(PageBreak())

        # --- PAGE 2: COMPOSITION & TRENDS ---
        elements.append(Paragraph("1. Composition & Categories", h1_style))
        if chart1:
            elements.append(chart1)
            elements.append(Spacer(1, 15))
        if chart2:
            elements.append(chart2)
            elements.append(Spacer(1, 15))
        if chart3:
            elements.append(chart3)
        elements.append(PageBreak())

        # --- PAGE 3: PERFORMANCE TRENDS ---
        elements.append(Paragraph("2. Financial Performance Trends", h1_style))
        if chart4: 
            elements.append(chart4)
            elements.append(Spacer(1, 10))
        if chart5: 
            elements.append(chart5)
            elements.append(Spacer(1, 10))
        if chart6: 
            elements.append(chart6)
        elements.append(PageBreak())

        # --- PAGE 4: DETAILED ANALYSIS ---
        elements.append(Paragraph("3. Comprehensive Category Analysis", h1_style))
        if chart6: 
            elements.append(Paragraph("All Categories Breakdown", styles['Heading2']))
            elements.append(chart6)
            elements.append(Spacer(1, 20))
        if chart7: 
            elements.append(chart7)
            elements.append(Spacer(1, 20))
        
        elements.append(Paragraph("Category Performance (Detailed Breakdown)", styles['Heading2']))
        cat_rows = [["Category", "Amount", "% of Total"]]
        for c, amt in sorted_cats_all[:15]:
            pct = (amt / total_expense * 100) if total_expense > 0 else 0
            safe_cat = xml_escape(str(c))
            cat_rows.append([safe_cat, f"${amt:,.0f}", f"{pct:.1f}%"])
        
        t_cats = Table(cat_rows, colWidths=[3*inch, 2*inch, 2*inch])
        t_cats.setStyle(TableStyle([
            ('BACKGROUND', (0,0), (-1,0), colors.HexColor('#475569')),
            ('TEXTCOLOR', (0,0), (-1,0), colors.white),
            ('GRID', (0,0), (-1,-1), 0.5, colors.HexColor('#e2e8f0')),
            ('ROWBACKGROUNDS', (0,1), (-1,-1), [colors.white, colors.HexColor('#f8fafc')])
        ]))
        elements.append(t_cats)
        elements.append(Spacer(1, 20))
        elements.append(PageBreak())

        # --- PAGE 5: ADVANCED INSIGHTS ---
        elements.append(Paragraph("4. Advanced Temporal & Behavioral Insights", h1_style))
        if chart8:
            elements.append(chart8)
            elements.append(Spacer(1, 15))
        if chart9:
            elements.append(chart9)
            elements.append(Spacer(1, 15))
        if chart10:
            elements.append(chart10)
            elements.append(Spacer(1, 20))
        
        if chart11:
            elements.append(Paragraph("Daily Variable Spending Trend", styles['Heading2']))
            elements.append(chart11)
            elements.append(Spacer(1, 20))

        # --- PAGE 5.5: FINANCIAL FORECASTING (AI-Powered) ---
        if forecast_data:
            elements.append(PageBreak())
            elements.append(Paragraph("4.5 Predictive Financial Intelligence (AI)", h1_style))
            elements.append(Paragraph("Our specialized algorithms have analyzed your historical spending velocity to engineer a 30-day projection.", styles['Normal']))
            elements.append(Spacer(1, 15))
            
            # Summary Metrics for Forecast
            total_predicted = sum(d['amount'] for d in forecast_data)
            avg_predicted = total_predicted / len(forecast_data) if forecast_data else 0
            
            forecast_metrics = [
                [Paragraph("30-DAY PREDICTED TOTAL", metric_label_style), Paragraph("DAILY PREDICTED AVG", metric_label_style)],
                [Paragraph(f"<font color='#3b82f6'>${total_predicted:,.2f}</font>", metric_value_int_style), 
                 Paragraph(f"<font color='#64748b'>${avg_predicted:,.2f}</font>", metric_value_int_style)]
            ]
            t_f_metrics = Table(forecast_metrics, colWidths=[3.5*inch, 3.5*inch])
            t_f_metrics.setStyle(TableStyle([
                ('BACKGROUND', (0,0), (-1,-1), colors.HexColor('#f8fafc')),
                ('BOX', (0,0), (-1,-1), 0.5, colors.HexColor('#e2e8f0')),
                ('VALIGN', (0,0), (-1,-1), 'MIDDLE'),
                ('TOPPADDING', (0,0), (-1,-1), 10),
                ('BOTTOMPADDING', (0,0), (-1,-1), 10),
            ]))
            elements.append(t_f_metrics)
            elements.append(Spacer(1, 20))
            
            elements.append(Paragraph("Predicted Daily High-Impact Windows", styles['Heading2']))
            f_rows = [["Date", "Predicted Amount", "Confidence Interval (Low-High Range)"]]
            # Show every day for maximum detail and page volume
            for i, d in enumerate(forecast_data):
                f_rows.append([
                    xml_escape(d['date']),
                    f"${d['amount']:,.2f}",
                    f"${d['low']:,.0f} - ${d['high']:,.0f}"
                ])
            
            t_f_detail = Table(f_rows, colWidths=[1.5*inch, 2*inch, 3.5*inch])
            t_f_detail.setStyle(TableStyle([
                ('BACKGROUND', (0,0), (-1,0), colors.HexColor('#1e293b')),
                ('TEXTCOLOR', (0,0), (-1,0), colors.white),
                ('GRID', (0,0), (-1,-1), 0.5, colors.HexColor('#e2e8f0')),
                ('ROWBACKGROUNDS', (0,1), (-1,-1), [colors.white, colors.HexColor('#f8fafc')]),
                ('FONTSIZE', (0,0), (-1,-1), 10),
                ('ALIGN', (1,0), (-1,-1), 'CENTER'),
            ]))
            elements.append(t_f_detail)
            elements.append(Spacer(1, 10))
            elements.append(Paragraph("<i>*Predictions are based on statistical regression and seasonal averaging. Accuracy increases with data density.</i>", 
                                     ParagraphStyle('FinePrint', parent=styles['Normal'], fontSize=8, textColor=colors.gray)))

        # --- PAGE 6: APPENDIX DATA ---
        elements.append(PageBreak())
        elements.append(Paragraph("5. Appendix: Detailed Data", h1_style))
        elements.append(Paragraph("Monthly Trends Data", styles['Heading2']))
        elements.append(Spacer(1, 10))
        
        trend_header = ["Month", "Income", "Expense", "Net"]
        trend_rows = [trend_header]
        for m in sorted_months:
            inc = monthly_map[m]['income']
            exp = monthly_map[m]['expense']
            net = inc - exp
            net_col = colors.green if net >= 0 else colors.red
            safe_month = xml_escape(str(m))
            trend_rows.append([
                safe_month,
                f"${inc:,.0f}",
                f"${exp:,.0f}",
                Paragraph(f"<font color='{net_col}'>${net:,.0f}</font>", styles['Normal'])
            ])
            
        t_trend = Table(trend_rows, colWidths=[2*inch, 1.5*inch, 1.5*inch, 1.5*inch])
        t_trend.setStyle(TableStyle([
            ('BACKGROUND', (0, 0), (-1, 0), colors.HexColor('#475569')),
            ('TEXTCOLOR', (0, 0), (-1, 0), colors.white),
            ('ALIGN', (1, 0), (-1, -1), 'RIGHT'),
            ('GRID', (0, 0), (-1, -1), 0.5, colors.HexColor('#e2e8f0')),
        ]))
        elements.append(t_trend)
        elements.append(Spacer(1, 20))

        # --- DAILY OPERATIONAL JOURNAL (Last 30 Days Only) ---
        elements.append(Paragraph("Daily Operational Journal (Last 30 Days)", styles['Heading2']))
        elements.append(Paragraph("Focused day-by-day record of recent operational outflows and liquidity events.", styles['Normal']))
        elements.append(Spacer(1, 10))
        
        # Calculate daily totals
        daily_map = {}
        for r in data:
            d_str = str(r.get('Date', ''))
            if d_str not in daily_map: daily_map[d_str] = 0
            if r['Amount'] < 0:
                daily_map[d_str] += abs(r['Amount'])
        
        # Limit to last 30 days
        import datetime as dt_mod
        thirty_days_ago = (dt_mod.datetime.now() - dt_mod.timedelta(days=30)).strftime('%Y-%m-%d')
        sorted_days = sorted([d for d in daily_map.keys() if d >= thirty_days_ago], reverse=True)
        
        journal_rows = [["Date", "Daily Net Outflow", "Status"]]
        for d_str in sorted_days:
            amt = daily_map[d_str]
            status = "CALM" if amt < 100 else "ELEVATED" if amt < 500 else "HIGH-IMPACT"
            journal_rows.append([xml_escape(d_str), f"${amt:,.2f}", status])
            
        if len(journal_rows) > 1:
            t_journal = Table(journal_rows, colWidths=[2*inch, 2.5*inch, 1.5*inch], repeatRows=1)
            t_journal.setStyle(TableStyle([
                ('BACKGROUND', (0,0), (-1,0), colors.HexColor('#334155')),
                ('TEXTCOLOR', (0,0), (-1,0), colors.white),
                ('GRID', (0,0), (-1,-1), 0.5, colors.HexColor('#e2e8f0')),
                ('ROWBACKGROUNDS', (0,1), (-1,-1), [colors.white, colors.HexColor('#f1f5f9')]),
                ('ALIGN', (1,0), (1,-1), 'RIGHT'),
            ]))
            elements.append(t_journal)
        else:
            elements.append(Paragraph("<i>No operational data identified within the last 30-day window.</i>", styles['Normal']))

        elements.append(PageBreak())
        elements.append(Paragraph("Full Transaction History", h1_style))
        elements.append(Paragraph("<i>(Showing last 200 transactions for audit verification.)</i>", styles['Normal']))
        elements.append(Spacer(1, 10))
        
        table_header = ["Date", "Type", "Category", "Title", "Amount"]
        table_rows = [table_header]
        for r in data[:200]: 
            date_str = str(r.get('Date', ''))
            type_str = str(r.get('Type', ''))
            title_text = xml_escape(str(r['Title'] or "No Title"))
            category_text = xml_escape(str(r['Category'] or "Uncategorized"))
            table_rows.append([
                date_str,
                type_str,
                category_text[:15],
                Paragraph(title_text[:60], styles['Normal']), 
                f"${r['Amount']:,.0f}"
            ])

        t_data = Table(table_rows, colWidths=[1.2*inch, 0.8*inch, 1.5*inch, 2.8*inch, 1.2*inch], repeatRows=1)
        t_data.setStyle(TableStyle([
            ('BACKGROUND', (0, 0), (-1, 0), colors.HexColor('#0f172a')),
            ('TEXTCOLOR', (0, 0), (-1, 0), colors.white),
            ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'),
            ('ALIGN', (0, 0), (-1, 0), 'CENTER'),
            ('BOTTOMPADDING', (0, 0), (-1, 0), 10),
            ('BACKGROUND', (0, 1), (-1, -1), colors.white),
            ('GRID', (0, 0), (-1, -1), 0.5, colors.HexColor('#e2e8f0')),
            ('ALIGN', (-1, 1), (-1, -1), 'RIGHT'),
            ('FONTSIZE', (0, 0), (-1, -1), 9),
            ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'),
        ]))
        
        for i, row in enumerate(data[:200]):
            bg_color = colors.HexColor('#f8fafc') if i % 2 == 0 else colors.white
            t_data.setStyle(TableStyle([('BACKGROUND', (0, i+1), (-1, i+1), bg_color)]))
            text_color = colors.HexColor('#ef4444') if row['Amount'] < 0 else colors.HexColor('#16a34a')
            t_data.setStyle(TableStyle([('TEXTCOLOR', (-1, i+1), (-1, i+1), text_color)]))

        elements.append(t_data)
        doc.build(elements)
        print(f"DEBUG: Total PDF generation took {time.time() - start_time:.2f} seconds.")
        return buffer.getvalue()

    def fetch_data(self, user_id, data_type):
        db = MongoDBClient.get_client()
        # user_id should already be a valid ObjectId (or whatever provided by get_user_db_id)
        user = db.users.find_one({'_id': user_id}, {'financial_data': 1})
        incomes = []
        expenses = []
        budgets = []
        
        if user and 'financial_data' in user:
             incomes = user['financial_data'].get('incomes', [])
             expenses = user['financial_data'].get('expenses', [])
             budgets = user['financial_data'].get('budgets', [])
        
        rows = []
        for i in incomes:
            date_val = i.get('date')
            try:
                if isinstance(date_val, str):
                    date_obj = datetime.strptime(date_val, '%Y-%m-%d')
                else:
                    date_obj = date_val
            except:
                date_obj = datetime.min
            
            rows.append({
                "Type": "Income", 
                "DateObj": date_obj,
                "Date": date_obj.strftime('%Y-%m-%d') if isinstance(date_obj, datetime) else str(date_val), 
                "Title": i.get('title'),
                "Amount": float(i.get('amount', 0)), 
                "Category": i.get('category', 'Income'), 
                "Details": i.get('description', '')
            })
            
        for e in expenses:
            date_val = e.get('date')
            try:
                if isinstance(date_val, str):
                    date_obj = datetime.strptime(date_val, '%Y-%m-%d')
                else:
                    date_obj = date_val
            except:
                date_obj = datetime.min

            rows.append({
                "Type": "Expense", 
                "DateObj": date_obj,
                "Date": date_obj.strftime('%Y-%m-%d') if isinstance(date_obj, datetime) else str(date_val), 
                "Title": e.get('title'),
                "Amount": -float(e.get('amount', 0)), 
                "Category": e.get('category', 'Uncategorized'), 
                "Details": e.get('description', '')
            })
            
        # Sort by Date Descending (Newest first)
        rows.sort(key=lambda x: x['DateObj'], reverse=True)
        
        return rows, budgets

    def head(self, request):
        """
        Quickly returns headers for PDF metadata without full generation.
        Prevents 'Broken pipe' on curl -I or similar HEAD checks.
        """
        filename = f"Ultimate_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.pdf"
        response = HttpResponse(content_type='application/pdf')
        response['Content-Disposition'] = f'attachment; filename="{filename}"'
        response['Content-Length'] = '0' # We don't know the exact size yet
        return response

    def post(self, request):
        """
        Initiates background PDF generation. Returns a task_id.
        """
        # --- CLEANUP OLD TASKS ---
        now = datetime.now()
        expired_tasks = [tid for tid, t in REPORT_TASKS.items() if (now - t['created_at']).total_seconds() > 3600]
        for tid in expired_tasks:
            del REPORT_TASKS[tid]
            
        user_id = request.user.id
        report_type = request.query_params.get('type', 'pdf')
        provider = request.data.get('provider')
        
        task_id = str(uuid.uuid4())
        REPORT_TASKS[task_id] = {
            'status': 'pending',
            'created_at': datetime.now(),
            'data': None,
            'type': report_type,
            'provider': provider
        }
        
        # Start background thread
        thread = threading.Thread(target=self.bg_generate_pdf_task, args=(task_id, request.user))
        thread.daemon = True
        thread.start()
        
        return Response({"task_id": task_id, "status": "pending"}, status=status.HTTP_202_ACCEPTED)

    def bg_generate_pdf_task(self, task_id, user):
        """Helper to run in thread"""
        try:
            mongo_id = get_user_db_id(user)
            data, budgets = self.fetch_data(mongo_id, 'all')
            provider = REPORT_TASKS[task_id].get('provider')
            pdf_bytes, ai_data = self.generate_pdf_bytes(data, budgets, mongo_id=mongo_id, provider=provider)
            REPORT_TASKS[task_id]['data'] = pdf_bytes
            REPORT_TASKS[task_id]['ai_insight'] = ai_data
            REPORT_TASKS[task_id]['status'] = 'completed'
        except Exception as e:
            print(f"Background PDF Generation Error: {str(e)}")
            REPORT_TASKS[task_id]['status'] = 'failed'
            REPORT_TASKS[task_id]['error'] = str(e)

    def get(self, request):
        """
        Handles both legacy synchronous generation and new background polling/downloading.
        """
        task_id = request.query_params.get('task_id')
        if task_id:
            task = REPORT_TASKS.get(task_id)
            if not task:
                return Response({"error": "Task not found"}, status=status.HTTP_404_NOT_FOUND)
            
            if task['status'] == 'completed':
                if request.query_params.get('download') == 'true':
                    response = HttpResponse(task['data'], content_type='application/pdf')
                    response['Content-Disposition'] = f'attachment; filename="pro_financial_report_{task_id[:8]}.pdf"'
                    return response
                return Response({"status": "completed", "ready": True})
            
            return Response({"status": task['status'], "ready": False, "error": task.get('error')})

        # Legacy Synchronous Path
        data_type = request.query_params.get('type', 'all') 
        try:
            mongo_id = get_user_db_id(request.user)
            data, budgets = self.fetch_data(mongo_id, data_type)
        except Exception as e:
            return Response({"error": f"Data fetch failed: {str(e)}"}, status=500)
        
        if data_type == 'csv':
            try:
                import csv
                output = io.StringIO()
                writer = csv.writer(output)
                writer.writerow(['Date', 'Type', 'Category', 'Title', 'Amount'])
                for row in data:
                    writer.writerow([row.get('Date', ''), row.get('Type', ''), row.get('Category', ''), row.get('Title', ''), row.get('Amount', 0)])
                output.seek(0)
                csv_content = output.getvalue()
                response = HttpResponse(csv_content, content_type='text/csv')
                response['Content-Disposition'] = f'attachment; filename="Transactions_Export_{datetime.now().strftime("%Y%m%d")}.csv"'
                return response
            except Exception as e:
                return Response({"error": f"CSV Generation failed: {str(e)}"}, status=500)
        
        filename = f"Ultimate_Report_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
        try:
            file_content, _ = self.generate_pdf_bytes(data, budgets, mongo_id)
            response = HttpResponse(file_content, content_type='application/pdf')
            response['Content-Disposition'] = f'attachment; filename="{filename}.pdf"'
            return response
        except Exception as e:
            return Response({"error": str(e)}, status=500)

    def generate_pdf_bytes(self, data, budgets, mongo_id=None, provider=None):
        data = self.enrich_data_for_pdf(data) # Ensure data is ready

        # --- PRE-CALCULATE METRICS FOR AI (Memory Isolation) ---
        # We process this HERE to run the AI before loading Matplotlib (which is heavy).
        try:
            total_income = sum(r['Amount'] for r in data if r['Amount'] > 0)
            total_expense = sum(abs(r['Amount']) for r in data if r['Amount'] < 0)
            net_savings = total_income - total_expense

            # --- PRE-CALCULATE ENRICHED DATA FOR AI ---
            # Monthly Trends (last 12 months)
            monthly_map = {}
            for r in data:
                date_str = str(r.get('Date', ''))
                month = date_str[:7] # YYYY-MM
                if month not in monthly_map: monthly_map[month] = {'income': 0, 'expense': 0}
                if r['Amount'] > 0:
                    monthly_map[month]['income'] += r['Amount']
                else:
                    monthly_map[month]['expense'] += abs(r['Amount'])
            
            # Largest Outlier
            max_txn = max(data, key=lambda x: abs(x['Amount'])) if data else None
            max_txn_desc = f"{max_txn['Title']} (${abs(max_txn['Amount']):,.2f})" if max_txn else "N/A"
            
            # Category Analysis (Expanded for AI)
            cat_map = {}
            for r in data:
                if r['Amount'] < 0:
                    cat = r['Category']
                    cat_map[cat] = cat_map.get(cat, 0) + abs(r['Amount'])
            sorted_cats_full = sorted(cat_map.items(), key=lambda x: x[1], reverse=True)

            # Budget Analysis (Full)
            budget_health_full = []
            for b in budgets:
                category = b.get('category', 'Uncategorized')
                limit = float(b.get('amount', 0))
                spent = cat_map.get(category, 0)
                budget_health_full.append({
                    'category': category, 'limit': limit, 'spent': spent
                })
            
            # --- RUN AI (Before Charts) ---
            print("INFO: Starting Enriched AI Generation (Isolated)...")
            ai_data = None
            if provider != "None" and (total_income > 0 or total_expense > 0):
                try:
                    ai_data = self.generate_ai_insight(
                        total_income, total_expense, net_savings, 
                        sorted_cats_full, budget_health_full,
                        monthly_trends=monthly_map,
                        max_txn_desc=max_txn_desc,
                        provider=provider
                    )
                except Exception as e:
                    print(f"AI Gen Failed (Isolated): {e}")
            
            # --- FETCH FORECAST DATA ---
            forecast_data = None
            if mongo_id:
                try:
                    print(f"INFO: Fetching forecast data for user {mongo_id}...")
                    forecast_data = get_forecast(mongo_id)
                except Exception as e:
                    print(f"Forecast Fetch Failed: {e}")

            # Force Memory Clear
            import gc
            gc.collect()
            print("INFO: AI & Forecast Done. Starting PDF Generation...")
            
            # --- LOW MEMORY OPTIMIZATION for Render ---
            if os.getenv('ENVIRONMENT') == 'production':
                 print("DEBUG: Production mode detected. applying aggressive memory settings.")
                 # Clear any heavy objects before Matplotlib
                 gc.collect()

        except Exception as e:
            print(f"Pre-calc error: {e}")
            ai_data = None
            forecast_data = None

        return self.generate_ultimate_pdf_bytes(data, budgets, ai_text_precalculated=ai_data, forecast_data=forecast_data), ai_data

class EmailReportView(ExportDataView):
    """
    Generates an AI-powered financial report and sends it to a specified email address.
    """
    permission_classes = [permissions.IsAuthenticated]

    def post(self, request):
        # --- CLEANUP OLD TASKS ---
        now = datetime.now()
        expired_tasks = [tid for tid, t in REPORT_TASKS.items() if (now - t['created_at']).total_seconds() > 3600]
        for tid in expired_tasks:
            del REPORT_TASKS[tid]
            
        email = request.data.get('email')
        provider = request.data.get('provider')
        if not email:
            return Response({"error": "Recipient email is required"}, status=400)

        import threading
        
        # Extract user info for the background thread (don't pass request object itself)
        # RESOLVE MONGO ID NOW to avoid int/ObjectId mismatch in thread
        mongo_id = get_user_db_id(request.user)
        username = request.user.username

        # Generate Task ID for polling
        task_id = str(uuid.uuid4())
        REPORT_TASKS[task_id] = {
            'status': 'pending',
            'created_at': datetime.now(),
            'type': 'email'
        }

        def background_email_task():
            try:
                print(f"DEBUG: [Thread] Starting background email report generation for user {username}")
                
                # 1. Fetch Data
                data, budgets = self.fetch_data(mongo_id, 'all')
                
                # 2. Generate PDF
                print("DEBUG: [Thread] Generating PDF Attachment...")
                # We pre-calculate AI insight inside generate_pdf_bytes or pass None
                pdf_bytes, ai_insight = self.generate_pdf_bytes(data, budgets, mongo_id=mongo_id, provider=provider)
                
                # Use AI insight summary or dynamic data summary for the email body
                if ai_insight and ai_insight.get('summary'):
                    email_body = ai_insight['summary']
                else:
                    total_inc = sum(r.get('Amount', 0) for r in data if r.get('Amount', 0) > 0)
                    total_exp = sum(abs(r.get('Amount', 0)) for r in data if r.get('Amount', 0) < 0)
                    net_sav = total_inc - total_exp
                    
                    email_body = f"""Hello {username},

We hope this email finds you well. 

Please find attached your comprehensive **Financial Intelligence Report** for {datetime.now().strftime('%B %Y')}. We have carefully compiled your recent financial data to help you track your progress and maintain control over your financial journey.

### 📊 Monthly Snapshot
- **Total Income Received:** ${total_inc:,.2f}
- **Total Expenses Incurred:** ${total_exp:,.2f}
- **Net Position (Savings):** ${net_sav:,.2f}

Your attached, encrypted PDF document contains an in-depth breakdown of your transactions, budget utilization heatmaps, and categorical analysis to provide full transparency into your monthly cash flow.

Thank you for choosing FinMK to manage your financial future.

Warm regards,
**The FinMK Team**"""

                # 3. Send Email
                subject = f"Your Financial Report - {datetime.now().strftime('%B %Y')}"
                success, message = send_financial_report_email(
                    receiver_email=email,
                    subject=subject,
                    body_text=email_body,
                    pdf_content=pdf_bytes,
                    pdf_filename=f"FinMK_Report_{datetime.now().strftime('%Y%m%d')}.pdf"
                )

                if success:
                    print(f"DEBUG: [Thread] Email report sent successfully to {email}")
                    REPORT_TASKS[task_id]['status'] = 'completed'
                else:
                    print(f"ERROR: [Thread] Failed to send email to {email}: {message}")
                    REPORT_TASKS[task_id]['status'] = 'failed'
                    REPORT_TASKS[task_id]['error'] = message

            except Exception as thread_e:
                print(f"CRITICAL ERROR in Email Task Thread: {thread_e}")
                traceback.print_exc()
                REPORT_TASKS[task_id]['status'] = 'failed'
                REPORT_TASKS[task_id]['error'] = str(thread_e)

        # Start the background thread
        thread = threading.Thread(target=background_email_task)
        thread.start()

        return Response({
            "message": "Report generation started in the background! You will receive an email shortly.",
            "status": "processing",
            "task_id": task_id
        })

    # Note: format_text_for_pdf, enrich_data_for_pdf, and generate_fallback_insight
    # are inherited from ExportDataView.