Update UI styling and message formatting improvements

check_chroma_data.py

@@ -0,0 +1,39 @@
+"""Quick script to check what's stored in ChromaDB metadata collection."""
+
+from services.chroma_service import ChromaService
+
+def check_metadata():
+    print("Connecting to ChromaDB...")
+    c = ChromaService()
+    
+    data = c.metadata_collection.get()
+    total = len(data['ids'])
+    print(f"Total entries in document_metadata: {total}")
+    
+    if total > 0:
+        print("\n--- Sample entry (first) ---")
+        meta = data['metadatas'][0]
+        for key, value in sorted(meta.items()):
+            print(f"  {key}: {value}")
+        
+        # Find Feb 2026 renewals
+        print("\n--- Entries with Feb 2026 renewal ---")
+        feb_count = 0
+        for meta in data['metadatas']:
+            rd = str(meta.get('renewal_date', ''))
+            ry = meta.get('renewal_year', 0)
+            # Check for Feb 2026
+            if ry == 2026 and '-02-' in rd:
+                feb_count += 1
+                print(f"  {meta.get('document_title')}: renewal_date={rd}, renewal_year={ry}")
+        
+        print(f"\nTotal Feb 2026 renewals found: {feb_count}")
+        
+        # Show all unique renewal years
+        years = set(meta.get('renewal_year', 0) for meta in data['metadatas'])
+        print(f"\nAll renewal years in data: {sorted(years)}")
+    else:
+        print("No data found!")
+
+if __name__ == "__main__":
+    check_metadata()

check_users.py

@@ -0,0 +1,30 @@
+"""Check what users exist in the system."""
+
+from services.chroma_service import ChromaService
+
+def check_users():
+    print("Connecting to ChromaDB...")
+    c = ChromaService()
+    
+    # Get all users
+    users = c.users_collection.get()
+    print(f"\nTotal users: {len(users['ids'])}")
+    
+    for i, uid in enumerate(users['ids']):
+        meta = users['metadatas'][i]
+        print(f"  User ID: {uid}")
+        print(f"    Username: {meta.get('username')}")
+        print(f"    Role: {meta.get('role')}")
+        print()
+    
+    # Check buckets for first user
+    if users['ids']:
+        first_user = users['ids'][0]
+        buckets = c.buckets_collection.get(where={"user_id": first_user})
+        print(f"\nBuckets for user {first_user}:")
+        for i, bid in enumerate(buckets['ids']):
+            print(f"  Bucket ID: {bid}")
+            print(f"    Name: {buckets['metadatas'][i].get('name')}")
+
+if __name__ == "__main__":
+    check_users()

clear_and_remigrate.py

@@ -0,0 +1,355 @@
+"""
+Script to clear document_metadata collection and re-run migration
+with a different user_id and no bucket_id
+"""
+
+import re
+import hashlib
+import time
+from services.chroma_service import ChromaService
+
+
+def parse_markdown_table(filepath: str) -> tuple[list[str], list[dict]]:
+    """
+    Parse the markdown table from table.md and extract all rows.
+    
+    Returns:
+        - headers: list of column names
+        - rows: list of dicts with column names as keys
+    """
+    with open(filepath, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    lines = content.strip().split('\n')
+    
+    # Parse header row (line 1)
+    header_line = lines[0]
+    # Extract column names from header
+    raw_headers = [h.strip().replace('\t', '').strip() for h in header_line.split('|')]
+    headers = [h for h in raw_headers if h and h != ':--------:']
+    
+    print(f"Found {len(headers)} columns in header:")
+    for i, h in enumerate(headers):
+        print(f"  {i+1}. {h}")
+    
+    rows = []
+    
+    # Parse data rows (skip header and separator lines)
+    for line_num, line in enumerate(lines[2:], start=3):
+        if not line.strip():
+            continue
+        
+        # Split by pipe and clean values
+        raw_cells = [c.strip().replace('\t', '').strip() for c in line.split('|')]
+        cells = [c for c in raw_cells if c != '']
+        
+        if not cells:
+            continue
+        
+        # Create dict with header keys and cell values
+        row = {}
+        for i, header in enumerate(headers):
+            if i < len(cells):
+                value = cells[i]
+                # Clean up special characters and artifacts
+                if value == "..'" or value == "..'":
+                    value = ""
+                row[header] = value
+            else:
+                row[header] = ""
+        
+        rows.append(row)
+    
+    print(f"\nParsed {len(rows)} data rows from table")
+    return headers, rows
+
+
+def parse_date_to_iso(date_str: str) -> str:
+    """
+    Convert date from DD-MM-YYYY format to YYYY-MM-DD format.
+    Returns empty string if parsing fails.
+    """
+    if not date_str or date_str in ["..'" , "..'", ""]:
+        return ""
+    
+    date_str = date_str.strip()
+    
+    # Handle DD-MM-YYYY format
+    match = re.match(r'(\d{1,2})-(\d{1,2})-(\d{4})', date_str)
+    if match:
+        day, month, year = match.groups()
+        return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
+    
+    return date_str
+
+
+def extract_year_from_date(date_str: str) -> int:
+    """Extract year from a date string."""
+    if not date_str:
+        return 0
+    
+    # Try to find a 4-digit year
+    match = re.search(r'(\d{4})', date_str)
+    if match:
+        return int(match.group(1))
+    
+    return 0
+
+
+def parse_premium(value: str) -> float:
+    """Parse premium value to float."""
+    if not value or value in ["..'" , "..'", ""]:
+        return 0.0
+    
+    try:
+        # Remove commas and currency symbols
+        cleaned = re.sub(r'[^\d.]', '', str(value).replace(',', ''))
+        if cleaned:
+            return float(cleaned)
+    except:
+        pass
+    
+    return 0.0
+
+
+def parse_int_value(value: str) -> int:
+    """Parse integer value."""
+    if not value or value in ["..'" , "..'", ""]:
+        return 0
+    
+    try:
+        cleaned = re.sub(r'[^\d]', '', str(value))
+        if cleaned:
+            return int(cleaned)
+    except:
+        pass
+    
+    return 0
+
+
+def generate_doc_id(row: dict) -> str:
+    """Generate a unique document ID based on row data."""
+    pl_number = row.get('PL/EN #', row.get('PL/EN', ''))
+    policy_number = row.get('Policy Number', '')
+    client_name = row.get('Client Name', '')
+    
+    unique_string = f"{pl_number}_{policy_number}_{client_name}"
+    return hashlib.sha256(unique_string.encode()).hexdigest()[:16]
+
+
+def convert_row_to_metadata(row: dict, all_headers: list[str]) -> dict:
+    """
+    Convert a parsed table row into the metadata format expected by ChromaDB.
+    
+    ALL columns from the table are preserved.
+    """
+    # Parse dates
+    policy_start = parse_date_to_iso(row.get('Policy Start Date', ''))
+    policy_end = parse_date_to_iso(row.get('Policy End Date', ''))
+    
+    # Use policy_end_date as renewal_date
+    renewal_date = policy_end
+    renewal_year = extract_year_from_date(policy_end)
+    
+    # Parse premium amounts
+    premium_paid = parse_premium(row.get('Premium Paid', ''))
+    gross_premium = parse_premium(row.get('Gross Premium', ''))
+    net_premium = parse_premium(row.get('Premium', ''))
+    
+    # Use the best available premium value
+    premium_amount = premium_paid or gross_premium or net_premium
+    
+    # Build metadata dict matching ChromaDB structure (required fields)
+    metadata = {
+        # Basic document info
+        "document_type": str(row.get('Type', 'Policy')),
+        "document_title": str(row.get('PL/EN #', '')),
+        
+        # Policy details
+        "policy_number": str(row.get('Policy Number', '')),
+        "policy_type": str(row.get('Policy', '')),
+        
+        # Parties involved
+        "insurer_name": str(row.get('Insurer', '')),
+        "insured_name": str(row.get('Insured Name', row.get('Client Name', ''))),
+        "broker_name": "",
+        
+        # Financial
+        "sum_insured": 0.0,
+        "premium_amount": premium_amount,
+        
+        # Dates
+        "policy_start_date": policy_start,
+        "policy_end_date": policy_end,
+        "renewal_date": renewal_date,
+        "renewal_year": renewal_year,
+        
+        # Location
+        "city": "",
+        "state": "",
+        "pincode": "",
+        "property_address": "",
+        
+        # Classification
+        "industry": "",
+        "is_manufacturing": False,
+        
+        # Arrays stored as JSON
+        "coverage_type": [],
+        "keywords": [],
+        
+        # Tracking
+        "created_at": time.time()
+    }
+    
+    # Now add ALL columns from the table as additional metadata
+    # These will be stored as extra_* fields
+    for header in all_headers:
+        # Create a safe key name
+        safe_key = header.replace(' ', '_').replace('.', '').replace('#', 'num')
+        safe_key = re.sub(r'[^a-zA-Z0-9_]', '', safe_key)
+        safe_key = safe_key.lower()
+        
+        # Skip if already mapped above
+        if safe_key in ['policy_number', 'policy', 'insurer', 'insured_name', 
+                        'client_name', 'premium_paid', 'gross_premium', 'premium',
+                        'policy_start_date', 'policy_end_date', 'type', 'plen_num']:
+            continue
+        
+        value = row.get(header, '')
+        
+        # Store the raw value as a string (ChromaDB requires primitive types)
+        if value and value not in ["..'" , "..'", ""]:
+            metadata[f"col_{safe_key}"] = str(value)[:500]  # Limit string length
+    
+    return metadata
+
+
+def print_progress_bar(current: int, total: int, bar_length: int = 50):
+    """Print a progress bar using ASCII characters."""
+    percent = current / total
+    filled = int(bar_length * percent)
+    bar = '#' * filled + '-' * (bar_length - filled)
+    print(f'\r  Progress: |{bar}| {current}/{total} ({percent*100:.1f}%)', end='', flush=True)
+
+
+def clear_and_remigrate():
+    """
+    Clear all data from document_metadata collection and re-run migration
+    with new user_id and no bucket_id.
+    """
+    print("=" * 60)
+    print("Clearing document_metadata and re-running migration")
+    print("=" * 60)
+    
+    # Initialize ChromaDB service
+    print("\n[1] Connecting to ChromaDB...")
+    chroma = ChromaService()
+    
+    # Check current state
+    current_data = chroma.metadata_collection.get()
+    print(f"Current document_metadata collection has {len(current_data['ids'])} entries")
+    
+    # Delete all data from document_metadata collection
+    print("\n[2] Deleting all data from document_metadata collection...")
+    if current_data['ids']:
+        chroma.metadata_collection.delete(ids=current_data['ids'])
+        print(f"Deleted {len(current_data['ids'])} entries")
+    else:
+        print("Collection was already empty")
+    
+    # Verify deletion
+    verify_data = chroma.metadata_collection.get()
+    print(f"After deletion: {len(verify_data['ids'])} entries remain")
+    
+    # Parse the markdown table
+    print("\n[3] Parsing table.md...")
+    table_path = "table.md"
+    headers, rows = parse_markdown_table(table_path)
+    
+    if not rows:
+        print("ERROR: No data rows found in table.md")
+        return
+    
+    print(f"\nSample row data (first row, first 10 columns):")
+    sample = rows[0]
+    for key in list(sample.keys())[:10]:
+        print(f"  {key}: {sample[key][:50] if len(sample[key]) > 50 else sample[key]}")
+    
+    # NEW: Use different user_id (Nishant) and NO bucket_id
+    user_id = "55c0893720ef38eb"  # Nishant's user ID
+    bucket_id = ""  # No bucket ID specified
+    
+    print(f"\n[4] Migrating {len(rows)} rows to ChromaDB...")
+    print(f"Using user_id: {user_id} (Nishant)")
+    print(f"Using bucket_id: (empty - no bucket specified)")
+    print(f"Batch size: 10 entries")
+    print()
+    
+    success_count = 0
+    error_count = 0
+    
+    BATCH_SIZE = 10
+    total_batches = (len(rows) + BATCH_SIZE - 1) // BATCH_SIZE
+    
+    for batch_num in range(total_batches):
+        start_idx = batch_num * BATCH_SIZE
+        end_idx = min(start_idx + BATCH_SIZE, len(rows))
+        batch_rows = rows[start_idx:end_idx]
+        
+        for row in batch_rows:
+            try:
+                # Generate unique doc_id
+                doc_id = generate_doc_id(row)
+                
+                # Convert row to metadata format (include all headers)
+                metadata = convert_row_to_metadata(row, headers)
+                
+                # Store in ChromaDB
+                result = chroma.store_document_metadata(
+                    doc_id=doc_id,
+                    user_id=user_id,
+                    bucket_id=bucket_id,
+                    metadata=metadata
+                )
+                
+                if result.get("status") == "stored":
+                    success_count += 1
+                else:
+                    error_count += 1
+            
+            except Exception as e:
+                print(f"\n  Error on row: {str(e)[:100]}")
+                error_count += 1
+        
+        # Update progress bar
+        print_progress_bar(end_idx, len(rows))
+        
+        # Small delay between batches to not overload the API
+        if batch_num < total_batches - 1:
+            time.sleep(0.1)
+    
+    print()  # New line after progress bar
+    
+    # Final status
+    print("\n" + "=" * 60)
+    print("Migration Complete!")
+    print("=" * 60)
+    print(f"Successfully migrated: {success_count} entries")
+    print(f"Errors: {error_count}")
+    
+    # Verify final state
+    final_data = chroma.metadata_collection.get()
+    print(f"\nFinal document_metadata collection has {len(final_data['ids'])} entries")
+    
+    # Show sample of stored data
+    if final_data['ids']:
+        print("\nSample stored metadata (first entry):")
+        sample_meta = final_data['metadatas'][0]
+        for key, value in list(sample_meta.items())[:15]:
+            print(f"  {key}: {value}")
+        print(f"  ... and {len(sample_meta) - 15} more fields")
+
+
+if __name__ == "__main__":
+    clear_and_remigrate()

clear_old_migration.py

@@ -0,0 +1,25 @@
+"""Clear all entries with wrong user_id from document_metadata collection."""
+
+from services.chroma_service import ChromaService
+
+def clear_old_data():
+    print("Connecting to ChromaDB...")
+    c = ChromaService()
+    
+    # Get all entries with the wrong user_id
+    data = c.metadata_collection.get(where={"user_id": "jashdoshi"})
+    count = len(data['ids'])
+    
+    print(f"Found {count} entries with user_id='jashdoshi' (old migration)")
+    
+    if count > 0 and data['ids']:
+        print("Deleting these entries...")
+        c.metadata_collection.delete(ids=data['ids'])
+        print(f"Deleted {count} entries.")
+    
+    # Verify
+    remaining = c.metadata_collection.get()
+    print(f"Remaining entries in collection: {len(remaining['ids'])}")
+
+if __name__ == "__main__":
+    clear_old_data()

find_buckets.py

@@ -1,25 +1,38 @@
-"""Helper script to find buckets for a user"""
-import sys
-sys.path.insert(0, '.')
+"""Check buckets for all users."""
 
-from services.chroma_service import chroma_service
+from services.chroma_service import ChromaService
 
-# Check both users
-user_ids = ['55c0893720ef38eb', '7ac2ed69d52d2010']
-
-for user_id in user_ids:
-    print(f"\nUser: {user_id}")
-    print("-" * 40)
+def check_all_buckets():
+    print("Connecting to ChromaDB...")
+    c = ChromaService()
+    
+    # Get all users
+    users = c.users_collection.get()
+    print(f"\nTotal users: {len(users['ids'])}")
     
-    # Get documents
-    docs = chroma_service.get_user_documents(user_id)
-    print(f"Documents: {len(docs)}")
+    for i, uid in enumerate(users['ids']):
+        meta = users['metadatas'][i]
+        print(f"\n=== User: {meta.get('username')} (ID: {uid}) ===")
+        
+        # Get buckets for this user
+        buckets = c.buckets_collection.get(where={"user_id": uid})
+        print(f"Buckets: {len(buckets['ids'])}")
+        
+        for j, bid in enumerate(buckets['ids']):
+            print(f"  - {buckets['metadatas'][j].get('name')} (ID: {bid})")
     
-    # Get buckets
-    buckets = chroma_service.get_user_buckets(user_id)
-    if buckets:
-        print("Buckets:")
-        for b in buckets:
-            print(f"  - {b['name']} (ID: {b['bucket_id']}, Docs: {b.get('doc_count', 0)})")
-    else:
-        print("No buckets found")
+    # Also check what the metadata is stored with
+    print("\n=== Metadata collection user/bucket combos ===")
+    data = c.metadata_collection.get()
+    if data['ids']:
+        # Get unique user_id/bucket_id combinations
+        combos = set()
+        for meta in data['metadatas']:
+            combos.add((meta.get('user_id'), meta.get('bucket_id')))
+        
+        for user_id, bucket_id in combos:
+            count = sum(1 for m in data['metadatas'] if m.get('user_id') == user_id and m.get('bucket_id') == bucket_id)
+            print(f"  user_id={user_id}, bucket_id={bucket_id}: {count} entries")
+
+if __name__ == "__main__":
+    check_all_buckets()

migrate_table_to_chroma.py

@@ -0,0 +1,352 @@
+"""
+Migration script to populate document_metadata collection from table.md
+
+This script reads the markdown table and migrates data to ChromaDB's document_metadata
+collection. No AI or API keys are used - just straightforward data parsing.
+
+All columns from the table are preserved in the metadata.
+"""
+
+import re
+import hashlib
+import time
+import json
+from services.chroma_service import ChromaService
+
+
+def parse_markdown_table(filepath: str) -> tuple[list[str], list[dict]]:
+    """
+    Parse the markdown table from table.md and extract all rows.
+    
+    Returns:
+        - headers: list of column names
+        - rows: list of dicts with column names as keys
+    """
+    with open(filepath, 'r', encoding='utf-8') as f:
+        content = f.read()
+    
+    lines = content.strip().split('\n')
+    
+    # Parse header row (line 1)
+    header_line = lines[0]
+    # Extract column names from header
+    raw_headers = [h.strip().replace('\t', '').strip() for h in header_line.split('|')]
+    headers = [h for h in raw_headers if h and h != ':--------:']
+    
+    print(f"Found {len(headers)} columns in header:")
+    for i, h in enumerate(headers):
+        print(f"  {i+1}. {h}")
+    
+    rows = []
+    
+    # Parse data rows (skip header and separator lines)
+    for line_num, line in enumerate(lines[2:], start=3):
+        if not line.strip():
+            continue
+        
+        # Split by pipe and clean values
+        raw_cells = [c.strip().replace('\t', '').strip() for c in line.split('|')]
+        cells = [c for c in raw_cells if c != '']
+        
+        if not cells:
+            continue
+        
+        # Create dict with header keys and cell values
+        row = {}
+        for i, header in enumerate(headers):
+            if i < len(cells):
+                value = cells[i]
+                # Clean up special characters and artifacts
+                if value == "..'" or value == "..'":
+                    value = ""
+                row[header] = value
+            else:
+                row[header] = ""
+        
+        rows.append(row)
+    
+    print(f"\nParsed {len(rows)} data rows from table")
+    return headers, rows
+
+
+def parse_date_to_iso(date_str: str) -> str:
+    """
+    Convert date from DD-MM-YYYY format to YYYY-MM-DD format.
+    Returns empty string if parsing fails.
+    """
+    if not date_str or date_str in ["..'" , "..'", ""]:
+        return ""
+    
+    date_str = date_str.strip()
+    
+    # Handle DD-MM-YYYY format
+    match = re.match(r'(\d{1,2})-(\d{1,2})-(\d{4})', date_str)
+    if match:
+        day, month, year = match.groups()
+        return f"{year}-{month.zfill(2)}-{day.zfill(2)}"
+    
+    return date_str
+
+
+def extract_year_from_date(date_str: str) -> int:
+    """Extract year from a date string."""
+    if not date_str:
+        return 0
+    
+    # Try to find a 4-digit year
+    match = re.search(r'(\d{4})', date_str)
+    if match:
+        return int(match.group(1))
+    
+    return 0
+
+
+def parse_premium(value: str) -> float:
+    """Parse premium value to float."""
+    if not value or value in ["..'" , "..'", ""]:
+        return 0.0
+    
+    try:
+        # Remove commas and currency symbols
+        cleaned = re.sub(r'[^\d.]', '', str(value).replace(',', ''))
+        if cleaned:
+            return float(cleaned)
+    except:
+        pass
+    
+    return 0.0
+
+
+def parse_int_value(value: str) -> int:
+    """Parse integer value."""
+    if not value or value in ["..'" , "..'", ""]:
+        return 0
+    
+    try:
+        cleaned = re.sub(r'[^\d]', '', str(value))
+        if cleaned:
+            return int(cleaned)
+    except:
+        pass
+    
+    return 0
+
+
+def generate_doc_id(row: dict) -> str:
+    """Generate a unique document ID based on row data."""
+    pl_number = row.get('PL/EN #', row.get('PL/EN', ''))
+    policy_number = row.get('Policy Number', '')
+    client_name = row.get('Client Name', '')
+    
+    unique_string = f"{pl_number}_{policy_number}_{client_name}"
+    return hashlib.sha256(unique_string.encode()).hexdigest()[:16]
+
+
+def convert_row_to_metadata(row: dict, all_headers: list[str]) -> dict:
+    """
+    Convert a parsed table row into the metadata format expected by ChromaDB.
+    
+    ALL columns from the table are preserved.
+    """
+    # Parse dates
+    policy_start = parse_date_to_iso(row.get('Policy Start Date', ''))
+    policy_end = parse_date_to_iso(row.get('Policy End Date', ''))
+    
+    # Use policy_end_date as renewal_date
+    renewal_date = policy_end
+    renewal_year = extract_year_from_date(policy_end)
+    
+    # Parse premium amounts
+    premium_paid = parse_premium(row.get('Premium Paid', ''))
+    gross_premium = parse_premium(row.get('Gross Premium', ''))
+    net_premium = parse_premium(row.get('Premium', ''))
+    
+    # Use the best available premium value
+    premium_amount = premium_paid or gross_premium or net_premium
+    
+    # Build metadata dict matching ChromaDB structure (required fields)
+    metadata = {
+        # Basic document info
+        "document_type": str(row.get('Type', 'Policy')),
+        "document_title": str(row.get('PL/EN #', '')),
+        
+        # Policy details
+        "policy_number": str(row.get('Policy Number', '')),
+        "policy_type": str(row.get('Policy', '')),
+        
+        # Parties involved
+        "insurer_name": str(row.get('Insurer', '')),
+        "insured_name": str(row.get('Insured Name', row.get('Client Name', ''))),
+        "broker_name": "",
+        
+        # Financial
+        "sum_insured": 0.0,
+        "premium_amount": premium_amount,
+        
+        # Dates
+        "policy_start_date": policy_start,
+        "policy_end_date": policy_end,
+        "renewal_date": renewal_date,
+        "renewal_year": renewal_year,
+        
+        # Location
+        "city": "",
+        "state": "",
+        "pincode": "",
+        "property_address": "",
+        
+        # Classification
+        "industry": "",
+        "is_manufacturing": False,
+        
+        # Arrays stored as JSON
+        "coverage_type": [],
+        "keywords": [],
+        
+        # Tracking
+        "created_at": time.time()
+    }
+    
+    # Now add ALL columns from the table as additional metadata
+    # These will be stored as extra_* fields
+    for header in all_headers:
+        # Create a safe key name
+        safe_key = header.replace(' ', '_').replace('.', '').replace('#', 'num')
+        safe_key = re.sub(r'[^a-zA-Z0-9_]', '', safe_key)
+        safe_key = safe_key.lower()
+        
+        # Skip if already mapped above
+        if safe_key in ['policy_number', 'policy', 'insurer', 'insured_name', 
+                        'client_name', 'premium_paid', 'gross_premium', 'premium',
+                        'policy_start_date', 'policy_end_date', 'type', 'plen_num']:
+            continue
+        
+        value = row.get(header, '')
+        
+        # Store the raw value as a string (ChromaDB requires primitive types)
+        if value and value not in ["..'" , "..'", ""]:
+            metadata[f"col_{safe_key}"] = str(value)[:500]  # Limit string length
+    
+    return metadata
+
+
+def print_progress_bar(current: int, total: int, bar_length: int = 50):
+    """Print a progress bar using ASCII characters."""
+    percent = current / total
+    filled = int(bar_length * percent)
+    bar = '#' * filled + '-' * (bar_length - filled)
+    print(f'\r  Progress: |{bar}| {current}/{total} ({percent*100:.1f}%)', end='', flush=True)
+
+
+def migrate_table_to_chroma():
+    """
+    Main migration function.
+    
+    Reads table.md, parses all rows, and inserts them into ChromaDB in batches.
+    """
+    print("=" * 60)
+    print("Starting migration from table.md to ChromaDB")
+    print("=" * 60)
+    
+    # Initialize ChromaDB service
+    print("\n[1] Connecting to ChromaDB...")
+    chroma = ChromaService()
+    
+    # Check current state of the metadata collection
+    current_data = chroma.metadata_collection.get()
+    print(f"Current document_metadata collection has {len(current_data['ids'])} entries")
+    
+    # Parse the markdown table
+    print("\n[2] Parsing table.md...")
+    table_path = "table.md"
+    headers, rows = parse_markdown_table(table_path)
+    
+    if not rows:
+        print("ERROR: No data rows found in table.md")
+        return
+    
+    print(f"\nSample row data (first row, first 10 columns):")
+    sample = rows[0]
+    for key in list(sample.keys())[:10]:
+        print(f"  {key}: {sample[key][:50] if len(sample[key]) > 50 else sample[key]}")
+    
+    # Fixed user_id and bucket_id for all entries
+    # IMPORTANT: These should match the actual hashed IDs in the system
+    # Check with: python check_users.py
+    # User "jash" has ID: 7ac2ed69d52d2010
+    # Bucket "2025 policy sibro" has ID: ee449d7c04e92039
+    user_id = "7ac2ed69d52d2010"  # jash's user ID
+    bucket_id = "ee449d7c04e92039"  # 2025 policy sibro bucket
+    
+    print(f"\n[3] Migrating {len(rows)} rows to ChromaDB...")
+    print(f"Using user_id: {user_id}, bucket_id: {bucket_id}")
+    print(f"Batch size: 10 entries")
+    print()
+    
+    success_count = 0
+    error_count = 0
+    
+    BATCH_SIZE = 10
+    total_batches = (len(rows) + BATCH_SIZE - 1) // BATCH_SIZE
+    
+    for batch_num in range(total_batches):
+        start_idx = batch_num * BATCH_SIZE
+        end_idx = min(start_idx + BATCH_SIZE, len(rows))
+        batch_rows = rows[start_idx:end_idx]
+        
+        for row in batch_rows:
+            try:
+                # Generate unique doc_id
+                doc_id = generate_doc_id(row)
+                
+                # Convert row to metadata format (include all headers)
+                metadata = convert_row_to_metadata(row, headers)
+                
+                # Store in ChromaDB
+                result = chroma.store_document_metadata(
+                    doc_id=doc_id,
+                    user_id=user_id,
+                    bucket_id=bucket_id,
+                    metadata=metadata
+                )
+                
+                if result.get("status") == "stored":
+                    success_count += 1
+                else:
+                    error_count += 1
+            
+            except Exception as e:
+                print(f"\n  Error on row: {str(e)[:100]}")
+                error_count += 1
+        
+        # Update progress bar
+        print_progress_bar(end_idx, len(rows))
+        
+        # Small delay between batches to not overload the API
+        if batch_num < total_batches - 1:
+            time.sleep(0.1)
+    
+    print()  # New line after progress bar
+    
+    # Final status
+    print("\n" + "=" * 60)
+    print("Migration Complete!")
+    print("=" * 60)
+    print(f"Successfully migrated: {success_count} entries")
+    print(f"Errors: {error_count}")
+    
+    # Verify final state
+    final_data = chroma.metadata_collection.get()
+    print(f"\nFinal document_metadata collection has {len(final_data['ids'])} entries")
+    
+    # Show sample of stored data
+    if final_data['ids']:
+        print("\nSample stored metadata (first entry):")
+        sample_meta = final_data['metadatas'][0]
+        for key, value in list(sample_meta.items())[:15]:
+            print(f"  {key}: {value}")
+        print(f"  ... and {len(sample_meta) - 15} more fields")
+
+
+if __name__ == "__main__":
+    migrate_table_to_chroma()

services/chroma_service.py

@@ -410,25 +410,23 @@ class ChromaService:
         
         IMPORTANT: When bucket_id is provided, ONLY chunks from that bucket are returned.
         This ensures strict bucket isolation for multi-bucket deployments.
+        When doc_ids is also provided, it filters to specific documents within the bucket.
         """
-        # Build where clause with strict bucket isolation
+        # Build where clause with strict bucket isolation and optional doc_id filtering
+        conditions = [{"user_id": user_id}]
+        
         if bucket_id:
-            where_clause = {
-                "$and": [
-                    {"user_id": user_id},
-                    {"bucket_id": bucket_id}
-                ]
-            }
+            conditions.append({"bucket_id": bucket_id})
             print(f"[CHROMA] Strict bucket isolation: searching only bucket '{bucket_id}'")
-        elif doc_ids:
-            where_clause = {
-                "$and": [
-                    {"user_id": user_id},
-                    {"doc_id": {"$in": doc_ids}}
-                ]
-            }
+        
+        if doc_ids:
+            conditions.append({"doc_id": {"$in": doc_ids}})
+            print(f"[CHROMA] Filtering to {len(doc_ids)} specific documents")
+        
+        if len(conditions) > 1:
+            where_clause = {"$and": conditions}
         else:
-            where_clause = {"user_id": user_id}
+            where_clause = conditions[0]
         
         results = self.chunks_collection.query(
             query_texts=[query],

services/rag_service.py

@@ -234,6 +234,9 @@ CRITICAL RULES:
 2. When multiple industries are mentioned (e.g., "manufacturing and healthcare"), combine them with comma: "manufacturing, healthcare"
 3. When user asks for "top N" of something, set both limit AND sort_by appropriately
 4. Keywords like "manufacturing", "healthcare", "retail", "IT", "construction" are INDUSTRIES - put them in filters
+5. COMPANY NAME EXTRACTION: When user mentions a company name (e.g., "ABC Corp", "XYZ Industries", "Company Name"), extract it to insured_name filter. Extract the company name as mentioned in the query, even if it's partial. The system will handle name variations (case, spacing, suffixes like "Pvt Ltd", singular/plural) automatically.
+6. TYPO HANDLING: If user makes typos (e.g., "policie" -> "policies", "polciy" -> "policy"), still extract the correct intent and filters. The system is forgiving of spelling errors.
+7. COMPANY vs INDIVIDUAL: When user mentions a company name with business keywords (e.g., "ABC Chemical", "XYZ Industries", "Company Corp"), they want COMPANY policies, not individual person policies. The system will automatically filter out individual person names when company keywords are detected.
 
 FORMAT DETECTION (NEW):
 1. Detect if user explicitly asks for a specific format:
@@ -290,7 +293,22 @@ Query: "list all fire policies in bullet points"
 {"intent":"list","needs_metadata":true,"filters":{"policy_type":"fire"},"sort_by":null,"sort_order":"desc","limit":null,"calculation":null,"calculation_field":null,"format_preference":"bullets","is_format_change":false}
 
 Query: "top 5 health policies by sum insured as a table"
-{"intent":"rank","needs_metadata":true,"filters":{"policy_type":"health"},"sort_by":"sum_insured","sort_order":"desc","limit":5,"calculation":null,"calculation_field":null,"format_preference":"table","is_format_change":false}"""
+{"intent":"rank","needs_metadata":true,"filters":{"policy_type":"health"},"sort_by":"sum_insured","sort_order":"desc","limit":5,"calculation":null,"calculation_field":null,"format_preference":"table","is_format_change":false}
+
+Query: "renewals in march 2026"
+{"intent":"list","needs_metadata":true,"filters":{"renewal_year":2026,"renewal_month":"march"},"sort_by":"renewal_date","sort_order":"asc","limit":null,"calculation":null,"calculation_field":null,"format_preference":null,"is_format_change":false}
+
+Query: "renewals in march 2026 also list the renewal date"
+{"intent":"list","needs_metadata":true,"filters":{"renewal_year":2026,"renewal_month":"march"},"sort_by":"renewal_date","sort_order":"asc","limit":null,"calculation":null,"calculation_field":null,"format_preference":"table","is_format_change":false}
+
+Query: "policies expiring in april 2025 with premium details"
+{"intent":"list","needs_metadata":true,"filters":{"renewal_year":2025,"renewal_month":"april"},"sort_by":"renewal_date","sort_order":"asc","limit":null,"calculation":null,"calculation_field":null,"format_preference":null,"is_format_change":false}
+
+Query: "list all ABC Corp policies"
+{"intent":"list","needs_metadata":true,"filters":{"insured_name":"ABC Corp"},"sort_by":null,"sort_order":"desc","limit":null,"calculation":null,"calculation_field":null,"format_preference":null,"is_format_change":false}
+
+Query: "show me policies for XYZ Industries"
+{"intent":"list","needs_metadata":true,"filters":{"insured_name":"XYZ Industries"},"sort_by":null,"sort_order":"desc","limit":null,"calculation":null,"calculation_field":null,"format_preference":null,"is_format_change":false}"""
 
         messages = [
             {"role": "system", "content": system_prompt},
@@ -457,25 +475,32 @@ Query: "top 5 health policies by sum insured as a table"
         Returns markdown-compatible formatting guidance.
         """
         format_map = {
-            "table": """FORMAT: Present data in a markdown table.
-- Use | column | headers | with |---| separator line
-- Keep columns aligned and consistent
-- Include all requested data in table rows""",
+            "table": """FORMAT: Present data as a complete markdown table.
+CRITICAL TABLE RULES:
+1. Include EVERY SINGLE item from the data - do NOT truncate or summarize
+2. Use these standard columns: | S.No | Document/Policy Name | Insured Name | Policy Type | Sum Insured | Premium | Renewal Date |
+3. Use | column | headers | with |---| separator line
+4. If there are 37 items, the table MUST have 37 rows (plus header)
+5. Use ₹ symbol for currency values with proper formatting""",
             
-            "list": """FORMAT: Present as a numbered list.
-1. Each item on its own line with number prefix
-2. Include key details after the number
-3. Use consistent formatting for all items""",
+            "list": """FORMAT: Present as a complete numbered list.
+CRITICAL LIST RULES:
+1. Include EVERY SINGLE item - do NOT skip any
+2. Number each item starting from 1
+3. Include key details: name, policy type, amounts, dates
+4. If there are 37 items, list ALL 37 items""",
             
-            "bullets": """FORMAT: Use bullet points.
-- Each item as a bullet point
-- Sub-details can be indented bullets
-- Keep bullets concise and scannable""",
+            "bullets": """FORMAT: Use bullet points for all items.
+CRITICAL BULLET RULES:
+1. Include EVERY SINGLE item as a bullet point
+2. Sub-details can be indented bullets
+3. Do NOT summarize or truncate the list
+4. If there are 37 items, show ALL 37 bullets""",
             
             "paragraph": """FORMAT: Write in flowing prose paragraphs.
 - Use complete sentences and natural language
 - Group related information into paragraphs
-- Avoid lists or tables unless absolutely necessary"""
+- Still mention ALL items, just in prose form"""
         }
         
         return format_map.get(format_preference, "")
@@ -928,9 +953,91 @@ Summary: {summary[:300] if summary else 'No summary available'}
                                   
                 elif field in ['city', 'state', 'insurer_name', 'insured_name', 'broker_name']:
                     # Handle comma-separated values (OR logic)
+                    # For name fields, use flexible matching to handle variations
                     filter_values = [v.strip().lower() for v in str(value).split(',')]
+                    
+                    def matches_name(metadata_value, filter_value):
+                        """Flexible name matching that handles variations"""
+                        if not metadata_value or not filter_value:
+                            return False
+                        
+                        meta_lower = str(metadata_value).lower()
+                        filter_lower = filter_value.lower()
+                        
+                        # Detect if filter is looking for a company (contains business keywords)
+                        # vs individual person (just a name)
+                        filter_is_company = any(keyword in filter_lower for keyword in 
+                                              ['chemical', 'chemicals', 'industries', 'industry', 
+                                               'corp', 'corporation', 'ltd', 'limited', 'pvt', 
+                                               'private', 'inc', 'incorporated', 'llc', 'company', 
+                                               'enterprises', 'group', 'holdings'])
+                        
+                        # Detect if metadata is a company (has business suffixes or keywords)
+                        meta_is_company = any(keyword in meta_lower for keyword in 
+                                            [' pvt ltd', ' pvt. ltd', ' ltd', ' ltd.', ' limited', 
+                                             ' inc', ' inc.', ' incorporated', ' llc', ' llc.', 
+                                             ' corporation', ' corp', ' corp.', ' industries', 
+                                             ' industry', ' company', ' enterprises', ' group'])
+                        
+                        # If filter is for a company but metadata is individual, skip
+                        # This prevents matching "Choksey Chemical" company with "Bharat Choksey" person
+                        if filter_is_company and not meta_is_company:
+                            # Check if metadata is clearly a person name (has first/middle/last name pattern)
+                            # Person names typically have 2-4 words, company names are usually longer or have suffixes
+                            meta_words = meta_lower.split()
+                            if len(meta_words) <= 4 and not any(char.isdigit() for char in meta_lower):
+                                # Likely a person name, skip if filter is for company
+                                return False
+                        
+                        # Remove common suffixes/prefixes for better matching
+                        # Remove "pvt ltd", "ltd", "inc", "llc", etc.
+                        meta_clean = meta_lower
+                        filter_clean = filter_lower
+                        
+                        for suffix in [' pvt ltd', ' pvt. ltd', ' pvt ltd.', ' pvt. ltd.', 
+                                      ' ltd', ' ltd.', ' limited', ' inc', ' inc.', 
+                                      ' incorporated', ' llc', ' llc.', ' corporation', ' corp', ' corp.',
+                                      ' industries', ' industry', ' company', ' enterprises', ' group']:
+                            meta_clean = meta_clean.replace(suffix, '')
+                            filter_clean = filter_clean.replace(suffix, '')
+                        
+                        # Remove extra spaces and punctuation
+                        import re
+                        meta_clean = re.sub(r'[^\w\s]', ' ', meta_clean)
+                        filter_clean = re.sub(r'[^\w\s]', ' ', filter_clean)
+                        meta_clean = ' '.join(meta_clean.split())
+                        filter_clean = ' '.join(filter_clean.split())
+                        
+                        # Check if filter value is a substring of metadata value
+                        if filter_clean in meta_clean:
+                            return True
+                        
+                        # Also check if all significant words from filter are in metadata
+                        # Handle singular/plural variations
+                        filter_words = [w for w in filter_clean.split() if len(w) > 2]
+                        if filter_words:
+                            meta_words = set(meta_clean.split())
+                            for word in filter_words:
+                                # Check exact match
+                                if word in meta_words:
+                                    continue
+                                # Check singular/plural variations
+                                word_singular = word.rstrip('s') if word.endswith('s') else word
+                                word_plural = word + 's' if not word.endswith('s') else word
+                                if word_singular in meta_words or word_plural in meta_words:
+                                    continue
+                                # Check if word is a substring of any metadata word
+                                if any(word in mw or mw in word for mw in meta_words if len(mw) > 3):
+                                    continue
+                                # If none match, this word doesn't match
+                                return False
+                            return True
+                        
+                        return False
+                    
+                    # Apply flexible matching
                     all_metadata = [m for m in all_metadata 
-                                  if any(fv in str(m.get(field, '')).lower() for fv in filter_values)]
+                                  if any(matches_name(m.get(field, ''), fv) for fv in filter_values)]
                     print(f"[METADATA QUERY] Filtered by {field} {filter_values}: {len(all_metadata)} remaining")
                                   
                 elif field == 'renewal_year':
@@ -1041,8 +1148,8 @@ Summary: {summary[:300] if summary else 'No summary available'}
 - Location: {meta.get('city', '')}, {meta.get('state', '')}
 """
                 else:
-                    # Compact format for large sets
-                    entry = f"{i}. {meta.get('document_title', 'Unknown')} | {meta.get('insured_name', 'N/A')} | ₹{meta.get('premium_amount', 0):,.0f} | {meta.get('policy_type', 'N/A')}"
+                    # Compact format for large sets - includes renewal date
+                    entry = f"{i}. {meta.get('document_title', 'Unknown')} | {meta.get('insured_name', 'N/A')} | ₹{meta.get('premium_amount', 0):,.0f} | {meta.get('policy_type', 'N/A')} | Renewal: {meta.get('renewal_date', 'N/A')}"
                 
                 context_parts.append(entry)
         
@@ -1059,6 +1166,397 @@ Summary: {summary[:300] if summary else 'No summary available'}
             'sources': {m.get('doc_id'): m.get('document_title') for m in all_metadata}
         }
 
+    def _get_rag_context_for_query(self, user_id: str, bucket_id: str, query: str,
+                                    filters: dict = None, is_fallback: bool = False,
+                                    doc_ids: list[str] = None) -> dict:
+        """
+        Get RAG context from chunk retrieval for a query.
+        Used as fallback when metadata filtering returns 0 results,
+        or to supplement metadata with detailed document content.
+        
+        Args:
+            user_id: User ID
+            bucket_id: Bucket ID
+            query: The search query
+            filters: Optional filters from parsed query (used to build search query)
+            is_fallback: If True, use more aggressive search (higher top_k, better query construction)
+            doc_ids: Optional list of specific document IDs to search (from document name detection)
+            
+        Returns:
+            dict with:
+            - context: Combined text from retrieved chunks
+            - sources: dict of doc_id -> filename
+            - chunk_count: Number of chunks retrieved
+            - chunks: Raw chunk data
+        """
+        print(f"[HYBRID RAG] Getting RAG context for query: {query[:50]}... (fallback={is_fallback})")
+        
+        # Step 0: Detect if user mentioned a specific document name in the query
+        if doc_ids is None:
+            user_docs = chroma_service.get_user_documents(user_id, bucket_id)
+            referenced_doc_ids = self._detect_document_reference(query, user_docs)
+            if referenced_doc_ids:
+                doc_ids = referenced_doc_ids
+                print(f"[HYBRID RAG] Detected document reference in query: {len(doc_ids)} documents")
+        
+        # Build enhanced search query from filters if available
+        search_query = query
+        if filters:
+            # Add filter values to improve semantic search
+            filter_terms = []
+            for field, value in filters.items():
+                if value and field in ['insured_name', 'insurer_name', 'broker_name', 
+                                       'policy_type', 'industry', 'city', 'state']:
+                    filter_terms.append(str(value))
+            if filter_terms:
+                search_query = f"{query} {' '.join(filter_terms)}"
+                print(f"[HYBRID RAG] Enhanced search query: {search_query[:80]}...")
+        
+        # For fallback searches, use more aggressive parameters
+        if is_fallback:
+            # Extract key terms from original query for better matching
+            # Split query into words and keep important terms
+            query_words = query.lower().split()
+            # Remove common stop words but keep entity names and numbers
+            stop_words = {'how', 'many', 'are', 'is', 'the', 'a', 'an', 'for', 'in', 'on', 'at', 'to', 'of'}
+            key_terms = [w for w in query_words if w not in stop_words and len(w) > 2]
+            
+            # If we have filters, prioritize those terms
+            if filters:
+                for field, value in filters.items():
+                    if value and field in ['insured_name', 'insurer_name', 'broker_name']:
+                        # Add the filter value as a key term
+                        value_words = str(value).lower().split()
+                        key_terms.extend([w for w in value_words if len(w) > 2])
+            
+            # Build a more focused search query for fallback
+            if key_terms:
+                # Use original query + key terms for better semantic matching
+                enhanced_query = f"{query} {' '.join(set(key_terms))}"
+                print(f"[HYBRID RAG] Fallback enhanced query: {enhanced_query[:100]}...")
+                search_query = enhanced_query
+        
+        # Perform semantic chunk search with higher top_k for fallback
+        # IMPORTANT: ChromaDB Cloud has a quota limit of 300 results per query
+        # Cap top_k to respect this limit
+        CHROMADB_MAX_RESULTS = 300
+        if is_fallback:
+            top_k_value = min(self.top_k * 4, CHROMADB_MAX_RESULTS)
+        else:
+            top_k_value = min(self.top_k * 2, CHROMADB_MAX_RESULTS)
+        print(f"[HYBRID RAG] Using top_k={top_k_value} for search (capped at {CHROMADB_MAX_RESULTS} for ChromaDB quota)")
+        
+        chunks = chroma_service.search_chunks(
+            user_id=user_id,
+            query=search_query,
+            bucket_id=bucket_id,
+            doc_ids=doc_ids,  # Pass doc_ids to filter by specific documents if detected
+            top_k=top_k_value
+        )
+        
+        if not chunks:
+            print("[HYBRID RAG] No chunks found from RAG search")
+            return {
+                'context': '',
+                'sources': {},
+                'chunk_count': 0,
+                'chunks': []
+            }
+        
+        print(f"[HYBRID RAG] Found {len(chunks)} chunks via semantic search")
+        
+        # Build context from chunks
+        context_parts = []
+        sources = {}
+        
+        for i, chunk in enumerate(chunks, 1):
+            doc_id = chunk['doc_id']
+            
+            # Get filename from chroma if not cached
+            if doc_id not in sources:
+                doc_info = chroma_service.get_document(doc_id, user_id)
+                filename = doc_info.get('filename', 'Document') if doc_info else 'Document'
+                sources[doc_id] = filename
+            
+            # Build context entry with document label
+            section = f"=== DOCUMENT: {sources[doc_id]} (Section {i}) ===\n{chunk['text']}"
+            context_parts.append(section)
+        
+        context = "\n\n".join(context_parts)
+        print(f"[HYBRID RAG] Built context: {len(context)} chars from {len(chunks)} chunks")
+        
+        return {
+            'context': context,
+            'sources': sources,
+            'chunk_count': len(chunks),
+            'chunks': chunks
+        }
+
+    def _combine_metadata_and_rag(self, metadata_result: dict, rag_result: dict) -> dict:
+        """
+        Combine metadata and RAG contexts for hybrid queries.
+        Provides structured metadata summary + detailed RAG content.
+        
+        Args:
+            metadata_result: Result from _handle_metadata_query
+            rag_result: Result from _get_rag_context_for_query
+            
+        Returns:
+            Combined context dict with merged sources
+        """
+        combined_parts = []
+        
+        # Add metadata summary section if available
+        if metadata_result.get('context') and metadata_result.get('total_documents', 0) > 0:
+            combined_parts.append("=== DOCUMENT METADATA (Structured Fields) ===")
+            combined_parts.append(metadata_result['context'])
+            combined_parts.append("")
+        
+        # Add RAG context section if available
+        if rag_result.get('context') and rag_result.get('chunk_count', 0) > 0:
+            combined_parts.append("=== DETAILED DOCUMENT CONTENT (From Text Search) ===")
+            combined_parts.append(rag_result['context'])
+        
+        # Merge sources
+        all_sources = {}
+        all_sources.update(metadata_result.get('sources', {}))
+        all_sources.update(rag_result.get('sources', {}))
+        
+        combined_context = "\n".join(combined_parts)
+        
+        print(f"[HYBRID] Combined context: metadata={metadata_result.get('total_documents', 0)} docs, "
+              f"rag={rag_result.get('chunk_count', 0)} chunks, total sources={len(all_sources)}")
+        
+        return {
+            'context': combined_context,
+            'sources': all_sources,
+            'total_documents': metadata_result.get('total_documents', 0),
+            'chunk_count': rag_result.get('chunk_count', 0),
+            'calculation': metadata_result.get('calculation'),
+            'total_before_filter': metadata_result.get('total_before_filter', 0)
+        }
+
+    def _stream_hybrid_query(self, user_id: str, bucket_id: str,
+                              query: str, parsed: dict, chat_id: str = ""):
+        """
+        Stream responses for HYBRID queries.
+        Combines metadata (structured fields) with RAG (detailed content) for comprehensive answers.
+        
+        Works for all query types: specific, compare, general, summarize, followup.
+        """
+        print(f"[HYBRID STREAM] Handling {parsed.get('intent')} query with metadata+RAG")
+        
+        # Get format preference
+        format_preference = parsed.get('format_preference')
+        format_instructions = self._get_format_instructions(format_preference) if format_preference else ""
+        
+        # Step 1: Get metadata context (may return 0 if filters don't match exactly)
+        metadata_result = self._handle_metadata_query(user_id, bucket_id, query, parsed)
+        print(f"[HYBRID STREAM] Metadata returned {metadata_result.get('total_documents', 0)} docs")
+        
+        # Step 2: Always get RAG context for detailed content
+        # If metadata returned 0, use fallback mode for more aggressive search
+        # Also detect document names in query for targeted search
+        metadata_has_results = metadata_result.get('total_documents', 0) > 0
+        rag_result = self._get_rag_context_for_query(
+            user_id, bucket_id, query,
+            filters=parsed.get('filters'),
+            is_fallback=not metadata_has_results,  # Use fallback mode if metadata failed
+            doc_ids=None  # Document name detection happens inside the method
+        )
+        print(f"[HYBRID STREAM] RAG returned {rag_result.get('chunk_count', 0)} chunks")
+        
+        # Step 3: Combine contexts
+        if metadata_result.get('total_documents', 0) > 0:
+            # Have metadata - combine with RAG
+            combined = self._combine_metadata_and_rag(metadata_result, rag_result)
+        elif rag_result.get('chunk_count', 0) > 0:
+            # No metadata match but RAG found content - use RAG only
+            print("[HYBRID STREAM] No metadata match, using RAG-only context")
+            combined = {
+                'context': rag_result['context'],
+                'sources': rag_result['sources'],
+                'total_documents': 0,
+                'chunk_count': rag_result['chunk_count'],
+                'calculation': None,
+                'total_before_filter': 0
+            }
+        else:
+            # Neither found anything
+            yield {
+                "type": "error",
+                "content": "No matching documents found. The document may not exist or try rephrasing your query."
+            }
+            return
+        
+        context = combined['context']
+        sources = combined['sources']
+        total_docs = combined.get('total_documents', 0) + combined.get('chunk_count', 0)
+        
+        # Send sources first
+        yield {
+            "type": "sources",
+            "sources": list(sources.keys()),
+            "source_files": list(sources.values())
+        }
+        
+        # Step 4: Build AI prompt based on intent
+        intent = parsed.get('intent', 'specific')
+        
+        if intent == 'compare':
+            system_prompt = f"""You are Iribl AI, a document analysis assistant answering a COMPARISON query.
+
+CRITICAL INSTRUCTIONS:
+1. You have BOTH structured metadata AND detailed document content.
+2. Use metadata for key fields: policy numbers, amounts, dates, companies.
+3. Use detailed content for specifics not in metadata.
+4. Create a clear comparison highlighting differences and similarities.
+5. Use a table format if comparing multiple attributes.
+
+{format_instructions}
+
+Do NOT say information is missing if it's in the provided context."""
+
+        elif intent == 'summarize':
+            system_prompt = f"""You are Iribl AI, a document analysis assistant providing a SUMMARY.
+
+CRITICAL INSTRUCTIONS:
+1. You have BOTH structured metadata AND detailed document content.
+2. Provide a concise but comprehensive summary.
+3. Include key facts: insured name, policy type, coverage, premium, dates.
+4. Highlight important terms or conditions from detailed content.
+5. Format with clear headers and bullet points.
+
+{format_instructions}
+
+Do NOT say information is missing - search through ALL provided context thoroughly."""
+
+        elif intent == 'specific':
+            system_prompt = f"""You are Iribl AI, a document analysis assistant answering a SPECIFIC query about a particular document or entity.
+
+CRITICAL INSTRUCTIONS:
+1. You have BOTH structured metadata AND detailed document content.
+2. Use metadata for: policy number, insured name, sum insured, premium, dates.
+3. Use detailed content for: coverage details, terms, conditions, exclusions.
+4. Provide a comprehensive answer covering all relevant information.
+5. Format clearly with headers and bullet points.
+
+{format_instructions}
+
+Do NOT say information is missing - search through ALL provided context thoroughly."""
+
+        else:  # general, followup, or any other
+            system_prompt = f"""You are Iribl AI, a document analysis assistant.
+
+CRITICAL INSTRUCTIONS:
+1. You have BOTH structured metadata AND detailed document content.
+2. Search thoroughly through ALL provided context before answering.
+3. Use metadata for structured fields like names, amounts, dates.
+4. Use detailed content for explanations, terms, conditions.
+5. Provide a complete and accurate answer based on the documents.
+6. Format clearly with headers and bullet points where appropriate.
+
+{format_instructions}
+
+Do NOT say information is missing - search through ALL provided context thoroughly."""
+
+        # Step 5: Load conversation history
+        stored_history = []
+        if chat_id:
+            try:
+                all_history = chroma_service.get_conversation_history(
+                    user_id=user_id,
+                    bucket_id=bucket_id,
+                    limit=50
+                )
+                stored_history = [msg for msg in all_history if msg.get('chat_id', '') == chat_id]
+                stored_history = stored_history[-self.max_history:]
+            except Exception as e:
+                print(f"[HYBRID STREAM] Failed to load history: {e}")
+        
+        # Step 6: Build messages
+        messages = [{"role": "system", "content": system_prompt}]
+        
+        for msg in stored_history:
+            messages.append({
+                "role": msg['role'],
+                "content": msg['content']
+            })
+        
+        format_reminder = f"\n\nRemember: Format response as {format_preference}." if format_preference else ""
+        
+        user_message = f"""Based on the following document data, answer my question comprehensively.
+
+DOCUMENT DATA:
+{context}
+
+QUESTION: {query}
+
+Instructions: Use both the structured metadata AND detailed content to provide a complete answer.{format_reminder}"""
+        
+        messages.append({"role": "user", "content": user_message})
+        
+        # Step 7: Stream response
+        full_response = ""
+        chunk_count = 0
+        
+        if self.use_deepseek:
+            print("[HYBRID STREAM] Using DeepSeek for response")
+            for chunk in self._call_deepseek_streaming(messages):
+                if "error" in chunk:
+                    break
+                if "chunk" in chunk:
+                    full_response += chunk["chunk"]
+                    chunk_count += 1
+                    yield {"type": "content", "content": chunk["chunk"]}
+        
+        # Fallback to OpenRouter if needed
+        if not full_response:
+            print("[HYBRID STREAM] Falling back to OpenRouter")
+            for model_key in self.fallback_order:
+                try:
+                    for chunk in self._call_ai_model_streaming(model_key, messages):
+                        if "error" in chunk:
+                            continue
+                        if "chunk" in chunk:
+                            full_response += chunk["chunk"]
+                            chunk_count += 1
+                            yield {"type": "content", "content": chunk["chunk"]}
+                    if full_response:
+                        break
+                except Exception as e:
+                    print(f"[HYBRID STREAM] Model {model_key} failed: {e}")
+                    continue
+        
+        # Step 8: Store conversation
+        if full_response and chat_id:
+            try:
+                chroma_service.store_conversation(
+                    user_id=user_id,
+                    role="user",
+                    content=query,
+                    bucket_id=bucket_id or "",
+                    chat_id=chat_id
+                )
+                chroma_service.store_conversation(
+                    user_id=user_id,
+                    role="assistant",
+                    content=full_response,
+                    bucket_id=bucket_id or "",
+                    chat_id=chat_id,
+                    format_preference=format_preference
+                )
+            except Exception as e:
+                print(f"[HYBRID STREAM] Failed to store conversation: {e}")
+        
+        yield {
+            "type": "done",
+            "query_type": "hybrid",
+            "intent": intent,
+            "metadata_docs": combined.get('total_documents', 0),
+            "rag_chunks": combined.get('chunk_count', 0)
+        }
+
     def _stream_metadata_query(self, user_id: str, bucket_id: str, 
                                 query: str, parsed: dict, chat_id: str = ""):
         """
@@ -1116,13 +1614,34 @@ Summary: {summary[:300] if summary else 'No summary available'}
             total_before = result.get('total_before_filter', 0)
             calculation = result.get('calculation')
         
-        # Check if we have any data
+        # Check if we have any data - if not, try RAG fallback
         if not context or total_docs == 0:
-            yield {
-                "type": "error", 
-                "content": "No document metadata found. Please run the migration script to extract metadata from your documents."
-            }
-            return
+            print(f"[HYBRID] Metadata returned 0 results, attempting RAG fallback...")
+            
+            # Try RAG fallback with the filters as search enhancement
+            # Use is_fallback=True for more aggressive search
+            # Also detect document names in query for targeted search
+            rag_result = self._get_rag_context_for_query(
+                user_id, bucket_id, query, 
+                filters=parsed.get('filters'),
+                is_fallback=True,  # Use more aggressive search parameters
+                doc_ids=None  # Document name detection happens inside the method
+            )
+            
+            if rag_result.get('context') and rag_result.get('chunk_count', 0) > 0:
+                # Use RAG context instead
+                context = rag_result['context']
+                sources = rag_result['sources']
+                total_docs = rag_result['chunk_count']
+                total_before = 0  # Not applicable for RAG
+                print(f"[HYBRID] RAG fallback successful: found {total_docs} chunks")
+            else:
+                # Both metadata and RAG failed
+                yield {
+                    "type": "error", 
+                    "content": "No matching documents found. The document may not exist in this collection, or try rephrasing your query."
+                }
+                return
         
         # Send sources first
         yield {
@@ -1139,7 +1658,30 @@ Summary: {summary[:300] if summary else 'No summary available'}
         conciseness_directive = "\n\nIMPORTANT: Be concise and direct. No preambles or verbose explanations. Get straight to the formatted answer." if format_preference else ""
         
         if intent == 'count':
-            system_prompt = f"""You are Iribl AI, a document analysis assistant answering a COUNT query.
+            # Check if we're using RAG fallback (metadata returned 0)
+            is_rag_fallback = (total_before == 0 and total_docs > 0)
+            
+            if is_rag_fallback:
+                # Using RAG content - need to extract count from document text
+                system_prompt = f"""You are Iribl AI, a document analysis assistant answering a COUNT query.
+
+CRITICAL INSTRUCTIONS:
+1. The user is asking for a COUNT/NUMBER that may not be in structured metadata.
+2. You have been provided with detailed document content from RAG search.
+3. CAREFULLY read through the document content to find the specific number/count requested.
+4. Look for numbers, counts, totals, or quantities related to the query.
+5. If the query asks "how many students", search for phrases like:
+   - "total students", "number of students", "students insured", "X students"
+   - Look for explicit numbers in the context
+6. State the count clearly and directly. If you find the number, present it confidently.
+7. If the count is not explicitly stated, say so clearly.{conciseness_directive}
+
+{format_instructions}
+
+IMPORTANT: The answer is in the provided document content. Read it carefully to extract the exact number."""
+            else:
+                # Using metadata - count is pre-computed
+                system_prompt = f"""You are Iribl AI, a document analysis assistant answering a COUNT query.
 
 CRITICAL INSTRUCTIONS:
 1. The count has been computed: {total_docs} documents match the criteria.
@@ -1191,18 +1733,31 @@ CRITICAL INSTRUCTIONS:
 {format_instructions if format_instructions else "FORMAT: Use tables or side-by-side format where helpful."}"""
 
         else:  # list, summarize, or other
-            system_prompt = f"""You are Iribl AI, a document analysis assistant. You are answering a query that requires information from {total_docs} documents.
-
-CRITICAL INSTRUCTIONS:
-1. You have been given metadata for {total_docs} documents (from {total_before} total).
-2. Your answer must be COMPREHENSIVE - include ALL relevant items from the data provided.
-3. For "list" queries, actually list ALL matching items with key details.
-4. Organize information logically (by type, by company, by date, etc.).
-5. For "summarize" queries, provide a concise overview with key statistics.{conciseness_directive}
+            system_prompt = f"""You are Iribl AI, a document analysis assistant. You are answering a query about {total_docs} documents.
+
+ABSOLUTELY CRITICAL - READ CAREFULLY:
+1. You have been given metadata for EXACTLY {total_docs} documents.
+2. When asked to list or format as table, you MUST include ALL {total_docs} items.
+3. Do NOT truncate, summarize, or skip ANY items.
+4. If there are {total_docs} documents, your response MUST contain {total_docs} entries.
+5. For TABLES: Include EVERY row - the table must have exactly {total_docs} data rows.
+6. For LISTS: Number 1 through {total_docs} - include every single one.
+
+METADATA COLUMNS AVAILABLE:
+- document_title (Policy/Document Name)
+- insured_name (Insured Company)
+- insurer_name (Insurance Company)
+- policy_type (Type of Policy)
+- sum_insured (Coverage Amount)
+- premium_amount (Premium)
+- renewal_date (Renewal Date)
+- renewal_year (Renewal Year)
+- policy_start_date, policy_end_date
+- city, state (Location)
 
 {format_instructions if format_instructions else "FORMAT: Use headers, bullet points, and bold text for clarity."}
 
-Do NOT say information is missing - you have the filtered list. Do NOT ask for more documents."""
+FAILURE TO INCLUDE ALL {total_docs} ITEMS IS UNACCEPTABLE. Do NOT say 'and X more' or truncate the list."""
 
         # Step 3: Load conversation history for memory (CRITICAL FOR CONTEXT)
         stored_history = []
@@ -1900,13 +2455,25 @@ Instructions: Synthesize from multiple documents if relevant. Be detailed but co
         print(f"[QUERY ROUTING] AI-parsed query: {parsed}")
         
         # Route based on AI-parsed intent
-        # needs_metadata = True means query requires aggregate data across all documents
-        if parsed.get('needs_metadata', False):
+        intent = parsed.get('intent', 'specific')
+        needs_metadata = parsed.get('needs_metadata', False)
+        
+        # HYBRID ROUTING LOGIC:
+        # 1. For aggregate/list/count/rank queries: Use metadata (with RAG fallback)
+        # 2. For ALL other queries: Use HYBRID (metadata + RAG together) for comprehensive answers
+        
+        if intent in ['list', 'count', 'rank', 'calculate'] and needs_metadata:
+            # Aggregate queries - metadata is primary, RAG is fallback (handled inside)
+            print(f"[QUERY ROUTING] Using METADATA path for {intent} query")
             yield from self._stream_metadata_query(user_id, bucket_id, query, parsed, chat_id)
             return
         
-        # For all other query types (specific, comparison, followup, general),
-        # continue with existing top-K chunk retrieval logic
+        else:
+            # ALL other queries (specific, compare, general, summarize, followup)
+            # Use HYBRID approach - both metadata AND RAG for comprehensive answers
+            print(f"[QUERY ROUTING] Using HYBRID path for {intent} query")
+            yield from self._stream_hybrid_query(user_id, bucket_id, query, parsed, chat_id)
+            return
         
         # Step 1: Expand query for better retrieval (handles "module 5" -> "module five", etc.)
         expanded_queries = self._expand_query(query)

static/css/styles.css

@@ -671,6 +671,7 @@ body {
     overflow: hidden;
     height: 100%;
     /* Ensure it takes full height */
+    transition: all var(--transition-smooth);
 }
 
 /* ==================== Chat Bucket Filter ==================== */
@@ -807,6 +808,21 @@ body {
     gap: 1rem;
     min-height: 0;
     /* Critical: allows scrolling to work */
+    transition: padding var(--transition-smooth);
+}
+
+/* Adjust chat when both sidebars are open for better space utilization */
+.main-content:has(.sidebar-left:not(.collapsed)):has(.sidebar-right:not(.collapsed)) .chat-messages {
+    padding: 0.875rem;
+    gap: 0.875rem;
+}
+
+.main-content:has(.sidebar-left:not(.collapsed)):has(.sidebar-right:not(.collapsed)) .message.assistant {
+    max-width: 85%;
+}
+
+.main-content:has(.sidebar-left:not(.collapsed)):has(.sidebar-right:not(.collapsed)) .message.user {
+    max-width: 75%;
 }
 
 /* Custom scrollbar for chat messages */
@@ -830,8 +846,10 @@ body {
 .message {
     display: flex;
     gap: 0.75rem;
-    max-width: 85%;
+    max-width: 100%;
+    width: 100%;
     animation: messageSlide 0.3s ease-out;
+    min-width: 0; /* Allow flex item to shrink below content size */
 }
 
 @keyframes messageSlide {
@@ -849,6 +867,12 @@ body {
 .message.user {
     align-self: flex-end;
     flex-direction: row-reverse;
+    max-width: 80%; /* User messages can be narrower */
+}
+
+.message.assistant {
+    align-self: flex-start;
+    max-width: 90%; /* Assistant messages use most of the width, leaving small margin */
 }
 
 .message-avatar {
@@ -868,20 +892,29 @@ body {
     padding: 1rem 1.25rem;
     border-radius: var(--radius-lg);
     font-size: 0.9rem;
-    line-height: 1.6;
+    line-height: 1.7;
+    max-width: 100%;
+    min-width: 0; /* Allow content to shrink */
+    word-wrap: break-word;
+    overflow-wrap: break-word;
+    word-break: break-word; /* Break long words if needed */
+    color: var(--text-secondary);
 }
 
 .message.user .message-content {
     background: var(--accent-primary);
-    color: var(--bg-darkest);
+    color: var(--bg-darkest) !important; /* Ensure dark text on white background */
     border-bottom-right-radius: 4px;
+    padding: 1rem 1.25rem; /* Slightly less padding for user messages */
+    font-weight: 500; /* Slightly bolder for better readability */
+    line-height: 1.7;
 }
 
 .message.assistant .message-content {
     background: linear-gradient(135deg, var(--bg-light) 0%, var(--bg-medium) 100%);
     border: 1px solid var(--glass-border);
     border-bottom-left-radius: 4px;
-    box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
+    box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2);
 }
 
 .message-sources {
@@ -905,54 +938,84 @@ body {
 .message-content h2,
 .message-content h3,
 .message-content h4,
+.message-content h5,
+.message-content h6,
 .message-content .msg-header {
-    font-weight: 600;
+    font-weight: 700;
     color: var(--text-primary);
-    margin: 1.25rem 0 0.6rem 0;
-    line-height: 1.4;
+    margin: 2rem 0 1rem 0;
+    line-height: 1.3;
+    letter-spacing: -0.01em;
 }
 
 .message-content h1 {
-    font-size: 1.25rem;
-    background: linear-gradient(90deg, var(--accent-primary), var(--accent-secondary));
+    font-size: 1.3rem;
+    background: linear-gradient(135deg, var(--accent-primary) 0%, #a855f7 100%);
     -webkit-background-clip: text;
     -webkit-text-fill-color: transparent;
     background-clip: text;
-    padding-bottom: 0.5rem;
-    border-bottom: 2px solid rgba(168, 85, 247, 0.3);
+    padding-bottom: 0.6rem;
+    border-bottom: 2px solid rgba(168, 85, 247, 0.4);
+    margin-top: 0;
+    margin-bottom: 1.25rem;
 }
 
 .message-content h2 {
-    font-size: 1.1rem;
-    color: var(--accent-secondary);
-    border-bottom: 1px solid rgba(168, 85, 247, 0.2);
-    padding-bottom: 0.4rem;
+    font-size: 1.15rem;
+    color: var(--accent-primary);
+    border-bottom: 1px solid rgba(168, 85, 247, 0.3);
+    padding-bottom: 0.45rem;
+    margin-top: 1.5rem;
+    margin-bottom: 0.875rem;
 }
 
 .message-content h3 {
-    font-size: 1rem;
+    font-size: 1.05rem;
     color: var(--info);
+    margin-top: 1.5rem;
+    margin-bottom: 0.75rem;
+    font-weight: 600;
 }
 
 .message-content h4 {
-    font-size: 0.95rem;
+    font-size: 1rem;
     font-weight: 600;
-    color: var(--text-secondary);
-    margin: 0.9rem 0 0.4rem 0;
+    color: var(--text-primary);
+    margin: 1.25rem 0 0.625rem 0;
+}
+
+.message-content h5,
+.message-content h6 {
+    font-size: 1rem;
+    font-weight: 600;
+    color: var(--text-primary);
+    margin: 1.25rem 0 0.625rem 0;
 }
 
 .message-content h1:first-child,
 .message-content h2:first-child,
 .message-content h3:first-child,
 .message-content h4:first-child,
+.message-content h5:first-child,
+.message-content h6:first-child,
 .message-content .msg-header:first-child {
     margin-top: 0;
 }
 
 .message-content p,
 .message-content .msg-para {
-    margin: 0.75rem 0;
+    margin: 0.875rem 0;
     line-height: 1.75;
+    color: var(--text-secondary);
+    font-size: 0.9rem;
+}
+
+.message-content p:first-child {
+    margin-top: 0;
+}
+
+.message-content p:last-child {
+    margin-bottom: 0;
 }
 
 .message-content p:first-child,
@@ -962,7 +1025,7 @@ body {
 
 /* ==================== Enhanced Lists ==================== */
 .message-content .formatted-list {
-    margin: 1rem 0;
+    margin: 1.25rem 0;
     padding-left: 0;
     list-style: none;
 }
@@ -973,17 +1036,20 @@ body {
 
 .message-content .formatted-list li {
     position: relative;
-    padding: 0.5rem 0.75rem 0.5rem 2.25rem;
-    margin: 0.35rem 0;
-    background: rgba(255, 255, 255, 0.02);
+    padding: 0.625rem 0.875rem 0.625rem 2.25rem;
+    margin: 0.4rem 0;
+    background: rgba(255, 255, 255, 0.03);
     border-radius: var(--radius-md);
     border-left: 3px solid transparent;
-    line-height: 1.65;
+    line-height: 1.7;
     transition: all 0.2s ease;
+    font-size: 0.9rem;
+    color: var(--text-secondary);
 }
 
 .message-content .formatted-list li:hover {
-    background: rgba(255, 255, 255, 0.04);
+    background: rgba(255, 255, 255, 0.06);
+    transform: translateX(2px);
 }
 
 .message-content .formatted-list li.numbered {
@@ -1032,14 +1098,16 @@ body {
 /* Legacy list support */
 .message-content ul,
 .message-content ol {
-    margin: 0.75rem 0;
-    padding-left: 1.5rem;
+    margin: 1.25rem 0;
+    padding-left: 1.75rem;
 }
 
 .message-content li {
-    margin: 0.4rem 0;
+    margin: 0.5rem 0;
     padding-left: 0.5rem;
-    line-height: 1.6;
+    line-height: 1.7;
+    font-size: 0.9rem;
+    color: var(--text-secondary);
 }
 
 .message-content ul li::marker {
@@ -1066,12 +1134,15 @@ body {
     overflow-x: auto;
     overflow-y: hidden;
     max-width: 100%;
+    width: 100%;
     box-shadow: 0 2px 12px rgba(0, 0, 0, 0.2);
     border: 1px solid rgba(255, 255, 255, 0.08);
+    -webkit-overflow-scrolling: touch; /* Smooth scrolling on iOS */
 }
 
 .message-content table {
     width: 100%;
+    min-width: 100%; /* Ensure table takes full width of wrapper */
     border-collapse: collapse;
     font-size: 0.8rem;
     background: rgba(0, 0, 0, 0.2);
@@ -1083,23 +1154,27 @@ body {
 }
 
 .message-content th {
-    padding: 0.6rem 0.75rem;
-    font-weight: 600;
+    padding: 0.75rem 1rem;
+    font-weight: 700;
     color: var(--text-primary);
     text-align: left;
-    border-bottom: 2px solid rgba(168, 85, 247, 0.3);
+    border-bottom: 2px solid rgba(168, 85, 247, 0.4);
     text-transform: uppercase;
-    font-size: 0.7rem;
-    letter-spacing: 0.3px;
+    font-size: 0.75rem;
+    letter-spacing: 0.5px;
     white-space: nowrap;
 }
 
 .message-content td {
-    padding: 0.5rem 0.75rem;
-    border-bottom: 1px solid rgba(255, 255, 255, 0.05);
+    padding: 0.75rem 1rem;
+    border-bottom: 1px solid rgba(255, 255, 255, 0.08);
     color: var(--text-secondary);
     word-break: break-word;
-    max-width: 200px;
+    overflow-wrap: break-word;
+    max-width: none; /* Remove max-width restriction */
+    min-width: 100px; /* Minimum width for readability */
+    line-height: 1.6;
+    font-size: 0.9rem;
 }
 
 .message-content tbody tr {
@@ -1159,12 +1234,17 @@ body {
 .message-content b {
     font-weight: 700;
     color: var(--text-primary);
+    background: linear-gradient(135deg, rgba(255, 255, 255, 0.1) 0%, rgba(168, 85, 247, 0.1) 100%);
+    padding: 0.1rem 0.3rem;
+    border-radius: 4px;
+    font-weight: 600;
 }
 
 .message-content em,
 .message-content i {
     font-style: italic;
-    color: var(--text-secondary);
+    color: var(--accent-secondary);
+    font-weight: 500;
 }
 
 /* ==================== Dividers ==================== */
@@ -1173,17 +1253,21 @@ body {
     height: 1px;
     background: linear-gradient(90deg, transparent, rgba(168, 85, 247, 0.4), transparent);
     margin: 1.5rem 0;
+    border-radius: 2px;
 }
 
 /* ==================== Blockquotes ==================== */
 .message-content blockquote {
     border-left: 4px solid var(--accent-primary);
-    margin: 1rem 0;
-    padding: 0.75rem 1.25rem;
-    background: linear-gradient(135deg, rgba(168, 85, 247, 0.08) 0%, rgba(96, 165, 250, 0.05) 100%);
+    margin: 1.25rem 0;
+    padding: 0.875rem 1.25rem;
+    background: linear-gradient(135deg, rgba(168, 85, 247, 0.12) 0%, rgba(96, 165, 250, 0.08) 100%);
     border-radius: 0 var(--radius-md) var(--radius-md) 0;
     font-style: italic;
     color: var(--text-secondary);
+    font-size: 0.9rem;
+    line-height: 1.75;
+    box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
 }
 
 /* ==================== Typing Indicator ==================== */

static/js/app.js

@@ -1326,7 +1326,8 @@ function renderMessages() {
 
     const html = state.messages.map((msg, i) => {
         const avatar = msg.role === 'user' ? (state.user?.username?.charAt(0).toUpperCase() || 'U') : '🧠';
-        return `<div class="message ${msg.role}"><div class="message-avatar">${avatar}</div><div class="message-content">${formatContent(msg.content)}</div></div>`;
+        const isUserMessage = msg.role === 'user';
+        return `<div class="message ${msg.role}"><div class="message-avatar">${avatar}</div><div class="message-content">${formatContent(msg.content, isUserMessage)}</div></div>`;
     }).join('');
 
     // Build full content with summary panel and welcome screen
@@ -1354,10 +1355,24 @@ function renderMessages() {
     elements.summaryClose.addEventListener('click', hideSummary);
 }
 
-function formatContent(content) {
+function formatContent(content, isUserMessage = false) {
     // Enhanced markdown parsing for beautiful formatting
     let html = content;
 
+    // For user messages, escape HTML and preserve line breaks
+    if (isUserMessage) {
+        // Escape HTML to prevent XSS
+        html = html
+            .replace(/&/g, '&amp;')
+            .replace(/</g, '&lt;')
+            .replace(/>/g, '&gt;')
+            .replace(/"/g, '&quot;')
+            .replace(/'/g, '&#039;');
+        // Convert line breaks to <br>
+        html = html.replace(/\n/g, '<br>');
+        return html;
+    }
+
     // Escape HTML special characters first (except for already parsed markdown)
     // Skip this if content looks like it's already HTML
     if (!html.includes('<table') && !html.includes('<div')) {