app/validator.py

"""
Core validation logic for medical document validator.
Handles document text extraction, image extraction, and multimodal LLM-based validation.
"""

import json
import os
import base64
import tempfile
import logging
import time
import shutil
import io
from io import BytesIO
from typing import Dict, List, Optional, Tuple, Any
from pathlib import Path
from dataclasses import dataclass, asdict
from datetime import datetime

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

import anthropic
from dotenv import load_dotenv
import fitz  # PyMuPDF
from docx import Document
from pptx import Presentation
from PIL import Image

# Disable DecompressionBombError for large images
Image.MAX_IMAGE_PIXELS = None

# Load environment variables
load_dotenv()

# Template file path
TEMPLATES_FILE = Path(__file__).parent / "templates.json"


@dataclass
class ExtractedImage:
    """Data structure for extracted images from documents."""
    id: str
    file_path: str
    page_number: int = 0
    role_hint: str = ""  # e.g., 'company logo', 'signature block', 'qr code'
    element_type: str = ""  # 'logo', 'signature_block', 'qr_code_or_image'


def load_templates() -> Dict:
    """Load and parse templates.json file."""
    try:
        with open(TEMPLATES_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    except FileNotFoundError:
        raise FileNotFoundError(f"Templates file not found: {TEMPLATES_FILE}")
    except json.JSONDecodeError as e:
        raise ValueError(f"Invalid JSON in templates file: {e}")


def get_template(template_key: str) -> Optional[Dict]:
    """Retrieve a specific template by its key."""
    templates_data = load_templates()
    for template in templates_data.get("templates", []):
        if template.get("template_key") == template_key:
            return template
    return None


def extract_text_with_claude_ocr(file_content: bytes) -> str:
    """
    Extract text from image-based PDF using Claude's vision API.
    Renders PDF pages as images and sends them to Claude for OCR.
    """
    logger.info("Starting Claude-based OCR for image PDF...")
    
    try:
        # Open PDF
        doc = fitz.open(stream=file_content, filetype="pdf")
        logger.info(f"Opened PDF with {len(doc)} page(s) for OCR")
        
        all_text = []
        
        # Process each page (limit to first 5 pages for performance)
        max_pages = min(5, len(doc))
        for page_num in range(max_pages):
            page = doc.load_page(page_num)
            
            # Render page to high-resolution image
            matrix = fitz.Matrix(2.0, 2.0)  # 2x scale for good quality
            pix = page.get_pixmap(matrix=matrix, alpha=False)
            
            # Convert to PNG bytes
            img_bytes = pix.pil_tobytes(format="PNG")
            
            # Encode to base64 for Claude
            img_base64 = base64.b64encode(img_bytes).decode('utf-8')
            
            logger.info(f"Sending page {page_num + 1} to Claude for OCR...")
            
            # Initialize Claude client
            client = load_llm_client()
            
            # Send to Claude for OCR - use a model that definitely exists
            message = client.messages.create(
                model="claude-3-opus-20240229",  # Use stable Claude 3 Opus
                max_tokens=4096,
                messages=[
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image",
                                "source": {
                                    "type": "base64",
                                    "media_type": "image/png",
                                    "data": img_base64
                                }
                            },
                            {
                                "type": "text",
                                "text": "Extract ALL text from this image. Return only the text content, preserving the original formatting and layout as much as possible. Include all text visible in the image, including headers, body text, and any Arabic text."
                            }
                        ]
                    }
                ]
            )
            
            # Extract text from response
            page_text = message.content[0].text if message.content else ""
            all_text.append(page_text)
            logger.info(f"Page {page_num + 1} OCR completed: {len(page_text)} characters extracted")
        
        doc.close()
        
        full_text = "\n\n".join(all_text)
        logger.info(f"OCR completed for {max_pages} page(s): {len(full_text)} total characters")
        
        return full_text
        
    except Exception as e:
        logger.error(f"Claude OCR failed with error: {type(e).__name__}: {str(e)}", exc_info=True)
        # Print to console as well for debugging
        print(f"[OCR ERROR] {type(e).__name__}: {str(e)}")
        # Don't raise - return empty string so validation can continue
        # The caller will handle empty text
        return ""


def extract_pdf_text(file_content: bytes) -> str:
    """
    Extract text content from a PDF file using PyMuPDF.
    If the PDF is image-based or has minimal text, use Claude OCR as fallback.
    """
    try:
        doc = fitz.open(stream=file_content, filetype="pdf")
        text_parts = []
        for page in doc:
            text_parts.append(page.get_text("text"))
        doc.close()
        extracted_text = "\n".join(text_parts).strip()
        
        # Check if extraction was successful (more than 50 characters)
        if len(extracted_text) < 50:
            logger.warning(f"Minimal text extracted ({len(extracted_text)} chars), PDF may be image-based. Attempting OCR...")
            
            # Try OCR using Claude vision
            try:
                extracted_text = extract_text_with_claude_ocr(file_content)
                logger.info(f"OCR successful: extracted {len(extracted_text)} characters")
            except Exception as ocr_error:
                logger.error(f"OCR failed: {str(ocr_error)}")
                # Return what we got, even if minimal
                if not extracted_text:
                    raise ValueError("No text could be extracted from PDF (may be empty or purely image-based without OCR)")
        
        return extracted_text
    except Exception as e:
        raise ValueError(f"Failed to extract text from PDF: {str(e)}")


def extract_pdf_images(file_content: bytes, temp_dir: Path) -> List[ExtractedImage]:
    """
    Extract images from PDF file using PyMuPDF (fitz) for reliable extraction.
    """
    extracted_images = []
    logger.info("Starting PDF image extraction using PyMuPDF...")
    try:
        doc = fitz.open(stream=file_content, filetype="pdf")
        total_pages = len(doc)
        logger.info(f"PDF has {total_pages} page(s)")
        
        for page_index, page in enumerate(doc):
            page_num = page_index + 1
            logger.info(f"Processing PDF page {page_num}/{total_pages}")
            
            # Extract images using PyMuPDF's robust method
            image_list = page.get_images(full=True)
            logger.info(f"Found {len(image_list)} image(s) on page {page_num}")
            
            for img_index, img_info in enumerate(image_list, start=1):
                try:
                    xref = img_info[0]
                    logger.info(f"Extracting image {img_index} (xref: {xref}) from page {page_num}")
                    
                    # Extract image data using PyMuPDF
                    base_image = doc.extract_image(xref)
                    
                    if base_image and 'image' in base_image:
                        image_bytes = base_image["image"]
                        image_ext = base_image.get("ext", "png")
                        logger.info(f"Image format: {image_ext}, size: {len(image_bytes)} bytes")
                        
                        # Create ExtractedImage with image data in memory (avoid file system issues)
                        image_name = f"page_{page_num}_img_{img_index}.{image_ext}"
                        image_path = temp_dir / image_name
                        
                        # Store image data in memory first
                        logger.info(f"Processing image {img_index} from page {page_num}, size: {len(image_bytes)} bytes")
                        
                        # Try to get image dimensions from memory
                        dimensions_info = "unknown"
                        try:
                            from io import BytesIO
                            img_io = BytesIO(image_bytes)
                            pil_img = Image.open(img_io)
                            pil_img.load()
                            dimensions_info = f"{pil_img.size[0]}x{pil_img.size[1]} pixels, mode: {pil_img.mode}"
                            pil_img.close()
                            img_io.close()
                        except Exception as e:
                            logger.warning(f"Could not read image properties from memory: {str(e)}")
                        
                        logger.info(f"Image dimensions: {dimensions_info}")
                        
                        # Write to file only when needed (for LLM processing later)
                        # Don't write now to avoid file locking issues during extraction
                        
                        # Create ExtractedImage with image data stored in memory
                        # We'll store the raw bytes and write to file only when needed for LLM
                        extracted_image = ExtractedImage(
                            id=f"pdf_img_xref_{xref}",
                            file_path=str(image_path),
                            page_number=page_num,
                            role_hint="Potential Logo, Signature, or other required image.",
                            element_type="image"
                        )
                        # Store image bytes as a custom attribute for later use
                        extracted_image._image_bytes = image_bytes
                        extracted_image._image_ext = image_ext
                        
                        extracted_images.append(extracted_image)
                        logger.info(f"Successfully extracted image {len(extracted_images)} from PDF page {page_num}")
                    else:
                        logger.warning(f"Image xref {xref} extraction returned no image data")
                except Exception as e:
                    logger.warning(f"Failed to extract image {img_index} from page {page_num}: {str(e)}")
                    continue
        
        doc.close()
        logger.info(f"PDF image extraction complete: found {len(extracted_images)} image(s)")
    except Exception as e:
        logger.error(f"PDF image extraction error: {str(e)}", exc_info=True)
    
    return extracted_images


def extract_docx_text(file_content: bytes) -> str:
    """Extract text content from a DOCX file."""
    try:
        docx_file = BytesIO(file_content)
        doc = Document(docx_file)
        text_parts = []
        for paragraph in doc.paragraphs:
            if paragraph.text.strip():
                text_parts.append(paragraph.text)
        # Also extract text from tables
        for table in doc.tables:
            for row in table.rows:
                row_text = " | ".join(cell.text.strip() for cell in row.cells)
                if row_text.strip():
                    text_parts.append(row_text)
        return "\n".join(text_parts)
    except Exception as e:
        raise ValueError(f"Failed to extract text from DOCX: {str(e)}")


def extract_docx_images(file_content: bytes, temp_dir: Path) -> List[ExtractedImage]:
    """
    Extract images from DOCX file.
    
    NOTE: This requires iterating through document parts to extract embedded images.
    The current implementation extracts from relationships, but more complex extraction
    may be needed for images embedded in different ways.
    """
    extracted_images = []
    logger.info("Starting DOCX image extraction...")
    try:
        docx_file = BytesIO(file_content)
        doc = Document(docx_file)
        
        total_rels = len(doc.part.rels)
        logger.info(f"DOCX has {total_rels} relationship(s)")
        
        # Extract images from document relationships
        image_count = 0
        for rel_id, rel in doc.part.rels.items():
            logger.debug(f"Checking relationship {rel_id}: {rel.target_ref}")
            if "image" in rel.target_ref:
                image_count += 1
                try:
                    image_part = rel.target_part
                    image_data = image_part.blob
                    logger.info(f"Found image relationship {rel_id}, data size: {len(image_data)} bytes")
                    
                    # Determine image format
                    ext = image_part.filename.split('.')[-1] if '.' in image_part.filename else 'png'
                    img_path = temp_dir / f"docx_img_{len(extracted_images)}.{ext}"
                    logger.info(f"Saving DOCX image {len(extracted_images) + 1} as {ext} format")
                    
                    # Save image
                    with open(img_path, 'wb') as f:
                        f.write(image_data)
                        f.flush()  # Ensure data is written
                        os.fsync(f.fileno())  # Force write to disk
                    # File handle is now closed
                    
                    # Small delay to ensure Windows releases the file handle
                    time.sleep(0.01)
                    
                    file_size = img_path.stat().st_size
                    logger.info(f"Saved image to {img_path}, file size: {file_size} bytes")
                    # Note: Image dimensions will be checked later when preparing for LLM
                    
                    # Create ExtractedImage with image data stored in memory
                    extracted_image = ExtractedImage(
                        id=f"docx_img_{len(extracted_images)}",
                        file_path=str(img_path),
                        page_number=0,  # DOCX doesn't have pages, use 0
                        role_hint="Potential Logo, Signature, or other required image.",
                        element_type="image"
                    )
                    # Store image bytes as a custom attribute for later use
                    extracted_image._image_bytes = image_data
                    extracted_image._image_ext = ext
                    
                    extracted_images.append(extracted_image)
                    logger.info(f"Successfully extracted image {len(extracted_images)} from DOCX")
                except Exception as e:
                    logger.warning(f"Failed to extract image from relationship {rel_id}: {str(e)}")
                    continue
        
        logger.info(f"Found {image_count} image relationship(s), successfully extracted {len(extracted_images)}")
    except Exception as e:
        logger.error(f"DOCX image extraction error: {str(e)}", exc_info=True)
    
    logger.info(f"DOCX image extraction complete: found {len(extracted_images)} image(s)")
    return extracted_images


def extract_pptx_text(file_content: bytes) -> str:
    """Extract text content from a PPTX file."""
    try:
        pptx_file = BytesIO(file_content)
        prs = Presentation(pptx_file)
        text_parts = []
        for slide_num, slide in enumerate(prs.slides, 1):
            text_parts.append(f"--- Slide {slide_num} ---")
            for shape in slide.shapes:
                if hasattr(shape, "text") and shape.text.strip():
                    text_parts.append(shape.text)
        return "\n".join(text_parts)
    except Exception as e:
        raise ValueError(f"Failed to extract text from PPTX: {str(e)}")


def extract_pptx_images(file_content: bytes, temp_dir: Path) -> List[ExtractedImage]:
    """
    Extract images from PPTX file.
    
    NOTE: This requires iterating through slides and shapes to extract embedded images.
    More complex extraction may be needed for images embedded in different ways.
    """
    extracted_images = []
    logger.info("Starting PPTX image extraction...")
    try:
        pptx_file = BytesIO(file_content)
        prs = Presentation(pptx_file)
        total_slides = len(prs.slides)
        logger.info(f"PPTX has {total_slides} slide(s)")
        
        for slide_num, slide in enumerate(prs.slides, 1):
            logger.info(f"Processing PPTX slide {slide_num}/{total_slides}")
            shape_count = len(slide.shapes)
            logger.info(f"Slide {slide_num} has {shape_count} shape(s)")
            
            for shape_idx, shape in enumerate(slide.shapes):
                if hasattr(shape, "image"):
                    try:
                        image = shape.image
                        image_data = image.blob
                        logger.info(f"Found image in shape {shape_idx + 1} on slide {slide_num}, data size: {len(image_data)} bytes")
                        
                        # Determine image format
                        ext = image.ext if hasattr(image, 'ext') else 'png'
                        img_path = temp_dir / f"pptx_slide{slide_num}_img_{len(extracted_images)}.{ext}"
                        logger.info(f"Saving PPTX image as {ext} format")
                        
                        # Process image in memory first
                        logger.info(f"Processing PPTX image from slide {slide_num}, size: {len(image_data)} bytes")
                        
                        # Try to get image dimensions from memory
                        dimensions_info = "unknown"
                        try:
                            from io import BytesIO
                            img_io = BytesIO(image_data)
                            pil_img = Image.open(img_io)
                            pil_img.load()
                            dimensions_info = f"{pil_img.size[0]}x{pil_img.size[1]} pixels"
                            pil_img.close()
                            img_io.close()
                        except Exception as e:
                            logger.warning(f"Could not read PPTX image dimensions from memory: {str(e)}")
                        
                        logger.info(f"PPTX image dimensions: {dimensions_info}")
                        
                        # Don't write to file yet to avoid locking issues
                        
                        # Create ExtractedImage with image data stored in memory
                        extracted_image = ExtractedImage(
                            id=f"pptx_img_{slide_num}_{len(extracted_images)}",
                            file_path=str(img_path),
                            page_number=slide_num,  # Use slide number as page number
                            role_hint="Potential Logo, Signature, or other required image.",
                            element_type="image"
                        )
                        # Store image bytes as a custom attribute for later use
                        extracted_image._image_bytes = image_data
                        extracted_image._image_ext = ext
                        
                        extracted_images.append(extracted_image)
                        logger.info(f"Successfully extracted image {len(extracted_images)} from PPTX")
                    except Exception as e:
                        logger.warning(f"Failed to extract image from shape {shape_idx + 1} on slide {slide_num}: {str(e)}")
                        continue
                else:
                    logger.debug(f"Shape {shape_idx + 1} on slide {slide_num} does not have image attribute")
    except Exception as e:
        logger.error(f"PPTX image extraction error: {str(e)}", exc_info=True)
    
    logger.info(f"PPTX image extraction complete: found {len(extracted_images)} image(s)")
    return extracted_images


def extract_images_from_document(
    file_content: bytes, 
    file_extension: str, 
    template_elements: List[Dict],
    temp_dir: Path
) -> Tuple[str, List[ExtractedImage]]:
    """
    Parses the document, extracts the plain text, and saves visual elements
    (logos, signatures, QR codes) as temporary image files for the LLM.
    
    Args:
        file_content: Binary content of the file
        file_extension: File extension (e.g., '.pdf', '.docx', '.pptx')
        template_elements: List of template elements to identify visual elements
        temp_dir: Temporary directory to save extracted images
    
    Returns:
        Tuple of (extracted_text, list of ExtractedImage objects)
    """
    extension = file_extension.lower().lstrip(".")
    logger.info(f"Extracting from {extension.upper()} file, size: {len(file_content)} bytes")
    
    # Extract text
    if extension == "pdf":
        extracted_text = extract_pdf_text(file_content)
        extracted_images = extract_pdf_images(file_content, temp_dir)
    elif extension == "docx":
        extracted_text = extract_docx_text(file_content)
        extracted_images = extract_docx_images(file_content, temp_dir)
    elif extension == "pptx":
        extracted_text = extract_pptx_text(file_content)
        extracted_images = extract_pptx_images(file_content, temp_dir)
    else:
        raise ValueError(f"Unsupported file format: {file_extension}")
    
    logger.info(f"Extracted text length: {len(extracted_text)} characters")
    logger.info(f"Extracted {len(extracted_images)} image(s) from document")
    
    # Map extracted images to template elements based on type
    visual_element_types = ['logo', 'signature_block', 'qr_code_or_image']
    visual_elements = [e for e in template_elements if e.get('type') in visual_element_types]
    logger.info(f"Template requires {len(visual_elements)} visual element(s): {[e.get('type') for e in visual_elements]}")
    
    # Try to match extracted images to template elements
    # For now, we'll use all extracted images and let the LLM classify them
    # In a more sophisticated implementation, you could use image analysis to match
    matched_images = []
    for idx, img in enumerate(extracted_images):
        logger.info(f"Processing extracted image {idx + 1}/{len(extracted_images)}: {img.id}")
        logger.info(f"  - File path: {img.file_path}")
        logger.info(f"  - Role hint: {img.role_hint}")
        logger.info(f"  - Element type: {img.element_type}")
        
        # Try to find matching element based on position or other heuristics
        # For now, assign based on available visual elements
        if visual_elements:
            # Assign role hints based on template
            for elem in visual_elements:
                if elem.get('type') == 'logo' and 'logo' in img.role_hint.lower():
                    img.role_hint = elem.get('label', 'logo')
                    img.element_type = elem.get('type', 'logo')
                    logger.info(f"  - Matched to template element: {elem.get('label')} ({elem.get('type')})")
                    break
                elif elem.get('type') == 'signature_block' and 'signature' in img.role_hint.lower():
                    img.role_hint = elem.get('label', 'signature')
                    img.element_type = elem.get('type', 'signature_block')
                    logger.info(f"  - Matched to template element: {elem.get('label')} ({elem.get('type')})")
                    break
        
        matched_images.append(img)
    
    logger.info(f"Final matched images: {len(matched_images)}")
    for img in matched_images:
        logger.info(f"  - {img.id}: {img.role_hint} ({img.element_type})")
    
    return extracted_text, matched_images


def extract_document_text(file_content: bytes, file_extension: str) -> str:
    """
    Router function to extract text based on file extension.
    
    Args:
        file_content: Binary content of the file
        file_extension: File extension (e.g., '.pdf', '.docx', '.pptx')
    
    Returns:
        Extracted text content as string
    
    Raises:
        ValueError: If file format is unsupported or extraction fails
    """
    extension = file_extension.lower().lstrip(".")
    
    if extension == "pdf":
        return extract_pdf_text(file_content)
    elif extension == "docx":
        return extract_docx_text(file_content)
    elif extension == "pptx":
        return extract_pptx_text(file_content)
    else:
        raise ValueError(f"Unsupported file format: {file_extension}. Supported formats: PDF, DOCX, PPTX")


def load_llm_client():
    """
    Initializes and returns the Multimodal LLM client.
    
    Returns:
        anthropic.Anthropic: Configured Anthropic client for Claude models
        
    Raises:
        ValueError: If LLM_API_KEY is not found in environment variables
    """
    api_key = os.getenv("LLM_API_KEY")
    if not api_key:
        raise ValueError("LLM_API_KEY not found in .env file. Please set your Anthropic API key.")
    
    return anthropic.Anthropic(api_key=api_key)


class Validator:
    """Document validator using multimodal LLM for context-aware validation."""
    
    def __init__(self):
        """Initialize the validator."""
        # Initialize Anthropic client using the helper function
        self.client = load_llm_client()
        # Use Claude Opus 4 which supports multimodal (images)
        self.model = "claude-opus-4-20250514"

    async def check_links(self, links: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """
        Check health of extracted links using HTTP HEAD/GET requests.
        """
        import aiohttp
        import asyncio
        
        results = []
        if not links:
            return results
        
        # Add headers to avoid being blocked as a bot
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
        
        # Increase timeout to 10 seconds for slower sites
        async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10), headers=headers) as session:
            for link in links:
                url = link["url"]
                # Handle www without protocol
                check_url = url
                if url.startswith("www."):
                    check_url = "https://" + url
                    
                status = "unknown"
                message = ""
                status_code = 0
                
                try:
                    if check_url.startswith("mailto:"):
                        status = "valid"  # Assume mailto is valid format
                        message = "Email link"
                        status_code = 200
                    else:
                        try:
                            # Try GET directly (skip HEAD since many sites block it)
                            async with session.get(check_url, allow_redirects=True, ssl=False) as response:
                                status_code = response.status
                                if 200 <= status_code < 400:
                                    status = "valid"
                                    message = "OK"
                                else:
                                    status = "broken"
                                    message = f"HTTP {status_code}"
                        except aiohttp.ClientError as e:
                            # More specific error message
                            status = "broken"
                            message = f"Connection error: {type(e).__name__}"
                            status_code = 0
                except asyncio.TimeoutError:
                    status = "broken"
                    message = "Timeout (>10s)"
                    status_code = 408
                except Exception as e:
                    status = "broken" 
                    message = f"Error: {type(e).__name__}"
                    status_code = 0
                    
                results.append({
                    "url": url,
                    "status": status,
                    "status_code": status_code,
                    "message": message,
                    "page": str(link.get("page", "Unknown"))
                })
                
        return results

    async def compare_documents(
        self,
        file1_content: bytes,
        file1_extension: str,
        file1_name: str,
        file2_content: bytes,
        file2_extension: str,
        file2_name: str
    ) -> Dict[str, Any]:
        """
        Compare two document versions using LLM to identify semantic changes.
        
        Args:
            file1_content: Binary content of first document
            file1_extension: File extension of first document
            file1_name: Filename of first document
            file2_content: Binary content of second document
            file2_extension: File extension of second document
            file2_name: Filename of second document
        
        Returns:
            Dictionary with comparison results including summary and detailed changes
        """
        logger.info(f"Starting comparison: {file1_name} vs {file2_name}")
        
        # Normalize text to handle PDF ligatures and encoding differences
        def normalize_text(text: str) -> str:
            """Normalize Unicode ligatures and common PDF text artifacts."""
            import unicodedata
            # Common ligature replacements
            ligatures = {
                'fi': 'fi',
                'fl': 'fl',
                'ff': 'ff',
                'ffi': 'ffi',
                'ffl': 'ffl',
                'Ꜳ': 'AA',
                'ꜳ': 'aa',
                'Æ': 'AE',
                'æ': 'ae',
                'Œ': 'OE',
                'œ': 'oe',
                '\u00AD': '',  # Soft hyphen
                '\u200B': '',  # Zero-width space
                '\u200C': '',  # Zero-width non-joiner
                '\u200D': '',  # Zero-width joiner
                '\uFEFF': '',  # BOM
            }
            for lig, replacement in ligatures.items():
                text = text.replace(lig, replacement)
            # Normalize Unicode to NFC form
            text = unicodedata.normalize('NFC', text)
            return text
        
        # Extract text from both documents
        text1 = normalize_text(extract_document_text(file1_content, file1_extension))
        text2 = normalize_text(extract_document_text(file2_content, file2_extension))
        
        logger.info(f"Extracted text - File 1: {len(text1)} chars, File 2: {len(text2)} chars")
        
        if not text1 and not text2:
            raise ValueError("Both documents appear to be empty or contain no extractable text")
        
        # Build LLM prompt for comparison
        comparison_prompt = f"""You are comparing two versions of a document to identify MEANINGFUL content changes.

DOCUMENT 1 (Original - {file1_name}):
{text1[:10000]}

DOCUMENT 2 (Modified - {file2_name}):
{text2[:10000]}

Please analyze the differences between these two documents and provide:

1. A natural language summary of the main changes (2-3 sentences)
2. A detailed list of UNIQUE, MEANINGFUL changes only

IMPORTANT RULES:
- Do NOT report duplicate changes - each change should appear only once
- IGNORE font/encoding differences (like ligatures, special characters that look the same)
- IGNORE minor whitespace or formatting differences
- Focus on actual CONTENT changes (text additions, deletions, modifications)
- Group similar changes together instead of listing each instance separately

Format your response as a JSON object with this structure:
{{
    "summary": "Brief summary of changes...",
    "changes": [
        {{
            "type": "addition|deletion|modification",
            "section": "Section name where change occurred",
            "description": "Description of the change"
        }}
    ]
}}

If the documents are essentially identical (only minor formatting/encoding differences), return:
{{
    "summary": "Documents are essentially identical with no meaningful content changes.",
    "changes": []
}}
"""

        try:
            # Call LLM API
            logger.info("Calling LLM for document comparison...")
            
            message = self.client.messages.create(
                model="claude-opus-4-20250514",
                max_tokens=4096,
                temperature=0.1,
                messages=[
                    {
                        "role": "user",
                        "content": comparison_prompt
                    }
                ]
            )
            
            response_text = message.content[0].text if message.content else ""
            logger.info(f"Received comparison response ({len(response_text)} chars)")
            
            # Parse JSON response - extract from markdown code blocks if present
            import json
            import re
            
            # Try to extract JSON from markdown code blocks
            json_text = response_text
            
            # Check for ```json ... ``` or ``` ... ``` patterns
            code_block_match = re.search(r'```(?:json)?\s*([\s\S]*?)```', response_text)
            if code_block_match:
                json_text = code_block_match.group(1).strip()
                logger.info("Extracted JSON from markdown code block")
            else:
                # Try to find raw JSON object
                json_match = re.search(r'\{[\s\S]*\}', response_text)
                if json_match:
                    json_text = json_match.group(0)
                    logger.info("Extracted raw JSON object from response")
            
            try:
                comparison_data = json.loads(json_text)
            except json.JSONDecodeError as json_err:
                logger.error(f"JSON parsing failed. Raw response: {response_text[:500]}...")
                raise ValueError(f"Failed to parse comparison response as JSON: {str(json_err)}")
            
            # Deduplicate changes based on description
            if 'changes' in comparison_data:
                seen_descriptions = set()
                unique_changes = []
                for change in comparison_data['changes']:
                    desc = change.get('description', '').lower().strip()
                    if desc and desc not in seen_descriptions:
                        seen_descriptions.add(desc)
                        unique_changes.append(change)
                comparison_data['changes'] = unique_changes
                logger.info(f"After deduplication: {len(unique_changes)} unique changes")
            
            logger.info(f"Comparison complete: {len(comparison_data.get('changes', []))} changes detected")
            
            return comparison_data
            
        except Exception as e:
            logger.error(f"Comparison failed: {str(e)}", exc_info=True)
            raise ValueError(f"Failed to compare documents: {str(e)}")

    async def bulk_validate_certificates(
        self,
        excel_content: bytes,
        name_column: str,
        certificate_data: List[Tuple[str, bytes, str]]
    ) -> Dict[str, Any]:
        """
        Validate multiple certificates against Excel name list with fuzzy matching.
        
        Args:
            excel_content: Binary content of Excel file
            name_column: Column name containing names
            certificate_data: List of (filename, content, extension) tuples
        
        Returns:
            Dictionary with validation results including exact/fuzzy matches
        """
        logger.info(f"Starting bulk validation: {len(certificate_data)} certificates")
        
        try:
            import openpyxl
            from io import BytesIO
            from difflib import SequenceMatcher
            
            # Parse Excel and extract names
            wb = openpyxl.load_workbook(BytesIO(excel_content))
            ws = wb.active
            
            # Find column index
            headers = [str(cell.value) for cell in ws[1] if cell.value]
            if name_column not in headers:
                raise ValueError(f"Column '{name_column}' not found in Excel file")
            
            col_idx = headers.index(name_column) + 1
            
            # Extract names from Excel (skip header row)
            excel_names = []
            for row in ws.iter_rows(min_row=2, min_col=col_idx, max_col=col_idx):
                if row[0].value:
                    excel_names.append(str(row[0].value).strip())
            
            logger.info(f"Extracted {len(excel_names)} names from Excel")
            
            # Extract names from certificates (parallel processing)
            cert_names = {}
            for filename, content, ext in certificate_data:
                try:
                    text = extract_document_text(content, ext)
                    # Store extracted text for this certificate
                    cert_names[filename] = text
                except Exception as e:
                    logger.warning(f"Failed to extract from {filename}: {str(e)}")
                    cert_names[filename] = ""
            
            logger.info(f"Extracted text from {len(cert_names)} certificates")
            
            # Match names
            results = {
                "total_names": len(excel_names),
                "total_certificates": len(certificate_data),
                "exact_matches": 0,
                "fuzzy_matches": 0,
                "missing": 0,
                "extras": 0,
                "details": []
            }
            
            matched_certs = set()
            
            # Check each Excel name against certificates
            for name in excel_names:
                found = False
                best_match = None
                best_similarity = 0
                
                for cert_file, cert_text in cert_names.items():
                    # Exact match
                    if name.lower() in cert_text.lower():
                        results["exact_matches"] += 1
                        results["details"].append({
                            "name": name,
                            "status": "exact_match",
                            "certificate_file": cert_file,
                            "similarity": 100
                        })
                        matched_certs.add(cert_file)
                        found = True
                        break
                    
                    # Fuzzy match
                    similarity = SequenceMatcher(None, name.lower(), cert_text.lower()).ratio() * 100
                    if similarity >= 90 and similarity > best_similarity:
                        best_similarity = similarity
                        best_match = cert_file
                
                if not found:
                    if best_match and best_similarity >= 90:
                        # Fuzzy match found
                        results["fuzzy_matches"] += 1
                        results["details"].append({
                            "name": name,
                            "status": "fuzzy_match",
                            "certificate_file": best_match,
                            "similarity": int(best_similarity)
                        })
                        matched_certs.add(best_match)
                    else:
                        # Missing
                        results["missing"] += 1
                        results["details"].append({
                            "name": name,
                            "status": "missing",
                            "certificate_file": None,
                            "similarity": None
                        })
            
            # Find extra certificates (not matched to any Excel name)
            for cert_file in cert_names.keys():
                if cert_file not in matched_certs:
                    results["extras"] += 1
                    results["details"].append({
                        "name": f"[Certificate: {cert_file}]",
                        "status": "extra",
                        "certificate_file": cert_file,
                        "similarity": None
                    })
            
            logger.info(f"Bulk validation complete: {results['exact_matches']} exact, "
                       f"{results['fuzzy_matches']} fuzzy, {results['missing']} missing, "
                       f"{results['extras']} extra")
            
            return results
            
        except Exception as e:
            logger.error(f"Bulk validation failed: {str(e)}", exc_info=True)
            raise ValueError(f"Failed to validate certificates: {str(e)}")

    def extract_links(self, file_content: bytes, file_extension: str) -> List[Dict[str, Any]]:
        """
        Extract links from PDF, DOCX, or PPTX files.
        """
        links = []
        logger.info(f"Extracting links from {file_extension} document (size: {len(file_content)} bytes)")
        
        try:
            if file_extension == ".pdf":
                with fitz.open(stream=file_content, filetype="pdf") as doc:
                    logger.info(f"PDF page count: {len(doc)}")
                    for page_num, page in enumerate(doc):
                        page_links = page.get_links()
                        logger.info(f"Page {page_num+1} has {len(page_links)} link objects")
                        for link in page_links:
                            if "uri" in link:
                                logger.info(f"  Found PDF URI: {link['uri']}")
                                links.append({
                                    "url": link["uri"],
                                    "page": page_num + 1,
                                    "source": "page_link"
                                })
                                
            elif file_extension == ".docx":
                # For DOCX, we need to inspect the relationship files in the zip
                from zipfile import ZipFile
                from lxml import etree
                
                logger.info("Processing DOCX for links...")
                with io.BytesIO(file_content) as docx_file:
                    with ZipFile(docx_file) as zip_ref:
                        # List all files for debugging
                        # logger.info(f"Files in DOCX: {zip_ref.namelist()}")
                        
                        # Find all relationship files
                        rel_files = [f for f in zip_ref.namelist() if f.endswith(".rels")]
                        logger.info(f"Found {len(rel_files)} relationship files: {rel_files}")
                        
                        for rel_file in rel_files:
                            try:
                                with zip_ref.open(rel_file) as f:
                                    tree = etree.parse(f)
                                    root = tree.getroot()
                                    namespaces = {'rel': 'http://schemas.openxmlformats.org/package/2006/relationships'}
                                    
                                    rels = root.findall(".//rel:Relationship", namespaces)
                                    logger.info(f"  Scanning {rel_file}: found {len(rels)} relationships")
                                    
                                    for rel in rels:
                                        target = rel.get("Target")
                                        type_attr = rel.get("Type")
                                        
                                        if type_attr and "hyperlink" in type_attr and target:
                                            logger.info(f"  Found DOCX Hyperlink: {target}")
                                            links.append({
                                                "url": target,
                                                "page": "Unknown",  # DOCX doesn't have fixed pages
                                                "source": "document_link"
                                            })
                            except Exception as e:
                                logger.error(f"Error parsing {rel_file}: {e}")
                                continue
                                
            elif file_extension == ".pptx":
                from pptx import Presentation
                logger.info("Processing PPTX for links...")
                with io.BytesIO(file_content) as ppt_file:
                    prs = Presentation(ppt_file)
                    for slide_num, slide in enumerate(prs.slides):
                        logger.info(f"Scanning Slide {slide_num+1} with {len(slide.shapes)} shapes")
                        for shape in slide.shapes:
                            # Check shape click action
                            try:
                                if shape.click_action and shape.click_action.hyperlink and shape.click_action.hyperlink.address:
                                    url = shape.click_action.hyperlink.address
                                    logger.info(f"  Found PPTX Shape Link: {url}")
                                    links.append({
                                        "url": url,
                                        "page": f"Slide {slide_num + 1}",
                                        "source": "shape_link"
                                    })
                            except AttributeError:
                                pass
                                
                            # Check text runs
                            if hasattr(shape, "text_frame"):
                                try:
                                    for paragraph in shape.text_frame.paragraphs:
                                        for run in paragraph.runs:
                                            if run.hyperlink and run.hyperlink.address:
                                                url = run.hyperlink.address
                                                logger.info(f"  Found PPTX Text Link: {url}")
                                                links.append({
                                                    "url": url,
                                                    "page": f"Slide {slide_num + 1}",
                                                    "source": "text_link"
                                                })
                                except AttributeError:
                                    pass

        except Exception as e:
            logger.error(f"Error extracting links: {str(e)}", exc_info=True)
            
        # Deduplicate links
        unique_links = []
        seen_urls = set()
        logger.info(f"Total raw links found: {len(links)}")
        
        for link in links:
            url = link["url"].strip()
            # Relaxed filtering logic for debugging: accept everything that looks like a potential link
            # We'll filter strictly later if needed, but for now we want to see what we rejected
            is_valid_format = (url.startswith("http") or url.startswith("mailto:") or url.startswith("www."))
            
            if not is_valid_format:
                 logger.warning(f"Rejected link format: '{url}'")
            
            if url and url not in seen_urls and is_valid_format:
                seen_urls.add(url)
                unique_links.append(link)
                
        logger.info(f"Unique valid links returned: {len(unique_links)}")
        return unique_links
    
    def _generate_image_catalog(self, extracted_images: List[ExtractedImage]) -> str:
        """
        Generate a catalog of extracted images with unique IDs for LLM reference.
        
        Args:
            extracted_images: List of extracted images
            
        Returns:
            Formatted image catalog string
        """
        logger.info(f"Generating image catalog for {len(extracted_images)} images")
        
        image_catalog = "### [IMAGE_CATALOG]\n"
        image_catalog += f"The document has {len(extracted_images)} extracted visual elements. The images are provided below the prompt in this exact order. Use the assigned ID to report findings.\n\n"
        
        for idx, img_data in enumerate(extracted_images):
            # Create a unique ID for the LLM to reference based on its position and original ID
            unique_id = f"IMAGE_{idx+1}_REF_{img_data.id}"
            
            # Get additional context about the image
            context_info = []
            if hasattr(img_data, 'page_number') and img_data.page_number:
                context_info.append(f"Page {img_data.page_number}")
            if img_data.role_hint:
                context_info.append(f"Hint: {img_data.role_hint}")
            if img_data.element_type:
                context_info.append(f"Type: {img_data.element_type}")
            
            context_str = " | ".join(context_info) if context_info else "No additional context"
            
            image_catalog += f"- **{unique_id}**: {context_str} (Appears as visual element #{idx+1} in the contents list)\n"
            
            logger.info(f"  - Cataloged image {idx+1}: {unique_id} - {context_str}")
        
        image_catalog += "\n**CRITICAL**: When reporting visual elements as FOUND, you MUST reference the specific IMAGE_ID from this catalog.\n\n"
        
        logger.info(f"Generated image catalog with {len(extracted_images)} entries")
        return image_catalog
    
    def _build_multimodal_prompt(
        self, 
        document_text: str, 
        template: Dict, 
        extracted_images: List[ExtractedImage],
        image_catalog: str
    ) -> List:
        """
        Build a multimodal prompt for the LLM including text and images.
        
        Args:
            document_text: Extracted text from the document
            template: Template configuration dictionary
            extracted_images: List of extracted images
        
        Returns:
            List of content blocks (text and images) for the LLM
        """
        template_name = template.get("friendly_name", template.get("template_key"))
        elements = template.get("elements", [])
        
        # CONSTRUCT THE MASTER PROMPT (Using the user's exact instructions with image catalog)
        MASTER_PROMPT_INSTRUCTION = f"""# DOCUMENT VALIDATION SYSTEM — MASTER PROMPT

You are a Document Template Validator.
Your job is to strictly adhere to the following rules and the provided JSON structure.

[INPUT DATA]
1. Template Name: {template_name}
2. Required Template Elements (Elements to validate):
{json.dumps(template["elements"], indent=2)}

3. Extracted Document Text:
{document_text}

{image_catalog}

[DETECTION RULES]

📌 LOGO & SIGNATURE DETECTION (Visual Elements)
You must check whether the document contains any image that corresponds to the required visual element (Company Logo, Event Logo, Signature Block).
- If found: FOUND — The 'details' field MUST specify: "Detected a logo/signature image corresponding to **[UNIQUE_IMAGE_ID]**." (You must use the ID from the IMAGE_CATALOG above).
- If no match appears in the images provided: MISSING — The 'details' field must explain why none of the {len(extracted_images)} images match the requirement and list what was found instead.

📌 DATE DETECTION
Accept all valid date formats: DD/MM/YYYY, Month YYYY, YYYY-MM-DD, Verbal dates.
Return FOUND if any valid date appears where expected.

📌 NAME DETECTION
Treat any of the following as names: Dr. X, Prof. X, First + Last, Company names, Full faculty names.

📌 CODE DETECTION
Detect any alphanumeric code when expected (DHA, Approval, RCP Codes, etc.).

📌 PLACEHOLDER EQUIVALENCE
A placeholder should be considered correctly replaced if it contains the correct type of data.

[OUTPUT FORMAT]
You MUST return your results in JSON, structured EXACTLY as defined below. Do not include any text, headers, or markdown outside of the JSON block.

Required JSON Structure:
{{
  "template_type": "{template_name}",
  "validation_results": [
    {{
      "element": "Element Name/Label",
      "status": "FOUND" | "MISSING" | "DIFFERENT",
      "details": "Explanation of why this status was assigned"
    }}
  ],
  "overall_summary": "High-level summary describing completeness and issues"
}}

CRITICAL REQUIREMENTS:
1. Retrieve the required elements from the template definition.
2. Parse the uploaded document text AND analyze all {len(extracted_images)} provided images using the IMAGE_CATALOG.
3. For each required element:
   - If matching → FOUND
   - If no match → MISSING
   - If placeholder replaced with wrong type → DIFFERENT
4. Return structured JSON ONLY.
5. For visual elements (logos, signatures): You MUST analyze the provided images and reference the specific IMAGE_ID from the catalog in your details.
6. Use comprehensive detection for dates, names, codes, and placeholders as specified above.
7. **MANDATORY**: When reporting visual elements as FOUND, include the exact IMAGE_ID (e.g., "IMAGE_1_REF_pdf_img_xref_123") in your details.

RESPOND WITH JSON ONLY - NO ADDITIONAL TEXT OR MARKDOWN."""
        
        # Build content list with text and images
        content = [{"type": "text", "text": MASTER_PROMPT_INSTRUCTION}]
        
        # Add images to content - load as PIL.Image objects first, then convert to base64
        logger.info(f"Preparing {len(extracted_images)} image(s) for LLM")
        images_added = 0
        
        for idx, img in enumerate(extracted_images):
            pil_image = None
            try:
                logger.info(f"Loading image {idx + 1}/{len(extracted_images)}: {img.file_path}")
                logger.info(f"  - Page number: {img.page_number}")
                logger.info(f"  - Role hint: {img.role_hint}")
                
                # Check if file exists
                img_path = Path(img.file_path)
                if not img_path.exists():
                    logger.warning(f"Image file not found: {img.file_path}")
                    continue
                
                file_size = img_path.stat().st_size
                logger.info(f"  - File size: {file_size} bytes")
                
                # Load and optimize image data
                optimized_image_data = None
                original_size = None
                optimized_size = None
                
                # Check if image data is stored in memory (new approach)
                if hasattr(img, '_image_bytes') and img._image_bytes:
                    # Use image data from memory
                    image_data = img._image_bytes
                    logger.info(f"  - Using image data from memory: {len(image_data)} bytes")
                    
                    # Load and optimize image from memory
                    try:
                        from io import BytesIO
                        img_io = BytesIO(image_data)
                        pil_image = Image.open(img_io)
                        pil_image.load()
                        original_size = pil_image.size
                        logger.info(f"  - Original dimensions: {original_size[0]}x{original_size[1]} pixels")
                        logger.info(f"  - Image mode: {pil_image.mode}")
                        logger.info(f"  - Image format: {pil_image.format}")
                        
                        # --- IMAGE OPTIMIZATION LOGIC ---
                        max_size = 2048  # Max dimension (pixels). Standard multimodal models handle this well.
                        
                        if max(pil_image.size) > max_size:
                            # Calculate new size, maintaining aspect ratio
                            ratio = max_size / max(pil_image.size)
                            new_size = (int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio))
                            
                            # Resize using a high-quality filter
                            from PIL.Image import Resampling
                            pil_image = pil_image.resize(new_size, Resampling.LANCZOS)
                            optimized_size = new_size
                            logger.info(f"  - Resized to: {new_size[0]}x{new_size[1]} pixels (ratio: {ratio:.3f})")
                        else:
                            optimized_size = original_size
                            logger.info(f"  - No resizing needed (within {max_size}px limit)")
                        
                        # Convert optimized image back to bytes
                        output_io = BytesIO()
                        # Determine format for saving
                        save_format = pil_image.format if pil_image.format in ['PNG', 'JPEG'] else 'PNG'
                        if save_format == 'JPEG' and pil_image.mode in ('RGBA', 'LA', 'P'):
                            # Convert to RGB for JPEG
                            pil_image = pil_image.convert('RGB')
                        pil_image.save(output_io, format=save_format, quality=95 if save_format == 'JPEG' else None)
                        optimized_image_data = output_io.getvalue()
                        
                        # Cleanup
                        pil_image.close()
                        img_io.close()
                        output_io.close()
                        pil_image = None
                        
                    except Exception as e:
                        logger.warning(f"  - Could not optimize image from memory: {str(e)}")
                        # Fallback to original data
                        optimized_image_data = image_data
                        
                else:
                    # Fallback: read from file (old approach)
                    logger.info(f"  - Reading image from file: {img.file_path}")
                    try:
                        # Load and optimize image from file
                        pil_image = Image.open(img.file_path)
                        pil_image.load()
                        original_size = pil_image.size
                        logger.info(f"  - Original dimensions: {original_size[0]}x{original_size[1]} pixels")
                        logger.info(f"  - Image mode: {pil_image.mode}")
                        logger.info(f"  - Image format: {pil_image.format}")
                        
                        # --- IMAGE OPTIMIZATION LOGIC ---
                        max_size = 2048  # Max dimension (pixels)
                        
                        if max(pil_image.size) > max_size:
                            # Calculate new size, maintaining aspect ratio
                            ratio = max_size / max(pil_image.size)
                            new_size = (int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio))
                            
                            # Resize using a high-quality filter
                            from PIL.Image import Resampling
                            pil_image = pil_image.resize(new_size, Resampling.LANCZOS)
                            optimized_size = new_size
                            logger.info(f"  - Resized to: {new_size[0]}x{new_size[1]} pixels (ratio: {ratio:.3f})")
                        else:
                            optimized_size = original_size
                            logger.info(f"  - No resizing needed (within {max_size}px limit)")
                        
                        # Convert optimized image to bytes
                        output_io = BytesIO()
                        save_format = pil_image.format if pil_image.format in ['PNG', 'JPEG'] else 'PNG'
                        if save_format == 'JPEG' and pil_image.mode in ('RGBA', 'LA', 'P'):
                            pil_image = pil_image.convert('RGB')
                        pil_image.save(output_io, format=save_format, quality=95 if save_format == 'JPEG' else None)
                        optimized_image_data = output_io.getvalue()
                        
                        # Cleanup
                        pil_image.close()
                        output_io.close()
                        pil_image = None
                        
                    except Exception as e:
                        logger.error(f"  - Could not optimize image from file {img.file_path}: {str(e)}")
                        # Try to read raw file data as fallback
                        try:
                            with open(img.file_path, "rb") as f:
                                optimized_image_data = f.read()
                            logger.info(f"  - Using raw file data as fallback: {len(optimized_image_data)} bytes")
                        except Exception as e2:
                            logger.error(f"  - Could not read raw file data: {str(e2)}")
                            continue
                
                if not optimized_image_data:
                    logger.error(f"  - No image data available for {img.file_path}")
                    continue
                
                # Log optimization results
                if original_size and optimized_size:
                    original_pixels = original_size[0] * original_size[1]
                    optimized_pixels = optimized_size[0] * optimized_size[1]
                    reduction_ratio = optimized_pixels / original_pixels if original_pixels > 0 else 1.0
                    logger.info(f"  - Pixel reduction: {reduction_ratio:.3f}x ({original_pixels:,} → {optimized_pixels:,} pixels)")
                
                # Convert to base64 for Anthropic API
                image_base64 = base64.b64encode(optimized_image_data).decode('utf-8')
                base64_size = len(image_base64)
                logger.info(f"  - Base64 encoded size: {base64_size} characters")
                
                # Determine media type from file extension or stored extension
                if hasattr(img, '_image_ext') and img._image_ext:
                    ext = f".{img._image_ext.lower()}"
                else:
                    ext = img_path.suffix.lower()
                
                media_type_map = {
                    '.png': 'image/png',
                    '.jpg': 'image/jpeg',
                    '.jpeg': 'image/jpeg',
                    '.gif': 'image/gif',
                    '.webp': 'image/webp'
                }
                media_type = media_type_map.get(ext, 'image/png')
                logger.info(f"  - Media type: {media_type} (from extension: {ext})")
                
                content.append({
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": image_base64
                    }
                })
                images_added += 1
                logger.info(f"  - Successfully added image {images_added} to LLM content")
                    
            except Exception as e:
                logger.error(f"Failed to load image {img.file_path}: {str(e)}", exc_info=True)
                # Ensure PIL image is closed even on error
                if pil_image:
                    try:
                        pil_image.close()
                    except:
                        pass
                continue
            finally:
                # Final cleanup of PIL image if still open
                if pil_image:
                    try:
                        pil_image.close()
                    except:
                        pass
        
        logger.info(f"Added {images_added}/{len(extracted_images)} image(s) to LLM prompt")
        
        return content
    
    def _build_rendered_page_prompt(
        self,
        document_text: str,
        template: Dict,
        rendered_image_path: str
    ) -> List:
        """
        Build a multimodal prompt for the LLM using a single rendered page image.
        This approach captures all visual elements including vector graphics and backgrounds.
        
        Args:
            document_text: Extracted text from the document
            template: Template configuration dictionary
            rendered_image_path: Path to the rendered page image
            
        Returns:
            List of content blocks for the LLM API
        """
        template_name = template.get("friendly_name", template.get("template_key"))
        
        MASTER_PROMPT_INSTRUCTION = f"""
# DOCUMENT VALIDATION SYSTEM — MASTER PROMPT (FULL PAGE RENDERING)

You are a Document Template Validator.
Your job is to strictly adhere to the following rules and the provided JSON structure.

[INPUT DATA]
1. Template Name: {template_name}
2. Required Template Elements (Elements to validate):
{json.dumps(template.get("elements", []), indent=2)}

3. Extracted Document Text (for text validation):
{document_text}

4. Rendered Document Page: A **single high-resolution image** of the entire document page is provided. You MUST use this image for all visual validation tasks (logos, signatures, QR codes, etc.).

[DETECTION RULES]

📌 LOGO & SIGNATURE DETECTION (Visual Elements)
You must analyze the SINGLE provided full-page image to locate the required visual elements (Company Logo, Event Logo, Signature Block, QR Code/Barcode).
Since this is a complete page render, ALL visual elements should be detectable if present.
- If found: FOUND — The 'details' field MUST specify: "Detected [logo/signature/QR code] within the full page image at [approximate location]. Content: [Brief visual description]"
- If not found: MISSING — The 'details' field must explain why the element is not visible on the rendered page (e.g., "No company logo visible anywhere on the rendered page").

📌 DATE DETECTION
Accept all valid date formats: DD/MM/YYYY, Month YYYY, YYYY-MM-DD, Verbal dates, and any other recognizable date format.
Return FOUND if any valid date appears where expected.

📌 NAME DETECTION
Treat any of the following as names: Dr. X, Prof. X, First + Last, Company names, Full faculty names, Single names when appropriate for context.

📌 CODE DETECTION
Detect any alphanumeric code when expected (DHA, Approval, RCP Codes, etc.). Example formats: 123456, DHA-2025-001, XX9999. Codes must exist in the correct document section to be marked FOUND.

📌 PLACEHOLDER EQUIVALENCE
A placeholder should be considered correctly replaced if it contains the correct type of data (e.g., <<Event Date>> replaced with "30/11/2025"). If the placeholder is replaced with text of the wrong type (e.g., venue replaced with a person's name), mark as DIFFERENT.

[ANALYSIS PROCESS]
1. Examine the provided full-page image carefully for all visual elements
2. Cross-reference text content with template requirements
3. For each required element:
   - If matching → FOUND
   - If no match → MISSING
   - If placeholder replaced with wrong type → DIFFERENT
4. Return structured JSON ONLY
5. For visual elements: You MUST analyze the provided full-page image and describe what you see

[OUTPUT FORMAT]
You MUST return your results in JSON, structured EXACTLY as defined below. Do not include any text, headers, or markdown outside of the JSON block.

{{
  "template_type": "{template_name}",
  "validation_results": [
    {{
      "element": "element_id_from_template",
      "status": "FOUND|MISSING|DIFFERENT",
      "details": "Specific explanation of findings with visual descriptions for image elements"
    }}
  ],
  "overall_summary": "Brief summary of validation results"
}}

RESPOND WITH JSON ONLY - NO ADDITIONAL TEXT OR MARKDOWN.
"""
        
        # Build content list with text and the single rendered image
        content = [{"type": "text", "text": MASTER_PROMPT_INSTRUCTION}]
        
        # Load and optimize the rendered image
        logger.info(f"Loading rendered page image: {rendered_image_path}")
        try:
            # Check if file exists
            img_path = Path(rendered_image_path)
            if not img_path.exists():
                logger.error(f"Rendered image file not found: {rendered_image_path}")
                raise FileNotFoundError(f"Rendered image not found: {rendered_image_path}")
            
            file_size = img_path.stat().st_size
            logger.info(f"Rendered image file size: {file_size} bytes")
            
            # Load and optimize the image
            pil_image = Image.open(rendered_image_path)
            pil_image.load()
            original_size = pil_image.size
            logger.info(f"Original rendered image dimensions: {original_size[0]}x{original_size[1]} pixels")
            
            # Apply image optimization (same logic as before)
            max_size = 2048  # Max dimension for API compatibility
            optimized_size = original_size
            
            if max(pil_image.size) > max_size:
                # Calculate new size, maintaining aspect ratio
                ratio = max_size / max(pil_image.size)
                new_size = (int(pil_image.size[0] * ratio), int(pil_image.size[1] * ratio))
                
                # Resize using high-quality filter
                pil_image = pil_image.resize(new_size, Image.Resampling.LANCZOS)
                optimized_size = new_size
                logger.info(f"Resized to: {new_size[0]}x{new_size[1]} pixels (ratio: {ratio:.3f})")
            else:
                logger.info(f"No resizing needed (within {max_size}px limit)")
            
            # Convert to bytes for base64 encoding
            output_io = BytesIO()
            save_format = 'PNG'  # Always use PNG for rendered pages to preserve quality
            pil_image.save(output_io, format=save_format)
            image_data = output_io.getvalue()
            
            # Convert to base64 for Anthropic API
            image_base64 = base64.b64encode(image_data).decode('utf-8')
            base64_size = len(image_base64)
            logger.info(f"Base64 encoded size: {base64_size} characters")
            
            # Log optimization results
            if original_size != optimized_size:
                original_pixels = original_size[0] * original_size[1]
                optimized_pixels = optimized_size[0] * optimized_size[1]
                reduction_ratio = optimized_pixels / original_pixels if original_pixels > 0 else 1.0
                logger.info(f"Pixel reduction: {reduction_ratio:.3f}x ({original_pixels:,} → {optimized_pixels:,} pixels)")
            
            content.append({
                "type": "image",
                "source": {
                    "type": "base64",
                    "media_type": "image/png",
                    "data": image_base64
                }
            })
            
            # Cleanup
            pil_image.close()
            output_io.close()
            
            logger.info("Successfully added rendered page image to LLM content")
            
        except Exception as e:
            logger.error(f"Failed to load rendered page image {rendered_image_path}: {str(e)}", exc_info=True)
            raise ValueError(f"Failed to process rendered page image: {str(e)}")
        
        return content
    
    def _build_text_only_prompt(
        self,
        document_text: str,
        template: Dict
    ) -> List:
        """
        Build a text-only prompt for the LLM when no images are available.
        
        Args:
            document_text: Extracted text from the document
            template: Template configuration dictionary
            
        Returns:
            List of content blocks for the LLM API
        """
        template_name = template.get("friendly_name", template.get("template_key"))
        
        TEXT_ONLY_PROMPT = f"""
# DOCUMENT VALIDATION SYSTEM — TEXT-ONLY MODE

You are a Document Template Validator operating in TEXT-ONLY mode.
Your job is to validate text-based elements only, as no visual content is available.

[INPUT DATA]
1. Template Name: {template_name}
2. Required Template Elements (Elements to validate):
{json.dumps(template.get("elements", []), indent=2)}

3. Extracted Document Text:
{document_text}

[DETECTION RULES - TEXT ONLY]

📌 VISUAL ELEMENTS (LIMITATION)
For visual elements (logos, signatures, QR codes), you MUST mark them as MISSING with the explanation:
"Visual element validation not available - text-only mode"

📌 DATE DETECTION
Accept all valid date formats in the text content.

📌 NAME DETECTION
Detect names in the text content.

📌 CODE DETECTION
Detect alphanumeric codes in the text content.

📌 PLACEHOLDER EQUIVALENCE
Check if placeholders are replaced with appropriate text content.

[OUTPUT FORMAT]
{{
  "template_type": "{template_name}",
  "validation_results": [
    {{
      "element": "element_id_from_template",
      "status": "FOUND|MISSING|DIFFERENT",
      "details": "Text-based validation results or visual limitation notice"
    }}
  ],
  "overall_summary": "Text-only validation completed - visual elements not validated"
}}

RESPOND WITH JSON ONLY - NO ADDITIONAL TEXT OR MARKDOWN.
"""
        
        return [{"type": "text", "text": TEXT_ONLY_PROMPT}]
    
    def _parse_llm_response(self, response_text: str) -> Dict:
        """
        Parse the LLM response and extract JSON with enhanced validation.
        
        Args:
            response_text: Raw response text from LLM
        
        Returns:
            Parsed and validated JSON dictionary
            
        Raises:
            ValueError: If response cannot be parsed or doesn't match expected schema
        """
        logger.info(f"Parsing LLM response (length: {len(response_text)} chars)")
        
        # Try to extract JSON from the response
        # Remove markdown code blocks if present
        response_text = response_text.strip()
        if response_text.startswith("```json"):
            response_text = response_text[7:]
        if response_text.startswith("```"):
            response_text = response_text[3:]
        if response_text.endswith("```"):
            response_text = response_text[:-3]
        response_text = response_text.strip()
        
        # Log the cleaned response for debugging
        logger.debug(f"Cleaned response text: {response_text[:500]}...")
        
        try:
            parsed_response = json.loads(response_text)
            logger.info("Successfully parsed JSON response")
        except json.JSONDecodeError as e:
            logger.warning(f"Initial JSON parsing failed: {str(e)}")
            # If JSON parsing fails, try to find JSON object in the text
            import re
            json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
            if json_match:
                try:
                    parsed_response = json.loads(json_match.group())
                    logger.info("Successfully extracted JSON from response text")
                except json.JSONDecodeError:
                    logger.error("Failed to parse extracted JSON")
                    raise ValueError(f"Failed to parse LLM response as JSON: {e}")
            else:
                logger.error("No JSON object found in response")
                raise ValueError(f"No valid JSON found in LLM response: {e}")
        
        # Handle both old and new JSON formats
        if "template_type" in parsed_response and "validation_results" in parsed_response:
            # New format from master prompt - convert to old format
            logger.info("Converting new master prompt format to internal format")
            converted_response = {
                "template_key": parsed_response.get("template_type", "unknown"),
                "status": "PASS",  # Will be determined by validation logic
                "summary": parsed_response.get("overall_summary", "Validation completed"),
                "elements_report": []
            }
            
            # Convert validation_results to elements_report
            for result in parsed_response.get("validation_results", []):
                element_report = {
                    "id": result.get("element", "unknown"),
                    "label": result.get("element", "unknown"),
                    "required": True,  # Will be updated by template validation
                    "is_present": result.get("status") == "FOUND",
                    "reason": result.get("details", "No details provided")
                }
                converted_response["elements_report"].append(element_report)
            
            parsed_response = converted_response
            logger.info(f"Converted {len(parsed_response['elements_report'])} validation results to elements_report")
        
        # Validate response structure (both old and converted formats)
        required_fields = ["template_key", "status", "summary", "elements_report"]
        for field in required_fields:
            if field not in parsed_response:
                logger.warning(f"Missing required field '{field}' in LLM response")
                # Add default values for missing fields
                if field == "template_key":
                    parsed_response[field] = "unknown"
                elif field == "status":
                    parsed_response[field] = "FAIL"
                elif field == "summary":
                    parsed_response[field] = "Validation completed with parsing issues"
                elif field == "elements_report":
                    parsed_response[field] = []
        
        # Validate status field
        if parsed_response.get("status") not in ["PASS", "FAIL"]:
            logger.warning(f"Invalid status value: {parsed_response.get('status')}, defaulting to FAIL")
            parsed_response["status"] = "FAIL"
        
        # Validate elements_report structure
        if not isinstance(parsed_response.get("elements_report"), list):
            logger.warning("elements_report is not a list, creating empty list")
            parsed_response["elements_report"] = []
        
        # Validate each element in elements_report
        for i, element in enumerate(parsed_response["elements_report"]):
            if not isinstance(element, dict):
                logger.warning(f"Element {i} in elements_report is not a dictionary")
                continue
            
            # Ensure required fields exist in each element
            element_required_fields = ["id", "label", "required", "is_present", "reason"]
            for field in element_required_fields:
                if field not in element:
                    logger.warning(f"Missing field '{field}' in element {i}")
                    # Add default values
                    if field == "id":
                        element[field] = f"element_{i}"
                    elif field == "label":
                        element[field] = f"Element {i}"
                    elif field == "required":
                        element[field] = False
                    elif field == "is_present":
                        element[field] = False
                    elif field == "reason":
                        element[field] = "Analysis incomplete"
        
        logger.info(f"Validated response with {len(parsed_response.get('elements_report', []))} elements")
        return parsed_response
    
    def _validate_final_report(self, report: Dict, template: Dict) -> Dict:
        """
        Validate and enhance the final validation report.
        
        Args:
            report: Parsed LLM response
            template: Template configuration
            
        Returns:
            Enhanced and validated report
        """
        logger.info("Validating final report structure")
        
        # Ensure all template elements are covered in the report
        template_elements = template.get("elements", [])
        report_elements = {elem.get("id"): elem for elem in report.get("elements_report", [])}
        
        # Check for missing elements and add them
        for template_elem in template_elements:
            elem_id = template_elem.get("id")
            if elem_id not in report_elements:
                logger.warning(f"Template element '{elem_id}' missing from LLM report, adding default")
                missing_element = {
                    "id": elem_id,
                    "label": template_elem.get("label", elem_id),
                    "required": template_elem.get("required", False),
                    "is_present": False,
                    "reason": "Element not analyzed by LLM - marked as missing"
                }
                report["elements_report"].append(missing_element)
        
        # Validate status logic based on required elements
        required_missing = []
        for element in report["elements_report"]:
            if element.get("required", False) and not element.get("is_present", False):
                required_missing.append(element.get("label", element.get("id")))
        
        # Update status based on missing required elements
        if required_missing:
            report["status"] = "FAIL"
            if not report.get("summary") or "parsing issues" in report.get("summary", ""):
                report["summary"] = f"Validation failed: {len(required_missing)} required element(s) missing: {', '.join(required_missing[:3])}"
                if len(required_missing) > 3:
                    report["summary"] += f" and {len(required_missing) - 3} more"
        else:
            if report.get("status") != "PASS":
                logger.info("All required elements present, updating status to PASS")
                report["status"] = "PASS"
                if not report.get("summary") or "parsing issues" in report.get("summary", ""):
                    report["summary"] = "All required elements validated successfully"
        
        # Sort elements_report by required status (required first) then by id
        report["elements_report"] = sorted(
            report["elements_report"],
            key=lambda x: (not x.get("required", False), x.get("id", ""))
        )
        
        logger.info(f"Final report validation complete:")
        logger.info(f"  - Status: {report.get('status')}")
        logger.info(f"  - Total elements: {len(report.get('elements_report', []))}")
        logger.info(f"  - Required missing: {len(required_missing)}")
        
        return report
    
    async def validate_document(
        self, 
        file_content: bytes, 
        file_extension: str, 
        template_key: str,
        custom_prompt: Optional[str] = None
    ) -> Dict:
        """
        Validate a document against a template using multimodal LLM.
        
        Args:
            file_content: Binary content of the document file
            file_extension: File extension (e.g., '.pdf', '.docx', '.pptx')
            template_key: Template key to validate against
            custom_prompt: Optional custom instructions to adapt validation
        
        Returns:
            Validation report dictionary with status and element reports
        
        Raises:
            ValueError: If template not found, extraction fails, or validation fails
        """
        logger.info(f"Starting validation for {file_extension} document against template {template_key}")

        # 1. Extract Links & Check Health (Async)
        logger.info("======================================")
        logger.info("STARTING LINK VALIDATION")
        logger.info("======================================")
        logger.info("Extracting and checking links...")
        try:
            extracted_links = self.extract_links(file_content, file_extension)
            logger.info(f"✓ extract_links returned {len(extracted_links)} links")
            if extracted_links:
                logger.info(f"  Links: {[link.get('url') for link in extracted_links]}")
            
            link_validation_results = await self.check_links(extracted_links)
            logger.info(f"✓ check_links returned {len(link_validation_results)} results")
        except Exception as e:
            logger.error(f"❌ Link validation failed with exception: {e}", exc_info=True)
            link_validation_results = []
        
        logger.info(f"Final link_validation_results count: {len(link_validation_results)}")
        logger.info("======================================")

        # Load template
        template = get_template(template_key)
        if not template:
            raise ValueError(f"Template not found: {template_key}")
        
        # Create temporary directory for extracted images
        # Note: We'll manually manage cleanup to ensure all file handles are closed
        temp_dir = tempfile.mkdtemp()
        temp_path = Path(temp_dir)
        logger.info(f"Created temporary directory: {temp_dir}")
        
        # --- START NEW IMAGE LOGGING SETUP ---
        # Create persistent log directory for troubleshooting
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]  # Remove last 3 digits of microseconds
        # Use temp directory for logs to avoid PermissionError in read-only environments
        log_dir_name = os.path.join(tempfile.gettempdir(), "extracted_images_log", timestamp)
        
        # Create the persistent log folder
        os.makedirs(log_dir_name, exist_ok=True)
        logger.info(f"Saving extracted images for troubleshooting to: {log_dir_name}")
        print(f"[LOG] Persistent image log directory: {log_dir_name}")
        # --- END NEW IMAGE LOGGING SETUP ---
        
        try:
            # --- NEW: PDF RENDER LOGIC (Replaces extract_images_from_document) ---
            logger.info("=" * 60)
            logger.info("STARTING PDF PAGE RENDERING")
            logger.info("=" * 60)
            
            extracted_text = ""
            rendered_image_path = None
            
            if file_extension.lower() == ".pdf":
                try:
                    # Extract text using OCR-enabled function
                    extracted_text = extract_pdf_text(file_content)
                    logger.info(f"Extracted text length: {len(extracted_text)} characters")
                    
                    # Open the PDF file using PyMuPDF for rendering
                    doc = fitz.open(stream=file_content, filetype="pdf")
                    logger.info(f"PDF opened successfully - {len(doc)} page(s)")
                    
                    # Load the first page (most certificates are single page)
                    page = doc.load_page(0)
                    logger.info("Loading first page for rendering...")
                    
                    # Render the page to a high-resolution Pixmap (300 DPI equivalent, scale 3.0)
                    # Render the page to a high-resolution Pixmap (200 DPI equivalent, scale 2.0)
                    logger.info("Rendering page to high-resolution image (scale 2.0)...")
                    matrix = fitz.Matrix(2.0, 2.0)  # 2x scale for good quality without hitting size limits
                    pix = page.get_pixmap(matrix=matrix, alpha=False)
                    
                    # Define file paths
                    rendered_image_filename = "page_1_rendered.png"
                    temp_render_path = os.path.join(temp_dir, rendered_image_filename)
                    persistent_render_path = os.path.join(log_dir_name, rendered_image_filename)
                    
                    # Save the rendered image to temporary path
                    pix.save(temp_render_path)
                    rendered_image_path = temp_render_path
                    
                    # Copy to persistent log directory for troubleshooting
                    shutil.copy2(temp_render_path, persistent_render_path)
                    
                    # Log rendering details
                    image_size = os.path.getsize(temp_render_path)
                    logger.info(f"Page rendered successfully:")
                    logger.info(f"  - Dimensions: {pix.width}x{pix.height} pixels")
                    logger.info(f"  - File size: {image_size} bytes")
                    logger.info(f"  - Temp path: {temp_render_path}")
                    logger.info(f"  - Persistent path: {persistent_render_path}")
                    
                    print(f"[RENDER] Page rendered: {rendered_image_filename} ({pix.width}x{pix.height} pixels, {image_size} bytes)")
                    
                    # Create metadata for the rendered page
                    metadata_path = os.path.join(log_dir_name, "page_1_rendered_metadata.json")
                    metadata = {
                        "type": "full_page_render",
                        "filename": rendered_image_filename,
                        "temp_path": temp_render_path,
                        "persistent_path": persistent_render_path,
                        "dimensions": {"width": pix.width, "height": pix.height},
                        "scale_factor": 3.0,
                        "file_size": image_size,
                        "extraction_timestamp": timestamp,
                        "template_key": template_key,
                        "text_length": len(extracted_text)
                    }
                    
                    with open(metadata_path, 'w') as f:
                        json.dump(metadata, f, indent=2)
                    
                    # Cleanup PyMuPDF objects
                    pix = None
                    doc.close()
                    
                except Exception as e:
                    logger.error(f"PDF rendering failed: {str(e)}", exc_info=True)
                    raise ValueError(f"PDF rendering failed: {str(e)}")
            else:
                # For non-PDF files, fall back to text extraction only
                logger.warning(f"File extension {file_extension} not supported for rendering. Using text-only validation.")
                if file_extension.lower() == ".docx":
                    from docx import Document
                    doc = Document(BytesIO(file_content))
                    extracted_text = "\n".join([paragraph.text for paragraph in doc.paragraphs])
                elif file_extension.lower() == ".pptx":
                    from pptx import Presentation
                    prs = Presentation(BytesIO(file_content))
                    extracted_text = ""
                    for slide in prs.slides:
                        for shape in slide.shapes:
                            if hasattr(shape, "text"):
                                extracted_text += shape.text + "\n"
                else:
                    raise ValueError(f"Unsupported file format: {file_extension}")
            
            logger.info("=" * 60)
            logger.info("RENDERING SUMMARY")
            logger.info(f"Text length: {len(extracted_text)} characters")
            logger.info(f"Rendered image: {'Yes' if rendered_image_path else 'No (text-only)'}")
            if rendered_image_path:
                logger.info(f"  - Image path: {rendered_image_path}")
            logger.info("=" * 60)
            
            if (not extracted_text or not extracted_text.strip()) and not rendered_image_path:
                logger.warning("Document appears to be empty or contains no extractable text")
                raise ValueError("Document appears to be empty or contains no extractable text")
            elif (not extracted_text or not extracted_text.strip()) and rendered_image_path:
                logger.warning("No text extracted, but rendered image available. Proceeding with visual validation.")
                extracted_text = "[NO TEXT EXTRACTED - RELYING ON VISUAL VALIDATION]"
            
            
            # Build multimodal prompt for rendered page
            logger.info("Building multimodal prompt for LLM...")
            if rendered_image_path:
                # Use single rendered page approach
                content = self._build_rendered_page_prompt(extracted_text, template, rendered_image_path)
                logger.info(f"Prompt contains {len(content)} content block(s) (1 text + 1 rendered page image)")
            else:
                # Fallback to text-only validation
                content = self._build_text_only_prompt(extracted_text, template)
                logger.info(f"Prompt contains {len(content)} content block(s) (text-only validation)")
            
            # Append custom instructions if provided
            if custom_prompt and custom_prompt.strip():
                custom_instruction_block = {
                    "type": "text",
                    "text": f"\n\n ADDITIONAL USER INSTRUCTIONS:\n{custom_prompt.strip()}\n\nPlease incorporate these additional instructions into your validation process."
                }
                content.append(custom_instruction_block)
                logger.info(f"Added custom instructions to prompt ({len(custom_prompt)} characters)")
            
            # Call LLM API with fallback models (all support multimodal)
            models_to_try = [
                "claude-opus-4-20250514",
                "claude-3-opus-latest",
                "claude-3-5-sonnet-latest"
            ]
            
            last_error = None
            validation_report = None
            
            for model_name in models_to_try:
                try:
                    logger.info(f"Attempting validation with model: {model_name}")
                    logger.info(f"Sending {len(content)} content blocks to LLM")
                    
                    # Call the multimodal LLM with enhanced configuration
                    message = self.client.messages.create(
                        model=model_name,
                        max_tokens=4096,
                        temperature=0.1,  # Low temperature for consistent JSON output
                        messages=[
                            {
                                "role": "user",
                                "content": content
                            }
                        ]
                    )
                    
                    # Extract response text
                    response_text = message.content[0].text if message.content else ""
                    logger.info(f"Received response from {model_name} (length: {len(response_text)} chars)")
                    
                    if not response_text:
                        raise ValueError("Empty response from LLM")
                    
                    # Parse and validate response
                    validation_report = self._parse_llm_response(response_text)
                    
                    # Validate and enhance the final report
                    validation_report = self._validate_final_report(validation_report, template)
                    
                    # Ensure template_key matches
                    validation_report["template_key"] = template_key
                    
                    # Add metadata about the validation process
                    validation_report["_metadata"] = {
                        "model_used": model_name,
                        "images_analyzed": 1 if rendered_image_path else 0,
                        "text_length": len(extracted_text),
                        "extraction_method": "full_page_rendering" if rendered_image_path else "text_only",
                        "timestamp": int(time.time()),
                        "persistent_log_directory": log_dir_name,
                        "extraction_timestamp": timestamp,
                        "rendered_page": bool(rendered_image_path)
                    }
                    
                    # Update model for future use
                    self.model = model_name
                    logger.info(f"Validation completed successfully using {model_name}")
                    logger.info(f"Final status: {validation_report.get('status')}")
                    logger.info(f"Elements validated: {len(validation_report.get('elements_report', []))}")
                    break  # Success, exit loop
                    
                except anthropic.APIError as e:
                    last_error = e
                    logger.warning(f"API error with model {model_name}: {str(e)}")
                    # If it's a 404 (model not found), try next model
                    if hasattr(e, 'status_code') and e.status_code == 404:
                        logger.info(f"Model {model_name} not found, trying next model")
                        continue
                    # For other API errors, raise immediately
                    logger.error(f"Critical API error with {model_name}: {str(e)}")
                    raise ValueError(f"LLM API error: {str(e)}")
                except Exception as e:
                    # For non-API errors, raise immediately
                    logger.error(f"Validation error with {model_name}: {str(e)}", exc_info=True)
                    raise ValueError(f"Validation failed: {str(e)}")
            
            # If all models failed
            if validation_report is None:
                if last_error:
                    raise ValueError(f"LLM API error: All model attempts failed. Last error: {str(last_error)}")
                else:
                    raise ValueError("LLM API error: Unable to connect to any Claude model")
            
            # Add link report to result
            validation_report["link_report"] = link_validation_results
            
            return validation_report
            
        finally:
            # Ensure all file handles are closed before cleanup
            import gc
            gc.collect()  # Force garbage collection to close any lingering file handles
            
            # Clean up temporary directory
            if temp_dir and os.path.exists(temp_dir):
                try:
                    # Try to remove files individually first
                    for root, dirs, files in os.walk(temp_dir):
                        for file in files:
                            file_path = os.path.join(root, file)
                            try:
                                os.remove(file_path)
                            except Exception as e:
                                logger.warning(f"Could not remove file {file_path}: {str(e)}")
                    
                    # Remove the directory
                    shutil.rmtree(temp_dir, ignore_errors=True)
                    logger.info(f"Cleaned up temporary directory: {temp_dir}")
                except Exception as e:
                    logger.warning(f"Error cleaning up temporary directory {temp_dir}: {str(e)}")
                    # Try one more time after a short delay
                    time.sleep(0.1)
                    try:
                        shutil.rmtree(temp_dir, ignore_errors=True)
                    except:
                        pass

    def check_spelling(self, document_text: str, language_context: str = "English with Arabic names") -> Dict:
        """
        Check spelling in document text using Claude LLM with context-aware name detection.
        
        Args:
            document_text: Text content to check for spelling errors
            language_context: Language context for spell checking (default: "English with Arabic names")
        
        Returns:
            Dictionary with spell check results including errors, suggestions, and summary
        """
        # Check for empty text or the fallback placeholder
        if not document_text or not document_text.strip() or document_text == "[NO TEXT EXTRACTED - RELYING ON VISUAL VALIDATION]":
            logger.warning("Skipping spell check: No text extracted")
            return {
                "total_errors": 0,
                "errors": [],
                "summary": "Could not extract text for spell checking (Visual validation only)"
            }
            
        logger.info(f"Starting spell check for text ({len(document_text)} characters)")
        
        
        # Prepare prompt for quality checking (spelling, grammar, formatting)
        prompt = f"""
Analyze the following text from a medical document for spelling, grammar, and formatting consistency issues.

TEXT TO ANALYZE:
---
{document_text}
---

INSTRUCTIONS:
1. **Spelling & Arabic Support**:
   - Check both English and Arabic text for spelling errors.
   - IGNORE proper names (including common Arabic names like Mohammed, Ahmed, etc.), locations, and medical terminology.
   - IGNORE brand names or specialized abbreviations.

2. **Grammar**:
   - Identify grammatical errors, awkward phrasing, or punctuation issues.
   - Ensure the tone remains professional.

3. **Formatting Consistency (CRITICAL)**:
   - **AM/PM Consistency**: Strictly only uppercase "AM" and "PM" are permitted. 
   - Flag ANY variation such as "Am", "am", "aM", "Pm", "pm", "pM" as a "formatting" error.
   - Example: if you see "10:00am" or "10:00 Am", flag it and suggest "10:00 AM".
   - **Date Consistency**: Check for inconsistent date formats (e.g., mixing MM/DD/YYYY and DD.MM.YYYY).

4. **Output Format**:
Return your findings STRICTLY as a JSON object with this structure:
{{
  "total_errors": number,
  "errors": [
    {{
      "word": "the specific word or phrase with the issue",
      "context": "a short snippet of the surrounding text (about 5 words before and after)",
      "suggestions": ["suggestion1", "suggestion2"],
      "error_type": "spelling" | "grammar" | "formatting",
      "confidence": 0.0 to 1.0
    }}
  ],
  "summary": "a brief 1-2 sentence overview of the issues found"
}}

If no errors are found, return exactly:
{{
  "total_errors": 0,
  "errors": [],
  "summary": "No spelling, grammar, or formatting issues found."
}}

RESPOND WITH JSON ONLY - NO ADDITIONAL TEXT OR MARKDOWN.
"""
        
        try:
            logger.info("Sending text to Claude for spell checking...")
            
            # Call Claude API for spell checking
            message = self.client.messages.create(
                model="claude-opus-4-20250514",  # Use Claude Opus 4
                max_tokens=4096,
                temperature=0.1,  # Low temperature for consistent output
                messages=[
                    {
                        "role": "user",
                        "content": [{"type": "text", "text": prompt}]
                    }
                ]
            )
            
            # Extract response text
            response_text = message.content[0].text if message.content else ""
            logger.info(f"Received spell check response ({len(response_text)} chars)")
            
            if not response_text:
                raise ValueError("Empty response from LLM")
            
            # Parse JSON response
            response_text = response_text.strip()
            if response_text.startswith("```json"):
                response_text = response_text[7:]
            if response_text.startswith("```"):
                response_text = response_text[3:]
            if response_text.endswith("```"):
                response_text = response_text[:-3]
            response_text = response_text.strip()
            
            try:
                spell_check_result = json.loads(response_text)
                logger.info(f"Successfully parsed spell check JSON: {spell_check_result.get('total_errors', 0)} errors found")
            except json.JSONDecodeError as e:
                logger.error(f"Failed to parse spell check JSON response: {str(e)}")
                # Try to extract JSON from text
                import re
                json_match = re.search(r'\{.*\}', response_text, re.DOTALL)
                if json_match:
                    spell_check_result = json.loads(json_match.group())
                    logger.info("Extracted JSON from response text")
                else:
                    raise ValueError(f"Failed to parse spell check response as JSON: {e}")
            
            # Validate response structure
            if "total_errors" not in spell_check_result:
                spell_check_result["total_errors"] = len(spell_check_result.get("errors", []))
            
            if "errors" not in spell_check_result:
                spell_check_result["errors"] = []
            
            if "summary" not in spell_check_result:
                error_count = spell_check_result.get("total_errors", 0)
                if error_count == 0:
                    spell_check_result["summary"] = "No spelling errors found"
                elif error_count == 1:
                    spell_check_result["summary"] = "Found 1 spelling error"
                else:
                    spell_check_result["summary"] = f"Found {error_count} spelling errors"
            
            # Validate each error has required fields
            validated_errors = []
            for error in spell_check_result.get("errors", []):
                if not isinstance(error, dict):
                    continue
                
                # Ensure all required fields exist
                validated_error = {
                    "word": error.get("word", ""),
                    "context": error.get("context", ""),
                    "suggestions": error.get("suggestions", []),
                    "error_type": error.get("error_type", "spelling"),
                    "confidence": error.get("confidence", 0.8)
                }
                
                # Only include errors with actual content
                if validated_error["word"]:
                    validated_errors.append(validated_error)
            
            spell_check_result["errors"] = validated_errors
            spell_check_result["total_errors"] = len(validated_errors)
            
            logger.info(f"Spell check completed: {spell_check_result['total_errors']} errors found")
            return spell_check_result
            
        except Exception as e:
            logger.error(f"Spell check failed: {str(e)}", exc_info=True)
            # Return empty result on error
            return {
                "total_errors": 0,
                "errors": [],
                "summary": f"Spell check failed: {str(e)}"
            }