Upload 3 files

finalsite.py

@@ -0,0 +1,1254 @@
+import os
+import cv2
+import numpy as np
+import gradio as gr
+from ultralytics import YOLO
+from pdf2image import convert_from_path
+from PIL import Image
+import easyocr
+import uuid
+import re
+import difflib
+import math
+from faker import Faker
+import datetime
+import random
+import csv
+import tempfile
+from io import StringIO
+
+# Initialize Faker
+fake = Faker()
+
+# Conditional import for LLM
+try:
+    from llama_cpp import Llama
+    LLAMA_AVAILABLE = True
+except ImportError:
+    print("Warning: llama_cpp not available. LLM functionality will be disabled.")
+    LLAMA_AVAILABLE = False
+
+# --- Configuration ---
+# Folders for temporary files and results
+UPLOAD_FOLDER = 'static/uploads/'
+RESULTS_FOLDER = 'static/results/'
+
+# --- Model Paths (Update these paths if necessary) ---
+CUSTOM_MODEL_PATH = 'best.pt'
+PRETRAINED_MODEL_PATH = 'yolov10s.pt'
+SIGNATURE_MODEL_PATH = 'yolov8s.pt'
+LLAMA_MODEL_PATH = "unsloth.F16.gguf"
+
+# Detection Parameters
+YOLO_CONFIDENCE_THRESHOLD = 0.5
+OCR_CONFIDENCE_THRESHOLD = 0.5
+
+# Create directories if they don't exist
+os.makedirs(UPLOAD_FOLDER, exist_ok=True)
+os.makedirs(RESULTS_FOLDER, exist_ok=True)
+
+# --- Global Model Placeholders ---
+custom_model, pretrained_model, signature_model, reader, llama_model = None, None, None, None, None
+
+def load_models():
+    """
+    Loads all AI models into the global scope. This function is called on the first
+    analysis request to avoid startup conflicts. It ensures models are only loaded once.
+    """
+    global custom_model, pretrained_model, signature_model, reader, llama_model
+    # If models are already loaded, do nothing.
+    if reader is not None and (llama_model is not None or not LLAMA_AVAILABLE):
+        print("Models already loaded.")
+        return
+    print("=== Loading Models (this may take a moment) ===")
+    
+    # Helper function to check for model files
+    def check_model_path(path, name):
+        if not os.path.exists(path):
+            print(f"✗ WARNING: {name} model not found at '{path}'. The application may not function correctly.")
+            return False
+        return True
+    
+    # YOLO Models
+    if check_model_path(CUSTOM_MODEL_PATH, "Custom YOLO"):
+        try:
+            custom_model = YOLO(CUSTOM_MODEL_PATH)
+            print("✓ Custom YOLO model loaded.")
+        except Exception as e:
+            print(f"✗ Error loading custom model: {e}")
+    if check_model_path(PRETRAINED_MODEL_PATH, "Pre-trained YOLO"):
+        try:
+            pretrained_model = YOLO(PRETRAINED_MODEL_PATH)
+            print("✓ Pre-trained YOLO model loaded.")
+        except Exception as e:
+            print(f"✗ Error loading pre-trained model: {e}")
+    if check_model_path(SIGNATURE_MODEL_PATH, "Signature YOLO"):
+        try:
+            signature_model = YOLO(SIGNATURE_MODEL_PATH)
+            print("✓ Signature YOLO model loaded.")
+        except Exception as e:
+            print(f"✗ Error loading signature model: {e}")
+    
+    # OCR Model
+    try:
+        reader = easyocr.Reader(['en'], gpu=True)
+        print("✓ EasyOCR model loaded.")
+    except Exception as e:
+        print(f"✗ Error loading EasyOCR: {e}. Text detection will be unavailable.")
+    
+    # LLM Model - Only load if available
+    if LLAMA_AVAILABLE and check_model_path(LLAMA_MODEL_PATH, "LLM"):
+        try:
+            llama_model = Llama(
+                model_path=LLAMA_MODEL_PATH,
+                n_gpu_layers=-1, n_ctx=4096, chat_format="llama-3", verbose=False
+            )
+            print("✓ LLM model loaded.")
+        except Exception as e:
+            print(f"✗ Error loading LLM model: {e}. Text analysis will be unavailable.")
+    print("=== All Models Initialized ===")
+
+# YOLO Class Mappings
+CUSTOM_CLASS_NAMES = {0: 'face', 1: 'qr', 2: 'signature'}
+PRETRAINED_CLASS_MAP = {0: 'face'}
+
+# --- Core Detection & Processing Functions ---
+def detect_visual_pii(image_data):
+    """Runs the three-stage YOLO detection on a single image."""
+    all_boxes = []
+    all_classes = []
+    if custom_model is None:
+        print("Custom model not available for visual detection")
+        return all_boxes, all_classes
+    
+    # Pass 1: Custom Model
+    custom_results = custom_model.predict(source=image_data, conf=YOLO_CONFIDENCE_THRESHOLD, verbose=False)[0]
+    detected_custom_classes = {CUSTOM_CLASS_NAMES[int(cls)] for cls in custom_results.boxes.cls}
+    for box, cls in zip(custom_results.boxes.xyxy.cpu().numpy().astype(int), custom_results.boxes.cls):
+        all_boxes.append(box)
+        all_classes.append(CUSTOM_CLASS_NAMES[int(cls)])
+    
+    # Pass 2: Pre-trained Model (Face Fallback)
+    if 'face' not in detected_custom_classes and pretrained_model is not None:
+        print(" Custom model missed 'face'. Trying pre-trained model as fallback.")
+        pretrained_results = pretrained_model.predict(source=image_data, conf=YOLO_CONFIDENCE_THRESHOLD, verbose=False)[0]
+        for box in pretrained_results.boxes:
+            if int(box.cls[0]) in PRETRAINED_CLASS_MAP:
+                all_boxes.append(box.xyxy.cpu().numpy().astype(int)[0])
+                all_classes.append("face (fallback)")
+    
+    # Pass 3: Specialized Model (Signature Fallback)
+    if 'signature' not in detected_custom_classes and signature_model is not None:
+        print(" Custom model missed 'signature'. Trying specialized signature model as fallback.")
+        signature_results = signature_model.predict(source=image_data, conf=YOLO_CONFIDENCE_THRESHOLD, verbose=False)[0]
+        for box in signature_results.boxes:
+            all_boxes.append(box.xyxy.cpu().numpy().astype(int)[0])
+            all_classes.append("signature (fallback)")
+    
+    return all_boxes, all_classes
+
+# --- OCR + LLM Functions ---
+def calculate_distance(bbox1, bbox2):
+    """Calculates the Euclidean distance between the centers of two bounding boxes."""
+    c1_x = (bbox1[0] + bbox1[2]) / 2
+    c1_y = (bbox1[1] + bbox1[3]) / 2
+    c2_x = (bbox2[0] + bbox2[2]) / 2
+    c2_y = (bbox2[1] + bbox2[3]) / 2
+    return math.sqrt((c2_x - c1_x)**2 + (c2_y - c1_y)**2)
+
+def refine_pii_flags(ocr_results, isolation_threshold=150):
+    """Post-processing step to unmark short, isolated PII detections."""
+    pii_indices = [i for i, result in enumerate(ocr_results) if result["is_pii"]]
+    if len(pii_indices) <= 1:
+        return ocr_results
+    indices_to_unmark = []
+    for i in pii_indices:
+        current_result = ocr_results[i]
+        normalized_text = re.sub(r'[^a-zA-Z0-9]', '', current_result["text"])
+    
+        if len(normalized_text) <= 3:
+            min_dist_to_neighbor = float('inf')
+        
+            for j in pii_indices:
+                if i == j:
+                    continue
+            
+                other_result = ocr_results[j]
+                dist = calculate_distance(current_result["bbox"], other_result["bbox"])
+                if dist < min_dist_to_neighbor:
+                    min_dist_to_neighbor = dist
+            
+            if min_dist_to_neighbor > isolation_threshold:
+                print(f" - Refining PII: Unmarking short ('{current_result['text']}') and isolated (min_dist: {min_dist_to_neighbor:.2f}px) PII.")
+                indices_to_unmark.append(i)
+
+    for i in indices_to_unmark:
+        ocr_results[i]["is_pii"] = False
+        
+    return ocr_results
+
+def parse_pii_output(generated_text):
+    """Parse the new curly braces format PII output"""
+    pii_list = []
+    try:
+        match = re.search(r'\{([^}]*)\}', generated_text)
+        if match:
+            content = match.group(1)
+            items = re.findall(r'"([^"]*)"', content)
+            pii_list = [item.strip() for item in items if item.strip()]
+    except Exception as e:
+        print(f"Error parsing PII output: {e}")
+        pii_list = []
+    return pii_list
+
+def normalize_text(text):
+    """Comprehensive text normalization for better matching"""
+    if not text:
+        return ""
+    normalized = re.sub(r'[.,;:!?()"\'\-_/\\]', '', text)
+    ocr_corrections = {
+        '0': 'o', 'O': '0', '1': 'l', 'l': '1', '5': 's', 'S': '5',
+        '8': 'b', 'B': '8', 'rn': 'm', 'RN': 'M', 'vv': 'w', 'VV': 'W',
+        'cl': 'd', 'CL': 'D',
+    }
+    for wrong, correct in ocr_corrections.items():
+        normalized = normalized.replace(wrong, correct)
+    normalized = ' '.join(normalized.split()).lower()
+    return normalized
+
+def fuzzy_match_score(text1, text2, threshold=0.8):
+    """Calculate fuzzy matching score between two strings"""
+    if not text1 or not text2:
+        return False
+    return difflib.SequenceMatcher(None, text1.lower(), text2.lower()).ratio() >= threshold
+
+def levenshtein_distance(s1, s2):
+    """Calculate Levenshtein distance between two strings"""
+    if len(s1) < len(s2):
+        return levenshtein_distance(s2, s1)
+    if len(s2) == 0:
+        return len(s1)
+    previous_row = list(range(len(s2) + 1))
+    for i, c1 in enumerate(s1):
+        current_row = [i + 1]
+        for j, c2 in enumerate(s2):
+            insertions = previous_row[j + 1] + 1
+            deletions = current_row[j] + 1
+            substitutions = previous_row[j] + (c1 != c2)
+            current_row.append(min(insertions, deletions, substitutions))
+        previous_row = current_row
+    return previous_row[-1]
+
+def is_similar_by_edit_distance(text1, text2, max_distance=2):
+    """Check if two texts are similar within edit distance threshold"""
+    if not text1 or not text2:
+        return False
+    distance = levenshtein_distance(text1.lower(), text2.lower())
+    max_len = max(len(text1), len(text2))
+    if max_len <= 3:
+        threshold = 1
+    elif max_len <= 6:
+        threshold = 2
+    else:
+        threshold = min(max_distance, max_len // 3)
+    return distance <= threshold
+
+def extract_sentence_text(ocr_results):
+    """Extract sentence-based text for LLM input"""
+    paragraph_text = ""
+    for item in ocr_results:
+        if len(item) == 3:
+            _, text, _ = item
+        elif len(item) == 2:
+            _, text = item
+        else:
+            print(f"Unexpected OCR result format: {item}")
+            continue
+        if text.strip():
+            paragraph_text += text + " "
+    return paragraph_text.strip()
+
+def extract_word_bboxes_improved(ocr_results):
+    """Improved word extraction with better handling of punctuation and spacing"""
+    word_bbox_map = []
+    for item in ocr_results:
+        if len(item) == 3:
+            bbox, text, confidence = item
+        elif len(item) == 2:
+            bbox, text = item
+            confidence = 1.0
+        else:
+            print(f"Unexpected OCR result format: {item}")
+            continue
+    
+        original_text = text.strip()
+        if not original_text:
+            continue
+    
+        if isinstance(bbox[0], (list, tuple)):
+            x_coords = [point[0] for point in bbox]
+            y_coords = [point[1] for point in bbox]
+            line_x1, line_y1 = min(x_coords), min(y_coords)
+            line_x2, line_y2 = max(x_coords), max(y_coords)
+        else:
+            line_x1, line_y1, line_x2, line_y2 = bbox
+    
+        tokens = re.findall(r'\S+', original_text)
+        if len(tokens) <= 1:
+            padding = 1
+            word_bbox_map.append({
+                "word": original_text,
+                "bbox": [
+                    max(0, int(line_x1 - padding)),
+                    max(0, int(line_y1 - padding)),
+                    int(line_x2 + padding),
+                    int(line_y2 + padding)
+                ],
+                "confidence": confidence,
+                "original_line": original_text
+            })
+            continue
+    
+        full_width = line_x2 - line_x1
+        text_without_spaces = original_text.replace(' ', '')
+        total_chars = len(text_without_spaces)
+    
+        char_position = 0
+        for i, token in enumerate(tokens):
+            token_start_ratio = char_position / total_chars if total_chars > 0 else 0
+            char_position += len(token)
+            token_end_ratio = char_position / total_chars if total_chars > 0 else 1
+        
+            token_x1 = line_x1 + (full_width * token_start_ratio)
+            token_x2 = line_x1 + (full_width * token_end_ratio)
+        
+            padding = 1
+            word_bbox = [
+                max(0, int(token_x1 - padding)),
+                max(0, int(line_y1 - padding)),
+                int(min(token_x2 + padding, line_x2)),
+                int(line_y2 + padding)
+            ]
+        
+            word_bbox_map.append({
+                "word": token,
+                "bbox": word_bbox,
+                "confidence": confidence,
+                "original_line": original_text
+            })
+    return sorted(word_bbox_map, key=lambda x: (x['bbox'][1], x['bbox'][0]))
+
+def advanced_match_pii_to_words(pii_list, word_bbox_map):
+    """Advanced multi-strategy PII matching with comprehensive fallbacks"""
+    ocr_results_for_template = []
+    words = [info['word'] for info in word_bbox_map]
+    bboxes = [info['bbox'] for info in word_bbox_map]
+    is_pii_flags = [False] * len(words)
+    # Pre-process all words with different normalization strategies
+    normalized_words = [normalize_text(word) for word in words]
+    print(f"Processing {len(pii_list)} PII items against {len(words)} OCR words")
+    for pii_idx, pii_item in enumerate(pii_list):
+        if not pii_item.strip():
+            continue
+    
+        print(f"Processing PII item {pii_idx + 1}: '{pii_item}'")
+    
+        # Normalize the PII item
+        normalized_pii = normalize_text(pii_item)
+        pii_words = normalized_pii.split()
+    
+        if not pii_words:
+            continue
+    
+        matched = False
+    
+        # Strategy 1: Exact matching after normalization
+        if len(pii_words) == 1:
+            pii_word = pii_words[0]
+            for idx, norm_word in enumerate(normalized_words):
+                if norm_word == pii_word and not is_pii_flags[idx]:
+                    is_pii_flags[idx] = True
+                    matched = True
+                    print(f" ✓ Exact match: '{words[idx]}' -> '{pii_item}'")
+        else:
+            # Multi-word exact matching
+            pii_len = len(pii_words)
+            start_idx = 0
+            while start_idx < len(normalized_words) - pii_len + 1:
+                exact_match = True
+                for j in range(pii_len):
+                    if normalized_words[start_idx + j] != pii_words[j]:
+                        exact_match = False
+                        break
+                
+                if exact_match:
+                    # Check spatial proximity
+                    spatial_ok = True
+                    for j in range(1, pii_len):
+                        prev_bbox = bboxes[start_idx + j - 1]
+                        curr_bbox = bboxes[start_idx + j]
+                    
+                        horizontal_distance = curr_bbox[0] - prev_bbox[2]
+                        vertical_alignment = (abs(prev_bbox[1] - curr_bbox[1]) < 30 and
+                                              abs(prev_bbox[3] - curr_bbox[3]) < 30)
+                    
+                        if not (vertical_alignment and horizontal_distance <= 150):
+                            spatial_ok = False
+                            break
+                    
+                    if spatial_ok:
+                        for j in range(pii_len):
+                            if not is_pii_flags[start_idx + j]:
+                                is_pii_flags[start_idx + j] = True
+                        matched = True
+                        matched_text = ' '.join(words[start_idx:start_idx + pii_len])
+                        print(f" ✓ Multi-word exact: '{matched_text}' -> '{pii_item}'")
+                        start_idx += pii_len
+                        continue
+                start_idx += 1
+    
+        # Strategy 2: Fuzzy matching if exact matching failed
+        if not matched:
+            if len(pii_words) == 1:
+                pii_word = pii_words[0]
+                for idx, norm_word in enumerate(normalized_words):
+                    if (not is_pii_flags[idx] and
+                        (fuzzy_match_score(norm_word, pii_word, 0.9) or
+                         is_similar_by_edit_distance(norm_word, pii_word, 2))):
+                        is_pii_flags[idx] = True
+                        matched = True
+                        print(f" ✓ Fuzzy match: '{words[idx]}' -> '{pii_item}'")
+            else:
+                # Multi-word fuzzy matching
+                pii_len = len(pii_words)
+                start_idx = 0
+                while start_idx < len(normalized_words) - pii_len + 1:
+                    fuzzy_match = True
+                    for j in range(pii_len):
+                        if not (fuzzy_match_score(normalized_words[start_idx + j], pii_words[j], 0.85) or
+                                is_similar_by_edit_distance(normalized_words[start_idx + j], pii_words[j], 2)):
+                            fuzzy_match = False
+                            break
+                    
+                    if fuzzy_match:
+                        # Check spatial proximity
+                        spatial_ok = True
+                        for j in range(1, pii_len):
+                            prev_bbox = bboxes[start_idx + j - 1]
+                            curr_bbox = bboxes[start_idx + j]
+                        
+                            horizontal_distance = curr_bbox[0] - prev_bbox[2]
+                            vertical_alignment = (abs(prev_bbox[1] - curr_bbox[1]) < 30 and
+                                                  abs(prev_bbox[3] - curr_bbox[3]) < 30)
+                        
+                            if not (vertical_alignment and horizontal_distance <= 150):
+                                spatial_ok = False
+                                break
+                        
+                        if spatial_ok:
+                            for j in range(pii_len):
+                                if not is_pii_flags[start_idx + j]:
+                                    is_pii_flags[start_idx + j] = True
+                            matched = True
+                            matched_text = ' '.join(words[start_idx:start_idx + pii_len])
+                            print(f" ✓ Multi-word fuzzy: '{matched_text}' -> '{pii_item}'")
+                            start_idx += pii_len
+                            continue
+                    start_idx += 1
+    
+        # Strategy 3: Substring and partial matching
+        if not matched:
+            full_normalized_text = ' '.join(normalized_words)
+        
+            pos = 0
+            while True:
+                start_pos = full_normalized_text.find(normalized_pii, pos)
+                if start_pos == -1:
+                    break
+                end_pos = start_pos + len(normalized_pii)
+                
+                char_count = 0
+                start_word_idx = None
+                end_word_idx = None
+                
+                for idx, norm_word in enumerate(normalized_words):
+                    word_start = char_count
+                    word_end = char_count + len(norm_word)
+                    
+                    if start_word_idx is None and word_end > start_pos:
+                        start_word_idx = idx
+                    
+                    if word_start < end_pos:
+                        end_word_idx = idx
+                    
+                    char_count += len(norm_word) + 1
+                
+                if start_word_idx is not None and end_word_idx is not None and end_word_idx - start_word_idx + 1 >= len(pii_words):
+                    spatial_ok = True
+                    for j in range(start_word_idx, end_word_idx):
+                        if j + 1 <= end_word_idx:
+                            prev_bbox = bboxes[j]
+                            next_bbox = bboxes[j + 1]
+                        
+                            horizontal_distance = next_bbox[0] - prev_bbox[2]
+                            vertical_alignment = (abs(prev_bbox[1] - next_bbox[1]) < 30 and
+                                                  abs(prev_bbox[3] - next_bbox[3]) < 30)
+                        
+                            if not (vertical_alignment and horizontal_distance <= 200):
+                                spatial_ok = False
+                                break
+                    
+                    if spatial_ok:
+                        for j in range(start_word_idx, end_word_idx + 1):
+                            if not is_pii_flags[j]:
+                                is_pii_flags[j] = True
+                        matched = True
+                        matched_text = ' '.join(words[start_word_idx:end_word_idx + 1])
+                        print(f" ✓ Substring match: '{matched_text}' -> '{pii_item}'")
+                pos = end_pos
+    
+        # Strategy 4: Individual word matching with relaxed criteria
+        if not matched:
+            for pii_word in pii_words:
+                if len(pii_word) < 3:
+                    continue
+                
+                for idx, norm_word in enumerate(normalized_words):
+                    if not is_pii_flags[idx]:
+                        if (norm_word == pii_word or
+                            fuzzy_match_score(norm_word, pii_word, 0.8) or
+                            is_similar_by_edit_distance(norm_word, pii_word, 2) or
+                            (len(pii_word) > 5 and (pii_word in norm_word or norm_word in pii_word))):
+                            is_pii_flags[idx] = True
+                            print(f" ✓ Individual word match: '{words[idx]}' -> '{pii_word}' from '{pii_item}'")
+    
+        if not matched:
+            print(f" ✗ No match found for: '{pii_item}'")
+    for idx, word_info in enumerate(word_bbox_map):
+        ocr_results_for_template.append({
+            "text": word_info["word"],
+            "bbox": word_info["bbox"],
+            "is_pii": is_pii_flags[idx],
+            "confidence": word_info.get("confidence", 1.0)
+        })
+    ocr_results_for_template = merge_horizontal_pii_boxes_improved(ocr_results_for_template)
+    return ocr_results_for_template
+
+def merge_horizontal_pii_boxes_improved(ocr_results, merge_distance=50):
+    """Improved merging with better spatial awareness and tighter boxes"""
+    if not ocr_results:
+        return ocr_results
+    merged_results = []
+    i = 0
+    while i < len(ocr_results):
+        current_word = ocr_results[i]
+    
+        if not current_word["is_pii"]:
+            merged_results.append(current_word)
+            i += 1
+            continue
+    
+        merge_group = [current_word]
+        j = i + 1
+    
+        while j < len(ocr_results):
+            next_word = ocr_results[j]
+        
+            if not next_word["is_pii"]:
+                break
+        
+            current_bbox = merge_group[-1]["bbox"]
+            next_bbox = next_word["bbox"]
+        
+            y_center_current = (current_bbox[1] + current_bbox[3]) / 2
+            y_center_next = (next_bbox[1] + next_bbox[3]) / 2
+            y_overlap = abs(y_center_current - y_center_next) < 20
+        
+            horizontal_distance = next_bbox[0] - current_bbox[2]
+        
+            if y_overlap and horizontal_distance <= merge_distance:
+                merge_group.append(next_word)
+                j += 1
+            else:
+                break
+    
+        if len(merge_group) > 1:
+            min_x = min(word["bbox"][0] for word in merge_group)
+            min_y = min(word["bbox"][1] for word in merge_group)
+            max_x = max(word["bbox"][2] for word in merge_group)
+            max_y = max(word["bbox"][3] for word in merge_group)
+        
+            merged_text = " ".join(word["text"] for word in merge_group)
+        
+            merged_word = {
+                "text": merged_text,
+                "bbox": [min_x, min_y, max_x, max_y],
+                "is_pii": True,
+                "confidence": max(word.get("confidence", 1.0) for word in merge_group)
+            }
+            merged_results.append(merged_word)
+            print(f" ✓ Merged PII box: '{merged_text}' at [{min_x},{min_y},{max_x},{max_y}]")
+        else:
+            merged_results.append(current_word)
+    
+        i = j
+    return merged_results
+
+def post_process_pii_detection(ocr_results_for_template, pii_list):
+    """Post-process to catch any missed PII using relaxed matching"""
+    words = [result["text"] for result in ocr_results_for_template]
+    for pii_item in pii_list:
+        normalized_pii = normalize_text(pii_item)
+        pii_words = normalized_pii.split()
+    
+        if not pii_words:
+            continue
+    
+        pii_detected = False
+        for result in ocr_results_for_template:
+            if result["is_pii"]:
+                result_normalized = normalize_text(result["text"])
+                if (normalized_pii in result_normalized or
+                    result_normalized in normalized_pii or
+                    fuzzy_match_score(result_normalized, normalized_pii, 0.7)):
+                    pii_detected = True
+                    break
+    
+        if not pii_detected:
+            print(f" ⚠ PII not detected, trying fallback matching: '{pii_item}'")
+        
+            for idx, result in enumerate(ocr_results_for_template):
+                if result["is_pii"]:
+                    continue
+                
+                word_normalized = normalize_text(result["text"])
+                
+                for pii_word in pii_words:
+                    if (len(pii_word) > 3 and
+                        (pii_word in word_normalized or
+                         word_normalized in pii_word or
+                         fuzzy_match_score(word_normalized, pii_word, 0.6) or
+                         is_similar_by_edit_distance(word_normalized, pii_word, 3))):
+                    
+                        ocr_results_for_template[idx]["is_pii"] = True
+                        print(f" ✓ Fallback match: '{result['text']}' -> '{pii_word}' from '{pii_item}'")
+                        break
+    return ocr_results_for_template
+
+def detect_pii_from_combined_text(combined_text):
+    """Detect PII from combined multi-page text using LLM"""
+    if llama_model is None:
+        print("LLM model not available for PII detection")
+        return [], "LLM model not available"
+    instruction = (
+        "Extract all Personally Identifiable Information (PII) of the main subject from the given text. "
+        "Include data like Name, Date of Birth, Gender, Address, Phone Number, Email, Social Security Number (SSN), Member ID, Group Number, or any other PII data available. "
+        "Ignore any information about doctors, staff, providers, colleagues, organizations, companies, hospitals, educational institutes, or facilities. "
+        "Return the results strictly as a flat set of strings enclosed in { } without labels."
+    )
+    prompt_content = f"{instruction}\n{combined_text}"
+    pii_list = []
+    llama_raw_output = ""
+    try:
+        messages = [{"role": "user", "content": prompt_content}]
+        response = llama_model.create_chat_completion(
+            messages=messages,
+            max_tokens=512,
+            temperature=0.1,
+        )
+        llama_raw_output = response['choices'][0]['message']['content']
+        pii_list = parse_pii_output(llama_raw_output)
+        print(f"LLM detected {len(pii_list)} PII items from combined text: {pii_list}")
+    except Exception as e:
+        print(f"Error during Llama PII detection: {e}")
+        llama_raw_output = f"Error: {str(e)}"
+        pii_list = []
+    return pii_list, llama_raw_output
+
+# --- Combined Processing Function ---
+def process_page_combined(img_cv, global_pii_list):
+    """Process a single page with both YOLO and OCR+LLM detection"""
+    all_detections = []
+    # Step 1: YOLO Visual Detection
+    print(" Running YOLO visual detection...")
+    visual_boxes, visual_classes = detect_visual_pii(img_cv)
+    for box, cls in zip(visual_boxes, visual_classes):
+        all_detections.append({
+            "text": cls,
+            "bbox": box.tolist() if hasattr(box, 'tolist') else box,
+            "is_pii": True,
+            "confidence": 1.0,
+            "detection_type": "visual"
+        })
+    print(f" YOLO detected {len(visual_boxes)} visual elements")
+    # Step 2: OCR + LLM Text Detection
+    if reader is not None:
+        print(" Running OCR text extraction...")
+        word_ocr_results = reader.readtext(img_cv, paragraph=False, width_ths=0.7, height_ths=0.7)
+    
+        if word_ocr_results:
+            word_bbox_map = extract_word_bboxes_improved(word_ocr_results)
+        
+            ocr_results_for_template = advanced_match_pii_to_words(global_pii_list, word_bbox_map)
+            ocr_results_for_template = post_process_pii_detection(ocr_results_for_template, global_pii_list)
+            ocr_results_for_template = refine_pii_flags(ocr_results_for_template)
+            ocr_results_for_template = merge_horizontal_pii_boxes_improved(ocr_results_for_template)
+        
+            for result in ocr_results_for_template:
+                if result["is_pii"]:
+                    result["detection_type"] = "text"
+                    all_detections.append(result)
+            
+            print(f" OCR detected {sum(1 for r in ocr_results_for_template if r['is_pii'])} text PII elements")
+    return all_detections
+
+def classify_pii(text):
+    text = text.strip()
+    clean_text = re.sub(r'\s+', '', text)
+    if re.match(r'^\d{3}-\d{2}-\d{4}$', text) or re.match(r'^\d{3}-\d{2}-\d{4}$', clean_text):
+        return 'ssn'
+    elif re.match(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text) or re.match(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', clean_text) or '@' in text:
+        return 'email'
+    elif re.match(r'^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}$', text) or re.match(r'^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}$', clean_text):
+        return 'phone'
+    elif re.match(r'^\d{1,2}/\d{1,2}/\d{4}$', text) or re.match(r'^\d{4}-\d{2}-\d{2}$', text) or re.match(r'^\d{1,2}/\d{1,2}/\d{4}$', clean_text) or re.match(r'^\d{1,2}-\d{1,2}-\d{4}$', text) or re.match(r'^\d{1,2}-\d{1,2}-\d{2}$', text) or re.match(r'^\d{2}/\d{2}/\d{4}$', text) or re.match(r'^\d{2}-\d{2}-\d{4}$', text) or re.match(r'^(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},\s+\d{4}$', text, re.IGNORECASE) or re.match(r'^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},\s+\d{4}$', text, re.IGNORECASE) or re.match(r'^\d{1,2}\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{4}$', text, re.IGNORECASE) or re.match(r'^\d{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{4}$', text, re.IGNORECASE):
+        return 'dob'
+    elif re.match(r'^(?=.*\d)[A-Za-z0-9]+$', text) and len(text) > 5:
+        return 'id'
+    elif ',' in text or 'St' in text or 'Ave' in text or re.search(r'\d{5}', text):
+        return 'address'
+    elif re.match(r'^(male|female|m|f|transgender|nonbinary|non-binary|other|unknown|u|o)$', text.lower()):
+        return 'gender'
+    else:
+        return 'name'
+
+def format_gender(base_gender, original_text):
+    orig = original_text.strip()
+    orig_lower = orig.lower()
+    if orig_lower not in ['male', 'female', 'm', 'f', 'transgender', 'nonbinary', 'non-binary', 'other', 'unknown', 'u', 'o']:
+        return base_gender.capitalize()
+    if len(orig) == 1:
+        char = 'M' if base_gender == 'male' else 'F'
+        return char.lower() if orig.islower() else char
+    else:
+        if orig.isupper():
+            return base_gender.upper()
+        elif orig.islower():
+            return base_gender.lower()
+        else:
+            return base_gender.capitalize()
+
+def generate_fake(pii_type, length, original=None):
+    max_attempts = 100
+    if pii_type == 'dob' and original:
+        formats = [
+            '%m/%d/%Y', '%m/%d/%y', '%d/%m/%Y', '%d/%m/%y', '%Y-%m-%d', '%y-%m-%d', '%m-%d-%Y', '%m-%d-%y',
+            '%d-%m-%Y', '%d-%m-%y', '%Y/%m/%d', '%y/%m/%d', '%d.%m.%Y', '%m.%d.%Y', '%B %d, %Y', '%b %d, %Y',
+            '%d %B %Y', '%d %b %Y', '%B %d %Y', '%b %d %Y'
+        ]
+        for fmt in formats:
+            try:
+                datetime.datetime.strptime(original.strip(), fmt)
+                fake_dt = fake.date_object()
+                return fake_dt.strftime(fmt)
+            except ValueError:
+                pass
+        return fake.date(pattern='%m/%d/%Y')
+    elif pii_type == 'gender' and original:
+        return format_gender(random.choice(['male', 'female']), original)
+    elif pii_type in ['name', 'email', 'phone', 'address']:
+        for _ in range(max_attempts):
+            if pii_type == 'name': f = fake.name()
+            elif pii_type == 'email': f = fake.email()
+            elif pii_type == 'phone': f = fake.phone_number()
+            elif pii_type == 'address': f = fake.address().replace('\n', ', ')
+            if len(f) == length:
+                return f
+        
+        closest = None
+        min_diff = float('inf')
+        for _ in range(50):
+            if pii_type == 'name': f = fake.name()
+            elif pii_type == 'email': f = fake.email()
+            elif pii_type == 'phone': f = fake.phone_number()
+            elif pii_type == 'address': f = fake.address().replace('\n', ', ')
+            diff = abs(len(f) - length)
+            if diff < min_diff:
+                min_diff, closest = diff, f
+        return closest
+    elif pii_type == 'ssn':
+        return fake.ssn()
+    else: # id and others
+        return fake.lexify(text='?' * length)
+
+def detect_format(text):
+    text = text.strip().strip('"')
+    if text.startswith('ISA*'):
+        return 'edi_x12'
+    else:
+        return 'plain'
+
+def convert_to_readable(text, format_type):
+    if format_type == 'edi_x12':
+        parsed_transactions = parse_edi_fallback(text)
+        output = StringIO()
+        format_output(parsed_transactions, file=output)
+        return output.getvalue()
+    else:
+        return text
+
+def redact_text(original_text, pii_list):
+    redacted = original_text
+    for pii in pii_list:
+        redacted = re.sub(re.escape(pii), '[REDACTED]', redacted, flags=re.IGNORECASE)
+    return redacted
+
+def anonymize_text(original_text, pii_list):
+    anonymized = original_text
+    pii_map = {}
+    for pii in pii_list:
+        normalized = normalize_text(pii)
+        pii_type = classify_pii(pii)
+        if normalized not in pii_map:
+            if pii_type == 'gender':
+                base_gender = random.choice(['male', 'female'])
+                fake_val = format_gender(base_gender, pii)
+                pii_map[normalized] = fake_val
+            else:
+                fake_val = generate_fake(pii_type, len(pii), pii)
+                pii_map[normalized] = fake_val
+        else:
+            fake_val = pii_map[normalized]
+        anonymized = re.sub(re.escape(pii), fake_val, anonymized, flags=re.IGNORECASE)
+    return anonymized
+
+def parse_edi_fallback(edi_content):
+    """
+    Fallback parser that combines address parts into a single line for easier redaction.
+    """
+    print("Using fallback parser...")
+    
+    segments = edi_content.replace('~', '\n').split('\n')
+    segments = [seg.strip() for seg in segments if seg.strip()]
+    
+    parsed_data = {
+        'transaction_info': {}, 'patient_info': {}, 'provider_info': {},
+        'service_info': {}, 'diagnosis_info': {}
+    }
+    
+    for segment in segments:
+        elements = segment.split('*')
+        segment_id = elements[0]
+        
+        if segment_id == 'ST':
+            parsed_data['transaction_info']['transaction_type'] = elements[1]
+        elif segment_id == 'NM1':
+            entity_type = elements[1]
+            if entity_type == 'IL':  # Patient
+                last_name = elements[3] if len(elements) > 3 else ''
+                first_name = elements[4] if len(elements) > 4 else ''
+                parsed_data['patient_info']['name'] = f"{first_name} {last_name}".strip()
+                if len(elements) > 8:
+                    parsed_data['patient_info']['id'] = elements[9]
+            elif entity_type == 'SJ':  # Provider
+                last_name = elements[3] if len(elements) > 3 else ''
+                first_name = elements[4] if len(elements) > 4 else ''
+                parsed_data['provider_info']['name'] = f"{first_name} {last_name}".strip()
+                if len(elements) > 8:
+                    parsed_data['provider_info']['npi'] = elements[9]
+        elif segment_id == 'N3':
+            # Store the first line of the address
+            parsed_data['patient_info']['address'] = elements[1]
+        elif segment_id == 'N4':
+            # Combine City, State, and Zip with the address line
+            city = elements[1] if len(elements) > 1 else ''
+            state = elements[2] if len(elements) > 2 else ''
+            zip_code = elements[3] if len(elements) > 3 else ''
+            
+            full_address_parts = [city, state, zip_code]
+            
+            # If an address line already exists from N3, prepend it
+            if 'address' in parsed_data['patient_info']:
+                full_address_parts.insert(0, parsed_data['patient_info']['address'])
+            
+            # Join all parts with ", " and filter out any empty parts
+            parsed_data['patient_info']['address'] = ", ".join(filter(None, full_address_parts))
+
+        elif segment_id == 'DMG':
+            parsed_data['patient_info']['dob'] = elements[2]
+            parsed_data['patient_info']['gender'] = 'Female' if elements[3] == 'F' else 'Male'
+        elif segment_id == 'UM':
+            parsed_data['service_info']['service_type'] = elements[1]
+            parsed_data['service_info']['request_category'] = elements[2]
+            parsed_data['service_info']['service_code'] = elements[3]
+            if len(elements) > 4:
+                parsed_data['service_info']['quantity'] = elements[4]
+        elif segment_id == 'HI':
+            diagnosis_info = elements[1].split(':')
+            if len(diagnosis_info) > 1:
+                parsed_data['diagnosis_info']['code_qualifier'] = diagnosis_info[0]
+                parsed_data['diagnosis_info']['diagnosis_code'] = diagnosis_info[1]
+    
+    return [{
+        'transaction_type': parsed_data['transaction_info'].get('transaction_type', 'Unknown'),
+        'parsed_data': {
+            'description': 'Health Care Services Review',
+            'patient_info': parsed_data['patient_info'],
+            'provider_info': parsed_data['provider_info'],
+            'service_info': parsed_data['service_info'],
+            'diagnosis_info': parsed_data['diagnosis_info']
+        }
+    }]
+
+def format_output(parsed_transactions, file=None):
+    """Format parsed data for display"""
+    output_lines = []
+    for i, transaction in enumerate(parsed_transactions):
+        output_lines.append(f"\n=== TRANSACTION {i+1} ===")
+        output_lines.append(f"Transaction Type: {transaction['transaction_type']}")
+        
+        if 'parsed_data' in transaction:
+            data = transaction['parsed_data']
+            
+            if 'description' in data:
+                output_lines.append(f"Description: {data['description']}")
+            
+            # Patient Information
+            if 'patient_info' in data and data['patient_info']:
+                output_lines.append("\nPATIENT INFORMATION:")
+                for key, value in data['patient_info'].items():
+                    output_lines.append(f"  {key.replace('_', ' ').title()}: {value}")
+            
+            # Provider Information
+            if 'provider_info' in data and data['provider_info']:
+                output_lines.append("\nPROVIDER INFORMATION:")
+                for key, value in data['provider_info'].items():
+                    output_lines.append(f"  {key.replace('_', ' ').title()}: {value}")
+            
+            # Service Information
+            if 'service_info' in data and data['service_info']:
+                output_lines.append("\nSERVICE INFORMATION:")
+                for key, value in data['service_info'].items():
+                    output_lines.append(f"  {key.replace('_', ' ').title()}: {value}")
+            
+            # Diagnosis Information
+            if 'diagnosis_info' in data and data['diagnosis_info']:
+                output_lines.append("\nDIAGNOSIS INFORMATION:")
+                for key, value in data['diagnosis_info'].items():
+                    output_lines.append(f"  {key.replace('_', ' ').title()}: {value}")
+    output_str = '\n'.join(output_lines)
+    if file:
+        file.write(output_str)
+    else:
+        print(output_str)
+    return output_str
+
+# --- Main Gradio Processing Function ---
+def analyze_document(file, progress=gr.Progress()):
+    """
+    This function takes an uploaded file, processes it through the PII detection pipeline,
+    and returns the annotated images, a redacted PDF, and a summary report.
+    """
+    load_models()
+    if file is None:
+        return None, None, None, None, None, "Please upload a document to begin."
+    
+    unique_id = uuid.uuid4().hex
+    if hasattr(file, 'name'):
+        filepath = file.name
+    else:
+        filepath = str(file)
+    filename = os.path.basename(filepath)
+    extension = os.path.splitext(filename)[1]
+    
+    if extension.lower() == '.csv':
+        rows = []
+        with open(filepath, 'r', newline='') as csvfile:
+            csv_reader = csv.reader(csvfile)
+            for row in csv_reader:
+                if row:
+                    rows.append(row[0])
+        total_rows = len(rows)
+        print(f"Processing {total_rows} rows for job {unique_id}...")
+        progress(0.1, desc="Reading CSV rows...")
+        report = f"## 🔍 Analysis Report for CSV\n**Total Rows:** {total_rows}\n\n---\n"
+        redacted_rows = []
+        anonymized_rows = []
+        for i, text in enumerate(rows):
+            progress(0.4 + (i / total_rows * 0.5), desc=f"Processing Row {i+1}/{total_rows}...")
+            format_type = detect_format(text)
+            readable_text = convert_to_readable(text, format_type)
+            pii_list, llama_raw_output = detect_pii_from_combined_text(readable_text)
+            redacted_text = redact_text(readable_text, pii_list)
+            anonymized_text = anonymize_text(readable_text, pii_list)
+            redacted_rows.append(redacted_text)
+            anonymized_rows.append(anonymized_text)
+            report += f"### 📄 Row {i+1}\n- **Text Detections:** {len(pii_list)}\n- **PII Found:** {', '.join(pii_list) if pii_list else 'None'}\n\n"
+        progress(0.9, desc="Generating final CSVs...")
+        redacted_csv_path = os.path.join(RESULTS_FOLDER, f"redacted_{unique_id}.csv")
+        with open(redacted_csv_path, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            for txt in redacted_rows:
+                writer.writerow([txt])
+        anonymized_csv_path = os.path.join(RESULTS_FOLDER, f"anonymized_{unique_id}.csv")
+        with open(anonymized_csv_path, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            for txt in anonymized_rows:
+                writer.writerow([txt])
+        progress(1, desc="Complete!")
+        print("Processing Complete.")
+        return [], [], [], redacted_csv_path, anonymized_csv_path, report
+    
+    progress(0, desc="Converting document to images...")
+    images_to_process = []
+    try:
+        if extension.lower() == '.pdf':
+            try:
+                images_to_process = [cv2.cvtColor(np.array(page), cv2.COLOR_RGB2BGR) for page in convert_from_path(filepath, dpi=300)]
+            except Exception as e:
+                print(f"PDF conversion error: {e}. Trying fallback method...")
+                try:
+                    images_to_process = [cv2.cvtColor(np.array(page), cv2.COLOR_RGB2BGR) for page in convert_from_path(filepath, dpi=150)]
+                except Exception as e2:
+                    return None, None, None, None, None, f"🔴 **Error:** Could not process PDF. Please ensure Poppler is installed.\nDetails: {e2}"
+        else:
+            img = cv2.imread(filepath)
+            if img is not None:
+                images_to_process.append(img)
+    except Exception as e:
+        return None, None, None, None, None, f"🔴 **Error:** Could not process file. Details: {e}"
+    
+    if not images_to_process:
+        return None, None, None, None, None, "🔴 **Error:** No pages could be extracted from the document."
+    
+    total_pages = len(images_to_process)
+    print(f"Processing {total_pages} pages for job {unique_id}...")
+    
+    progress(0.1, desc="Extracting text from all pages (OCR)...")
+    combined_text, all_pages_data = "", []
+    for i, img_cv in enumerate(images_to_process):
+        page_text = ""
+        if reader:
+            page_text = extract_sentence_text(reader.readtext(img_cv, paragraph=True))
+            combined_text += f"\n--- Page {i+1} ---\n{page_text}\n"
+        all_pages_data.append({"img_cv": img_cv, "page_num": i + 1})
+    
+    progress(0.4, desc="Analyzing text for PII with LLM...")
+    global_pii_list, llama_raw_output = detect_pii_from_combined_text(combined_text)
+    
+    annotated_paths, redacted_paths, anonymized_paths = [], [], []
+    redacted_pils, anonymized_pils = [], []
+    report = f"## 🔍 Analysis Report\n**Global PII Found:** `{', '.join(global_pii_list) if global_pii_list else 'None'}`\n\n---\n"
+    pii_map = {}
+    
+    for i, page_info in enumerate(all_pages_data):
+        progress(0.5 + (i / total_pages * 0.4), desc=f"Processing Page {i+1}/{total_pages} (Visual & Text)...")
+        img_cv, page_num = page_info["img_cv"], page_info["page_num"]
+        detections = process_page_combined(img_cv, global_pii_list)
+        
+        annotated_img = img_cv.copy()
+        redacted_img = img_cv.copy()
+        anonymized_img = img_cv.copy()
+        visual_count = sum(1 for d in detections if d["detection_type"] == "visual")
+        text_count = sum(1 for d in detections if d.get("detection_type") == "text")
+        
+        for d in detections:
+            bbox = d.get("bbox", [])
+            if not bbox: continue
+            x1, y1, x2, y2 = map(int, bbox)
+            color = (0, 255, 0) if d.get("detection_type") == "visual" else (0, 0, 255)
+            cv2.rectangle(annotated_img, (x1, y1), (x2, y2), color, 3)
+            
+            if d["detection_type"] == "visual":
+                cv2.rectangle(redacted_img, (x1, y1), (x2, y2), (0, 0, 0), -1)
+                cv2.rectangle(anonymized_img, (x1, y1), (x2, y2), (0, 0, 0), -1)
+            else:
+                bg_color = (255, 255, 255)
+                height, width = img_cv.shape[:2]
+                if x2 + 20 < width:
+                    sample = img_cv[y1:y2, x2:x2+20]
+                    if sample.size > 0:
+                        bg_color = tuple(map(int, np.mean(sample, axis=(0,1))))
+                else:
+                    if x1 > 20:
+                        sample = img_cv[y1:y2, x1-20:x1]
+                        if sample.size > 0:
+                            bg_color = tuple(map(int, np.mean(sample, axis=(0,1))))
+                
+                cv2.rectangle(redacted_img, (x1, y1), (x2, y2), (0, 0, 0), -1)
+                cv2.rectangle(anonymized_img, (x1, y1), (x2, y2), bg_color, -1)
+                
+                original_text = d["text"]
+                pii_type = classify_pii(original_text)
+                normalized = normalize_text(original_text)
+                key = 'gender' if pii_type == 'gender' else normalized
+                
+                if key not in pii_map:
+                    if pii_type == 'gender':
+                        base_gender = random.choice(['male', 'female'])
+                        fake_text = format_gender(base_gender, original_text)
+                        pii_map[key] = base_gender
+                    else:
+                        fake_text = generate_fake(pii_type, len(original_text), original_text)
+                        pii_map[key] = fake_text
+                else:
+                    if pii_type == 'gender':
+                        base_gender = pii_map[key]
+                        fake_text = format_gender(base_gender, original_text)
+                    else:
+                        fake_text = pii_map[key]
+                
+                font = cv2.FONT_HERSHEY_SIMPLEX
+                font_scale = (y2 - y1) / 40.0
+                thickness = 2
+                text_size, _ = cv2.getTextSize(fake_text, font, font_scale, thickness)
+                box_width = x2 - x1
+                if text_size[0] > box_width - 10:
+                    font_scale *= (box_width - 10) / text_size[0]
+                    text_size, _ = cv2.getTextSize(fake_text, font, font_scale, thickness)
+                text_x = x1 + (box_width - text_size[0]) // 2
+                text_y = y1 + ((y2 - y1) + text_size[1]) // 2
+                bg_brightness = 0.299 * bg_color[2] + 0.587 * bg_color[1] + 0.114 * bg_color[0]
+                text_color = (0, 0, 0) if bg_brightness > 128 else (255, 255, 255)
+                cv2.putText(anonymized_img, fake_text, (text_x, text_y), font, font_scale, text_color, thickness)
+
+        # Save all three versions of the image for the galleries
+        annotated_path = os.path.join(RESULTS_FOLDER, f"annotated_{unique_id}_{page_num}.jpg")
+        redacted_path = os.path.join(RESULTS_FOLDER, f"redacted_preview_{unique_id}_{page_num}.jpg")
+        anonymized_path = os.path.join(RESULTS_FOLDER, f"anonymized_preview_{unique_id}_{page_num}.jpg")
+
+        cv2.imwrite(annotated_path, annotated_img)
+        cv2.imwrite(redacted_path, redacted_img)
+        cv2.imwrite(anonymized_path, anonymized_img)
+
+        annotated_paths.append(annotated_path)
+        redacted_paths.append(redacted_path)
+        anonymized_paths.append(anonymized_path)
+        
+        redacted_pils.append(Image.fromarray(cv2.cvtColor(redacted_img, cv2.COLOR_BGR2RGB)))
+        anonymized_pils.append(Image.fromarray(cv2.cvtColor(anonymized_img, cv2.COLOR_BGR2RGB)))
+        
+        report += f"### 📄 Page {page_num}\n- **Visual Detections (🟩 Green):** {visual_count}\n- **Text Detections (🟥 Red):** {text_count}\n"
+    
+    progress(0.9, desc="Generating final PDFs...")
+    redacted_pdf_path, anonymized_pdf_path = None, None
+    if redacted_pils:
+        pdf_path = os.path.join(RESULTS_FOLDER, f"redacted_{unique_id}.pdf")
+        redacted_pils[0].save(pdf_path, "PDF", resolution=100.0, save_all=True, append_images=redacted_pils[1:])
+        redacted_pdf_path = pdf_path
+    if anonymized_pils:
+        pdf_path = os.path.join(RESULTS_FOLDER, f"anonymized_{unique_id}.pdf")
+        anonymized_pils[0].save(pdf_path, "PDF", resolution=100.0, save_all=True, append_images=anonymized_pils[1:])
+        anonymized_pdf_path = pdf_path
+        
+    progress(1, desc="Complete!")
+    print("Processing Complete.")
+    
+    return annotated_paths, redacted_paths, anonymized_paths, redacted_pdf_path, anonymized_pdf_path, report
+
+# --- Gradio Interface Definition ---
+title = "🔒 Combined PII Detection System"
+description = """
+### Advanced Multi-Modal PII Detection
+This system uses a combination of visual and textual analysis to detect and redact Personally Identifiable Information from your documents.
+- **🖼️ Visual Detection (YOLO):** Detects Faces, QR Codes, and Signatures.
+- **📝 Text Detection (OCR + LLM):** Detects Names, Addresses, Phone Numbers, IDs, and other contextual PII.
+**How to Use:**
+1. Upload a document (PDF or image format).
+2. The system will process each page and display annotated previews with colored boxes.
+3. A fully redacted PDF with blacked-out PII is generated for you to download.
+4. An analysis report summarizes the findings for each page.
+"""
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    gr.Markdown(f"<h1 style='margin-bottom: 0.25rem;'>{title}</h1>")
+    gr.Markdown(
+        "<p style='color:#475569; line-height:1.6;'>"
+        "Upload a PDF or image to detect and redact PII using visual detectors (🟩) and text analysis (🟥). "
+        "Fixed for local environment with proper YOLO support."
+        "</p>"
+    )
+    with gr.Accordion("About this tool", open=False):
+        gr.Markdown(description)
+    with gr.Tabs():
+        with gr.Tab("Run"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    file_input = gr.File(
+                        label="Upload Document",
+                        file_types=['.pdf', '.jpg', '.jpeg', '.png', '.bmp', '.csv'],
+                        file_count="single",
+                        height=100
+                    )
+                    submit_btn = gr.Button("🚀 Analyze Document", variant="primary")
+                    with gr.Accordion("Tips", open=False):
+                        gr.Markdown(
+                            "- Prefer high-resolution files for better OCR results (300 DPI for PDFs).\n"
+                            "- For PDFs, ensure Poppler is installed on your system.\n"
+                            "- Visual detections are drawn in green; text-based detections are in red.\n"
+                            "- Use the Previews tab to inspect annotated pages and the Report tab to download the redacted PDF."
+                        )
+                with gr.Column(scale=1):
+                    gr.Markdown("### What happens during analysis")
+                    gr.Markdown(
+                        "- Convert pages to images\n"
+                        "- Run global OCR to build combined text\n"
+                        "- Use LLM to extract possible PII strings\n"
+                        "- Match PII back to words and merge boxes\n"
+                        "- Render annotated previews and build a redacted PDF"
+                    )
+                    clear_btn = gr.Button("🧹 Clear Results", variant="secondary")
+        
+        with gr.Tab("Annotated Preview (Detection)"):
+            gr.Markdown("### Annotated Previews (🟩 Visual, 🟥 Text)")
+            annotated_gallery_output = gr.Gallery(
+                label="Annotated Pages", show_label=False, elem_id="gallery_annotated",
+                columns=[2], rows=[1], object_fit="contain", height=480
+            )
+            
+        with gr.Tab("Redacted Preview"):
+            gr.Markdown("### Redacted Previews (Blacked Out)")
+            redacted_gallery_output = gr.Gallery(
+                label="Redacted Pages", show_label=False, elem_id="gallery_redacted",
+                columns=[2], rows=[1], object_fit="contain", height=480
+            )
+
+        with gr.Tab("Anonymized Preview"):
+            gr.Markdown("### Anonymized Previews (Fake Data)")
+            anonymized_gallery_output = gr.Gallery(
+                label="Anonymized Pages", show_label=False, elem_id="gallery_anonymized",
+                columns=[2], rows=[1], object_fit="contain", height=480
+            )
+
+        with gr.Tab("Report & Download"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.Markdown("### Downloads")
+                    redacted_file_output = gr.File(label="Redacted PDF (Blacked Out)")
+                    anonymized_file_output = gr.File(label="Anonymized PDF (Fake Data)")
+                with gr.Column(scale=2):
+                    gr.Markdown("### Analysis Report")
+                    report_output = gr.Markdown(label="Analysis Report")
+
+    outputs_list = [
+        annotated_gallery_output, 
+        redacted_gallery_output, 
+        anonymized_gallery_output,
+        redacted_file_output, 
+        anonymized_file_output, 
+        report_output
+    ]
+
+    submit_btn.click(
+        fn=analyze_document,
+        inputs=file_input,
+        outputs=outputs_list
+    )
+    
+    clear_btn.click(
+        fn=lambda: ([], [], [], None, None, "Ready. Upload a document and click Analyze."),
+        inputs=None,
+        outputs=outputs_list
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch(server_port=8000)

requirements.txt

@@ -0,0 +1,10 @@
+gradio==3.50.2
+ultralytics==8.1.34
+pdf2image==1.17.0
+easyocr==1.7.2
+llama-cpp-python==0.3.16
+pydantic==1.10.13
+opencv-python
+torch
+torchvision
+Pillow

unsloth.F16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:411f2d808152dd6e9f5d350242dce3317bb8b6055eedbc04f7f155430b63c356
+size 2479591968