Update working_yolo_pipeline.py

working_yolo_pipeline.py

@@ -91,51 +91,6 @@ def sanitize_text(text: Optional[str]) -> str:
 
 
 
-# def get_latex_from_base64(base64_string: str) -> str:
-#     """
-#     Decodes a Base64 image string and uses the pre-initialized TrOCR/ORT model
-#     to recognize the formula. It cleans the output by removing spaces and
-#     crucially, replacing double backslashes with single backslashes for correct LaTeX.
-#     """
-#     if ort_model is None or processor is None:
-#         return "[MODEL_ERROR: Model not initialized]"
-
-#     try:
-#         # 1. Decode Base64 to Image
-#         image_data = base64.b64decode(base64_string)
-#         # We must ensure the image is RGB format for the model input
-#         image = Image.open(io.BytesIO(image_data)).convert('RGB')
-        
-#         # 2. Preprocess the image
-#         pixel_values = processor(images=image, return_tensors="pt").pixel_values
-        
-#         # 3. Text Generation (OCR)
-#         generated_ids = ort_model.generate(pixel_values)
-#         raw_generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)
-        
-#         if not raw_generated_text:
-#             return "[OCR_WARNING: No formula found]"
-            
-#         latex_string = raw_generated_text[0]
-        
-#         # --- 4. Post-processing and Cleanup ---
-        
-#         # A. Remove all spaces/line breaks
-#         cleaned_latex = re.sub(r'\s+', '', latex_string)
-        
-#         # B. CRITICAL FIX: Replace double backslashes with single backslashes.
-      
-#         return cleaned_latex
-
-
-#     except Exception as e:
-#         # Catch any unexpected errors
-#         print(f"  ❌ TR-OCR Recognition failed: {e}")
-#         return f"[TR_OCR_ERROR: Recognition failed: {e}]"
-
-
-
-
 
 def get_latex_from_base64(base64_string: str) -> str:
     """
@@ -622,26 +577,6 @@ def pixmap_to_numpy(pix: fitz.Pixmap) -> np.ndarray:
     return img
 
 
-# def extract_native_words_and_convert(fitz_page, scale_factor: float = 2.0) -> list:
-#     raw_word_data = fitz_page.get_text("words")
-#     converted_ocr_output = []
-#     DEFAULT_CONFIDENCE = 99.0
-
-#     for x1, y1, x2, y2, word, *rest in raw_word_data:
-#         if not word.strip(): continue
-#         x1_pix = int(x1 * scale_factor)
-#         y1_pix = int(y1 * scale_factor)
-#         x2_pix = int(x2 * scale_factor)
-#         y2_pix = int(y2 * scale_factor)
-#         converted_ocr_output.append({
-#             'type': 'text',
-#             'word': word,
-#             'confidence': DEFAULT_CONFIDENCE,
-#             'bbox': [x1_pix, y1_pix, x2_pix, y2_pix],
-#             'y0': y1_pix, 'x0': x1_pix
-#         })
-#     return converted_ocr_output
-
 
 
 def extract_native_words_and_convert(fitz_page, scale_factor: float = 2.0) -> list:
@@ -672,265 +607,6 @@ def extract_native_words_and_convert(fitz_page, scale_factor: float = 2.0) -> li
 
     
 
-# def preprocess_and_ocr_page(original_img: np.ndarray, model, pdf_path: str,
-#                             page_num: int, fitz_page: fitz.Page,
-#                             pdf_name: str) -> Tuple[List[Dict[str, Any]], Optional[int]]:
-#     """
-#     OPTIMIZED FLOW:
-#     1. Run YOLO to find Equations/Tables.
-#     2. Mask raw text with YOLO boxes.
-#     3. Run Column Detection on the MASKED data.
-#     4. Proceed with OCR (Native or High-Res Tesseract Fallback) and Output.
-#     """
-#     global GLOBAL_FIGURE_COUNT, GLOBAL_EQUATION_COUNT
-
-#     start_time_total = time.time()
-
-#     if original_img is None:
-#         print(f"  ❌ Invalid image for page {page_num}.")
-#         return None, None
-
-#     # ====================================================================
-#     # --- STEP 1: YOLO DETECTION ---
-#     # ====================================================================
-#     start_time_yolo = time.time()
-#     results = model.predict(source=original_img, conf=CONF_THRESHOLD, imgsz=640, verbose=False)
-
-#     relevant_detections = []
-#     if results and results[0].boxes:
-#         for box in results[0].boxes:
-#             class_id = int(box.cls[0])
-#             class_name = model.names[class_id]
-#             if class_name in TARGET_CLASSES:
-#                 x1, y1, x2, y2 = box.xyxy[0].cpu().numpy().astype(int)
-#                 relevant_detections.append(
-#                     {'coords': (x1, y1, x2, y2), 'y1': y1, 'class': class_name, 'conf': float(box.conf[0])}
-#                 )
-
-#     merged_detections = merge_overlapping_boxes(relevant_detections, IOU_MERGE_THRESHOLD)
-#     print(f"    [LOG] YOLO found {len(merged_detections)} objects in {time.time() - start_time_yolo:.3f}s.")
-
-#     # ====================================================================
-#     # --- STEP 2: PREPARE DATA FOR COLUMN DETECTION (MASKING) ---
-#     # ====================================================================
-#     # Note: This uses the updated 'get_word_data_for_detection' which has its own optimizations
-#     raw_words_for_layout = get_word_data_for_detection(
-#         fitz_page, pdf_path, page_num,
-#         top_margin_percent=0.10, bottom_margin_percent=0.10
-#     )
-
-#     masked_word_data = merge_yolo_into_word_data(raw_words_for_layout, merged_detections, scale_factor=2.0)
-
-#     # ====================================================================
-#     # --- STEP 3: COLUMN DETECTION ---
-#     # ====================================================================
-#     page_width_pdf = fitz_page.rect.width
-#     page_height_pdf = fitz_page.rect.height
-
-#     column_detection_params = {
-#         'cluster_bin_size': 2, 'cluster_smoothing': 2,
-#         'cluster_min_width': 10, 'cluster_threshold_percentile': 85,
-#     }
-
-#     separators = calculate_x_gutters(masked_word_data, column_detection_params, page_height_pdf)
-
-#     page_separator_x = None
-#     if separators:
-#         central_min = page_width_pdf * 0.35
-#         central_max = page_width_pdf * 0.65
-#         central_separators = [s for s in separators if central_min <= s <= central_max]
-
-#         if central_separators:
-#             center_x = page_width_pdf / 2
-#             page_separator_x = min(central_separators, key=lambda x: abs(x - center_x))
-#             print(f"      ✅ Column Split Confirmed at X={page_separator_x:.1f}")
-#         else:
-#             print("      ⚠️ Gutter found off-center. Ignoring.")
-#     else:
-#         print("      -> Single Column Layout Confirmed.")
-
-#     # ====================================================================
-#     # --- STEP 4: COMPONENT EXTRACTION (Save Images) ---
-#     # ====================================================================
-#     start_time_components = time.time()
-#     component_metadata = []
-#     fig_count_page = 0
-#     eq_count_page = 0
-
-#     for detection in merged_detections:
-#         x1, y1, x2, y2 = detection['coords']
-#         class_name = detection['class']
-
-#         if class_name == 'figure':
-#             GLOBAL_FIGURE_COUNT += 1
-#             counter = GLOBAL_FIGURE_COUNT
-#             component_word = f"FIGURE{counter}"
-#             fig_count_page += 1
-#         elif class_name == 'equation':
-#             GLOBAL_EQUATION_COUNT += 1
-#             counter = GLOBAL_EQUATION_COUNT
-#             component_word = f"EQUATION{counter}"
-#             eq_count_page += 1
-#         else:
-#             continue
-
-#         component_crop = original_img[y1:y2, x1:x2]
-#         component_filename = f"{pdf_name}_page{page_num}_{class_name}{counter}.png"
-#         cv2.imwrite(os.path.join(FIGURE_EXTRACTION_DIR, component_filename), component_crop)
-
-#         y_midpoint = (y1 + y2) // 2
-#         component_metadata.append({
-#             'type': class_name, 'word': component_word,
-#             'bbox': [int(x1), int(y1), int(x2), int(y2)],
-#             'y0': int(y_midpoint), 'x0': int(x1)
-#         })
-
-#     # ====================================================================
-#     # --- STEP 5: HYBRID OCR (Native Text + Cached Tesseract Fallback) ---
-#     # ====================================================================
-#     raw_ocr_output = []
-#     scale_factor = 2.0  # Pipeline standard scale
-
-#     try:
-#         # Try getting native text first
-#         raw_ocr_output = extract_native_words_and_convert(fitz_page, scale_factor=scale_factor)
-#     except Exception as e:
-#         print(f"  ❌ Native text extraction failed: {e}")
-
-#     # If native text is missing, fall back to OCR
-#     if not raw_ocr_output:
-#         if _ocr_cache.has_ocr(pdf_path, page_num):
-#             print(f"  ⚡ Using cached Tesseract OCR for page {page_num}")
-#             cached_word_data = _ocr_cache.get_ocr(pdf_path, page_num)
-#             for word_tuple in cached_word_data:
-#                 word_text, x1, y1, x2, y2 = word_tuple
-
-#                 # Scale from PDF points to Pipeline Pixels (2.0)
-#                 x1_pix = int(x1 * scale_factor)
-#                 y1_pix = int(y1 * scale_factor)
-#                 x2_pix = int(x2 * scale_factor)
-#                 y2_pix = int(y2 * scale_factor)
-
-#                 raw_ocr_output.append({
-#                     'type': 'text', 'word': word_text, 'confidence': 95.0,
-#                     'bbox': [x1_pix, y1_pix, x2_pix, y2_pix],
-#                     'y0': y1_pix, 'x0': x1_pix
-#                 })
-#         else:
-#             # === START OF OPTIMIZED OCR BLOCK ===
-#             try:
-#                 # 1. Re-render Page at High Resolution (Zoom 4.0 = ~300 DPI)
-#                 # We do this specifically for OCR accuracy, separate from the pipeline image
-#                 ocr_zoom = 4.0
-#                 pix_ocr = fitz_page.get_pixmap(matrix=fitz.Matrix(ocr_zoom, ocr_zoom))
-
-#                 # Convert PyMuPDF Pixmap to OpenCV format
-#                 img_ocr_np = np.frombuffer(pix_ocr.samples, dtype=np.uint8).reshape(pix_ocr.height, pix_ocr.width,
-#                                                                                     pix_ocr.n)
-#                 if pix_ocr.n == 3:
-#                     img_ocr_np = cv2.cvtColor(img_ocr_np, cv2.COLOR_RGB2BGR)
-#                 elif pix_ocr.n == 4:
-#                     img_ocr_np = cv2.cvtColor(img_ocr_np, cv2.COLOR_RGBA2BGR)
-
-#                 # 2. Preprocess (Binarization)
-#                 # Ensure 'preprocess_image_for_ocr' is defined at top of file!
-#                 processed_img = preprocess_image_for_ocr(img_ocr_np)
-
-#                 # 3. Run Tesseract with Optimized Configuration
-#                 # --oem 3: Default LSTM engine
-#                 # --psm 6: Assume a single uniform block of text (Critical for lists/questions)
-#                 custom_config = r'--oem 3 --psm 6'
-
-#                 hocr_data = pytesseract.image_to_data(
-#                     processed_img,
-#                     output_type=pytesseract.Output.DICT,
-#                     config=custom_config
-#                 )
-
-#                 for i in range(len(hocr_data['level'])):
-#                     text = hocr_data['text'][i].strip()
-#                     if text and hocr_data['conf'][i] > -1:
-#                         # 4. Coordinate Mapping
-#                         # We scanned at Zoom 4.0, but our pipeline expects Zoom 2.0.
-#                         # Scale Factor = (Target 2.0) / (Source 4.0) = 0.5
-#                         scale_adjustment = scale_factor / ocr_zoom
-
-#                         x1 = int(hocr_data['left'][i] * scale_adjustment)
-#                         y1 = int(hocr_data['top'][i] * scale_adjustment)
-#                         w = int(hocr_data['width'][i] * scale_adjustment)
-#                         h = int(hocr_data['height'][i] * scale_adjustment)
-#                         x2 = x1 + w
-#                         y2 = y1 + h
-
-#                         raw_ocr_output.append({
-#                             'type': 'text',
-#                             'word': text,
-#                             'confidence': float(hocr_data['conf'][i]),
-#                             'bbox': [x1, y1, x2, y2],
-#                             'y0': y1,
-#                             'x0': x1
-#                         })
-#             except Exception as e:
-#                 print(f"  ❌ Tesseract OCR Error: {e}")
-#             # === END OF OPTIMIZED OCR BLOCK ===
-
-#     # ====================================================================
-#     # --- STEP 6: OCR CLEANING AND MERGING ---
-#     # ====================================================================
-#     items_to_sort = []
-
-#     for ocr_word in raw_ocr_output:
-#         is_suppressed = False
-#         for component in component_metadata:
-#             # Do not include words that are inside figure/equation boxes
-#             ioa = calculate_ioa(ocr_word['bbox'], component['bbox'])
-#             if ioa > IOA_SUPPRESSION_THRESHOLD:
-#                 is_suppressed = True
-#                 break
-#         if not is_suppressed:
-#             items_to_sort.append(ocr_word)
-
-#     # Add figures/equations back into the flow as "words"
-#     items_to_sort.extend(component_metadata)
-
-#     # ====================================================================
-#     # --- STEP 7: LINE-BASED SORTING ---
-#     # ====================================================================
-#     items_to_sort.sort(key=lambda x: (x['y0'], x['x0']))
-#     lines = []
-
-#     for item in items_to_sort:
-#         placed = False
-#         for line in lines:
-#             y_ref = min(it['y0'] for it in line)
-#             if abs(y_ref - item['y0']) < LINE_TOLERANCE:
-#                 line.append(item)
-#                 placed = True
-#                 break
-#         if not placed and item['type'] in ['equation', 'figure']:
-#             for line in lines:
-#                 y_ref = min(it['y0'] for it in line)
-#                 if abs(y_ref - item['y0']) < 20:
-#                     line.append(item)
-#                     placed = True
-#                     break
-#         if not placed:
-#             lines.append([item])
-
-#     for line in lines:
-#         line.sort(key=lambda x: x['x0'])
-
-#     final_output = []
-#     for line in lines:
-#         for item in line:
-#             data_item = {"word": item["word"], "bbox": item["bbox"], "type": item["type"]}
-#             if 'tag' in item: data_item['tag'] = item['tag']
-#             final_output.append(data_item)
-
-#     return final_output, page_separator_x
-
-
-
 
 
 
@@ -1648,15 +1324,6 @@ def run_inference_and_get_raw_words(pdf_path: str, model_path: str,
 
 
 
-
-
-
-
-
-
-
-
-
 # ============================================================================
 # --- PHASE 3: BIO TO STRUCTURED JSON DECODER ---
 # ============================================================================
@@ -1863,133 +1530,6 @@ def calculate_similarity(doc1: str, doc2: str) -> float:
         return 0.0
 
 
-# def process_context_linking(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-#     """
-#     Links questions to passages based on 'passage' flow vs 'new_passage' priority.
-#     Includes 'Decay Logic': If 2 consecutive questions fail to match the active passage,
-#     the passage context is dropped to prevent false positives downstream.
-#     """
-#     print("\n" + "=" * 80)
-#     print("--- STARTING CONTEXT LINKING (WITH DECAY LOGIC) ---")
-#     print("=" * 80)
-
-#     if not data: return []
-
-#     # --- PHASE 1: IDENTIFY PASSAGE DEFINERS ---
-#     passage_definer_indices = []
-#     for i, entry in enumerate(data):
-#         if entry.get("passage") and entry["passage"].strip():
-#             passage_definer_indices.append(i)
-#         if entry.get("new_passage") and entry["new_passage"].strip():
-#             if i not in passage_definer_indices:
-#                 passage_definer_indices.append(i)
-
-#     # --- PHASE 2: CONTEXT TRANSFER & LINKING ---
-#     current_passage_text = None
-#     current_new_passage_text = None
-
-#     # NEW: Counter to track consecutive linking failures
-#     consecutive_failures = 0
-#     MAX_CONSECUTIVE_FAILURES = 2
-
-#     for i, entry in enumerate(data):
-#         item_type = entry.get("type", "Question")
-
-#         # A. UNCONDITIONALLY UPDATE CONTEXTS (And Reset Decay Counter)
-#         if entry.get("passage") and entry["passage"].strip():
-#             current_passage_text = entry["passage"]
-#             consecutive_failures = 0  # Reset because we have fresh explicit context
-#             # print(f"  [Flow] Updated Standard Context from Item {i}")
-
-#         if entry.get("new_passage") and entry["new_passage"].strip():
-#             current_new_passage_text = entry["new_passage"]
-#             # We don't necessarily reset standard failures here as this is a local override
-
-#         # B. QUESTION LINKING
-#         if entry.get("question") and item_type != "METADATA":
-#             combined_query = create_query_text(entry)
-
-#             # Skip if query is too short (noise)
-#             if len(combined_query.strip()) < 5:
-#                 continue
-
-#             # Calculate scores
-#             score_old = calculate_similarity(current_passage_text, combined_query) if current_passage_text else 0.0
-#             score_new = calculate_similarity(current_new_passage_text,
-#                                              combined_query) if current_new_passage_text else 0.0
-
-#             q_preview = entry['question'][:30] + '...'
-
-#             # RESOLUTION LOGIC
-#             linked = False
-
-#             # 1. Prefer New Passage if significantly better
-#             if current_new_passage_text and (score_new > score_old + RESOLUTION_MARGIN) and (
-#                     score_new >= SIMILARITY_THRESHOLD):
-#                 entry["passage"] = current_new_passage_text
-#                 print(f"  [Linker] 🚀 Q{i} ('{q_preview}') -> NEW PASSAGE (Score: {score_new:.3f})")
-#                 linked = True
-#                 # Note: We do not reset 'consecutive_failures' for the standard passage here,
-#                 # because we matched the *new* passage, not the standard one.
-
-#             # 2. Otherwise use Standard Passage if it meets threshold
-#             elif current_passage_text and (score_old >= SIMILARITY_THRESHOLD):
-#                 entry["passage"] = current_passage_text
-#                 print(f"  [Linker] ✅ Q{i} ('{q_preview}') -> STANDARD PASSAGE (Score: {score_old:.3f})")
-#                 linked = True
-#                 consecutive_failures = 0  # Success! Reset the kill switch.
-
-#             if not linked:
-#                 # 3. DECAY LOGIC
-#                 if current_passage_text:
-#                     consecutive_failures += 1
-#                     print(
-#                         f"  [Linker] ⚠️ Q{i} NOT LINKED. (Failures: {consecutive_failures}/{MAX_CONSECUTIVE_FAILURES})")
-
-#                     if consecutive_failures >= MAX_CONSECUTIVE_FAILURES:
-#                         print(f"  [Linker] 🗑️  Context dropped due to {consecutive_failures} consecutive misses.")
-#                         current_passage_text = None
-#                         consecutive_failures = 0
-#                 else:
-#                     print(f"  [Linker] ⚠️ Q{i} NOT LINKED (No active context).")
-
-#     # --- PHASE 3: CLEANUP AND INTERPOLATION ---
-#     print("  [Linker] Running Cleanup & Interpolation...")
-
-#     # 3A. Self-Correction (Remove weak links)
-#     for i in passage_definer_indices:
-#         entry = data[i]
-#         if entry.get("question") and entry.get("type") != "METADATA":
-#             passage_to_check = entry.get("passage") or entry.get("new_passage")
-#             if passage_to_check:
-#                 self_sim = calculate_similarity(passage_to_check, create_query_text(entry))
-#                 if self_sim < SIMILARITY_THRESHOLD:
-#                     entry["passage"] = ""
-#                     if "new_passage" in entry: entry["new_passage"] = ""
-#                     print(f"  [Cleanup] Removed weak link for Q{i}")
-
-#     # 3B. Interpolation (Fill gaps)
-#     # We only interpolate if the gap is strictly 1 question wide to avoid undoing the decay logic
-#     for i in range(1, len(data) - 1):
-#         current_entry = data[i]
-#         is_gap = current_entry.get("question") and not current_entry.get("passage")
-#         if is_gap:
-#             prev_p = data[i - 1].get("passage")
-#             next_p = data[i + 1].get("passage")
-#             if prev_p and next_p and (prev_p == next_p) and prev_p.strip():
-#                 current_entry["passage"] = prev_p
-#                 print(f"  [Linker] 🥪 Q{i} Interpolated from neighbors.")
-
-#     return data
-
-
-
-
-
-
-
-
-
 
 
 
@@ -2162,20 +1702,6 @@ def correct_misaligned_options(structured_data: List[Dict[str, Any]]) -> List[Di
     return structured_data
 
 
-# ============================================================================
-# --- PHASE 4: IMAGE EMBEDDING (Base64) ---
-# ============================================================================
-
-# def get_base64_for_file(filepath: str) -> str:
-#     try:
-#         with open(filepath, 'rb') as f:
-#             return base64.b64encode(f.read()).decode('utf-8')
-#     except Exception as e:
-#         print(f"  ❌ Error encoding file {filepath}: {e}")
-#         return ""
-
-
-
 
 def get_base64_for_file(filepath: str) -> Optional[str]:
     """Reads a file and returns its Base64 encoded string without the data URI prefix."""
@@ -2191,121 +1717,6 @@ def get_base64_for_file(filepath: str) -> Optional[str]:
 
 
 
-# def embed_images_as_base64_in_memory(structured_data: List[Dict[str, Any]], figure_extraction_dir: str) -> List[
-#     Dict[str, Any]]:
-#     print("\n" + "=" * 80)
-#     print("--- 4. STARTING IMAGE EMBEDDING (Base64) ---")
-#     print("=" * 80)
-#     if not structured_data: return []
-#     image_files = glob.glob(os.path.join(figure_extraction_dir, "*.png"))
-#     image_lookup = {}
-#     tag_regex = re.compile(r'(figure|equation)(\d+)', re.IGNORECASE)
-#     for filepath in image_files:
-#         filename = os.path.basename(filepath)
-#         match = re.search(r'_(figure|equation)(\d+)\.png$', filename, re.IGNORECASE)
-#         if match:
-#             key = f"{match.group(1).upper()}{match.group(2)}"
-#             image_lookup[key] = filepath
-#     print(f"  -> Found {len(image_lookup)} image components.")
-#     final_structured_data = []
-#     for item in structured_data:
-#         text_fields = [item.get('question', ''), item.get('passage', '')]
-#         if 'options' in item:
-#             for opt_val in item['options'].values(): text_fields.append(opt_val)
-#         if 'new_passage' in item: text_fields.append(item['new_passage'])
-#         unique_tags_to_embed = set()
-#         for text in text_fields:
-#             if not text: continue
-#             for match in tag_regex.finditer(text):
-#                 tag = match.group(0).upper()
-#                 if tag in image_lookup: unique_tags_to_embed.add(tag)
-#         for tag in sorted(list(unique_tags_to_embed)):
-#             filepath = image_lookup[tag]
-#             base64_code = get_base64_for_file(filepath)
-#             base_key = tag.replace(' ', '').lower()
-#             item[base_key] = base64_code
-#         final_structured_data.append(item)
-#     print(f"✅ Image embedding complete.")
-#     return final_structured_data
-
-
-
-
-# def embed_images_as_base64_in_memory(structured_data: List[Dict[str, Any]], figure_extraction_dir: str) -> List[
-#     Dict[str, Any]]:
-#     print("\n" + "=" * 80)
-#     print("--- 4. STARTING IMAGE EMBEDDING (Base64) / EQUATION TO LATEX CONVERSION ---")
-#     print("=" * 80)
-#     if not structured_data: return []
-#     image_files = glob.glob(os.path.join(figure_extraction_dir, "*.png"))
-#     image_lookup = {}
-#     tag_regex = re.compile(r'(figure|equation)(\d+)', re.IGNORECASE)
-#     for filepath in image_files:
-#         filename = os.path.basename(filepath)
-#         match = re.search(r'_(figure|equation)(\d+)\.png$', filename, re.IGNORECASE)
-#         if match:
-#             key = f"{match.group(1).upper()}{match.group(2)}"
-#             image_lookup[key] = filepath
-#     print(f"  -> Found {len(image_lookup)} image components.")
-    
-#     final_structured_data = []
-    
-#     for item in structured_data:
-#         text_fields = [item.get('question', ''), item.get('passage', '')]
-#         if 'options' in item:
-#             for opt_val in item['options'].values(): text_fields.append(opt_val)
-#         if 'new_passage' in item: text_fields.append(item['new_passage'])
-            
-#         unique_tags_to_embed = set()
-#         for text in text_fields:
-#             if not text: continue
-#             for match in tag_regex.finditer(text):
-#                 tag = match.group(0).upper()
-#                 if tag in image_lookup: unique_tags_to_embed.add(tag)
-                
-#         # List of tags that were successfully converted to LaTeX
-#         tags_converted_to_latex = set()
-        
-#         for tag in sorted(list(unique_tags_to_embed)):
-#             filepath = image_lookup[tag]
-#             # Get the base64 code for processing, whether we embed it or convert it to LaTeX
-#             base64_code = get_base64_for_file(filepath)
-            
-#             # --- PIX2TEXT/EQUATION CONVERSION LOGIC START ---
-#             if tag.startswith('EQUATION') and p2t is not None:
-#                 print(f"  -> Converting EQUATION {tag} to LaTeX...")
-#                 latex_code = get_latex_from_base64(base64_code)
-                
-#                 # Replace the original tag (e.g., EQUATION1) in the item's text fields with LaTeX
-#                 for key in ['question', 'passage', 'new_passage']:
-#                     if item.get(key) and tag in item[key]:
-#                         item[key] = item[key].replace(tag, latex_code)
-                
-#                 if 'options' in item:
-#                     for opt_key, opt_val in item['options'].items():
-#                         if tag in opt_val:
-#                             item['options'][opt_key] = opt_val.replace(tag, latex_code)
-
-#                 tags_converted_to_latex.add(tag)
-#                 # Skip the embedding of the Base64 code for equations
-#                 continue
-#             # --- PIX2TEXT/EQUATION CONVERSION LOGIC END ---
-
-#             # Original logic (for figures): Embed the base64 code
-#             base_key = tag.replace(' ', '').lower()
-#             item[base_key] = base64_code
-            
-#         final_structured_data.append(item)
-    
-#     print(f"✅ Image embedding complete. {len(tags_converted_to_latex)} equations converted to LaTeX.")
-#     return final_structured_data
-
-
-
-
-
-
-
 
 def embed_images_as_base64_in_memory(structured_data: List[Dict[str, Any]], figure_extraction_dir: str) -> List[ Dict[str, Any]]:
     print("\n" + "=" * 80)
@@ -2476,91 +1887,6 @@ def run_document_pipeline( input_pdf_path: str, layoutlmv3_model_path: str, stru
 
 
 
-# if __name__ == "__main__":
-#     parser = argparse.ArgumentParser(description="Complete Pipeline")
-#     parser.add_argument("--input_pdf", type=str, required=True, help="Input PDF")
-#     parser.add_argument("--layoutlmv3_model_path", type=str, default=DEFAULT_LAYOUTLMV3_MODEL_PATH, help="Model Path")
-
-#     # --- ADDED ARGUMENT FOR DEBUGGING ---
-#     parser.add_argument("--raw_preds_path", type=str, default='BIO_debug.json',
-#                         help="Debug path for raw BIO tag predictions (JSON).")
-#     # ------------------------------------
-#     args = parser.parse_args()
-
-#     pdf_name = os.path.splitext(os.path.basename(args.input_pdf))[0]
-#     final_output_path = os.path.abspath(f"{pdf_name}_final_output_embedded.json")
-
-#     # --- CALCULATE RAW PREDICTIONS OUTPUT PATH ---
-#     # raw_predictions_output_path = os.path.abspath(
-#     #     args.raw_preds_path if args.raw_preds_path else f"{pdf_name}_raw_predictions_debug.json")
-#     # ---------------------------------------------
-
-#     # --- UPDATED FUNCTION CALL ---
-#     final_json_data = run_document_pipeline(
-#         args.input_pdf,
-#         args.layoutlmv3_model_path )
-#     # -----------------------------
-
-
-
-#     if final_json_data:
-#         with open(final_output_path, 'w', encoding='utf-8') as f:
-#             json.dump(final_json_data, f, indent=2, ensure_ascii=False)
-#         print(f"\n✅ Final Data Saved: {final_output_path}")
-#     else:
-#         print("\n❌ Pipeline Failed.")
-#         sys.exit(1)
-
-
-
-
-# if __name__ == "__main__":
-#     parser = argparse.ArgumentParser(description="Complete Pipeline")
-#     parser.add_argument("--input_pdf", type=str, required=True, help="Input PDF")
-#     parser.add_argument("--layoutlmv3_model_path", type=str, default=DEFAULT_LAYOUTLMV3_MODEL_PATH, help="Model Path")
-
-#     # --- ADDED ARGUMENT FOR DEBUGGING ---
-#     parser.add_argument("--raw_preds_path", type=str, default='BIO_debug.json',
-#                         help="Debug path for raw BIO tag predictions (JSON).")
-#     # ------------------------------------
-#     args = parser.parse_args()
-
-#     pdf_name = os.path.splitext(os.path.basename(args.input_pdf))[0]
-#     final_output_path = os.path.abspath(f"{pdf_name}_final_output_embedded.json")
-
-#     # --- CALCULATE RAW PREDICTIONS OUTPUT PATH (Kept commented as per original script) ---
-#     # raw_predictions_output_path = os.path.abspath(
-#     #     args.raw_preds_path if args.raw_preds_path else f"{pdf_name}_raw_predictions_debug.json")
-#     # ---------------------------------------------
-
-#     # --- UPDATED FUNCTION CALL ---
-#     final_json_data = run_document_pipeline(
-#         args.input_pdf,
-#         args.layoutlmv3_model_path )
-#     # -----------------------------
-
-#     # 🛑 CRITICAL FINAL FIX: CUSTOM JSON SAVING TO REMOVE ESCAPING 🛑
-#     if final_json_data:
-#         # 1. Dump the Python object to a standard JSON string.
-#         # This converts the in-memory single backslash ('\') into the JSON escaped double backslash ('\\').
-#         json_str = json.dumps(final_json_data, indent=2, ensure_ascii=False)
-
-#         # 2. **UNDO ESCAPING:** The string currently contains '\\' where you need '\' for LaTeX.
-#         # We replace the JSON-escaped double backslash ('\\') with a single literal backslash ('\').
-#         final_output_content = json_str.replace('\\\\', '\\')
-
-#         # 3. Write the corrected string content to the file.
-#         with open(final_output_path, 'w', encoding='utf-8') as f:
-#             f.write(final_output_content)
-            
-#         print(f"\n✅ Final Data Saved: {final_output_path}")
-#     else:
-#         print("\n❌ Pipeline Failed.")
-#         sys.exit(1)
-
-
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Complete Pipeline")
     parser.add_argument("--input_pdf", type=str, required=True, help="Input PDF")