Update working_yolo_pipeline.py

working_yolo_pipeline.py

@@ -256,8 +256,6 @@ def merge_yolo_into_word_data(raw_word_data: list, yolo_detections: list, scale_
 
 
 
-
-
 # ============================================================================
 # --- MISSING HELPER FUNCTION ---
 # ============================================================================
@@ -317,65 +315,6 @@ def calculate_vertical_gap_coverage(word_data: list, sep_x: int, page_height: fl
     return coverage_ratio
 
 
-# def calculate_x_gutters(word_data: list, params: Dict, page_height: float) -> List[int]:
-#     """Calculates X-axis histogram and validates using BRIDGING CHECK and Vertical Coverage."""
-#     if not word_data: return []
-
-#     x_points = []
-#     for _, x1, _, x2, *rest in word_data:
-#         x_points.extend([x1, x2])
-
-#     if not x_points: return []
-#     max_x = max(x_points)
-
-#     bin_size = params.get('cluster_bin_size', 5)
-#     smoothing = params.get('cluster_smoothing', 5)
-#     min_width = params.get('cluster_min_width', 20)
-#     threshold_percentile = params.get('cluster_threshold_percentile', 85)
-
-#     num_bins = int(np.ceil(max_x / bin_size))
-#     hist, bin_edges = np.histogram(x_points, bins=num_bins, range=(0, max_x))
-
-#     smoothed_hist = gaussian_filter1d(hist.astype(float), sigma=smoothing)
-#     inverted_signal = np.max(smoothed_hist) - smoothed_hist
-
-#     peaks, properties = find_peaks(
-#         inverted_signal,
-#         height=np.max(inverted_signal) - np.percentile(smoothed_hist, threshold_percentile),
-#         distance=min_width / bin_size
-#     )
-
-#     if not peaks.size: return []
-
-#     separator_x_coords = [int(bin_edges[p]) for p in peaks]
-#     final_separators = []
-    
-#     for x_coord in separator_x_coords:
-#         # 1. BRIDGING CHECK: The "Do Not Cut Words" Constraint
-#         # Count how many words/blocks physically cross this specific X coordinate.
-#         bridging_count = 0
-#         for _, wx1, _, wx2, _ in word_data:
-#             # Strictly check if a word physically sits on this line
-#             if wx1 < x_coord and wx2 > x_coord:
-#                 bridging_count += 1
-        
-#         # Strict Threshold: If more than 2 items (allow for noise) cross, REJECT.
-#         if bridging_count > 2:
-#             print(f"      ❌ Separator X={x_coord} REJECTED: Cuts through {bridging_count} words/blocks.")
-#             continue
-
-#         # 2. VERTICAL COVERAGE CHECK
-#         # The gap must exist for > 65% of the text height of the page.
-#         coverage = calculate_vertical_gap_coverage(word_data, x_coord, page_height, gutter_width=min_width)
-        
-#         if coverage >= 0.65:
-#             final_separators.append(x_coord)
-#             print(f"      -> Separator X={x_coord} ACCEPTED (Coverage: {coverage:.1%}, Bridging: {bridging_count})")
-#         else:
-#             print(f"      ❌ Separator X={x_coord} REJECTED (Coverage: {coverage:.1%}, Bridging: {bridging_count})")
-
-#     return sorted(final_separators)
-
 
 
 def calculate_x_gutters(word_data: list, params: Dict, page_height: float) -> List[int]:
@@ -456,39 +395,6 @@ def calculate_x_gutters(word_data: list, params: Dict, page_height: float) -> Li
     return sorted(final_separators)
 
 
-# def get_word_data_for_detection(page: fitz.Page, pdf_path: str, page_num: int,
-#                                 top_margin_percent=0.10, bottom_margin_percent=0.10) -> list:
-#     """Extract word data with OCR caching to avoid redundant Tesseract runs."""
-#     word_data = page.get_text("words")
-
-#     if len(word_data) > 0:
-#         word_data = [(w[4], w[0], w[1], w[2], w[3]) for w in word_data]
-#     else:
-#         if _ocr_cache.has_ocr(pdf_path, page_num):
-#             word_data = _ocr_cache.get_ocr(pdf_path, page_num)
-#         else:
-#             try:
-#                 pix = page.get_pixmap(matrix=fitz.Matrix(3, 3))
-#                 img_bytes = pix.tobytes("png")
-#                 img = Image.open(io.BytesIO(img_bytes))
-#                 data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT)
-#                 full_word_data = []
-#                 for i in range(len(data['level'])):
-#                     if data['text'][i].strip():
-#                         x1, y1 = data['left'][i] / 3, data['top'][i] / 3
-#                         x2, y2 = x1 + data['width'][i] / 3, y1 + data['height'][i] / 3
-#                         full_word_data.append((data['text'][i], x1, y1, x2, y2))
-#                 word_data = full_word_data
-#                 _ocr_cache.set_ocr(pdf_path, page_num, word_data)
-#             except Exception:
-#                 return []
-
-#     # Apply margin filtering
-#     page_height = page.rect.height
-#     y_min = page_height * top_margin_percent
-#     y_max = page_height * (1 - bottom_margin_percent)
-#     return [d for d in word_data if d[2] >= y_min and d[4] <= y_max]
-
 
 
 
@@ -839,221 +745,6 @@ def preprocess_and_ocr_page(original_img: np.ndarray, model, pdf_path: str,
 
 
 
-
-
-# def preprocess_and_ocr_page(original_img: np.ndarray, model, pdf_path: str,
-#                             page_num: int, fitz_page: fitz.Page,
-#                             pdf_name: str) -> Tuple[List[Dict[str, Any]], Optional[int]]:
-#     """
-#     OPTIMIZED FLOW:
-#     1. Run YOLO to find Equations/Tables.
-#     2. Mask raw text with YOLO boxes.
-#     3. Run Column Detection on the MASKED data (Populates OCR cache).
-#     4. Proceed with Final OCR Output (Strictly using the cache).
-#     """
-#     global GLOBAL_FIGURE_COUNT, GLOBAL_EQUATION_COUNT
-
-#     start_time_total = time.time()
-
-#     if original_img is None:
-#         print(f"  ❌ Invalid image for page {page_num}.")
-#         return None, None
-
-#     # ====================================================================
-#     # --- STEP 1: YOLO DETECTION ---
-#     # ====================================================================
-#     start_time_yolo = time.time()
-#     results = model.predict(source=original_img, conf=CONF_THRESHOLD, imgsz=640, verbose=False)
-
-#     relevant_detections = []
-#     if results and results[0].boxes:
-#         for box in results[0].boxes:
-#             class_id = int(box.cls[0])
-#             class_name = model.names[class_id]
-#             if class_name in TARGET_CLASSES:
-#                 x1, y1, x2, y2 = box.xyxy[0].cpu().numpy().astype(int)
-#                 relevant_detections.append(
-#                     {'coords': (x1, y1, x2, y2), 'y1': y1, 'class': class_name, 'conf': float(box.conf[0])}
-#                 )
-
-#     merged_detections = merge_overlapping_boxes(relevant_detections, IOU_MERGE_THRESHOLD)
-#     print(f"    [LOG] YOLO found {len(merged_detections)} objects in {time.time() - start_time_yolo:.3f}s.")
-
-#     # ====================================================================
-#     # --- STEP 2: PREPARE DATA FOR COLUMN DETECTION (MASKING & CACHING) ---
-#     # This call to get_word_data_for_detection will execute Tesseract if
-#     # native words are missing, and save the result to the cache.
-#     # ====================================================================
-#     raw_words_for_layout = get_word_data_for_detection(
-#         fitz_page, pdf_path, page_num,
-#         top_margin_percent=0.10, bottom_margin_percent=0.10
-#     )
-
-#     masked_word_data = merge_yolo_into_word_data(raw_words_for_layout, merged_detections, scale_factor=2.0)
-
-#     # ====================================================================
-#     # --- STEP 3: COLUMN DETECTION ---
-#     # ====================================================================
-#     page_width_pdf = fitz_page.rect.width
-#     page_height_pdf = fitz_page.rect.height
-
-#     column_detection_params = {
-#         'cluster_bin_size': 2, 'cluster_smoothing': 2,
-#         'cluster_min_width': 10, 'cluster_threshold_percentile': 85,
-#     }
-
-#     separators = calculate_x_gutters(masked_word_data, column_detection_params, page_height_pdf)
-
-#     page_separator_x = None
-#     if separators:
-#         central_min = page_width_pdf * 0.35
-#         central_max = page_width_pdf * 0.65
-#         central_separators = [s for s in separators if central_min <= s <= central_max]
-
-#         if central_separators:
-#             center_x = page_width_pdf / 2
-#             page_separator_x = min(central_separators, key=lambda x: abs(x - center_x))
-#             print(f"      ✅ Column Split Confirmed at X={page_separator_x:.1f}")
-#         else:
-#             print("      ⚠️ Gutter found off-center. Ignoring.")
-#     else:
-#         print("      -> Single Column Layout Confirmed.")
-
-#     # ====================================================================
-#     # --- STEP 4: COMPONENT EXTRACTION (Save Images) ---
-#     # ====================================================================
-#     start_time_components = time.time()
-#     component_metadata = []
-#     fig_count_page = 0
-#     eq_count_page = 0
-
-#     for detection in merged_detections:
-#         x1, y1, x2, y2 = detection['coords']
-#         class_name = detection['class']
-
-#         if class_name == 'figure':
-#             GLOBAL_FIGURE_COUNT += 1
-#             counter = GLOBAL_FIGURE_COUNT
-#             component_word = f"FIGURE{counter}"
-#             fig_count_page += 1
-#         elif class_name == 'equation':
-#             GLOBAL_EQUATION_COUNT += 1
-#             counter = GLOBAL_EQUATION_COUNT
-#             component_word = f"EQUATION{counter}"
-#             eq_count_page += 1
-#         else:
-#             continue
-
-#         component_crop = original_img[y1:y2, x1:x2]
-#         component_filename = f"{pdf_name}_page{page_num}_{class_name}{counter}.png"
-#         cv2.imwrite(os.path.join(FIGURE_EXTRACTION_DIR, component_filename), component_crop)
-
-#         y_midpoint = (y1 + y2) // 2
-#         component_metadata.append({
-#             'type': class_name, 'word': component_word,
-#             'bbox': [int(x1), int(y1), int(x2), int(y2)],
-#             'y0': int(y_midpoint), 'x0': int(x1)
-#         })
-
-#     # ====================================================================
-#     # --- STEP 5: CACHED OCR RETRIEVAL (No Redundant Tesseract) ---
-#     # ====================================================================
-#     raw_ocr_output = []
-#     scale_factor = 2.0  # Pipeline standard scale
-
-#     if _ocr_cache.has_ocr(pdf_path, page_num):
-#         print(f"  ⚡ Using cached OCR (Native or Tesseract) for page {page_num}")
-#         cached_word_data = _ocr_cache.get_ocr(pdf_path, page_num)
-        
-#         for word_tuple in cached_word_data:
-#             # Cache stores: (text, x1, y1, x2, y2) in PDF points (see get_word_data_for_detection)
-#             word_text, x1, y1, x2, y2 = word_tuple 
-            
-#             # Scale from PDF points back to Pipeline Pixels (2.0)
-#             x1_pix = int(x1 * scale_factor)
-#             y1_pix = int(y1 * scale_factor)
-#             x2_pix = int(x2 * scale_factor)
-#             y2_pix = int(y2 * scale_factor)
-
-#             raw_ocr_output.append({
-#                 'type': 'text', 'word': word_text, 'confidence': 95.0, # 95.0 is a default/placeholder confidence
-#                 'bbox': [x1_pix, y1_pix, x2_pix, y2_pix],
-#                 'y0': y1_pix, 'x0': x1_pix
-#             })
-#     else:
-#         # This branch is hit only if the cache check in Step 2 failed to produce text,
-#         # meaning the page is genuinely textless or entirely composed of images/figures.
-#         print(f"  ⚠️ No text found in cache for page {page_num}. Proceeding without words.")
-
-
-#     # ====================================================================
-#     # --- STEP 6: OCR CLEANING AND MERGING (Original Logic Unchanged) ---
-#     # ====================================================================
-#     items_to_sort = []
-
-#     for ocr_word in raw_ocr_output:
-#         is_suppressed = False
-#         for component in component_metadata:
-#             # Do not include words that are inside figure/equation boxes
-#             ioa = calculate_ioa(ocr_word['bbox'], component['bbox'])
-#             if ioa > IOA_SUPPRESSION_THRESHOLD:
-#                 is_suppressed = True
-#                 break
-#         if not is_suppressed:
-#             items_to_sort.append(ocr_word)
-
-#     # Add figures/equations back into the flow as "words"
-#     items_to_sort.extend(component_metadata)
-
-#     # ====================================================================
-#     # --- STEP 7: LINE-BASED SORTING (Original Logic Unchanged) ---
-#     # ====================================================================
-#     items_to_sort.sort(key=lambda x: (x['y0'], x['x0']))
-#     lines = []
-
-#     for item in items_to_sort:
-#         placed = False
-#         for line in lines:
-#             y_ref = min(it['y0'] for it in line)
-#             if abs(y_ref - item['y0']) < LINE_TOLERANCE:
-#                 line.append(item)
-#                 placed = True
-#                 break
-#         if not placed and item['type'] in ['equation', 'figure']:
-#             for line in lines:
-#                 y_ref = min(it['y0'] for it in line)
-#                 if abs(y_ref - item['y0']) < 20:
-#                     line.append(item)
-#                     placed = True
-#                     break
-#         if not placed:
-#             lines.append([item])
-
-#     for line in lines:
-#         line.sort(key=lambda x: x['x0'])
-
-#     final_output = []
-#     for line in lines:
-#         for item in line:
-#             data_item = {"word": item["word"], "bbox": item["bbox"], "type": item["type"]}
-#             if 'tag' in item: data_item['tag'] = item['tag']
-#             final_output.append(data_item)
-
-#     return final_output, page_separator_x
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 def run_single_pdf_preprocessing(pdf_path: str, preprocessed_json_path: str) -> Optional[str]:
     global GLOBAL_FIGURE_COUNT, GLOBAL_EQUATION_COUNT
 
@@ -1406,158 +1097,6 @@ def convert_raw_predictions_to_label_studio(page_data_list, output_path: str):
 # --- PHASE 3: BIO TO STRUCTURED JSON DECODER ---
 # ============================================================================
 
-# def convert_bio_to_structured_json_relaxed(input_path: str, output_path: str) -> Optional[List[Dict[str, Any]]]:
-#     print("\n" + "=" * 80)
-#     print("--- 3. STARTING BIO TO STRUCTURED JSON DECODING ---")
-#     print("=" * 80)
-#     try:
-#         with open(input_path, 'r', encoding='utf-8') as f:
-#             predictions_by_page = json.load(f)
-#     except Exception as e:
-#         print(f"❌ Error loading raw prediction file: {e}")
-#         return None
-
-#     predictions = []
-#     for page_item in predictions_by_page:
-#         if isinstance(page_item, dict) and 'data' in page_item:
-#             predictions.extend(page_item['data'])
-
-#     structured_data = []
-#     current_item = None
-#     current_option_key = None
-#     current_passage_buffer = []
-#     current_text_buffer = []
-#     first_question_started = False
-#     last_entity_type = None
-#     just_finished_i_option = False
-#     is_in_new_passage = False
-
-#     def finalize_passage_to_item(item, passage_buffer):
-#         if passage_buffer:
-#             passage_text = re.sub(r'\s{2,}', ' ', ' '.join(passage_buffer)).strip()
-#             if item.get('passage'): item['passage'] += ' ' + passage_text
-#             else: item['passage'] = passage_text
-#         passage_buffer.clear()
-
-#     for item in predictions:
-#         word = item['word']
-#         label = item['predicted_label']
-#         entity_type = label[2:].strip() if label.startswith(('B-', 'I-')) else None
-#         current_text_buffer.append(word)
-#         previous_entity_type = last_entity_type
-#         is_passage_label = (entity_type == 'PASSAGE')
-
-#         if not first_question_started:
-#             if label != 'B-QUESTION' and not is_passage_label:
-#                 just_finished_i_option = False
-#                 is_in_new_passage = False
-#                 continue
-#             if is_passage_label:
-#                 current_passage_buffer.append(word)
-#                 last_entity_type = 'PASSAGE'
-#                 just_finished_i_option = False
-#                 is_in_new_passage = False
-#                 continue
-
-#         if label == 'B-QUESTION':
-#             if not first_question_started:
-#                 header_text = ' '.join(current_text_buffer[:-1]).strip()
-#                 if header_text or current_passage_buffer:
-#                     metadata_item = {'type': 'METADATA', 'passage': ''}
-#                     finalize_passage_to_item(metadata_item, current_passage_buffer)
-#                     if header_text: metadata_item['text'] = header_text
-#                     structured_data.append(metadata_item)
-#                 first_question_started = True
-#                 current_text_buffer = [word]
-
-#             if current_item is not None:
-#                 finalize_passage_to_item(current_item, current_passage_buffer)
-#                 current_item['text'] = ' '.join(current_text_buffer[:-1]).strip()
-#                 structured_data.append(current_item)
-#                 current_text_buffer = [word]
-
-#             current_item = {
-#                 'question': word, 'options': {}, 'answer': '', 'passage': '', 'text': ''
-#             }
-#             current_option_key = None
-#             last_entity_type = 'QUESTION'
-#             just_finished_i_option = False
-#             is_in_new_passage = False
-#             continue
-
-#         if current_item is not None:
-#             if is_in_new_passage:
-#                 current_item['new_passage'] += f' {word}'
-#                 if label.startswith('B-') or (label.startswith('I-') and entity_type != 'PASSAGE'):
-#                     is_in_new_passage = False
-#                 if label.startswith(('B-', 'I-')): last_entity_type = entity_type
-#                 continue
-#             is_in_new_passage = False
-
-#             if label.startswith('B-'):
-#                 if entity_type in ['QUESTION', 'OPTION', 'ANSWER', 'SECTION_HEADING']:
-#                     finalize_passage_to_item(current_item, current_passage_buffer)
-#                     current_passage_buffer = []
-#                 last_entity_type = entity_type
-#                 if entity_type == 'PASSAGE':
-#                     if previous_entity_type == 'OPTION' and just_finished_i_option:
-#                         current_item['new_passage'] = word
-#                         is_in_new_passage = True
-#                     else:
-#                         current_passage_buffer.append(word)
-#                 elif entity_type == 'OPTION':
-#                     current_option_key = word
-#                     current_item['options'][current_option_key] = word
-#                     just_finished_i_option = False
-#                 elif entity_type == 'ANSWER':
-#                     current_item['answer'] = word
-#                     current_option_key = None
-#                     just_finished_i_option = False
-#                 elif entity_type == 'QUESTION':
-#                     current_item['question'] += f' {word}'
-#                     just_finished_i_option = False
-
-#             elif label.startswith('I-'):
-#                 if entity_type == 'QUESTION':
-#                     current_item['question'] += f' {word}'
-#                 elif entity_type == 'PASSAGE':
-#                     if previous_entity_type == 'OPTION' and just_finished_i_option:
-#                         current_item['new_passage'] = word
-#                         is_in_new_passage = True
-#                     else:
-#                         if not current_passage_buffer: last_entity_type = 'PASSAGE'
-#                         current_passage_buffer.append(word)
-#                 elif entity_type == 'OPTION' and current_option_key is not None:
-#                     current_item['options'][current_option_key] += f' {word}'
-#                     just_finished_i_option = True
-#                 elif entity_type == 'ANSWER':
-#                     current_item['answer'] += f' {word}'
-#                 just_finished_i_option = (entity_type == 'OPTION')
-
-#             elif label == 'O':
-#                 if last_entity_type == 'QUESTION':
-#                     current_item['question'] += f' {word}'
-#                 just_finished_i_option = False
-
-#     if current_item is not None:
-#         finalize_passage_to_item(current_item, current_passage_buffer)
-#         current_item['text'] = ' '.join(current_text_buffer).strip()
-#         structured_data.append(current_item)
-    
-#     for item in structured_data:
-#         item['text'] = re.sub(r'\s{2,}', ' ', item['text']).strip()
-#         if 'new_passage' in item:
-#             item['new_passage'] = re.sub(r'\s{2,}', ' ', item['new_passage']).strip()
-
-#     try:
-#         with open(output_path, 'w', encoding='utf-8') as f:
-#             json.dump(structured_data, f, indent=2, ensure_ascii=False)
-#     except Exception: pass
-
-#     return structured_data
-
-
-
 
 
 def convert_bio_to_structured_json_relaxed(input_path: str, output_path: str) -> Optional[List[Dict[str, Any]]]:
@@ -1729,6 +1268,8 @@ def create_query_text(entry: Dict[str, Any]) -> str:
                     query_parts.append(value)
     return " ".join(query_parts)
 
+    
+
 def calculate_similarity(doc1: str, doc2: str) -> float:
     """Calculates Cosine Similarity between two text strings."""
     if not doc1 or not doc2:
@@ -1759,123 +1300,6 @@ def calculate_similarity(doc1: str, doc2: str) -> float:
 
 
 
-
-
-
-
-
-# def process_context_linking(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-#     """
-#     Links questions to passages based on 'passage' flow vs 'new_passage' priority.
-#     """
-#     print("\n" + "=" * 80)
-#     print("--- STARTING CONTEXT LINKING AND SELF-CORRECTION (DEBUG MODE) ---")
-#     print("=" * 80)
-    
-#     if not data: return []
-
-#     # --- PHASE 1: IDENTIFY PASSAGE DEFINERS ---
-#     passage_definer_indices = []
-#     for i, entry in enumerate(data):
-#         # We track metadata indices too now, as they are valid sources
-#         if entry.get("passage") and entry["passage"].strip():
-#             passage_definer_indices.append(i)
-#         if entry.get("new_passage") and entry["new_passage"].strip():
-#             if i not in passage_definer_indices:
-#                 passage_definer_indices.append(i)
-
-#     # --- PHASE 2: CONTEXT TRANSFER & LINKING ---
-#     current_passage_text = None
-#     current_new_passage_text = None
-
-#     # DEBUG: Check what the first item is offering
-#     if data and data[0].get('type') == 'METADATA':
-#         print(f"  [Debug] Found METADATA at start. Length of passage: {len(data[0].get('passage', ''))}")
-
-#     for i, entry in enumerate(data):
-#         item_type = entry.get("type", "Question")
-
-#         # A. UNCONDITIONALLY UPDATE CONTEXTS
-#         # FIX: Removed 'and entry.get("type") != "METADATA"'
-#         # We WANT Metadata to update the current_passage_text
-#         if entry.get("passage") and entry["passage"].strip():
-#             current_passage_text = entry["passage"]
-#             # print(f"  [Flow] Updated Standard Context from Item {i} ({item_type})")
-        
-#         if entry.get("new_passage") and entry["new_passage"].strip():
-#             current_new_passage_text = entry["new_passage"]
-#             # print(f"  [Flow] Updated New/Local Context from Item {i} ({item_type})")
-
-#         # B. QUESTION LINKING
-#         if entry.get("question") and item_type != "METADATA":
-#             combined_query = create_query_text(entry)
-            
-#             # Skip if query is too short (noise)
-#             if len(combined_query.strip()) < 5: 
-#                 continue
-
-#             # Calculate scores
-#             score_old = calculate_similarity(current_passage_text, combined_query) if current_passage_text else 0.0
-#             score_new = calculate_similarity(current_new_passage_text, combined_query) if current_new_passage_text else 0.0
-
-#             q_preview = entry['question'][:30] + '...'
-            
-#             # DEBUG PRINT to see why it might be failing
-#             # print(f"  [Check Q{i}] Old_Ctx_Len: {len(str(current_passage_text))} | Score: {score_old:.4f}")
-
-#             # RESOLUTION LOGIC
-#             linked = False
-            
-#             # 1. Prefer New Passage if significantly better
-#             if current_new_passage_text and (score_new > score_old + RESOLUTION_MARGIN) and (score_new >= SIMILARITY_THRESHOLD):
-#                 entry["passage"] = current_new_passage_text
-#                 print(f"  [Linker] 🚀 Q{i} ('{q_preview}') -> NEW PASSAGE (Score: {score_new:.3f})")
-#                 linked = True
-            
-#             # 2. Otherwise use Standard Passage if it meets threshold
-#             elif current_passage_text and (score_old >= SIMILARITY_THRESHOLD):
-#                 entry["passage"] = current_passage_text
-#                 print(f"  [Linker] ✅ Q{i} ('{q_preview}') -> STANDARD PASSAGE (Score: {score_old:.3f})")
-#                 linked = True
-            
-#             if not linked:
-#                 print(f"  [Linker] ⚠️ Q{i} NOT LINKED. Max Score: {max(score_old, score_new):.4f} < Threshold {SIMILARITY_THRESHOLD}")
-
-#     # --- PHASE 3: CLEANUP AND INTERPOLATION ---
-#     print("  [Linker] Running Cleanup & Interpolation...")
-    
-#     # 3A. Self-Correction (Remove weak links)
-#     for i in passage_definer_indices:
-#         entry = data[i]
-#         # Don't wipe out Metadata passages, only questions that got linked
-#         if entry.get("question") and entry.get("type") != "METADATA":
-#             passage_to_check = entry.get("passage") or entry.get("new_passage")
-#             if passage_to_check:
-#                 self_sim = calculate_similarity(passage_to_check, create_query_text(entry))
-#                 if self_sim < SIMILARITY_THRESHOLD:
-#                     entry["passage"] = ""
-#                     if "new_passage" in entry: entry["new_passage"] = ""
-#                     print(f"  [Cleanup] Removed weak link for Q{i}")
-
-#     # 3B. Interpolation (Fill gaps)
-#     for i in range(1, len(data) - 1):
-#         current_entry = data[i]
-#         is_gap = current_entry.get("question") and not current_entry.get("passage")
-#         if is_gap:
-#             prev_p = data[i - 1].get("passage")
-#             next_p = data[i + 1].get("passage")
-#             if prev_p and next_p and (prev_p == next_p):
-#                 current_entry["passage"] = prev_p
-#                 print(f"  [Linker] 🥪 Q{i} Interopolated from neighbors.")
-
-#     return data
-
-
-
-
-
-
-
 def process_context_linking(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     """
     Links questions to passages based on 'passage' flow vs 'new_passage' priority.
@@ -2024,8 +1448,6 @@ def correct_misaligned_options(structured_data: List[Dict[str, Any]]) -> List[Di
 
 
 
-
-
 # ============================================================================
 # --- PHASE 4: IMAGE EMBEDDING (Base64) ---
 # ============================================================================
@@ -2143,82 +1565,6 @@ def run_document_pipeline(input_pdf_path: str, layoutlmv3_model_path: str, label
     return final_result
 
 
-# def run_document_pipeline(input_pdf_path: str, layoutlmv3_model_path: str, label_studio_output_path: str) -> Optional[List[Dict[str, Any]]]:
-#     if not os.path.exists(input_pdf_path): return None
-    
-#     print("\n" + "#" * 80)
-#     print("### STARTING PIPELINE WITH DEBUGGING ENABLED ###")
-#     print("#" * 80)
-    
-#     pdf_name = os.path.splitext(os.path.basename(input_pdf_path))[0]
-#     # Save debug files in the CURRENT directory so you can find them easily
-#     debug_dir = os.path.abspath(os.path.dirname(input_pdf_path)) 
-    
-#     preprocessed_json_path = os.path.join(tempfile.gettempdir(), f"{pdf_name}_preprocessed.json")
-    
-#     # --- DEBUG FILE PATHS ---
-#     debug_step2_path = os.path.join(debug_dir, f"DEBUG_2_raw_model_predictions.json")
-#     debug_step3_path = os.path.join(debug_dir, f"DEBUG_3_bio_parsed_before_linking.json")
-#     final_output_path = os.path.join(debug_dir, f"{pdf_name}_final_output.json")
-
-#     final_result = None
-#     try:
-#         # Phase 1: Preprocessing
-#         preprocessed_json_path_out = run_single_pdf_preprocessing(input_pdf_path, preprocessed_json_path)
-#         if not preprocessed_json_path_out: return None
-        
-#         # Phase 2: Inference (Model Predictions)
-#         page_raw_predictions_list = run_inference_and_get_raw_words(
-#             input_pdf_path, layoutlmv3_model_path, preprocessed_json_path_out
-#         )
-#         if not page_raw_predictions_list: return None
-        
-#         # 🔍 DEBUG SAVE 1: RAW PREDICTIONS
-#         # Open this file to see if the model actually predicted "B-PASSAGE" or "I-PASSAGE"
-#         print(f"  [DEBUG] Saving Raw Model Predictions to: {debug_step2_path}")
-#         with open(debug_step2_path, 'w', encoding='utf-8') as f:
-#             json.dump(page_raw_predictions_list, f, indent=4)
-            
-#         # Phase 3: BIO Decoding
-#         structured_data_list = convert_bio_to_structured_json_relaxed(
-#             debug_step2_path, debug_step3_path 
-#         )
-#         if not structured_data_list: return None
-        
-#         # 🔍 DEBUG SAVE 2: STRUCTURED DATA (Before Context Linking)
-#         # Open this file to see if 'new_passage' exists BEFORE we run the linker
-#         print(f"  [DEBUG] Saving Parsed Data (Pre-Link) to: {debug_step3_path}")
-#         # (The function convert_bio... already saved to debug_step3_path, so we just use it)
-
-#         structured_data_list = correct_misaligned_options(structured_data_list)
-
-#         # Phase 3.5: Context Linking
-#         # We run this on the memory object 'structured_data_list'
-#         structured_data_list = process_context_linking(structured_data_list)
-        
-#         try:
-#             convert_raw_predictions_to_label_studio(page_raw_predictions_list, label_studio_output_path)
-#         except Exception as e:
-#             print(f"❌ Error during Label Studio conversion: {e}")
-            
-#         # Phase 4: Embedding
-#         final_result = embed_images_as_base64_in_memory(structured_data_list, FIGURE_EXTRACTION_DIR)
-        
-#     except Exception as e:
-#         print(f"❌ FATAL ERROR: {e}")
-#         import traceback
-#         traceback.print_exc()
-#         return None
-        
-#     print("\n" + "#" * 80)
-#     print("### PIPELINE COMPLETE ###")
-#     print("#" * 80)
-#     return final_result
-
-
-
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Complete Pipeline")
     parser.add_argument("--input_pdf", type=str, required=True, help="Input PDF")