Update app.py

Add 'x' to the hierarchy nodes

app.py

@@ -98,7 +98,7 @@ def filter_headers_outside_toc(headers, toc_pages):
 def headers_with_location(doc, llm_headers):
     """
     Converts LLM headers into:
-    [text, font_size, page, y, suggested_level, confidence]
+    [text, font_size, page, y, suggested_level, x]
     Always include all headers, even if location not found.
     """
     headersJson = []
@@ -131,7 +131,7 @@ def headers_with_location(doc, llm_headers):
                     loc["page"],
                     loc["y"],
                     h["suggested_level"],
-                    
+                    loc.get("x", 0),  # Add x coordinate
                 ]
                 if entry not in headersJson:
                     headersJson.append(entry)
@@ -148,10 +148,10 @@ def build_hierarchy_from_llm(headers):
     for h in headers:
         # print("headerrrrrrrrrrrrrrr", h)
 
-        if len(h) < 5:
+        if len(h) < 6:  # Changed from 5 to 6 since we added 'x'
             continue
 
-        text, size, page, y, level = h
+        text, size, page, y, level, x = h  # Unpack 'x' here
 
         if level is None:
             continue
@@ -164,6 +164,7 @@ def build_hierarchy_from_llm(headers):
         node = {
             "text": text,
             "page": page if page is not None else -1,
+            "x": x if x is not None else -1,  # Add this
             "y": y if y is not None else -1,
             "size": size,
             "bold": False,
@@ -1336,6 +1337,8 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
             heading_to_search = heading_to_searchDict['text']
             heading_to_searchPageNum = heading_to_searchDict['page']
             paths=heading_to_searchDict['path']        
+            xloc=heading_to_searchDict.get('x', 0)  # Use get() with default
+            yloc=heading_to_searchDict.get('y', 0)  # Use get() with default
         
             # Initialize variables
             headertoContinue1 = False
@@ -1368,31 +1371,31 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                 page=doc[page_num]
                 page_height = page.rect.height
                 blocks = page.get_text("dict")["blocks"]
-    
+
                 for block in blocks:
                     if break_collecting:
                         break
-    
+
                     lines = block.get("lines", [])
                     i = 0
                     while i < len(lines):
                         if break_collecting:
                             break
-    
+
                         spans = lines[i].get("spans", [])
                         if not spans:
                             i += 1
                             continue
-    
+
                         y0 = spans[0]["bbox"][1]
                         y1 = spans[0]["bbox"][3]
                         if y0 < top_margin or y1 > (page_height - bottom_margin):
                             i += 1
                             continue
-    
+
                         line_text = get_spaced_text_from_spans(spans).lower()
                         line_text_norm = normalize_text(line_text)
-    
+
                         # Combine with next line if available
                         if i + 1 < len(lines):
                             next_spans = lines[i + 1].get("spans", [])
@@ -1427,12 +1430,12 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                             ( combined_line_norm in allchildrenheaders_set or
                             combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
                         )
-    
+
                         # New word-based matching
                         current_line_words = set(combined_line_norm.split())
                         heading_words = set(heading_norm.split())
                         all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
-    
+
                         substring_match = (
                             heading_norm in combined_line_norm or
                             combined_line_norm in heading_norm or
@@ -1442,10 +1445,10 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                         #     heading_norm in combined_line_norm or
                         #     combined_line_norm in heading_norm
                         # )
-    
+
                         if (substring_match and existsfull and not collecting and
                             len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
-    
+
                             # Check header conditions more efficiently
                             # header_spans = [
                             #     span for span in spans
@@ -1458,18 +1461,18 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                 # if stringtowrite=='To be billed':
                                 #     Alltexttobebilled+='\n'
                                 # matched_header_font_size = max(span["size"] for span in header_spans)
-    
+
                                 # collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
-    
+
                                 if valid_spans:
                                     x0s = [span["bbox"][0] for span in valid_spans]
                                     x1s = [span["bbox"][2] for span in valid_spans]
                                     y0s = [span["bbox"][1] for span in valid_spans]
                                     y1s = [span["bbox"][3] for span in valid_spans]
-    
+
                                     header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
+
                                     if page_num in current_bbox:
                                         cb = current_bbox[page_num]
                                         current_bbox[page_num] = [
@@ -1482,36 +1485,36 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                         current_bbox[page_num] = header_bbox
                                     last_y1s[page_num] = header_bbox[3]
                                     x0, y0, x1, y1 = header_bbox
-    
+
                                     zoom = 200
                                     left = int(x0)
                                     top = int(y0)
                                     zoom_str = f"{zoom},{left},{top}"
                                     pageNumberFound = page_num + 1
-    
+
                                   # Build the query parameters
                                     params = {
                                         'pdfLink': pdf_path,  # Your PDF link
                                         'keyword': heading_to_search,  # Your keyword (could be a string or list)
                                     }
-    
+
                                     # URL encode each parameter
                                     encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-    
+
                                     # Construct the final encoded link
                                     encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-    
+
                                     # Correctly construct the final URL with page and zoom
                                     # final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-    
+
                                     # Get current date and time
                                     now = datetime.now()
-    
+
                                     # Format the output
                                     formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
                                     # Optionally, add the URL to a DataFrame
-    
-    
+
+
                                     data_entry = {
                                             "PDF Name":filename,
                                             "NBSLink": zoom_str,
@@ -1540,13 +1543,13 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                         else:
                             if (substring_match and not collecting and
                                 len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
-    
+
                                 # Calculate word match percentage
                                 word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
-    
+
                                 # Check if at least 70% of header words exist in this line
                                 meets_word_threshold = word_match_percent >= 100
-    
+
                                 # Check header conditions (including word threshold)
                                 # header_spans = [
                                 #     span for span in spans
@@ -1554,7 +1557,7 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                 #         # and span['size'] >= subsubheaderFontSize
                                 #         and span['size'] < mainHeaderFontSize)
                                 # ]
-    
+
                                 if  (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ) and stringtowrite.startswith('To'):
                                     collecting = True
                                     if stringtowrite=='To be billed':
@@ -1565,15 +1568,15 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                  
                                     collected_lines.append(line_text)
                                     valid_spans = [span for span in spans if span.get("bbox")]
-    
+
                                     if valid_spans:
                                         x0s = [span["bbox"][0] for span in valid_spans]
                                         x1s = [span["bbox"][2] for span in valid_spans]
                                         y0s = [span["bbox"][1] for span in valid_spans]
                                         y1s = [span["bbox"][3] for span in valid_spans]
-    
+
                                         header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
+
                                         if page_num in current_bbox:
                                             cb = current_bbox[page_num]
                                             current_bbox[page_num] = [
@@ -1584,7 +1587,7 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                             ]
                                         else:
                                             current_bbox[page_num] = header_bbox
-    
+
                                         last_y1s[page_num] = header_bbox[3]
                                         x0, y0, x1, y1 = header_bbox
                                         zoom = 200
@@ -1592,29 +1595,29 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                         top = int(y0)
                                         zoom_str = f"{zoom},{left},{top}"
                                         pageNumberFound = page_num + 1
-    
+
                                       # Build the query parameters
                                         params = {
                                             'pdfLink': pdf_path,  # Your PDF link
                                             'keyword': heading_to_search,  # Your keyword (could be a string or list)
                                         }
-    
+
                                         # URL encode each parameter
                                         encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-    
+
                                         # Construct the final encoded link
                                         encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-    
+
                                         # Correctly construct the final URL with page and zoom
                                         # final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-    
+
                                         # Get current date and time
                                         now = datetime.now()
-    
+
                                         # Format the output
                                         formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
                                         # Optionally, add the URL to a DataFrame
-    
+
                                         logger.info(f"Logging into table")
                                         data_entry = {
                                                 "PDF Name":filename,
@@ -1644,7 +1647,7 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                         continue
                         if collecting:
                             norm_line = normalize_text(line_text)
-    
+
                             # Optimized URL check
                             if url_pattern.match(norm_line):
                                 line_is_header = False
@@ -1666,7 +1669,7 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                     # is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
                                     len(line_text.strip()) > 2
                                 )
-    
+
                                 if (norm_line != matched_header_line_norm and
                                     norm_line != heading_norm and
                                     is_probably_real_header):
@@ -1679,37 +1682,37 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
                                           bbox[3] = last_y1s.get(page_num, bbox[3])
                                           page_highlights[page_num] = bbox
                                       highlight_boxes(docHighlights, page_highlights,stringtowrite)
-    
+
                                       break_collecting = True
                                       break
-    
-                            if break_collecting:
-                                break
-    
-                            collected_lines.append(line_text)
-                            valid_spans = [span for span in spans if span.get("bbox")]
-                            if valid_spans:
-                                x0s = [span["bbox"][0] for span in valid_spans]
-                                x1s = [span["bbox"][2] for span in valid_spans]
-                                y0s = [span["bbox"][1] for span in valid_spans]
-                                y1s = [span["bbox"][3] for span in valid_spans]
-    
-                                line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
-                                if page_num in current_bbox:
-                                    cb = current_bbox[page_num]
-                                    current_bbox[page_num] = [
-                                        min(cb[0], line_bbox[0]),
-                                        min(cb[1], line_bbox[1]),
-                                        max(cb[2], line_bbox[2]),
-                                        max(cb[3], line_bbox[3])
-                                    ]
-                                else:
-                                    current_bbox[page_num] = line_bbox
-    
-                                last_y1s[page_num] = line_bbox[3]
+
+                        if break_collecting:
+                            break
+
+                        collected_lines.append(line_text)
+                        valid_spans = [span for span in spans if span.get("bbox")]
+                        if valid_spans:
+                            x0s = [span["bbox"][0] for span in valid_spans]
+                            x1s = [span["bbox"][2] for span in valid_spans]
+                            y0s = [span["bbox"][1] for span in valid_spans]
+                            y1s = [span["bbox"][3] for span in valid_spans]
+
+                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                            if page_num in current_bbox:
+                                cb = current_bbox[page_num]
+                                current_bbox[page_num] = [
+                                    min(cb[0], line_bbox[0]),
+                                    min(cb[1], line_bbox[1]),
+                                    max(cb[2], line_bbox[2]),
+                                    max(cb[3], line_bbox[3])
+                                ]
+                            else:
+                                current_bbox[page_num] = line_bbox
+
+                            last_y1s[page_num] = line_bbox[3]
                         i += 1
-    
+
             if not done:
                 for page_num, bbox in current_bbox.items():
                     bbox[3] = last_y1s.get(page_num, bbox[3])
@@ -1810,8 +1813,8 @@ def testFunction(pdf_path, model,LLM_prompt):
         heading_to_search = heading_to_searchDict['text']
         heading_to_searchPageNum = heading_to_searchDict['page']
         paths=heading_to_searchDict['path']        
-        xloc=heading_to_searchDict['x']
-        yloc=heading_to_searchDict['y']
+        xloc=heading_to_searchDict.get('x', 0)  # Use get() with default
+        yloc=heading_to_searchDict.get('y', 0)  # Use get() with default
         
         # Initialize variables
         headertoContinue1 = False
@@ -2359,4 +2362,4 @@ iface = gr.Interface(
 )
 
 # Launch with debug=True to see errors in the console
-iface.launch(debug=True) 
+iface.launch(debug=True)