Update InitialMarkups.py

InitialMarkups.py

@@ -89,7 +89,6 @@ def add_span_to_nearest_group(span_y, grouped_dict, pageNum=None, threshold=0.5)
     return (pageNum, span_y)
 
 def extract_headers(doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin):
-    print("Font baseline:", most_common_font_size, most_common_color, most_common_font)
 
     grouped_headers = defaultdict(list)
     spans = []
@@ -318,7 +317,7 @@ def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_co
     # Step 2: Identify level 0 headers (largest and in TOC)
     # max_size = max(h['size'] for h in headers) if headers else 0
     max_size,subheaderSize,nbsheadersize=top_3_font_sizes
-    print(max_size)
+    
     toc_text_match=[]
     # Improved TOC matching with exact and substring matching
     toc_matches = []
@@ -347,7 +346,6 @@ def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_co
               toc_matches.append(h)
               toc_text_match.append(h['text'])
         elif matching_toc_texts and h['size'] < max_size * 0.9 and h['size'] > nbsheadersize : # h['size'] < max_size * 0.9 and h['size'] > max_size*0.75:
-             print(h['text'],matching_toc_texts)
              headers.remove(h)
              continue
 
@@ -365,8 +363,7 @@ def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_co
             # Update the header text with cleaned version
             h['text'] = cleaned_text
             unique_level0.append(h)
-            print(f"Added unique header: {cleaned_text} (normalized: {norm_cleaned_text})")
-
+ 
     # Step 3: Process headers under each level 0 to identify level 1 format
 
     # First, group headers by their level 0 parent
@@ -667,7 +664,7 @@ def extract_section_under_header(pdf_path):
 
     hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
     listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-    print('listofHeaderstoMarkup',listofHeaderstoMarkup)
+    
     # Precompute all children headers once
     allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
@@ -683,7 +680,7 @@ def extract_section_under_header(pdf_path):
         subHeaderFontSize= top_3_font_sizes[1]
         subsubheaderFontSize= top_3_font_sizes[1]
 
-    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
+    
 
     # Preload all pages to avoid repeated loading
     # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
@@ -692,8 +689,6 @@ def extract_section_under_header(pdf_path):
         heading_to_search = heading_to_searchDict['text']
         heading_to_searchPageNum = heading_to_searchDict['page']
 
-        print('headertosearch', heading_to_search)
-
         # Initialize variables
         headertoContinue1 = False
         headertoContinue2 = False
@@ -754,10 +749,10 @@ def extract_section_under_header(pdf_path):
 
                     # Check if we should continue processing
                     if combined_line_norm and combined_line_norm in paths[0]:
-                        print(combined_line_norm)
+                        
                         headertoContinue1 = combined_line_norm
                     if combined_line_norm and combined_line_norm in paths[-2]:
-                        print(combined_line_norm)
+                    
                         headertoContinue2 = combined_line_norm
                     if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
                       stringtowrite='Not to be billed'
@@ -797,7 +792,6 @@ def extract_section_under_header(pdf_path):
                         if header_spans:
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
-                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
 
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
@@ -869,7 +863,6 @@ def extract_section_under_header(pdf_path):
                                 # Convert list to JSON
                                 json_output = json.dumps(data_list_JSON, indent=4)
 
-                                print("Final URL:", final_url)
                                 i += 2
                                 continue
                     else:
@@ -893,9 +886,7 @@ def extract_section_under_header(pdf_path):
                             if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                                print(f"📥 Start collecting after header: {combined_line_norm} "
-                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
+                             
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -966,7 +957,7 @@ def extract_section_under_header(pdf_path):
                                     # Convert list to JSON
                                     json_output = json.dumps(data_list_JSON, indent=4)
 
-                                    print("Final URL:", final_url)
+                
                                     i += 2
                                     continue
                     if collecting:
@@ -990,7 +981,6 @@ def extract_section_under_header(pdf_path):
                                 norm_line != heading_norm and
                                 is_probably_real_header):
                                 if line_text not in heading_norm:
-                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
                                   collecting = False
                                   done = True
                                   headertoContinue1 = False
@@ -1044,7 +1034,6 @@ def extract_section_under_header(pdf_path):
 
     pdf_bytes = BytesIO()
     docHighlights.save(pdf_bytes)
-    print('JSONN',json_output)
     return pdf_bytes.getvalue(), docHighlights , json_output
 
 
@@ -1109,7 +1098,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
     hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
     listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-    # print('listofHeaderstoMarkup',listofHeaderstoMarkup)
     # Precompute all children headers once
     allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
@@ -1125,8 +1113,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         subHeaderFontSize= top_3_font_sizes[1]
         subsubheaderFontSize= top_3_font_sizes[1]
 
-    # print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
-
     # Preload all pages to avoid repeated loading
     # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
 
@@ -1134,8 +1120,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         heading_to_search = heading_to_searchDict['text']
         heading_to_searchPageNum = heading_to_searchDict['page']
 
-        # print('headertosearch', heading_to_search)
-
+  
         # Initialize variables
         headertoContinue1 = False
         headertoContinue2 = False
@@ -1196,10 +1181,8 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
                     # Check if we should continue processing
                     if combined_line_norm and combined_line_norm in paths[0]:
-                        print(combined_line_norm)
                         headertoContinue1 = combined_line_norm
                     if combined_line_norm and combined_line_norm in paths[-2]:
-                        print(combined_line_norm)
                         headertoContinue2 = combined_line_norm
                     if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
                       stringtowrite='Not to be billed'
@@ -1240,8 +1223,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                             Alltext_Tobebilled+=combined_line_norm
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
-                            # print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
-
+                        
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
                             
@@ -1312,7 +1294,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 # Convert list to JSON
                                 json_output = json.dumps(data_list_JSON, indent=4)
 
-                                # print("Final URL:", final_url)
                                 i += 2
                                 continue
                     else:
@@ -1337,9 +1318,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 Alltext_Tobebilled+=combined_line_norm
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                                # print(f"📥 Start collecting after header: {combined_line_norm} "
-                                #       f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
+                              
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -1410,7 +1389,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                     # Convert list to JSON
                                     json_output = json.dumps(data_list_JSON, indent=4)
 
-                                    # print("Final URL:", final_url)
                                     i += 2
                                     continue
                     if collecting:
@@ -1434,7 +1412,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 norm_line != heading_norm and
                                 is_probably_real_header):
                                 if line_text not in heading_norm:
-                                #   print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
                                   collecting = False
                                   done = True
                                   headertoContinue1 = False
@@ -1488,7 +1465,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
     pdf_bytes = BytesIO()
     docHighlights.save(pdf_bytes)
-    # print('JSONN',json_output)
     return pdf_bytes.getvalue(), docHighlights , json_output , Alltext_Tobebilled
 
 
@@ -1548,10 +1524,8 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
     else:
         for item in headingjson:
             listofheadingsfromrawan.append(normalize_text(item['Subject']))
-    print('hereeeeeeeeeeeeeee0',listofheadingsfromrawan)
     # Precompute all children headers once
     allchildrenheaders = listofheadingsfromrawan
-    print('hereeeeeeeeeeeeeee00',allchildrenheaders)
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
     
     df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
@@ -1564,7 +1538,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
         subHeaderFontSize= top_3_font_sizes[1]
         subsubheaderFontSize= top_3_font_sizes[1]
 
-    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
 
     # Preload all pages to avoid repeated loading
     # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
@@ -1578,7 +1551,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
             heading_to_searchPageNum = int(heading_to_searchDict['Page'])-1
             incomingheader = heading_to_searchDict['head above 1']
             
-        print('hereeeeeeeeeeeeeee0',heading_to_searchPageNum)
         done = False
         collecting = False
         collected_lines = []
@@ -1592,7 +1564,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
         heading_norm = normalize_text(heading_to_search)
 
         for page_num in range(heading_to_searchPageNum,len(doc)):
-            print('hereeeeeeeeeeeeeee1')
             if page_num in toc_pages:
               continue
             if break_collecting:
@@ -1662,7 +1633,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                         if header_spans:
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
-                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
 
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
@@ -1719,7 +1689,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                 if type(heading_to_searchDict) != str:
                                     heading_to_searchDict['NBSLink']=new_url
                                 newjsonList.append(heading_to_searchDict)
-                                print("Final URL:", final_url)
                                 i += 2
                                 continue
                     else:
@@ -1743,9 +1712,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                             if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                                print(f"📥 Start collecting after header: {combined_line_norm} "
-                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
+                       
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -1794,7 +1761,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                     if type(heading_to_searchDict) != str:
                                         heading_to_searchDict['NBSLink']=new_url
                                     newjsonList.append(heading_to_searchDict)                                
-                                    print("Final URL:", final_url)
                                     i += 2
                                     continue
                     if collecting:
@@ -1818,7 +1784,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                 norm_line != heading_norm and
                                 is_probably_real_header):
                                 if line_text not in heading_norm:
-                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
                                   collecting = False
                                   done = True
                                   headertoContinue1 = False
@@ -1882,438 +1847,438 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
  
 
 
-    top_margin = 70
-    bottom_margin = 50
-    headertoContinue1 = False
-    headertoContinue2=False
+    # top_margin = 70
+    # bottom_margin = 50
+    # headertoContinue1 = False
+    # headertoContinue2=False
     
-    parsed_url = urlparse(pdf_path)
-    filename = os.path.basename(parsed_url.path)
-    filename = unquote(filename)  # decode URL-encoded characters
-
-    # Optimized URL handling
-    if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
-        pdf_path = pdf_path.replace('dl=0', 'dl=1')
-
-    # Cache frequently used values
-    response = requests.get(pdf_path)
-    pdf_content = BytesIO(response.content)
-    if not pdf_content:
-        raise ValueError("No valid PDF content found.")
-
-    doc = fitz.open(stream=pdf_content, filetype="pdf")
-    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
-    most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
-
-    # Precompute regex patterns
-    dot_pattern = re.compile(r'\.{3,}')
-    url_pattern = re.compile(r'https?://\S+|www\.\S+')
-
-    def get_toc_page_numbers(doc, max_pages_to_check=15):
-        toc_pages = []
-        for page_num in range(min(len(doc), max_pages_to_check)):
-            page = doc.load_page(page_num)
-            blocks = page.get_text("dict")["blocks"]
-
-            dot_line_count = 0
-            for block in blocks:
-                for line in block.get("lines", []):
-                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
-                    if dot_pattern.search(line_text):
-                        dot_line_count += 1
-
-            if dot_line_count >= 3:
-                toc_pages.append(page_num)
-
-        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
-
-    toc_pages = get_toc_page_numbers(doc)
-
-    headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
-        doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
-    )
-
-    hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
-    listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-    print('listofHeaderstoMarkup',listofHeaderstoMarkup)
-    # Precompute all children headers once
-    allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
-    allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
-
-    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
-    dictionaryNBS={}
-    data_list_JSON = []
-
-    if len(top_3_font_sizes)==3:
-        mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
-    elif len(top_3_font_sizes)==2:
-        mainHeaderFontSize= top_3_font_sizes[0]
-        subHeaderFontSize= top_3_font_sizes[1]
-        subsubheaderFontSize= top_3_font_sizes[1]
-
-    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
-
-    # Preload all pages to avoid repeated loading
-    # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
-
-    for heading_to_searchDict, paths in listofHeaderstoMarkup:
-        heading_to_search = heading_to_searchDict['text']
-        heading_to_searchPageNum = heading_to_searchDict['page']
-
-        print('headertosearch', heading_to_search)
-
-        # Initialize variables
-        headertoContinue1 = False
-        headertoContinue2 = False
-        matched_header_line = None
-        done = False
-        collecting = False
-        collected_lines = []
-        page_highlights = {}
-        current_bbox = {}
-        last_y1s = {}
-        mainHeader = ''
-        subHeader = ''
-        matched_header_line_norm = heading_to_search
-        break_collecting = False
-        heading_norm = normalize_text(heading_to_search)
-        paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
-
-        for page_num in range(heading_to_searchPageNum,len(doc)):
-            if page_num in toc_pages:
-              continue
-            if break_collecting:
-                break
-            page=doc[page_num]
-            page_height = page.rect.height
-            blocks = page.get_text("dict")["blocks"]
-
-            for block in blocks:
-                if break_collecting:
-                    break
-
-                lines = block.get("lines", [])
-                i = 0
-                while i < len(lines):
-                    if break_collecting:
-                        break
-
-                    spans = lines[i].get("spans", [])
-                    if not spans:
-                        i += 1
-                        continue
-
-                    y0 = spans[0]["bbox"][1]
-                    y1 = spans[0]["bbox"][3]
-                    if y0 < top_margin or y1 > (page_height - bottom_margin):
-                        i += 1
-                        continue
-
-                    line_text = get_spaced_text_from_spans(spans).lower()
-                    line_text_norm = normalize_text(line_text)
-
-                    # Combine with next line if available
-                    if i + 1 < len(lines):
-                        next_spans = lines[i + 1].get("spans", [])
-                        next_line_text = get_spaced_text_from_spans(next_spans).lower()
-                        combined_line_norm = normalize_text(line_text + " " + next_line_text)
-                    else:
-                        combined_line_norm = line_text_norm
-
-                    # Check if we should continue processing
-                    if combined_line_norm and combined_line_norm in paths[0]:
-                        print(combined_line_norm)
-                        headertoContinue1 = combined_line_norm
-                    if combined_line_norm and combined_line_norm in paths[-2]:
-                        print(combined_line_norm)
-                        headertoContinue2 = combined_line_norm
-                    if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                      stringtowrite='Not to be billed'
-                    else:
-                      stringtowrite='To be billed'
-                    # Optimized header matching
-                    existsfull = (
-                        ( combined_line_norm in allchildrenheaders_set or
-                        combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
-                    )
-
-                    # New word-based matching
-                    current_line_words = set(combined_line_norm.split())
-                    heading_words = set(heading_norm.split())
-                    all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
-
-                    substring_match = (
-                        heading_norm in combined_line_norm or
-                        combined_line_norm in heading_norm or
-                        all_words_match  # Include the new word-based matching
-                    )
-                    # substring_match = (
-                    #     heading_norm in combined_line_norm or
-                    #     combined_line_norm in heading_norm
-                    # )
-
-                    if (substring_match and existsfull and not collecting and
-                        len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
-
-                        # Check header conditions more efficiently
-                        header_spans = [
-                            span for span in spans
-                            if (is_header(span, most_common_font_size, most_common_color, most_common_font)
-                                # and span['size'] >= subsubheaderFontSize
-                                and span['size'] < mainHeaderFontSize)
-                        ]
-                        if header_spans:
-                            collecting = True
-                            matched_header_font_size = max(span["size"] for span in header_spans)
-                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
-
-                            collected_lines.append(line_text)
-                            valid_spans = [span for span in spans if span.get("bbox")]
-
-                            if valid_spans:
-                                x0s = [span["bbox"][0] for span in valid_spans]
-                                x1s = [span["bbox"][2] for span in valid_spans]
-                                y0s = [span["bbox"][1] for span in valid_spans]
-                                y1s = [span["bbox"][3] for span in valid_spans]
-
-                                header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
-                                if page_num in current_bbox:
-                                    cb = current_bbox[page_num]
-                                    current_bbox[page_num] = [
-                                        min(cb[0], header_bbox[0]),
-                                        min(cb[1], header_bbox[1]),
-                                        max(cb[2], header_bbox[2]),
-                                        max(cb[3], header_bbox[3])
-                                    ]
-                                else:
-                                    current_bbox[page_num] = header_bbox
-                                last_y1s[page_num] = header_bbox[3]
-                                x0, y0, x1, y1 = header_bbox
-
-                                zoom = 200
-                                left = int(x0)
-                                top = int(y0)
-                                zoom_str = f"{zoom},{left},{top}"
-                                pageNumberFound = page_num + 1
-
-                              # Build the query parameters
-                                params = {
-                                    'pdfLink': pdf_path,  # Your PDF link
-                                    'keyword': heading_to_search,  # Your keyword (could be a string or list)
-                                }
-
-                                # URL encode each parameter
-                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-
-                                # Construct the final encoded link
-                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-
-                                # Correctly construct the final URL with page and zoom
-                                final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-
-                                # Get current date and time
-                                now = datetime.now()
-
-                                # Format the output
-                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
-                                # Optionally, add the URL to a DataFrame
-
-
-                                data_entry = {
-                                        "NBSLink": final_url,
-                                        "Subject": heading_to_search,
-                                        "Page": str(pageNumberFound),
-                                        "Author": "ADR",
-                                        "Creation Date": formatted_time,
-                                        "Layer": "Initial",
-                                        "Code": stringtowrite,
-                                        "head above 1":  paths[-2],
-                                        "head above 2":  paths[0],
-                                        "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
-                                    }
-                                data_list_JSON.append(data_entry)
-
-                                # Convert list to JSON
-                                json_output = json.dumps(data_list_JSON, indent=4)
-
-                                print("Final URL:", final_url)
-                                i += 2
-                                continue
-                    else:
-                        if (substring_match and not collecting and
-                            len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
-
-                            # Calculate word match percentage
-                            word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
-
-                            # Check if at least 70% of header words exist in this line
-                            meets_word_threshold = word_match_percent >= 100
-
-                            # Check header conditions (including word threshold)
-                            header_spans = [
-                                span for span in spans
-                                if (is_header(span, most_common_font_size, most_common_color, most_common_font)
-                                    # and span['size'] >= subsubheaderFontSize
-                                    and span['size'] < mainHeaderFontSize)
-                            ]
-
-                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
-                                collecting = True
-                                matched_header_font_size = max(span["size"] for span in header_spans)
-                                print(f"📥 Start collecting after header: {combined_line_norm} "
-                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
-                                collected_lines.append(line_text)
-                                valid_spans = [span for span in spans if span.get("bbox")]
-
-                                if valid_spans:
-                                    x0s = [span["bbox"][0] for span in valid_spans]
-                                    x1s = [span["bbox"][2] for span in valid_spans]
-                                    y0s = [span["bbox"][1] for span in valid_spans]
-                                    y1s = [span["bbox"][3] for span in valid_spans]
-
-                                    header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
-                                    if page_num in current_bbox:
-                                        cb = current_bbox[page_num]
-                                        current_bbox[page_num] = [
-                                            min(cb[0], header_bbox[0]),
-                                            min(cb[1], header_bbox[1]),
-                                            max(cb[2], header_bbox[2]),
-                                            max(cb[3], header_bbox[3])
-                                        ]
-                                    else:
-                                        current_bbox[page_num] = header_bbox
-
-                                    last_y1s[page_num] = header_bbox[3]
-                                    x0, y0, x1, y1 = header_bbox
-                                    zoom = 200
-                                    left = int(x0)
-                                    top = int(y0)
-                                    zoom_str = f"{zoom},{left},{top}"
-                                    pageNumberFound = page_num + 1
-
-                                  # Build the query parameters
-                                    params = {
-                                        'pdfLink': pdf_path,  # Your PDF link
-                                        'keyword': heading_to_search,  # Your keyword (could be a string or list)
-                                    }
-
-                                    # URL encode each parameter
-                                    encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-
-                                    # Construct the final encoded link
-                                    encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-
-                                    # Correctly construct the final URL with page and zoom
-                                    final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-
-                                    # Get current date and time
-                                    now = datetime.now()
-
-                                    # Format the output
-                                    formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
-                                    # Optionally, add the URL to a DataFrame
-
-
-                                    data_entry = {
-                                            "NBSLink": final_url,
-                                            "Subject": heading_to_search,
-                                            "Page": str(pageNumberFound),
-                                            "Author": "ADR",
-                                            "Creation Date": formatted_time,
-                                            "Layer": "Initial",
-                                            "Code": stringtowrite,
-                                            "head above 1":  paths[-2],
-                                            "head above 2":  paths[0],
-                                            "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
-                                        }
-                                    data_list_JSON.append(data_entry)
-
-                                    # Convert list to JSON
-                                    json_output = json.dumps(data_list_JSON, indent=4)
-
-                                    print("Final URL:", final_url)
-                                    i += 2
-                                    continue
-                    if collecting:
-                        norm_line = normalize_text(line_text)
-
-                        # Optimized URL check
-                        if url_pattern.match(norm_line):
-                            line_is_header = False
-                        else:
-                            line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
-
-                        if line_is_header:
-                            header_font_size = max(span["size"] for span in spans)
-                            is_probably_real_header = (
-                                header_font_size >= matched_header_font_size and
-                                is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
-                                len(line_text.strip()) > 2
-                            )
-
-                            if (norm_line != matched_header_line_norm and
-                                norm_line != heading_norm and
-                                is_probably_real_header):
-                                if line_text not in heading_norm:
-                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
-                                  collecting = False
-                                  done = True
-                                  headertoContinue1 = False
-                                  headertoContinue2=False
-                                  for page_num, bbox in current_bbox.items():
-                                      bbox[3] = last_y1s.get(page_num, bbox[3])
-                                      page_highlights[page_num] = bbox
-                                  highlight_boxes(docHighlights, page_highlights,stringtowrite)
-
-                                  break_collecting = True
-                                  break
-
-                        if break_collecting:
-                            break
-
-                        collected_lines.append(line_text)
-                        valid_spans = [span for span in spans if span.get("bbox")]
-                        if valid_spans:
-                            x0s = [span["bbox"][0] for span in valid_spans]
-                            x1s = [span["bbox"][2] for span in valid_spans]
-                            y0s = [span["bbox"][1] for span in valid_spans]
-                            y1s = [span["bbox"][3] for span in valid_spans]
-
-                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
-                            if page_num in current_bbox:
-                                cb = current_bbox[page_num]
-                                current_bbox[page_num] = [
-                                    min(cb[0], line_bbox[0]),
-                                    min(cb[1], line_bbox[1]),
-                                    max(cb[2], line_bbox[2]),
-                                    max(cb[3], line_bbox[3])
-                                ]
-                            else:
-                                current_bbox[page_num] = line_bbox
+    # parsed_url = urlparse(pdf_path)
+    # filename = os.path.basename(parsed_url.path)
+    # filename = unquote(filename)  # decode URL-encoded characters
+
+    # # Optimized URL handling
+    # if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
+    #     pdf_path = pdf_path.replace('dl=0', 'dl=1')
+
+    # # Cache frequently used values
+    # response = requests.get(pdf_path)
+    # pdf_content = BytesIO(response.content)
+    # if not pdf_content:
+    #     raise ValueError("No valid PDF content found.")
+
+    # doc = fitz.open(stream=pdf_content, filetype="pdf")
+    # docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
+    # most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
+
+    # # Precompute regex patterns
+    # dot_pattern = re.compile(r'\.{3,}')
+    # url_pattern = re.compile(r'https?://\S+|www\.\S+')
+
+    # def get_toc_page_numbers(doc, max_pages_to_check=15):
+    #     toc_pages = []
+    #     for page_num in range(min(len(doc), max_pages_to_check)):
+    #         page = doc.load_page(page_num)
+    #         blocks = page.get_text("dict")["blocks"]
+
+    #         dot_line_count = 0
+    #         for block in blocks:
+    #             for line in block.get("lines", []):
+    #                 line_text = get_spaced_text_from_spans(line["spans"]).strip()
+    #                 if dot_pattern.search(line_text):
+    #                     dot_line_count += 1
+
+    #         if dot_line_count >= 3:
+    #             toc_pages.append(page_num)
+
+    #     return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
+
+    # toc_pages = get_toc_page_numbers(doc)
+
+    # headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
+    #     doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
+    # )
+
+    # hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
+    # listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
+    # print('listofHeaderstoMarkup',listofHeaderstoMarkup)
+    # # Precompute all children headers once
+    # allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
+    # allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
 
-                            last_y1s[page_num] = line_bbox[3]
-                    i += 1
+    # df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+    # dictionaryNBS={}
+    # data_list_JSON = []
 
-        if not done:
-            for page_num, bbox in current_bbox.items():
-                bbox[3] = last_y1s.get(page_num, bbox[3])
-                page_highlights[page_num] = bbox
-            if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                stringtowrite='Not to be billed'
-            else:
-                stringtowrite='To be billed'
-            highlight_boxes(docHighlights, page_highlights,stringtowrite)
+    # if len(top_3_font_sizes)==3:
+    #     mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
+    # elif len(top_3_font_sizes)==2:
+    #     mainHeaderFontSize= top_3_font_sizes[0]
+    #     subHeaderFontSize= top_3_font_sizes[1]
+    #     subsubheaderFontSize= top_3_font_sizes[1]
 
-    # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
+    # print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
 
-    pdf_bytes = BytesIO()
-    docHighlights.save(pdf_bytes)
-    print('JSONN',json_output)
-    return pdf_bytes.getvalue(), docHighlights , json_output
+    # # Preload all pages to avoid repeated loading
+    # # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
+
+    # for heading_to_searchDict, paths in listofHeaderstoMarkup:
+    #     heading_to_search = heading_to_searchDict['text']
+    #     heading_to_searchPageNum = heading_to_searchDict['page']
+
+    #     print('headertosearch', heading_to_search)
+
+    #     # Initialize variables
+    #     headertoContinue1 = False
+    #     headertoContinue2 = False
+    #     matched_header_line = None
+    #     done = False
+    #     collecting = False
+    #     collected_lines = []
+    #     page_highlights = {}
+    #     current_bbox = {}
+    #     last_y1s = {}
+    #     mainHeader = ''
+    #     subHeader = ''
+    #     matched_header_line_norm = heading_to_search
+    #     break_collecting = False
+    #     heading_norm = normalize_text(heading_to_search)
+    #     paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
+
+    #     for page_num in range(heading_to_searchPageNum,len(doc)):
+    #         if page_num in toc_pages:
+    #           continue
+    #         if break_collecting:
+    #             break
+    #         page=doc[page_num]
+    #         page_height = page.rect.height
+    #         blocks = page.get_text("dict")["blocks"]
+
+    #         for block in blocks:
+    #             if break_collecting:
+    #                 break
+
+    #             lines = block.get("lines", [])
+    #             i = 0
+    #             while i < len(lines):
+    #                 if break_collecting:
+    #                     break
+
+    #                 spans = lines[i].get("spans", [])
+    #                 if not spans:
+    #                     i += 1
+    #                     continue
+
+    #                 y0 = spans[0]["bbox"][1]
+    #                 y1 = spans[0]["bbox"][3]
+    #                 if y0 < top_margin or y1 > (page_height - bottom_margin):
+    #                     i += 1
+    #                     continue
+
+    #                 line_text = get_spaced_text_from_spans(spans).lower()
+    #                 line_text_norm = normalize_text(line_text)
+
+    #                 # Combine with next line if available
+    #                 if i + 1 < len(lines):
+    #                     next_spans = lines[i + 1].get("spans", [])
+    #                     next_line_text = get_spaced_text_from_spans(next_spans).lower()
+    #                     combined_line_norm = normalize_text(line_text + " " + next_line_text)
+    #                 else:
+    #                     combined_line_norm = line_text_norm
+
+    #                 # Check if we should continue processing
+    #                 if combined_line_norm and combined_line_norm in paths[0]:
+    #                     print(combined_line_norm)
+    #                     headertoContinue1 = combined_line_norm
+    #                 if combined_line_norm and combined_line_norm in paths[-2]:
+    #                     print(combined_line_norm)
+    #                     headertoContinue2 = combined_line_norm
+    #                 if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+    #                   stringtowrite='Not to be billed'
+    #                 else:
+    #                   stringtowrite='To be billed'
+    #                 # Optimized header matching
+    #                 existsfull = (
+    #                     ( combined_line_norm in allchildrenheaders_set or
+    #                     combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
+    #                 )
+
+    #                 # New word-based matching
+    #                 current_line_words = set(combined_line_norm.split())
+    #                 heading_words = set(heading_norm.split())
+    #                 all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
+
+    #                 substring_match = (
+    #                     heading_norm in combined_line_norm or
+    #                     combined_line_norm in heading_norm or
+    #                     all_words_match  # Include the new word-based matching
+    #                 )
+    #                 # substring_match = (
+    #                 #     heading_norm in combined_line_norm or
+    #                 #     combined_line_norm in heading_norm
+    #                 # )
+
+    #                 if (substring_match and existsfull and not collecting and
+    #                     len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
+
+    #                     # Check header conditions more efficiently
+    #                     header_spans = [
+    #                         span for span in spans
+    #                         if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+    #                             # and span['size'] >= subsubheaderFontSize
+    #                             and span['size'] < mainHeaderFontSize)
+    #                     ]
+    #                     if header_spans:
+    #                         collecting = True
+    #                         matched_header_font_size = max(span["size"] for span in header_spans)
+    #                         print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
+
+    #                         collected_lines.append(line_text)
+    #                         valid_spans = [span for span in spans if span.get("bbox")]
+
+    #                         if valid_spans:
+    #                             x0s = [span["bbox"][0] for span in valid_spans]
+    #                             x1s = [span["bbox"][2] for span in valid_spans]
+    #                             y0s = [span["bbox"][1] for span in valid_spans]
+    #                             y1s = [span["bbox"][3] for span in valid_spans]
+
+    #                             header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+    #                             if page_num in current_bbox:
+    #                                 cb = current_bbox[page_num]
+    #                                 current_bbox[page_num] = [
+    #                                     min(cb[0], header_bbox[0]),
+    #                                     min(cb[1], header_bbox[1]),
+    #                                     max(cb[2], header_bbox[2]),
+    #                                     max(cb[3], header_bbox[3])
+    #                                 ]
+    #                             else:
+    #                                 current_bbox[page_num] = header_bbox
+    #                             last_y1s[page_num] = header_bbox[3]
+    #                             x0, y0, x1, y1 = header_bbox
+
+    #                             zoom = 200
+    #                             left = int(x0)
+    #                             top = int(y0)
+    #                             zoom_str = f"{zoom},{left},{top}"
+    #                             pageNumberFound = page_num + 1
+
+    #                           # Build the query parameters
+    #                             params = {
+    #                                 'pdfLink': pdf_path,  # Your PDF link
+    #                                 'keyword': heading_to_search,  # Your keyword (could be a string or list)
+    #                             }
+
+    #                             # URL encode each parameter
+    #                             encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+    #                             # Construct the final encoded link
+    #                             encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+    #                             # Correctly construct the final URL with page and zoom
+    #                             final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+    #                             # Get current date and time
+    #                             now = datetime.now()
+
+    #                             # Format the output
+    #                             formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+    #                             # Optionally, add the URL to a DataFrame
+
+
+    #                             data_entry = {
+    #                                     "NBSLink": final_url,
+    #                                     "Subject": heading_to_search,
+    #                                     "Page": str(pageNumberFound),
+    #                                     "Author": "ADR",
+    #                                     "Creation Date": formatted_time,
+    #                                     "Layer": "Initial",
+    #                                     "Code": stringtowrite,
+    #                                     "head above 1":  paths[-2],
+    #                                     "head above 2":  paths[0],
+    #                                     "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
+    #                                 }
+    #                             data_list_JSON.append(data_entry)
+
+    #                             # Convert list to JSON
+    #                             json_output = json.dumps(data_list_JSON, indent=4)
+
+    #                             print("Final URL:", final_url)
+    #                             i += 2
+    #                             continue
+    #                 else:
+    #                     if (substring_match and not collecting and
+    #                         len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
+
+    #                         # Calculate word match percentage
+    #                         word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
+
+    #                         # Check if at least 70% of header words exist in this line
+    #                         meets_word_threshold = word_match_percent >= 100
+
+    #                         # Check header conditions (including word threshold)
+    #                         header_spans = [
+    #                             span for span in spans
+    #                             if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+    #                                 # and span['size'] >= subsubheaderFontSize
+    #                                 and span['size'] < mainHeaderFontSize)
+    #                         ]
+
+    #                         if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+    #                             collecting = True
+    #                             matched_header_font_size = max(span["size"] for span in header_spans)
+    #                             print(f"📥 Start collecting after header: {combined_line_norm} "
+    #                                   f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
+
+    #                             collected_lines.append(line_text)
+    #                             valid_spans = [span for span in spans if span.get("bbox")]
+
+    #                             if valid_spans:
+    #                                 x0s = [span["bbox"][0] for span in valid_spans]
+    #                                 x1s = [span["bbox"][2] for span in valid_spans]
+    #                                 y0s = [span["bbox"][1] for span in valid_spans]
+    #                                 y1s = [span["bbox"][3] for span in valid_spans]
+
+    #                                 header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+    #                                 if page_num in current_bbox:
+    #                                     cb = current_bbox[page_num]
+    #                                     current_bbox[page_num] = [
+    #                                         min(cb[0], header_bbox[0]),
+    #                                         min(cb[1], header_bbox[1]),
+    #                                         max(cb[2], header_bbox[2]),
+    #                                         max(cb[3], header_bbox[3])
+    #                                     ]
+    #                                 else:
+    #                                     current_bbox[page_num] = header_bbox
+
+    #                                 last_y1s[page_num] = header_bbox[3]
+    #                                 x0, y0, x1, y1 = header_bbox
+    #                                 zoom = 200
+    #                                 left = int(x0)
+    #                                 top = int(y0)
+    #                                 zoom_str = f"{zoom},{left},{top}"
+    #                                 pageNumberFound = page_num + 1
+
+    #                               # Build the query parameters
+    #                                 params = {
+    #                                     'pdfLink': pdf_path,  # Your PDF link
+    #                                     'keyword': heading_to_search,  # Your keyword (could be a string or list)
+    #                                 }
+
+    #                                 # URL encode each parameter
+    #                                 encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+    #                                 # Construct the final encoded link
+    #                                 encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+    #                                 # Correctly construct the final URL with page and zoom
+    #                                 final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+    #                                 # Get current date and time
+    #                                 now = datetime.now()
+
+    #                                 # Format the output
+    #                                 formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+    #                                 # Optionally, add the URL to a DataFrame
+
+
+    #                                 data_entry = {
+    #                                         "NBSLink": final_url,
+    #                                         "Subject": heading_to_search,
+    #                                         "Page": str(pageNumberFound),
+    #                                         "Author": "ADR",
+    #                                         "Creation Date": formatted_time,
+    #                                         "Layer": "Initial",
+    #                                         "Code": stringtowrite,
+    #                                         "head above 1":  paths[-2],
+    #                                         "head above 2":  paths[0],
+    #                                         "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
+    #                                     }
+    #                                 data_list_JSON.append(data_entry)
+
+    #                                 # Convert list to JSON
+    #                                 json_output = json.dumps(data_list_JSON, indent=4)
+
+    #                                 print("Final URL:", final_url)
+    #                                 i += 2
+    #                                 continue
+    #                 if collecting:
+    #                     norm_line = normalize_text(line_text)
+
+    #                     # Optimized URL check
+    #                     if url_pattern.match(norm_line):
+    #                         line_is_header = False
+    #                     else:
+    #                         line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
+
+    #                     if line_is_header:
+    #                         header_font_size = max(span["size"] for span in spans)
+    #                         is_probably_real_header = (
+    #                             header_font_size >= matched_header_font_size and
+    #                             is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
+    #                             len(line_text.strip()) > 2
+    #                         )
+
+    #                         if (norm_line != matched_header_line_norm and
+    #                             norm_line != heading_norm and
+    #                             is_probably_real_header):
+    #                             if line_text not in heading_norm:
+    #                               print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
+    #                               collecting = False
+    #                               done = True
+    #                               headertoContinue1 = False
+    #                               headertoContinue2=False
+    #                               for page_num, bbox in current_bbox.items():
+    #                                   bbox[3] = last_y1s.get(page_num, bbox[3])
+    #                                   page_highlights[page_num] = bbox
+    #                               highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+    #                               break_collecting = True
+    #                               break
+
+    #                     if break_collecting:
+    #                         break
+
+    #                     collected_lines.append(line_text)
+    #                     valid_spans = [span for span in spans if span.get("bbox")]
+    #                     if valid_spans:
+    #                         x0s = [span["bbox"][0] for span in valid_spans]
+    #                         x1s = [span["bbox"][2] for span in valid_spans]
+    #                         y0s = [span["bbox"][1] for span in valid_spans]
+    #                         y1s = [span["bbox"][3] for span in valid_spans]
+
+    #                         line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+    #                         if page_num in current_bbox:
+    #                             cb = current_bbox[page_num]
+    #                             current_bbox[page_num] = [
+    #                                 min(cb[0], line_bbox[0]),
+    #                                 min(cb[1], line_bbox[1]),
+    #                                 max(cb[2], line_bbox[2]),
+    #                                 max(cb[3], line_bbox[3])
+    #                             ]
+    #                         else:
+    #                             current_bbox[page_num] = line_bbox
+
+    #                         last_y1s[page_num] = line_bbox[3]
+    #                 i += 1
+
+    #     if not done:
+    #         for page_num, bbox in current_bbox.items():
+    #             bbox[3] = last_y1s.get(page_num, bbox[3])
+    #             page_highlights[page_num] = bbox
+    #         if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+    #             stringtowrite='Not to be billed'
+    #         else:
+    #             stringtowrite='To be billed'
+    #         highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+    # # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
+
+    # pdf_bytes = BytesIO()
+    # docHighlights.save(pdf_bytes)
+    # print('JSONN',json_output)
+    # return pdf_bytes.getvalue(), docHighlights , json_output
 
 
 
@@ -2393,8 +2358,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         subHeaderFontSize= top_3_font_sizes[1]
         subsubheaderFontSize= top_3_font_sizes[1]
 
-    # print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
-
+   
     # Preload all pages to avoid repeated loading
     # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
 
@@ -2402,8 +2366,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         heading_to_search = heading_to_searchDict['text']
         heading_to_searchPageNum = heading_to_searchDict['page']
 
-        # print('headertosearch', heading_to_search)
-
+        
         # Initialize variables
         headertoContinue1 = False
         headertoContinue2 = False
@@ -2464,10 +2427,8 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
                     # Check if we should continue processing
                     if combined_line_norm and combined_line_norm in paths[0]:
-                        print(combined_line_norm)
                         headertoContinue1 = combined_line_norm
                     if combined_line_norm and combined_line_norm in paths[-2]:
-                        print(combined_line_norm)
                         headertoContinue2 = combined_line_norm
                     if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
                       stringtowrite='Not to be billed'
@@ -2508,8 +2469,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                             Alltext_tobebilled+=combined_line_norm
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
-                            # print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
-
+                      
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -2580,7 +2540,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 # Convert list to JSON
                                 json_output = json.dumps(data_list_JSON, indent=4)
 
-                                # print("Final URL:", final_url)
                                 i += 2
                                 continue
                     else:
@@ -2605,9 +2564,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 Alltext_tobebilled+=combined_line_norm
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                                # print(f"📥 Start collecting after header: {combined_line_norm} "
-                                #       f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
+                            
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -2678,7 +2635,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                     # Convert list to JSON
                                     json_output = json.dumps(data_list_JSON, indent=4)
 
-                                    print("Final URL:", final_url)
                                     i += 2
                                     continue
                     if collecting:
@@ -2702,7 +2658,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 norm_line != heading_norm and
                                 is_probably_real_header):
                                 if line_text not in heading_norm:
-                                #   print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
                                   collecting = False
                                   done = True
                                   headertoContinue1 = False
@@ -2756,7 +2711,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
     pdf_bytes = BytesIO()
     docHighlights.save(pdf_bytes)
-    # print('JSONN',json_output)
+
     return pdf_bytes.getvalue(), docHighlights , json_output , Alltext_tobebilled
 
 
@@ -2816,10 +2771,10 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
     else:
         for item in headingjson:
             listofheadingsfromrawan.append(normalize_text(item['Subject']))
-    print('hereeeeeeeeeeeeeee0',listofheadingsfromrawan)
+    # print('hereeeeeeeeeeeeeee0',listofheadingsfromrawan)
     # Precompute all children headers once
     allchildrenheaders = listofheadingsfromrawan
-    print('hereeeeeeeeeeeeeee00',allchildrenheaders)
+    # print('hereeeeeeeeeeeeeee00',allchildrenheaders)
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
     
     df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
@@ -2832,7 +2787,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
         subHeaderFontSize= top_3_font_sizes[1]
         subsubheaderFontSize= top_3_font_sizes[1]
 
-    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
+    # print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
 
     # Preload all pages to avoid repeated loading
     # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
@@ -2846,7 +2801,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
             heading_to_searchPageNum = int(heading_to_searchDict['Page'])-1
             incomingheader = heading_to_searchDict['head above 1']
             
-        print('hereeeeeeeeeeeeeee0',heading_to_searchPageNum)
+        # print('hereeeeeeeeeeeeeee0',heading_to_searchPageNum)
         done = False
         collecting = False
         collected_lines = []
@@ -2860,7 +2815,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
         heading_norm = normalize_text(heading_to_search)
 
         for page_num in range(heading_to_searchPageNum,len(doc)):
-            print('hereeeeeeeeeeeeeee1')
+            
             if page_num in toc_pages:
               continue
             if break_collecting:
@@ -2930,7 +2885,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                         if header_spans:
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
-                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
+                            # print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
 
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
@@ -2987,7 +2942,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                 if type(heading_to_searchDict) != str:
                                     heading_to_searchDict['NBSLink']=new_url
                                 newjsonList.append(heading_to_searchDict)
-                                print("Final URL:", final_url)
+                                # print("Final URL:", final_url)
                                 i += 2
                                 continue
                     else:
@@ -3011,9 +2966,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                             if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                                print(f"📥 Start collecting after header: {combined_line_norm} "
-                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
-
+                              
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -3062,7 +3015,6 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                     if type(heading_to_searchDict) != str:
                                         heading_to_searchDict['NBSLink']=new_url
                                     newjsonList.append(heading_to_searchDict)                                
-                                    print("Final URL:", final_url)
                                     i += 2
                                     continue
                     if collecting:
@@ -3086,7 +3038,7 @@ def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incominghea
                                 norm_line != heading_norm and
                                 is_probably_real_header):
                                 if line_text not in heading_norm:
-                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
+                                #   print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
                                   collecting = False
                                   done = True
                                   headertoContinue1 = False