Update InitialMarkups.py

InitialMarkups.py

@@ -13,7 +13,7 @@ tobebilledonlyLink='https://adr.trevorsadd.co.uk/api/view-pdf-tobebilled?'
 
     
 
-import datefinder
+
 from urllib.parse import urlparse, unquote
 import os
 from io import BytesIO
@@ -36,9 +36,8 @@ import tsadropboxretrieval
 
 import urllib.parse
 
-
-
-import urllib.parse
+top_margin = 70
+bottom_margin = 85
 
 def changepdflinks(json_data, pdf_path):
     print('ll , ' ,json_data,pdf_path)
@@ -74,7 +73,7 @@ def get_toc_page_numbers(doc, max_pages_to_check=15):
     # 2. NEW: Title Pattern (looking for specific headers)
     # ^ and $ ensure the line is JUST that word (ignoring "The contents of the bag...")
     # re.IGNORECASE makes it match "CONTENTS", "Contents", "Index", etc.
-    title_pattern = re.compile(r"^\s*(table of contents|contents|index|content)\s*$", re.IGNORECASE)
+    title_pattern = re.compile(r"^\s*(table of contents|contents|index)\s*$", re.IGNORECASE)
 
     for page_num in range(min(len(doc), max_pages_to_check)):
         page = doc.load_page(page_num)
@@ -108,13 +107,13 @@ def get_toc_page_numbers(doc, max_pages_to_check=15):
     # If we found TOC pages (e.g., [2, 3]), we return [0, 1, 2, 3]
     # This covers the cover page, inside cover, and the TOC itself.
     if toc_pages:
-        print('toccc',toc_pages)
         last_toc_page = toc_pages[0]
         return list(range(0, last_toc_page + 1))
 
     return [] # Return empty list if nothing found
 
 
+
 def get_regular_font_size_and_color(doc):
     font_sizes = []
     colors = []
@@ -311,7 +310,7 @@ def clean_toc_entry(toc_text):
     # Remove everything after last sequence of dots/whitespace followed by digits
     return re.sub(r'[\.\s]+\d+.*$', '', toc_text).strip('. ')
 
-def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin=70, bottom_margin=70):
+def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin=70, bottom_margin=85):
     # Extract headers with margin handling
     headers_list, top_3_font_sizes, smallest_font_size, spans = extract_headers(
         doc,
@@ -690,8 +689,6 @@ def same_start_word(s1, s2):
 def extract_section_under_header(multiplePDF_Paths):
     filenames=[]
     keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
-    top_margin = 70
-    bottom_margin = 50
     arrayofPDFS=multiplePDF_Paths.split(',')
     print(multiplePDF_Paths)
     print(arrayofPDFS,len(arrayofPDFS))
@@ -724,8 +721,7 @@ def extract_section_under_header(multiplePDF_Paths):
         dot_pattern = re.compile(r'\.{3,}')
         url_pattern = re.compile(r'https?://\S+|www\.\S+')
 
-
-
+       
         toc_pages = get_toc_page_numbers(doc)
 
         headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
@@ -1127,10 +1123,6 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
     alltextWithoutNotbilled=''
     # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
     
-
-    
-    top_margin = 70
-    bottom_margin = 50
     headertoContinue1 = False
     headertoContinue2=False
     
@@ -1161,24 +1153,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
     dot_pattern = re.compile(r'\.{3,}')
     url_pattern = re.compile(r'https?://\S+|www\.\S+')
 
-    def get_toc_page_numbers(doc, max_pages_to_check=15):
-        toc_pages = []
-        for page_num in range(min(len(doc), max_pages_to_check)):
-            page = doc.load_page(page_num)
-            blocks = page.get_text("dict")["blocks"]
-
-            dot_line_count = 0
-            for block in blocks:
-                for line in block.get("lines", []):
-                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
-                    if dot_pattern.search(line_text):
-                        dot_line_count += 1
-
-            if dot_line_count >= 1:
-                toc_pages.append(page_num)
-
-        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
-
+  
     toc_pages = get_toc_page_numbers(doc)
 
     headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
@@ -1581,8 +1556,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
     # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
 
     keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
-    top_margin = 70
-    bottom_margin = 50
+
     headertoContinue1 = False
     headertoContinue2=False
     Alltexttobebilled=''
@@ -1608,24 +1582,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
     dot_pattern = re.compile(r'\.{3,}')
     url_pattern = re.compile(r'https?://\S+|www\.\S+')
 
-    def get_toc_page_numbers(doc, max_pages_to_check=15):
-        toc_pages = []
-        for page_num in range(min(len(doc), max_pages_to_check)):
-            page = doc.load_page(page_num)
-            blocks = page.get_text("dict")["blocks"]
-
-            dot_line_count = 0
-            for block in blocks:
-                for line in block.get("lines", []):
-                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
-                    if dot_pattern.search(line_text):
-                        dot_line_count += 1
-
-            if dot_line_count >= 1:
-                toc_pages.append(page_num)
-
-        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
-
+   
     toc_pages = get_toc_page_numbers(doc)
 
     headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
@@ -2039,13 +1996,12 @@ def extract_section_under_header_tobebilled2(pdf_path):
 
 
 
+
 def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths):
-    baselink = "https://findconsole-initialmarkups.hf.space/view-pdf?"
     # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
     filenames=[]
     keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
-    top_margin = 70
-    bottom_margin = 50
+
     arrayofPDFS=multiplePDF_Paths.split(',')
     print(multiplePDF_Paths)
     print(arrayofPDFS)
@@ -2063,86 +2019,35 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
         # Optimized URL handling
         if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
             pdf_path = pdf_path.replace('dl=0', 'dl=1')
-
+    
         # Cache frequently used values
         response = requests.get(pdf_path)
         pdf_content = BytesIO(response.content)
         if not pdf_content:
             raise ValueError("No valid PDF content found.")
-
+    
         doc = fitz.open(stream=pdf_content, filetype="pdf")
         docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
         most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
-
+    
         # Precompute regex patterns
         dot_pattern = re.compile(r'\.{3,}')
         url_pattern = re.compile(r'https?://\S+|www\.\S+')
+    
 
-
-        def get_toc_page_numbers(doc, max_pages_to_check=15):
-            toc_pages = []
-
-            # 1. Existing Dot Pattern (looking for ".....")
-            dot_pattern = re.compile(r"\.{2,}")
-
-            # 2. NEW: Title Pattern (looking for specific headers)
-            # ^ and $ ensure the line is JUST that word (ignoring "The contents of the bag...")
-            # re.IGNORECASE makes it match "CONTENTS", "Contents", "Index", etc.
-            title_pattern = re.compile(r"^\s*(table of contents|contents|index)\s*$", re.IGNORECASE)
-
-            for page_num in range(min(len(doc), max_pages_to_check)):
-                page = doc.load_page(page_num)
-                blocks = page.get_text("dict")["blocks"]
-                skip_page_due_to_footer = False  
-                dot_line_count = 0
-                has_toc_title = False
-
-                for block in blocks:
-                    for line in block.get("lines", []):
-                        # Extract text from spans (mimicking get_spaced_text_from_spans)
-                        line_text = " ".join([span["text"] for span in line["spans"]]).strip()
-
-                        # CHECK A: Does the line have dots?
-                        if dot_pattern.search(line_text):
-                            dot_line_count += 1
-
-                        # CHECK B: Is this line a Title?
-                        # We check this early in the loop. If a page has a title "Contents",
-                        # we mark it immediately.
-                        if title_pattern.match(line_text):
-                            has_toc_title = True
-
-                # CONDITION:
-                # It is a TOC page if it has a Title OR if it has dot leaders.
-                # We use 'dot_line_count >= 1' to be sensitive to single-item lists.
-                if has_toc_title or dot_line_count >= 1:
-                    toc_pages.append(page_num)
-
-            # RETURN:
-            # If we found TOC pages (e.g., [2, 3]), we return [0, 1, 2, 3]
-            # This covers the cover page, inside cover, and the TOC itself.
-            if toc_pages:
-                last_toc_page = toc_pages[-1]
-                return list(range(0, last_toc_page + 1))
-
-            return [] # Return empty list if nothing found
-
-        # Usage
         toc_pages = get_toc_page_numbers(doc)
-
+    
         headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
             doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
         )
-
+    
         hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
         listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-        # print(listofHeaderstoMarkup)
-        for header, path in listofHeaderstoMarkup:
-          print(path)
+        
         # Precompute all children headers once
         allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
         allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
-
+    
         # df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
         dictionaryNBS={}
         data_list_JSON = []
@@ -2154,16 +2059,16 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
             mainHeaderFontSize= top_3_font_sizes[0]
             subHeaderFontSize= top_3_font_sizes[1]
             subsubheaderFontSize= top_3_font_sizes[1]
-
-
-
+    
+        
+    
         # Preload all pages to avoid repeated loading
         # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
-        skip_page_due_to_footer = False  
+    
         for heading_to_searchDict, paths in listofHeaderstoMarkup:
             heading_to_search = heading_to_searchDict['text']
             heading_to_searchPageNum = heading_to_searchDict['page']
-
+    
             # Initialize variables
             headertoContinue1 = False
             headertoContinue2 = False
@@ -2181,48 +2086,45 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
             heading_norm = normalize_text(heading_to_search)
             paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
             for page_num in range(heading_to_searchPageNum,len(doc)):
-                skip_page_due_to_footer = False
                 # print(heading_to_search)
                 if paths[0].strip().lower() != currentgroupname.strip().lower():
                     Alltexttobebilled+= paths[0] +'\n'
                     currentgroupname=paths[0]
                     # print(paths[0])
-
-
+    
+                    
                 if page_num in toc_pages:
                   continue
-
                 if break_collecting:
                     break
                 page=doc[page_num]
                 page_height = page.rect.height
                 blocks = page.get_text("dict")["blocks"]
-               
+    
                 for block in blocks:
                     if break_collecting:
                         break
-
+    
                     lines = block.get("lines", [])
                     i = 0
                     while i < len(lines):
                         if break_collecting:
                             break
-
+    
                         spans = lines[i].get("spans", [])
                         if not spans:
                             i += 1
                             continue
-
+    
                         y0 = spans[0]["bbox"][1]
                         y1 = spans[0]["bbox"][3]
                         if y0 < top_margin or y1 > (page_height - bottom_margin):
                             i += 1
                             continue
-
+    
                         line_text = get_spaced_text_from_spans(spans).lower()
                         line_text_norm = normalize_text(line_text)
-
-
+    
                         # Combine with next line if available
                         if i + 1 < len(lines):
                             next_spans = lines[i + 1].get("spans", [])
@@ -2230,14 +2132,16 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                             combined_line_norm = normalize_text(line_text + " " + next_line_text)
                         else:
                             combined_line_norm = line_text_norm
+    
                         # Check if we should continue processing
                         if combined_line_norm and combined_line_norm in paths[0]:
+                            
                             headertoContinue1 = combined_line_norm
                         if combined_line_norm and combined_line_norm in paths[-2]:
+                        
                             headertoContinue2 = combined_line_norm
                         # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
                         last_path = paths[-2].lower()
-
                         # if any(word in paths[-2].lower() for word in keywordstoSkip):
                         # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() or 'workmanship' in paths[-2].lower() or 'testing' in paths[-2].lower() or 'labeling' in paths[-2].lower():
                         if any(keyword in last_path for keyword in keywords):
@@ -2248,18 +2152,18 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                             # Alltexttobebilled+= combined_line_norm  #################################################
                             if matched_header_line_norm in combined_line_norm:
                                 Alltexttobebilled+='\n'
-                            Alltexttobebilled+= ' '+combined_line_norm
+                            Alltexttobebilled+= ' '+combined_line_norm 
                         # Optimized header matching
                         existsfull = (
                             ( combined_line_norm in allchildrenheaders_set or
                             combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
                         )
-
+    
                         # New word-based matching
                         current_line_words = set(combined_line_norm.split())
                         heading_words = set(heading_norm.split())
                         all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
-
+    
                         substring_match = (
                             heading_norm in combined_line_norm or
                             combined_line_norm in heading_norm or
@@ -2269,10 +2173,10 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                         #     heading_norm in combined_line_norm or
                         #     combined_line_norm in heading_norm
                         # )
-
+    
                         if (substring_match and existsfull and not collecting and
                             len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
-
+    
                             # Check header conditions more efficiently
                             header_spans = [
                                 span for span in spans
@@ -2280,23 +2184,23 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                     # and span['size'] >= subsubheaderFontSize
                                     and span['size'] < mainHeaderFontSize)
                             ]
-                            if header_spans and stringtowrite.startswith('To'):
+                            if header_spans and stringtowrite.startswith('To') and is_numbered(heading_to_search):
                                 collecting = True
                                 # if stringtowrite=='To be billed':
                                 #     Alltexttobebilled+='\n'
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-
-                                collected_lines.append(line_text)
+    
+                                # collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
-
+    
                                 if valid_spans:
                                     x0s = [span["bbox"][0] for span in valid_spans]
                                     x1s = [span["bbox"][2] for span in valid_spans]
                                     y0s = [span["bbox"][1] for span in valid_spans]
                                     y1s = [span["bbox"][3] for span in valid_spans]
-
+    
                                     header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
+    
                                     if page_num in current_bbox:
                                         cb = current_bbox[page_num]
                                         current_bbox[page_num] = [
@@ -2309,36 +2213,36 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                         current_bbox[page_num] = header_bbox
                                     last_y1s[page_num] = header_bbox[3]
                                     x0, y0, x1, y1 = header_bbox
-
+    
                                     zoom = 200
                                     left = int(x0)
                                     top = int(y0)
                                     zoom_str = f"{zoom},{left},{top}"
                                     pageNumberFound = page_num + 1
-
+    
                                   # Build the query parameters
                                     params = {
                                         'pdfLink': pdf_path,  # Your PDF link
                                         'keyword': heading_to_search,  # Your keyword (could be a string or list)
                                     }
-
+    
                                     # URL encode each parameter
                                     encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-
+    
                                     # Construct the final encoded link
                                     encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-
+    
                                     # Correctly construct the final URL with page and zoom
                                     final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-
+    
                                     # Get current date and time
                                     now = datetime.now()
-
+    
                                     # Format the output
                                     formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
                                     # Optionally, add the URL to a DataFrame
-
-
+    
+    
                                     data_entry = {
                                             "PDF Name":filename,
                                             "NBSLink": zoom_str,
@@ -2354,23 +2258,23 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                             "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                         }
                                     data_list_JSON.append(data_entry)
-
+    
                                     # Convert list to JSON
                                     # json_output = [data_list_JSON]
                                     # json_output = json.dumps(data_list_JSON, indent=4)
-
+    
                                     i += 2
                                     continue
                         else:
                             if (substring_match and not collecting and
                                 len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
-
+    
                                 # Calculate word match percentage
                                 word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
-
+    
                                 # Check if at least 70% of header words exist in this line
                                 meets_word_threshold = word_match_percent >= 100
-
+    
                                 # Check header conditions (including word threshold)
                                 header_spans = [
                                     span for span in spans
@@ -2378,7 +2282,7 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                         # and span['size'] >= subsubheaderFontSize
                                         and span['size'] < mainHeaderFontSize)
                                 ]
-
+    
                                 if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ) and stringtowrite.startswith('To'):
                                     collecting = True
                                     if stringtowrite=='To be billed':
@@ -2386,18 +2290,18 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                     # if stringtowrite=='To be billed':
                                     #     Alltexttobebilled+= ' '+ combined_line_norm
                                     matched_header_font_size = max(span["size"] for span in header_spans)
-                                    # if normalize_text(line_text)!=heading_norm:
+                                 
                                     collected_lines.append(line_text)
                                     valid_spans = [span for span in spans if span.get("bbox")]
-
+    
                                     if valid_spans:
                                         x0s = [span["bbox"][0] for span in valid_spans]
                                         x1s = [span["bbox"][2] for span in valid_spans]
                                         y0s = [span["bbox"][1] for span in valid_spans]
                                         y1s = [span["bbox"][3] for span in valid_spans]
-
+    
                                         header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
+    
                                         if page_num in current_bbox:
                                             cb = current_bbox[page_num]
                                             current_bbox[page_num] = [
@@ -2408,7 +2312,7 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                             ]
                                         else:
                                             current_bbox[page_num] = header_bbox
-
+    
                                         last_y1s[page_num] = header_bbox[3]
                                         x0, y0, x1, y1 = header_bbox
                                         zoom = 200
@@ -2416,30 +2320,30 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                         top = int(y0)
                                         zoom_str = f"{zoom},{left},{top}"
                                         pageNumberFound = page_num + 1
-
+    
                                       # Build the query parameters
                                         params = {
                                             'pdfLink': pdf_path,  # Your PDF link
                                             'keyword': heading_to_search,  # Your keyword (could be a string or list)
                                         }
-
+    
                                         # URL encode each parameter
                                         encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-
+    
                                         # Construct the final encoded link
                                         encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-
+    
                                         # Correctly construct the final URL with page and zoom
                                         final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-
+    
                                         # Get current date and time
                                         now = datetime.now()
-
+    
                                         # Format the output
                                         formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
                                         # Optionally, add the URL to a DataFrame
-
-
+    
+    
                                         data_entry = {
                                                 "PDF Name":filename,
                                                 "NBSLink": zoom_str,
@@ -2455,34 +2359,23 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                                 "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                             }
                                         data_list_JSON.append(data_entry)
-
+    
                                         # Convert list to JSON
                                         # json_output = [data_list_JSON]
                                         # json_output = json.dumps(data_list_JSON, indent=4)
-
-
+    
+                    
                                         i += 2
                                         continue
                         if collecting:
                             norm_line = normalize_text(line_text)
-            # ...
-                            matches = list(datefinder.find_dates(line_text_norm))
-
-                            if matches:  # If the list is not empty, a date was found
-                                # --- MODIFY THIS BLOCK ---
-                                print(f"Skipping rest of page {page_num} due to date: {line_text_norm}") # Optional
-                                skip_page_due_to_footer = True 
-                                i += 1
-                                break
-                                # --- END OF CHANGE ---
-            # ...
-                            print(norm_line)
+    
                             # Optimized URL check
                             if url_pattern.match(norm_line):
                                 line_is_header = False
                             else:
                                 line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
-
+    
                             if line_is_header:
                                 header_font_size = max(span["size"] for span in spans)
                                 is_probably_real_header = (
@@ -2490,7 +2383,7 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                     is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
                                     len(line_text.strip()) > 2
                                 )
-
+    
                                 if (norm_line != matched_header_line_norm and
                                     norm_line != heading_norm and
                                     is_probably_real_header):
@@ -2503,31 +2396,13 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                           bbox[3] = last_y1s.get(page_num, bbox[3])
                                           page_highlights[page_num] = bbox
                                       highlight_boxes(docHighlights, page_highlights,stringtowrite)
-
+    
                                       break_collecting = True
                                       break
-                        # ... (this is the end of your 'while' loop, after line 508)
-                                
-                                # --- ADD THIS BLOCK (after 'while' loop) ---
-                                # This breaks out of the 'for block...' loop
-                                # if skip_page_due_to_footer or break_collecting:
-                                #     break
-
-                            # --- End of 'for block' loop ---
-
-                            # --- ADD THIS BLOCK (after 'for block...' loop) ---
-                            # # This skips to the next page
-                            # if skip_page_due_to_footer:
-                            #     continue
-
-                            # This (existing) check stops collecting for this header
+    
                             if break_collecting:
                                 break
-                        # ...
-
-                            # if line_text.lower() != heading_norm.lower():
-                            #   print('checkk',line_text,heading_norm)
-                            #   collected_lines.append(line_text)
+    
                             collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
                             if valid_spans:
@@ -2535,9 +2410,9 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                 x1s = [span["bbox"][2] for span in valid_spans]
                                 y0s = [span["bbox"][1] for span in valid_spans]
                                 y1s = [span["bbox"][3] for span in valid_spans]
-
+    
                                 line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
+    
                                 if page_num in current_bbox:
                                     cb = current_bbox[page_num]
                                     current_bbox[page_num] = [
@@ -2548,13 +2423,10 @@ def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths)
                                     ]
                                 else:
                                     current_bbox[page_num] = line_bbox
-
+    
                                 last_y1s[page_num] = line_bbox[3]
                         i += 1
-                    if skip_page_due_to_footer:
-                        break # This breaks the 'for block...' loop
-                if skip_page_due_to_footer:
-                  continue # This skips to the next 'for page_num...'
+    
             if not done:
                 for page_num, bbox in current_bbox.items():
                     bbox[3] = last_y1s.get(page_num, bbox[3])