Update InitialMarkups.py

InitialMarkups.py

@@ -649,283 +649,200 @@ def same_start_word(s1, s2):
     return False
 
 
-def extract_section_under_header(pdf_path):
-    top_margin = 70
-    bottom_margin = 50
-    headertoContinue1 = False
-    headertoContinue2=False
-    
-    parsed_url = urlparse(pdf_path)
-    filename = os.path.basename(parsed_url.path)
-    filename = unquote(filename)  # decode URL-encoded characters
 
-    # Optimized URL handling
-    if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
-        pdf_path = pdf_path.replace('dl=0', 'dl=1')
 
-    # Cache frequently used values
-    response = requests.get(pdf_path)
-    pdf_content = BytesIO(response.content)
-    if not pdf_content:
-        raise ValueError("No valid PDF content found.")
+def extract_section_under_header(multiplePDF_Paths):
+    filenames=[]
+    keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
+    top_margin = 70
+    bottom_margin = 50
+    arrayofPDFS=multiplePDF_Paths.split(',')
+    print(multiplePDF_Paths)
+    print(arrayofPDFS)
+    docarray=[]
+    jsons=[]
+    df = pd.DataFrame(columns=["PDF Name","NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
+    for pdf_path in arrayofPDFS:
+        headertoContinue1 = False
+        headertoContinue2=False
+        Alltexttobebilled=''
+        parsed_url = urlparse(pdf_path)
+        filename = os.path.basename(parsed_url.path)
+        filename = unquote(filename)  # decode URL-encoded characters
+        filenames.append(filename)
+        # Optimized URL handling
+        if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
+            pdf_path = pdf_path.replace('dl=0', 'dl=1')
 
-    doc = fitz.open(stream=pdf_content, filetype="pdf")
-    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
-    most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
+        # Cache frequently used values
+        response = requests.get(pdf_path)
+        pdf_content = BytesIO(response.content)
+        if not pdf_content:
+            raise ValueError("No valid PDF content found.")
 
-    # Precompute regex patterns
-    dot_pattern = re.compile(r'\.{3,}')
-    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+        doc = fitz.open(stream=pdf_content, filetype="pdf")
+        docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
+        most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
 
-    def get_toc_page_numbers(doc, max_pages_to_check=15):
-        toc_pages = []
-        for page_num in range(min(len(doc), max_pages_to_check)):
-            page = doc.load_page(page_num)
-            blocks = page.get_text("dict")["blocks"]
+        # Precompute regex patterns
+        dot_pattern = re.compile(r'\.{3,}')
+        url_pattern = re.compile(r'https?://\S+|www\.\S+')
 
-            dot_line_count = 0
-            for block in blocks:
-                for line in block.get("lines", []):
-                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
-                    if dot_pattern.search(line_text):
-                        dot_line_count += 1
+        def get_toc_page_numbers(doc, max_pages_to_check=15):
+            toc_pages = []
+            for page_num in range(min(len(doc), max_pages_to_check)):
+                page = doc.load_page(page_num)
+                blocks = page.get_text("dict")["blocks"]
 
-            if dot_line_count >= 3:
-                toc_pages.append(page_num)
+                dot_line_count = 0
+                for block in blocks:
+                    for line in block.get("lines", []):
+                        line_text = get_spaced_text_from_spans(line["spans"]).strip()
+                        if dot_pattern.search(line_text):
+                            dot_line_count += 1
 
-        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
+                if dot_line_count >= 1:
+                    toc_pages.append(page_num)
 
-    toc_pages = get_toc_page_numbers(doc)
+            return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
 
-    headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
-        doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
-    )
+        toc_pages = get_toc_page_numbers(doc)
 
-    hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
-    listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-    
-    # Precompute all children headers once
-    allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
-    allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
+        headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
+            doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
+        )
 
-    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
-    dictionaryNBS={}
-    data_list_JSON = []
+        hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
+        listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
+        
+        # Precompute all children headers once
+        allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
+        allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
 
-    if len(top_3_font_sizes)==3:
-        mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
-    elif len(top_3_font_sizes)==2:
-        mainHeaderFontSize= top_3_font_sizes[0]
-        subHeaderFontSize= top_3_font_sizes[1]
-        subsubheaderFontSize= top_3_font_sizes[1]
+        df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+        dictionaryNBS={}
+        data_list_JSON = []
 
-    
+        if len(top_3_font_sizes)==3:
+            mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
+        elif len(top_3_font_sizes)==2:
+            mainHeaderFontSize= top_3_font_sizes[0]
+            subHeaderFontSize= top_3_font_sizes[1]
+            subsubheaderFontSize= top_3_font_sizes[1]
 
-    # Preload all pages to avoid repeated loading
-    # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
+        
 
-    for heading_to_searchDict, paths in listofHeaderstoMarkup:
-        heading_to_search = heading_to_searchDict['text']
-        heading_to_searchPageNum = heading_to_searchDict['page']
+        # Preload all pages to avoid repeated loading
+        # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
 
-        # Initialize variables
-        headertoContinue1 = False
-        headertoContinue2 = False
-        matched_header_line = None
-        done = False
-        collecting = False
-        collected_lines = []
-        page_highlights = {}
-        current_bbox = {}
-        last_y1s = {}
-        mainHeader = ''
-        subHeader = ''
-        matched_header_line_norm = heading_to_search
-        break_collecting = False
-        heading_norm = normalize_text(heading_to_search)
-        paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
+        for heading_to_searchDict, paths in listofHeaderstoMarkup:
+            heading_to_search = heading_to_searchDict['text']
+            heading_to_searchPageNum = heading_to_searchDict['page']
 
-        for page_num in range(heading_to_searchPageNum,len(doc)):
-            if page_num in toc_pages:
-              continue
-            if break_collecting:
-                break
-            page=doc[page_num]
-            page_height = page.rect.height
-            blocks = page.get_text("dict")["blocks"]
+            # Initialize variables
+            headertoContinue1 = False
+            headertoContinue2 = False
+            matched_header_line = None
+            done = False
+            collecting = False
+            collected_lines = []
+            page_highlights = {}
+            current_bbox = {}
+            last_y1s = {}
+            mainHeader = ''
+            subHeader = ''
+            matched_header_line_norm = heading_to_search
+            break_collecting = False
+            heading_norm = normalize_text(heading_to_search)
+            paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
 
-            for block in blocks:
+            for page_num in range(heading_to_searchPageNum,len(doc)):
+                if page_num in toc_pages:
+                    continue
                 if break_collecting:
                     break
+                page=doc[page_num]
+                page_height = page.rect.height
+                blocks = page.get_text("dict")["blocks"]
 
-                lines = block.get("lines", [])
-                i = 0
-                while i < len(lines):
+                for block in blocks:
                     if break_collecting:
                         break
 
-                    spans = lines[i].get("spans", [])
-                    if not spans:
-                        i += 1
-                        continue
-
-                    y0 = spans[0]["bbox"][1]
-                    y1 = spans[0]["bbox"][3]
-                    if y0 < top_margin or y1 > (page_height - bottom_margin):
-                        i += 1
-                        continue
-
-                    line_text = get_spaced_text_from_spans(spans).lower()
-                    line_text_norm = normalize_text(line_text)
-
-                    # Combine with next line if available
-                    if i + 1 < len(lines):
-                        next_spans = lines[i + 1].get("spans", [])
-                        next_line_text = get_spaced_text_from_spans(next_spans).lower()
-                        combined_line_norm = normalize_text(line_text + " " + next_line_text)
-                    else:
-                        combined_line_norm = line_text_norm
-
-                    # Check if we should continue processing
-                    if combined_line_norm and combined_line_norm in paths[0]:
-                        
-                        headertoContinue1 = combined_line_norm
-                    if combined_line_norm and combined_line_norm in paths[-2]:
-                    
-                        headertoContinue2 = combined_line_norm
-                    if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                      stringtowrite='Not to be billed'
-                    else:
-                      stringtowrite='To be billed'
-                    # Optimized header matching
-                    existsfull = (
-                        ( combined_line_norm in allchildrenheaders_set or
-                        combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
-                    )
-
-                    # New word-based matching
-                    current_line_words = set(combined_line_norm.split())
-                    heading_words = set(heading_norm.split())
-                    all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
-
-                    substring_match = (
-                        heading_norm in combined_line_norm or
-                        combined_line_norm in heading_norm or
-                        all_words_match  # Include the new word-based matching
-                    )
-                    # substring_match = (
-                    #     heading_norm in combined_line_norm or
-                    #     combined_line_norm in heading_norm
-                    # )
-
-                    if (substring_match and existsfull and not collecting and
-                        len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
-
-                        # Check header conditions more efficiently
-                        header_spans = [
-                            span for span in spans
-                            if (is_header(span, most_common_font_size, most_common_color, most_common_font)
-                                # and span['size'] >= subsubheaderFontSize
-                                and span['size'] < mainHeaderFontSize)
-                        ]
-                        if header_spans:
-                            collecting = True
-                            matched_header_font_size = max(span["size"] for span in header_spans)
-
-                            collected_lines.append(line_text)
-                            valid_spans = [span for span in spans if span.get("bbox")]
-
-                            if valid_spans:
-                                x0s = [span["bbox"][0] for span in valid_spans]
-                                x1s = [span["bbox"][2] for span in valid_spans]
-                                y0s = [span["bbox"][1] for span in valid_spans]
-                                y1s = [span["bbox"][3] for span in valid_spans]
-
-                                header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-
-                                if page_num in current_bbox:
-                                    cb = current_bbox[page_num]
-                                    current_bbox[page_num] = [
-                                        min(cb[0], header_bbox[0]),
-                                        min(cb[1], header_bbox[1]),
-                                        max(cb[2], header_bbox[2]),
-                                        max(cb[3], header_bbox[3])
-                                    ]
-                                else:
-                                    current_bbox[page_num] = header_bbox
-                                last_y1s[page_num] = header_bbox[3]
-                                x0, y0, x1, y1 = header_bbox
-
-                                zoom = 200
-                                left = int(x0)
-                                top = int(y0)
-                                zoom_str = f"{zoom},{left},{top}"
-                                pageNumberFound = page_num + 1
-
-                              # Build the query parameters
-                                params = {
-                                    'pdfLink': pdf_path,  # Your PDF link
-                                    'keyword': heading_to_search,  # Your keyword (could be a string or list)
-                                }
-
-                                # URL encode each parameter
-                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-
-                                # Construct the final encoded link
-                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-
-                                # Correctly construct the final URL with page and zoom
-                                final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+                    lines = block.get("lines", [])
+                    i = 0
+                    while i < len(lines):
+                        if break_collecting:
+                            break
 
-                                # Get current date and time
-                                now = datetime.now()
+                        spans = lines[i].get("spans", [])
+                        if not spans:
+                            i += 1
+                            continue
 
-                                # Format the output
-                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
-                                # Optionally, add the URL to a DataFrame
+                        y0 = spans[0]["bbox"][1]
+                        y1 = spans[0]["bbox"][3]
+                        if y0 < top_margin or y1 > (page_height - bottom_margin):
+                            i += 1
+                            continue
 
+                        line_text = get_spaced_text_from_spans(spans).lower()
+                        line_text_norm = normalize_text(line_text)
 
-                                data_entry = {
-                                        "NBSLink": final_url,
-                                        "Subject": heading_to_search,
-                                        "Page": str(pageNumberFound),
-                                        "Author": "ADR",
-                                        "Creation Date": formatted_time,
-                                        "Layer": "Initial",
-                                        "Code": stringtowrite,
-                                        "head above 1":  paths[-2],
-                                        "head above 2":  paths[0],
-                                        "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
-                                    }
-                                data_list_JSON.append(data_entry)
+                        # Combine with next line if available
+                        if i + 1 < len(lines):
+                            next_spans = lines[i + 1].get("spans", [])
+                            next_line_text = get_spaced_text_from_spans(next_spans).lower()
+                            combined_line_norm = normalize_text(line_text + " " + next_line_text)
+                        else:
+                            combined_line_norm = line_text_norm
 
-                                # Convert list to JSON
-                                json_output = json.dumps(data_list_JSON, indent=4)
+                        # Check if we should continue processing
+                        if combined_line_norm and combined_line_norm in paths[0]:
+                            
+                            headertoContinue1 = combined_line_norm
+                        if combined_line_norm and combined_line_norm in paths[-2]:
+                        
+                            headertoContinue2 = combined_line_norm
+                        if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                            stringtowrite='Not to be billed'
+                        else:
+                            stringtowrite='To be billed'
+                        # Optimized header matching
+                        existsfull = (
+                            ( combined_line_norm in allchildrenheaders_set or
+                            combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
+                        )
 
-                                i += 2
-                                continue
-                    else:
-                        if (substring_match and not collecting and
-                            len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
+                        # New word-based matching
+                        current_line_words = set(combined_line_norm.split())
+                        heading_words = set(heading_norm.split())
+                        all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
 
-                            # Calculate word match percentage
-                            word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
+                        substring_match = (
+                            heading_norm in combined_line_norm or
+                            combined_line_norm in heading_norm or
+                            all_words_match  # Include the new word-based matching
+                        )
+                        # substring_match = (
+                        #     heading_norm in combined_line_norm or
+                        #     combined_line_norm in heading_norm
+                        # )
 
-                            # Check if at least 70% of header words exist in this line
-                            meets_word_threshold = word_match_percent >= 100
+                        if (substring_match and existsfull and not collecting and
+                            len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
 
-                            # Check header conditions (including word threshold)
+                            # Check header conditions more efficiently
                             header_spans = [
                                 span for span in spans
                                 if (is_header(span, most_common_font_size, most_common_color, most_common_font)
                                     # and span['size'] >= subsubheaderFontSize
                                     and span['size'] < mainHeaderFontSize)
                             ]
-
-                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+                            if header_spans:
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
-                             
+
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -947,16 +864,16 @@ def extract_section_under_header(pdf_path):
                                         ]
                                     else:
                                         current_bbox[page_num] = header_bbox
-
                                     last_y1s[page_num] = header_bbox[3]
                                     x0, y0, x1, y1 = header_bbox
+
                                     zoom = 200
                                     left = int(x0)
                                     top = int(y0)
                                     zoom_str = f"{zoom},{left},{top}"
                                     pageNumberFound = page_num + 1
 
-                                  # Build the query parameters
+                                # Build the query parameters
                                     params = {
                                         'pdfLink': pdf_path,  # Your PDF link
                                         'keyword': heading_to_search,  # Your keyword (could be a string or list)
@@ -994,94 +911,191 @@ def extract_section_under_header(pdf_path):
                                     data_list_JSON.append(data_entry)
 
                                     # Convert list to JSON
-                                    json_output = json.dumps(data_list_JSON, indent=4)
+                                    # json_output = json.dumps(data_list_JSON, indent=4)
 
-                
                                     i += 2
                                     continue
-                    if collecting:
-                        norm_line = normalize_text(line_text)
-
-                        # Optimized URL check
-                        if url_pattern.match(norm_line):
-                            line_is_header = False
                         else:
-                            line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
+                            if (substring_match and not collecting and
+                                len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
 
-                        if line_is_header:
-                            header_font_size = max(span["size"] for span in spans)
-                            is_probably_real_header = (
-                                header_font_size >= matched_header_font_size and
-                                is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
-                                len(line_text.strip()) > 2
-                            )
+                                # Calculate word match percentage
+                                word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
 
-                            if (norm_line != matched_header_line_norm and
-                                norm_line != heading_norm and
-                                is_probably_real_header):
-                                if line_text not in heading_norm:
-                                  collecting = False
-                                  done = True
-                                  headertoContinue1 = False
-                                  headertoContinue2=False
-                                  for page_num, bbox in current_bbox.items():
-                                      bbox[3] = last_y1s.get(page_num, bbox[3])
-                                      page_highlights[page_num] = bbox
-                                  highlight_boxes(docHighlights, page_highlights,stringtowrite)
+                                # Check if at least 70% of header words exist in this line
+                                meets_word_threshold = word_match_percent >= 100
 
-                                  break_collecting = True
-                                  break
+                                # Check header conditions (including word threshold)
+                                header_spans = [
+                                    span for span in spans
+                                    if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+                                        # and span['size'] >= subsubheaderFontSize
+                                        and span['size'] < mainHeaderFontSize)
+                                ]
 
-                        if break_collecting:
-                            break
+                                if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+                                    collecting = True
+                                    matched_header_font_size = max(span["size"] for span in header_spans)
+                                
+                                    collected_lines.append(line_text)
+                                    valid_spans = [span for span in spans if span.get("bbox")]
 
-                        collected_lines.append(line_text)
-                        valid_spans = [span for span in spans if span.get("bbox")]
-                        if valid_spans:
-                            x0s = [span["bbox"][0] for span in valid_spans]
-                            x1s = [span["bbox"][2] for span in valid_spans]
-                            y0s = [span["bbox"][1] for span in valid_spans]
-                            y1s = [span["bbox"][3] for span in valid_spans]
+                                    if valid_spans:
+                                        x0s = [span["bbox"][0] for span in valid_spans]
+                                        x1s = [span["bbox"][2] for span in valid_spans]
+                                        y0s = [span["bbox"][1] for span in valid_spans]
+                                        y1s = [span["bbox"][3] for span in valid_spans]
 
-                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+                                        header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
 
-                            if page_num in current_bbox:
-                                cb = current_bbox[page_num]
-                                current_bbox[page_num] = [
-                                    min(cb[0], line_bbox[0]),
-                                    min(cb[1], line_bbox[1]),
-                                    max(cb[2], line_bbox[2]),
-                                    max(cb[3], line_bbox[3])
-                                ]
+                                        if page_num in current_bbox:
+                                            cb = current_bbox[page_num]
+                                            current_bbox[page_num] = [
+                                                min(cb[0], header_bbox[0]),
+                                                min(cb[1], header_bbox[1]),
+                                                max(cb[2], header_bbox[2]),
+                                                max(cb[3], header_bbox[3])
+                                            ]
+                                        else:
+                                            current_bbox[page_num] = header_bbox
+
+                                        last_y1s[page_num] = header_bbox[3]
+                                        x0, y0, x1, y1 = header_bbox
+                                        zoom = 200
+                                        left = int(x0)
+                                        top = int(y0)
+                                        zoom_str = f"{zoom},{left},{top}"
+                                        pageNumberFound = page_num + 1
+
+                                    # Build the query parameters
+                                        params = {
+                                            'pdfLink': pdf_path,  # Your PDF link
+                                            'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                        }
+
+                                        # URL encode each parameter
+                                        encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                        # Construct the final encoded link
+                                        encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                        # Correctly construct the final URL with page and zoom
+                                        final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+                                        # Get current date and time
+                                        now = datetime.now()
+
+                                        # Format the output
+                                        formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                        # Optionally, add the URL to a DataFrame
+
+
+                                        data_entry = {
+                                                "NBSLink": final_url,
+                                                "Subject": heading_to_search,
+                                                "Page": str(pageNumberFound),
+                                                "Author": "ADR",
+                                                "Creation Date": formatted_time,
+                                                "Layer": "Initial",
+                                                "Code": stringtowrite,
+                                                "head above 1":  paths[-2],
+                                                "head above 2":  paths[0],
+                                                "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
+                                            }
+                                        data_list_JSON.append(data_entry)
+
+                                        # Convert list to JSON
+                                        # json_output = json.dumps(data_list_JSON, indent=4)
+
+                    
+                                        i += 2
+                                        continue
+                        if collecting:
+                            norm_line = normalize_text(line_text)
+
+                            # Optimized URL check
+                            if url_pattern.match(norm_line):
+                                line_is_header = False
                             else:
-                                current_bbox[page_num] = line_bbox
+                                line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
 
-                            last_y1s[page_num] = line_bbox[3]
-                    i += 1
+                            if line_is_header:
+                                header_font_size = max(span["size"] for span in spans)
+                                is_probably_real_header = (
+                                    header_font_size >= matched_header_font_size and
+                                    is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
+                                    len(line_text.strip()) > 2
+                                )
 
-        if not done:
-            for page_num, bbox in current_bbox.items():
-                bbox[3] = last_y1s.get(page_num, bbox[3])
-                page_highlights[page_num] = bbox
-            if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                stringtowrite='Not to be billed'
-            else:
-                stringtowrite='To be billed'
-            highlight_boxes(docHighlights, page_highlights,stringtowrite)
+                                if (norm_line != matched_header_line_norm and
+                                    norm_line != heading_norm and
+                                    is_probably_real_header):
+                                    if line_text not in heading_norm:
+                                        collecting = False
+                                        done = True
+                                        headertoContinue1 = False
+                                        headertoContinue2=False
+                                        for page_num, bbox in current_bbox.items():
+                                            bbox[3] = last_y1s.get(page_num, bbox[3])
+                                            page_highlights[page_num] = bbox
+                                        highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+                                        break_collecting = True
+                                        break
 
-    # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
-    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
-    metadata = dbxTeam.sharing_get_shared_link_metadata(pdf_path)
-    dbPath = '/TSA JOBS/ADR Test/FIND/'
-    pdf_bytes = BytesIO()
-    docHighlights.save(pdf_bytes)
-    pdflink = tsadropboxretrieval.uploadanyFile(doc=docHighlights, path=dbPath, pdfname=filename)
-    json_output=changepdflinks(json_output,pdflink)
-    return pdf_bytes.getvalue(), docHighlights , json_output
+                            if break_collecting:
+                                break
+
+                            collected_lines.append(line_text)
+                            valid_spans = [span for span in spans if span.get("bbox")]
+                            if valid_spans:
+                                x0s = [span["bbox"][0] for span in valid_spans]
+                                x1s = [span["bbox"][2] for span in valid_spans]
+                                y0s = [span["bbox"][1] for span in valid_spans]
+                                y1s = [span["bbox"][3] for span in valid_spans]
+
+                                line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
 
+                                if page_num in current_bbox:
+                                    cb = current_bbox[page_num]
+                                    current_bbox[page_num] = [
+                                        min(cb[0], line_bbox[0]),
+                                        min(cb[1], line_bbox[1]),
+                                        max(cb[2], line_bbox[2]),
+                                        max(cb[3], line_bbox[3])
+                                    ]
+                                else:
+                                    current_bbox[page_num] = line_bbox
 
+                                last_y1s[page_num] = line_bbox[3]
+                        i += 1
 
+            if not done:
+                for page_num, bbox in current_bbox.items():
+                    bbox[3] = last_y1s.get(page_num, bbox[3])
+                    page_highlights[page_num] = bbox
+                if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                    stringtowrite='Not to be billed'
+                else:
+                    stringtowrite='To be billed'
+                highlight_boxes(docHighlights, page_highlights,stringtowrite)
 
+        docarray.append(docHighlights)
+        jsons.append(data_list_JSON)
+    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+    dbPath = '/TSA JOBS/ADR Test/FIND/'
+    jsonCombined=[] 
+    for i in range(len(arrayofPDFS)):
+        singlepdf=arrayofPDFS[i]
+        
+        metadata = dbxTeam.sharing_get_shared_link_metadata(singlepdf) 
+        pdf_bytes = BytesIO()
+        docHighlights.save(pdf_bytes)
+        pdflink = tsadropboxretrieval.uploadanyFile(doc=docarray[i], path=dbPath, pdfname=filenames[i])
+        json_output1=changepdflinks(jsons[i],pdflink)
+        jsonCombined.extend(json_output1)
+    combined_json_str = json.dumps(jsonCombined, indent=1)
+    return pdf_bytes.getvalue(), docHighlights , combined_json_str
 
 ########################################################################################################################################################
 ########################################################################################################################################################