Update InitialMarkups.py

InitialMarkups.py

@@ -6,12 +6,14 @@ Automatically generated by Colab.
 Original file is located at
     https://colab.research.google.com/drive/12XfVkmKmN3oVjHhLVE0_GgkftgArFEK2
 """
-baselink='https://findconsole-initialmarkups.hf.space/view-pdf?'
-  
-newlink='https://findconsole-initialmarkups.hf.space/view-highlight?'
-tobebilledonlyLink='https://findconsole-initialmarkups.hf.space/view-pdf-tobebilled?'
+baselink='https://adr.trevorsadd.co.uk/api/view-pdf?'
+
+newlink='https://adr.trevorsadd.co.uk/api/view-highlight?'
+tobebilledonlyLink='https://adr.trevorsadd.co.uk/api/view-pdf-tobebilled?'
+
+    
+
 
-import tsadropboxretrieval
 from urllib.parse import urlparse, unquote
 import os
 from io import BytesIO
@@ -29,13 +31,11 @@ from datetime import datetime
 from collections import defaultdict, Counter
 import difflib
 from fuzzywuzzy import fuzz
+import copy
+import tsadropboxretrieval
 
-def filteredJsons(pdf_path,filteredjsonsfromrawan):
-    # for heading in subjects:
-    extract_section_under_headerRawan (pdf_path=pdf_path,listofheadingsfromrawan=filteredjsonsfromrawan)
-
+    
         
-
 def changepdflinks(data_list_JSON, pdflink):
     print('henaaaa weee',data_list_JSON)
     
@@ -75,7 +75,6 @@ def changepdflinks(data_list_JSON, pdflink):
 
     return data_list_JSON
 
- 
 def get_regular_font_size_and_color(doc):
     font_sizes = []
     colors = []
@@ -239,7 +238,7 @@ def extract_headers(doc, toc_pages, most_common_font_size, most_common_color, mo
     font_size_counts = Counter(font_sizes)
 
       # Filter font sizes that appear at least 3 times
-    valid_font_sizes = [size for size, count in font_size_counts.items() if count >= 3]
+    valid_font_sizes = [size for size, count in font_size_counts.items() if count >= 1]
 
     # Sort in descending order
     valid_font_sizes_sorted = sorted(valid_font_sizes, reverse=True)
@@ -649,8 +648,6 @@ def same_start_word(s1, s2):
     return False
 
 
-
-
 def extract_section_under_header(multiplePDF_Paths):
     filenames=[]
     keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
@@ -1096,7 +1093,7 @@ def extract_section_under_header(multiplePDF_Paths):
         jsonCombined.extend(json_output1)
     combined_json_str = json.dumps(jsonCombined, indent=1)
     return pdf_bytes.getvalue(), docHighlights , combined_json_str
-
+ 
 ########################################################################################################################################################
 ########################################################################################################################################################
 
@@ -1105,6 +1102,10 @@ def extract_section_under_header(multiplePDF_Paths):
 def extract_section_under_header_tobebilledOnly(pdf_path):
     Alltexttobebilled=''
     alltextWithoutNotbilled=''
+    # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
+    
+
+    
     top_margin = 70
     bottom_margin = 50
     headertoContinue1 = False
@@ -1125,7 +1126,12 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         raise ValueError("No valid PDF content found.")
 
     doc = fitz.open(stream=pdf_content, filetype="pdf")
-    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
+    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")   
+    parsed_url = urlparse(pdf_path)
+    filename = os.path.basename(parsed_url.path)
+    filename = unquote(filename)  # decode URL-encoded characters
+
+
     most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
 
     # Precompute regex patterns
@@ -1145,7 +1151,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                     if dot_pattern.search(line_text):
                         dot_line_count += 1
 
-            if dot_line_count >= 3:
+            if dot_line_count >= 1:
                 toc_pages.append(page_num)
 
         return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
@@ -1163,7 +1169,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
     allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
 
-    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2",'BodyText'])
     dictionaryNBS={}
     data_list_JSON = []
 
@@ -1182,7 +1188,9 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
     for heading_to_searchDict, paths in listofHeaderstoMarkup:
         heading_to_search = heading_to_searchDict['text']
         heading_to_searchPageNum = heading_to_searchDict['page']
-
+        
+        
+        
         # Initialize variables
         headertoContinue1 = False
         headertoContinue2 = False
@@ -1199,7 +1207,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
         break_collecting = False
         heading_norm = normalize_text(heading_to_search)
         paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
-
+        
         for page_num in range(heading_to_searchPageNum,len(doc)):
             if page_num in toc_pages:
               continue
@@ -1240,7 +1248,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                         combined_line_norm = normalize_text(line_text + " " + next_line_text)
                     else:
                         combined_line_norm = line_text_norm
-
+   
                     # Check if we should continue processing
                     if combined_line_norm and combined_line_norm in paths[0]:
                         
@@ -1249,6 +1257,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                     
                         headertoContinue2 = combined_line_norm
                     if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                    # if any(word in paths[-2].lower() for word in keywordstoSkip):
                       stringtowrite='Not to be billed'
                     else:
                       stringtowrite='To be billed'
@@ -1289,7 +1298,8 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                             collecting = True
                             matched_header_font_size = max(span["size"] for span in header_spans)
                             Alltexttobebilled+= ' '+ combined_line_norm
-                            collected_lines.append(line_text)
+                            
+                            # collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
 
                             if valid_spans:
@@ -1352,7 +1362,9 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                         "Code": stringtowrite,
                                         "head above 1":  paths[-2],
                                         "head above 2":  paths[0],
+                                        "BodyText": collected_lines,
                                         "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
+
                                     }
                                 data_list_JSON.append(data_entry)
 
@@ -1383,6 +1395,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                 collecting = True
                                 matched_header_font_size = max(span["size"] for span in header_spans)
                                 Alltexttobebilled+= ' '+ combined_line_norm
+                                
                                 collected_lines.append(line_text)
                                 valid_spans = [span for span in spans if span.get("bbox")]
 
@@ -1446,6 +1459,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                             "Code": stringtowrite,
                                             "head above 1":  paths[-2],
                                             "head above 2":  paths[0],
+                                            "BodyText": collected_lines,
                                             "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                         }
                                     data_list_JSON.append(data_entry)
@@ -1458,7 +1472,7 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                                     continue
                     if collecting:
                         norm_line = normalize_text(line_text)
-
+                        
                         # Optimized URL check
                         if url_pattern.match(norm_line):
                             line_is_header = False
@@ -1492,7 +1506,9 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
                         if break_collecting:
                             break
 
+
                         collected_lines.append(line_text)
+                        
                         valid_spans = [span for span in spans if span.get("bbox")]
                         if valid_spans:
                             x0s = [span["bbox"][0] for span in valid_spans]
@@ -1528,12 +1544,20 @@ def extract_section_under_header_tobebilledOnly(pdf_path):
 
     # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
 
+    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+    metadata = dbxTeam.sharing_get_shared_link_metadata(pdf_path)
+    dbPath = '/TSA JOBS/ADR Test/FIND/'
     pdf_bytes = BytesIO()
     docHighlights.save(pdf_bytes)
-    return pdf_bytes.getvalue(), docHighlights , json_output , Alltexttobebilled , alltextWithoutNotbilled
+    pdflink = tsadropboxretrieval.uploadanyFile(doc=docHighlights, path=dbPath, pdfname=filename)
+    json_output=changepdflinks(json_output,pdflink)
+    return pdf_bytes.getvalue(), docHighlights , json_output , Alltexttobebilled , alltextWithoutNotbilled , filename
 
 
 def extract_section_under_header_tobebilled2(pdf_path):
+    # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
+
+    keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
     top_margin = 70
     bottom_margin = 50
     headertoContinue1 = False
@@ -1574,7 +1598,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
                     if dot_pattern.search(line_text):
                         dot_line_count += 1
 
-            if dot_line_count >= 3:
+            if dot_line_count >= 1:
                 toc_pages.append(page_num)
 
         return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
@@ -1592,7 +1616,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
     allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
     allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
 
-    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
     dictionaryNBS={}
     data_list_JSON = []
     currentgroupname=''
@@ -1683,7 +1707,11 @@ def extract_section_under_header_tobebilled2(pdf_path):
                     if combined_line_norm and combined_line_norm in paths[-2]:
                     
                         headertoContinue2 = combined_line_norm
-                    if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                    # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                    last_path = paths[-2].lower()
+                    # if any(word in paths[-2].lower() for word in keywordstoSkip):
+                    # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() or 'workmanship' in paths[-2].lower() or 'testing' in paths[-2].lower() or 'labeling' in paths[-2].lower():
+                    if any(keyword in last_path for keyword in keywords):
                       stringtowrite='Not to be billed'
                     else:
                       stringtowrite='To be billed'
@@ -1723,13 +1751,13 @@ def extract_section_under_header_tobebilled2(pdf_path):
                                 # and span['size'] >= subsubheaderFontSize
                                 and span['size'] < mainHeaderFontSize)
                         ]
-                        if header_spans:
+                        if header_spans and stringtowrite.startswith('To'):
                             collecting = True
                             # if stringtowrite=='To be billed':
                             #     Alltexttobebilled+='\n'
                             matched_header_font_size = max(span["size"] for span in header_spans)
 
-                            collected_lines.append(line_text)
+                            # collected_lines.append(line_text)
                             valid_spans = [span for span in spans if span.get("bbox")]
 
                             if valid_spans:
@@ -1792,6 +1820,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
                                         "Code": stringtowrite,
                                         "head above 1":  paths[-2],
                                         "head above 2":  paths[0],
+                                        "BodyText":collected_lines,
                                         "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                     }
                                 data_list_JSON.append(data_entry)
@@ -1819,7 +1848,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
                                     and span['size'] < mainHeaderFontSize)
                             ]
 
-                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ) and stringtowrite.startswith('To'):
                                 collecting = True
                                 if stringtowrite=='To be billed':
                                     Alltexttobebilled+='\n'
@@ -1890,6 +1919,7 @@ def extract_section_under_header_tobebilled2(pdf_path):
                                             "Code": stringtowrite,
                                             "head above 1":  paths[-2],
                                             "head above 2":  paths[0],
+                                            "BodyText":collected_lines,
                                             "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                         }
                                     data_list_JSON.append(data_entry)
@@ -1972,15 +2002,22 @@ def extract_section_under_header_tobebilled2(pdf_path):
 
     # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
 
+    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+    metadata = dbxTeam.sharing_get_shared_link_metadata(pdf_path)
+    dbPath = '/TSA JOBS/ADR Test/FIND/'
     pdf_bytes = BytesIO()
     docHighlights.save(pdf_bytes)
+    pdflink = tsadropboxretrieval.uploadanyFile(doc=docHighlights, path=dbPath, pdfname=filename)
+    json_output=changepdflinks(json_output,pdflink)
+    return pdf_bytes.getvalue(), docHighlights , json_output, Alltexttobebilled , filename
+
 
-    return pdf_bytes.getvalue(), docHighlights , json_output, Alltexttobebilled
 
 
 
 
-def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
+
+def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths):
     # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
     filenames=[]
     keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
@@ -1989,6 +2026,8 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
     arrayofPDFS=multiplePDF_Paths.split(',')
     print(multiplePDF_Paths)
     print(arrayofPDFS)
+    docarray=[]
+    jsons=[]
     df = pd.DataFrame(columns=["PDF Name","NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
     for pdf_path in arrayofPDFS:
         headertoContinue1 = False
@@ -2029,7 +2068,7 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
                         if dot_pattern.search(line_text):
                             dot_line_count += 1
     
-                if dot_line_count >= 3:
+                if dot_line_count >= 1:
                     toc_pages.append(page_num)
     
             return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
@@ -2050,6 +2089,7 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
         # df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
         dictionaryNBS={}
         data_list_JSON = []
+        json_output=[]
         currentgroupname=''
         if len(top_3_font_sizes)==3:
             mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
@@ -2258,7 +2298,8 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
                                     data_list_JSON.append(data_entry)
     
                                     # Convert list to JSON
-                                    json_output = json.dumps(data_list_JSON, indent=4)
+                                    # json_output = [data_list_JSON]
+                                    # json_output = json.dumps(data_list_JSON, indent=4)
     
                                     i += 2
                                     continue
@@ -2358,7 +2399,8 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
                                         data_list_JSON.append(data_entry)
     
                                         # Convert list to JSON
-                                        json_output = json.dumps(data_list_JSON, indent=4)
+                                        # json_output = [data_list_JSON]
+                                        # json_output = json.dumps(data_list_JSON, indent=4)
     
                     
                                         i += 2
@@ -2432,489 +2474,24 @@ def extract_section_under_header_tobebilled2marthe(multiplePDF_Paths):
                 else:
                     stringtowrite='To be billed'
                 highlight_boxes(docHighlights, page_highlights,stringtowrite)
-    
-        # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
-    
-    pdf_bytes = BytesIO()
-    docHighlights.save(pdf_bytes)
-    print(filenames)
-    return pdf_bytes.getvalue(), docHighlights , json_output, Alltexttobebilled , filenames
-
-
-
-def extract_section_under_header_tobebilledMultiplePDFSmarthe(multiplePDF_Paths):
-    # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
-    filenames=[]
-    keywords = {'installation', 'execution', 'miscellaneous items', 'workmanship', 'testing', 'labeling'}
-    top_margin = 70
-    bottom_margin = 50
-    arrayofPDFS=multiplePDF_Paths.split(',')
-    print(multiplePDF_Paths)
-    print(arrayofPDFS)
-    docarray=[]
-    jsons=[]
-    df = pd.DataFrame(columns=["PDF Name","NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
-    for pdf_path in arrayofPDFS:
-        headertoContinue1 = False
-        headertoContinue2=False
-        Alltexttobebilled=''
-        parsed_url = urlparse(pdf_path)
-        filename = os.path.basename(parsed_url.path)
-        filename = unquote(filename)  # decode URL-encoded characters
-        print(filename)
-        filenames.append(filename)
-        # Optimized URL handling
-        if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
-            pdf_path = pdf_path.replace('dl=0', 'dl=1')
-    
-        # Cache frequently used values
-        response = requests.get(pdf_path)
-        pdf_content = BytesIO(response.content)
-        if not pdf_content:
-            raise ValueError("No valid PDF content found.")
-    
-        doc = fitz.open(stream=pdf_content, filetype="pdf")
-        docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
-        most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
-    
-        # Precompute regex patterns
-        dot_pattern = re.compile(r'\.{3,}')
-        url_pattern = re.compile(r'https?://\S+|www\.\S+')
-    
-        def get_toc_page_numbers(doc, max_pages_to_check=15):
-            toc_pages = []
-            for page_num in range(min(len(doc), max_pages_to_check)):
-                page = doc.load_page(page_num)
-                blocks = page.get_text("dict")["blocks"]
-    
-                dot_line_count = 0
-                for block in blocks:
-                    for line in block.get("lines", []):
-                        line_text = get_spaced_text_from_spans(line["spans"]).strip()
-                        if dot_pattern.search(line_text):
-                            dot_line_count += 1
-    
-                if dot_line_count >= 3:
-                    toc_pages.append(page_num)
-    
-            return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
-    
-        toc_pages = get_toc_page_numbers(doc)
-    
-        headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
-            doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
-        )
-    
-        hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
-        listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
-        
-        # Precompute all children headers once
-        allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
-        allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
-    
-        # df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2","BodyText"])
-        dictionaryNBS={}
-        data_list_JSON = []
-        currentgroupname=''
-        if len(top_3_font_sizes)==3:
-            mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
-        elif len(top_3_font_sizes)==2:
-            mainHeaderFontSize= top_3_font_sizes[0]
-            subHeaderFontSize= top_3_font_sizes[1]
-            subsubheaderFontSize= top_3_font_sizes[1]
-    
+        docarray.append(docHighlights)
+        jsons.append(data_list_JSON)
+    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
+    dbPath = '/TSA JOBS/ADR Test/FIND/'
+    jsonCombined=[]
+    for i in range(len(arrayofPDFS)):
+        singlepdf=arrayofPDFS[i]
         
-    
-        # Preload all pages to avoid repeated loading
-        # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
-    
-        for heading_to_searchDict, paths in listofHeaderstoMarkup:
-            heading_to_search = heading_to_searchDict['text']
-            heading_to_searchPageNum = heading_to_searchDict['page']
-    
-            # Initialize variables
-            headertoContinue1 = False
-            headertoContinue2 = False
-            matched_header_line = None
-            done = False
-            collecting = False
-            collected_lines = []
-            page_highlights = {}
-            current_bbox = {}
-            last_y1s = {}
-            mainHeader = ''
-            subHeader = ''
-            matched_header_line_norm = heading_to_search
-            break_collecting = False
-            heading_norm = normalize_text(heading_to_search)
-            paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
-            for page_num in range(heading_to_searchPageNum,len(doc)):
-                # print(heading_to_search)
-                if paths[0].strip().lower() != currentgroupname.strip().lower():
-                    Alltexttobebilled+= paths[0] +'\n'
-                    currentgroupname=paths[0]
-                    # print(paths[0])
-    
-                    
-                if page_num in toc_pages:
-                  continue
-                if break_collecting:
-                    break
-                page=doc[page_num]
-                page_height = page.rect.height
-                blocks = page.get_text("dict")["blocks"]
-    
-                for block in blocks:
-                    if break_collecting:
-                        break
-    
-                    lines = block.get("lines", [])
-                    i = 0
-                    while i < len(lines):
-                        if break_collecting:
-                            break
-    
-                        spans = lines[i].get("spans", [])
-                        if not spans:
-                            i += 1
-                            continue
-    
-                        y0 = spans[0]["bbox"][1]
-                        y1 = spans[0]["bbox"][3]
-                        if y0 < top_margin or y1 > (page_height - bottom_margin):
-                            i += 1
-                            continue
-    
-                        line_text = get_spaced_text_from_spans(spans).lower()
-                        line_text_norm = normalize_text(line_text)
-    
-                        # Combine with next line if available
-                        if i + 1 < len(lines):
-                            next_spans = lines[i + 1].get("spans", [])
-                            next_line_text = get_spaced_text_from_spans(next_spans).lower()
-                            combined_line_norm = normalize_text(line_text + " " + next_line_text)
-                        else:
-                            combined_line_norm = line_text_norm
-    
-                        # Check if we should continue processing
-                        if combined_line_norm and combined_line_norm in paths[0]:
-                            
-                            headertoContinue1 = combined_line_norm
-                        if combined_line_norm and combined_line_norm in paths[-2]:
-                        
-                            headertoContinue2 = combined_line_norm
-                        # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                        last_path = paths[-2].lower()
-                        # if any(word in paths[-2].lower() for word in keywordstoSkip):
-                        # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() or 'workmanship' in paths[-2].lower() or 'testing' in paths[-2].lower() or 'labeling' in paths[-2].lower():
-                        if any(keyword in last_path for keyword in keywords):
-                          stringtowrite='Not to be billed'
-                        else:
-                          stringtowrite='To be billed'
-                        if stringtowrite=='To be billed':
-                            # Alltexttobebilled+= combined_line_norm  #################################################
-                            if matched_header_line_norm in combined_line_norm:
-                                Alltexttobebilled+='\n'
-                            Alltexttobebilled+= ' '+combined_line_norm 
-                        # Optimized header matching
-                        existsfull = (
-                            ( combined_line_norm in allchildrenheaders_set or
-                            combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
-                        )
-    
-                        # New word-based matching
-                        current_line_words = set(combined_line_norm.split())
-                        heading_words = set(heading_norm.split())
-                        all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
-    
-                        substring_match = (
-                            heading_norm in combined_line_norm or
-                            combined_line_norm in heading_norm or
-                            all_words_match  # Include the new word-based matching
-                        )
-                        # substring_match = (
-                        #     heading_norm in combined_line_norm or
-                        #     combined_line_norm in heading_norm
-                        # )
-    
-                        if (substring_match and existsfull and not collecting and
-                            len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
-    
-                            # Check header conditions more efficiently
-                            header_spans = [
-                                span for span in spans
-                                if (is_header(span, most_common_font_size, most_common_color, most_common_font)
-                                    # and span['size'] >= subsubheaderFontSize
-                                    and span['size'] < mainHeaderFontSize)
-                            ]
-                            if header_spans and stringtowrite.startswith('To'):
-                                collecting = True
-                                # if stringtowrite=='To be billed':
-                                #     Alltexttobebilled+='\n'
-                                matched_header_font_size = max(span["size"] for span in header_spans)
-    
-                                # collected_lines.append(line_text)
-                                valid_spans = [span for span in spans if span.get("bbox")]
-    
-                                if valid_spans:
-                                    x0s = [span["bbox"][0] for span in valid_spans]
-                                    x1s = [span["bbox"][2] for span in valid_spans]
-                                    y0s = [span["bbox"][1] for span in valid_spans]
-                                    y1s = [span["bbox"][3] for span in valid_spans]
-    
-                                    header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
-                                    if page_num in current_bbox:
-                                        cb = current_bbox[page_num]
-                                        current_bbox[page_num] = [
-                                            min(cb[0], header_bbox[0]),
-                                            min(cb[1], header_bbox[1]),
-                                            max(cb[2], header_bbox[2]),
-                                            max(cb[3], header_bbox[3])
-                                        ]
-                                    else:
-                                        current_bbox[page_num] = header_bbox
-                                    last_y1s[page_num] = header_bbox[3]
-                                    x0, y0, x1, y1 = header_bbox
-    
-                                    zoom = 200
-                                    left = int(x0)
-                                    top = int(y0)
-                                    zoom_str = f"{zoom},{left},{top}"
-                                    pageNumberFound = page_num + 1
-    
-                                  # Build the query parameters
-                                    params = {
-                                        'pdfLink': pdf_path,  # Your PDF link
-                                        'keyword': heading_to_search,  # Your keyword (could be a string or list)
-                                    }
-    
-                                    # URL encode each parameter
-                                    encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-    
-                                    # Construct the final encoded link
-                                    encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-    
-                                    # Correctly construct the final URL with page and zoom
-                                    final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-    
-                                    # Get current date and time
-                                    now = datetime.now()
-    
-                                    # Format the output
-                                    formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
-                                    # Optionally, add the URL to a DataFrame
-    
-    
-                                    data_entry = {
-                                            "PDF Name":filename,
-                                            "NBSLink": final_url,
-                                            "Subject": heading_to_search,
-                                            "Page": str(pageNumberFound),
-                                            "Author": "ADR",
-                                            "Creation Date": formatted_time,
-                                            "Layer": "Initial",
-                                            "Code": stringtowrite,
-                                            "head above 1":  paths[-2],
-                                            "head above 2":  paths[0],
-                                            "BodyText":collected_lines,
-                                            "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
-                                        }
-                                    data_list_JSON.append(data_entry)
-    
-                                    # Convert list to JSON
-                                    json_output = json.dumps(data_list_JSON, indent=4)
-    
-                                    i += 2
-                                    continue
-                        else:
-                            if (substring_match and not collecting and
-                                len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
-    
-                                # Calculate word match percentage
-                                word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
-    
-                                # Check if at least 70% of header words exist in this line
-                                meets_word_threshold = word_match_percent >= 100
-    
-                                # Check header conditions (including word threshold)
-                                header_spans = [
-                                    span for span in spans
-                                    if (is_header(span, most_common_font_size, most_common_color, most_common_font)
-                                        # and span['size'] >= subsubheaderFontSize
-                                        and span['size'] < mainHeaderFontSize)
-                                ]
-    
-                                if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ) and stringtowrite.startswith('To'):
-                                    collecting = True
-                                    if stringtowrite=='To be billed':
-                                        Alltexttobebilled+='\n'
-                                    # if stringtowrite=='To be billed':
-                                    #     Alltexttobebilled+= ' '+ combined_line_norm
-                                    matched_header_font_size = max(span["size"] for span in header_spans)
-                                 
-                                    collected_lines.append(line_text)
-                                    valid_spans = [span for span in spans if span.get("bbox")]
-    
-                                    if valid_spans:
-                                        x0s = [span["bbox"][0] for span in valid_spans]
-                                        x1s = [span["bbox"][2] for span in valid_spans]
-                                        y0s = [span["bbox"][1] for span in valid_spans]
-                                        y1s = [span["bbox"][3] for span in valid_spans]
-    
-                                        header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
-                                        if page_num in current_bbox:
-                                            cb = current_bbox[page_num]
-                                            current_bbox[page_num] = [
-                                                min(cb[0], header_bbox[0]),
-                                                min(cb[1], header_bbox[1]),
-                                                max(cb[2], header_bbox[2]),
-                                                max(cb[3], header_bbox[3])
-                                            ]
-                                        else:
-                                            current_bbox[page_num] = header_bbox
-    
-                                        last_y1s[page_num] = header_bbox[3]
-                                        x0, y0, x1, y1 = header_bbox
-                                        zoom = 200
-                                        left = int(x0)
-                                        top = int(y0)
-                                        zoom_str = f"{zoom},{left},{top}"
-                                        pageNumberFound = page_num + 1
-    
-                                      # Build the query parameters
-                                        params = {
-                                            'pdfLink': pdf_path,  # Your PDF link
-                                            'keyword': heading_to_search,  # Your keyword (could be a string or list)
-                                        }
-    
-                                        # URL encode each parameter
-                                        encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-    
-                                        # Construct the final encoded link
-                                        encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-    
-                                        # Correctly construct the final URL with page and zoom
-                                        final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-    
-                                        # Get current date and time
-                                        now = datetime.now()
-    
-                                        # Format the output
-                                        formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
-                                        # Optionally, add the URL to a DataFrame
-    
-    
-                                        data_entry = {
-                                                "PDF Name":filename,
-                                                "NBSLink": final_url,
-                                                "Subject": heading_to_search,
-                                                "Page": str(pageNumberFound),
-                                                "Author": "ADR",
-                                                "Creation Date": formatted_time,
-                                                "Layer": "Initial",
-                                                "Code": stringtowrite,
-                                                "head above 1":  paths[-2],
-                                                "head above 2":  paths[0],
-                                                "BodyText":collected_lines,
-                                                "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
-                                            }
-                                        data_list_JSON.append(data_entry)
-    
-                                        # Convert list to JSON
-                                        json_output = json.dumps(data_list_JSON, indent=4)
-    
-                    
-                                        i += 2
-                                        continue
-                        if collecting:
-                            norm_line = normalize_text(line_text)
-    
-                            # Optimized URL check
-                            if url_pattern.match(norm_line):
-                                line_is_header = False
-                            else:
-                                line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
-    
-                            if line_is_header:
-                                header_font_size = max(span["size"] for span in spans)
-                                is_probably_real_header = (
-                                    header_font_size >= matched_header_font_size and
-                                    is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
-                                    len(line_text.strip()) > 2
-                                )
-    
-                                if (norm_line != matched_header_line_norm and
-                                    norm_line != heading_norm and
-                                    is_probably_real_header):
-                                    if line_text not in heading_norm:
-                                      collecting = False
-                                      done = True
-                                      headertoContinue1 = False
-                                      headertoContinue2=False
-                                      for page_num, bbox in current_bbox.items():
-                                          bbox[3] = last_y1s.get(page_num, bbox[3])
-                                          page_highlights[page_num] = bbox
-                                      highlight_boxes(docHighlights, page_highlights,stringtowrite)
-    
-                                      break_collecting = True
-                                      break
-    
-                            if break_collecting:
-                                break
-    
-                            collected_lines.append(line_text)
-                            valid_spans = [span for span in spans if span.get("bbox")]
-                            if valid_spans:
-                                x0s = [span["bbox"][0] for span in valid_spans]
-                                x1s = [span["bbox"][2] for span in valid_spans]
-                                y0s = [span["bbox"][1] for span in valid_spans]
-                                y1s = [span["bbox"][3] for span in valid_spans]
-    
-                                line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
-    
-                                if page_num in current_bbox:
-                                    cb = current_bbox[page_num]
-                                    current_bbox[page_num] = [
-                                        min(cb[0], line_bbox[0]),
-                                        min(cb[1], line_bbox[1]),
-                                        max(cb[2], line_bbox[2]),
-                                        max(cb[3], line_bbox[3])
-                                    ]
-                                else:
-                                    current_bbox[page_num] = line_bbox
-    
-                                last_y1s[page_num] = line_bbox[3]
-                        i += 1
-    
-            if not done:
-                for page_num, bbox in current_bbox.items():
-                    bbox[3] = last_y1s.get(page_num, bbox[3])
-                    page_highlights[page_num] = bbox
-                if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
-                    stringtowrite='Not to be billed'
-                else:
-                    stringtowrite='To be billed'
-                highlight_boxes(docHighlights, page_highlights,stringtowrite)
-        docarray.append(docHighlights)
-        jsons.append(json_output)
-        # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
-    
-    dbxTeam = tsadropboxretrieval.ADR_Access_DropboxTeam('user')
-    dbPath = '/TSA JOBS/ADR Test/FIND/'
-    jsonCombined=[]
-    for i in range(len(arrayofPDFS)):
-        singlepdf=arrayofPDFS[i]
-        
-        metadata = dbxTeam.sharing_get_shared_link_metadata(singlepdf)
+        metadata = dbxTeam.sharing_get_shared_link_metadata(singlepdf) 
         pdf_bytes = BytesIO()
         docHighlights.save(pdf_bytes)
         pdflink = tsadropboxretrieval.uploadanyFile(doc=docarray[i], path=dbPath, pdfname=filenames[i])
+        # json_copy = copy.deepcopy(jsons[i])
+        # Update links for this JSON
+        # json_output1 = changepdflinks(json_copy, pdflink)
         json_output1=changepdflinks(jsons[i],pdflink)
         jsonCombined.extend(json_output1)
     combined_json_str = json.dumps(jsonCombined, indent=1)
     print(combined_json_str)
     return pdf_bytes.getvalue(), docHighlights , combined_json_str, Alltexttobebilled , filenames
-
-   
+