Update app.py

app.py

@@ -802,7 +802,38 @@ def openPDF(pdf_path):
 #     return out
 
 
+
+def process_document_in_chunks(
+    lengthofDoc,
+    pdf_path,
+    LLM_prompt,
+    model,
+    chunk_size=15,
+    
+):
+    total_pages = lengthofDoc
+    all_results = []
+
+    for start in range(0, total_pages, chunk_size):
+        end = start + chunk_size
+
+        logger.info(f"Processing pages {start + 1} → {min(end, total_pages)}")
+
+        result = identify_headers_with_openrouterNEWW(
+            pdf_path=pdf_path,
+            model=model,
+            LLM_prompt=LLM_prompt,
+            pages_to_check=(start, end)
+        )
+
+        if result:
+            all_results.extend(result)
+
+    return all_results
+
+
 def identify_headers_with_openrouterNEWW(pdf_path, model,LLM_prompt, pages_to_check=None, top_margin=0, bottom_margin=0):
+    
     """Ask an LLM (OpenRouter) to identify headers in the document.
     Returns a list of dicts: {text, page, suggested_level, confidence}.
     The function sends plain page-line strings to the LLM (including page numbers)
@@ -824,41 +855,150 @@ def identify_headers_with_openrouterNEWW(pdf_path, model,LLM_prompt, pages_to_ch
     lines_for_prompt = []
     # pgestoRun=20
     # logger.info(f"TOC pages to skip: {toc_pages}")
+    # logger.info(f"Total pages in document: {len(doc)}")
     logger.info(f"Total pages in document: {len(doc)}")
     
     # Collect text lines from pages (skip TOC pages)
     total_lines = 0
-    for pno in range(len(doc)):
+
+    ArrayofTextWithFormat = []
+    total_pages = len(doc)
+
+    if pages_to_check is None:
+        start_page = 0
+        end_page = min(15, total_pages)
+    else:
+        start_page, end_page = pages_to_check
+        end_page = min(end_page, total_pages)  # 🔑 CRITICAL LINE
+
+    for pno in range(start_page, end_page):
+        page = doc.load_page(pno)    
+    # # Collect text lines from pages (skip TOC pages)
+    # total_lines = 0
+    # for pno in range(len(doc)):
         # if pages_to_check and pno not in pages_to_check:
         #     continue
         # if pno in toc_pages:
         #     logger.debug(f"Skipping TOC page {pno}")
         #     continue
         
-        page = doc.load_page(pno)
+        # page = doc.load_page(pno)
+    #     page_height = page.rect.height
+    #     lines_on_page = 0
+    #     text_dict = page.get_text("dict")
+    #     lines = []
+    #     # y_tolerance = 0.2  # tweak if needed (1–3 usually works)
+    #     for block in text_dict["blocks"]:
+    #         if block["type"] != 0:
+    #             continue
+    #         for line in block["lines"]:
+    #             for span in line["spans"]:
+    #                 text = span["text"].strip()
+    #                 if not text:
+    #                     continue
+    #                 if text:
+    #                     # prefix with page for easier mapping back
+    #                     lines_for_prompt.append(f"PAGE {pno+1}: {text}")
+    #                     lines_on_page += 1
+
+    #     if lines_on_page > 0:
+    #         logger.debug(f"Page {pno}: collected {lines_on_page} lines")
+    #     total_lines += lines_on_page
+    
+    # logger.info(f"Total lines collected for LLM: {total_lines}")
         page_height = page.rect.height
         lines_on_page = 0
         text_dict = page.get_text("dict")
         lines = []
-        # y_tolerance = 0.2  # tweak if needed (1–3 usually works)
+        y_tolerance = 0.5  # tweak if needed (1–3 usually works)
+        
         for block in text_dict["blocks"]:
             if block["type"] != 0:
                 continue
             for line in block["lines"]:
                 for span in line["spans"]:
                     text = span["text"].strip()
-                    if not text:
+                    if not text:  # Skip empty text
                         continue
-                    if text:
-                        # prefix with page for easier mapping back
-                        lines_for_prompt.append(f"PAGE {pno+1}: {text}")
-                        lines_on_page += 1
+                    
+                    # Extract all formatting attributes
+                    font = span.get('font')
+                    size = span.get('size')
+                    color = span.get('color')
+                    flags = span.get('flags', 0)
+                    bbox = span.get("bbox", (0, 0, 0, 0))
+                    x0, y0, x1, y1 = bbox
+                  
+                    # Create text format dictionary
+                    text_format = {
+                        'Font': font,
+                        'Size': size,
+                        'Flags': flags,
+                        'Color': color,
+                        'Text': text,
+                        'BBox': bbox,
+                        'Page': pno + 1
+                    }
+                    
+                    # Add to ArrayofTextWithFormat
+                    ArrayofTextWithFormat.append(text_format)
+                    
+                    # For line grouping (keeping your existing logic)
+                    matched = False
+                    for l in lines:
+                        if abs(l["y"] - y0) <= y_tolerance:
+                            l["spans"].append((x0, text, font, size, color, flags))
+                            matched = True
+                            break
+                    if not matched:
+                        lines.append({
+                            "y": y0,
+                            "spans": [(x0, text, font, size, color, flags)]
+                        })
+        
+        lines.sort(key=lambda l: l["y"])
 
+        # Join text inside each line with formatting info
+        final_lines = []
+        for l in lines:
+            l["spans"].sort(key=lambda s: s[0])  # left → right
+            
+            # Collect all text and formatting for this line
+            line_text = " ".join(text for _, text, _, _, _, _ in l["spans"])
+            
+            # Get dominant formatting for the line (based on first span)
+            if l["spans"]:
+                _, _, font, size, color, flags = l["spans"][0]
+                
+                # Store line with its formatting
+                line_with_format = {
+                    'text': line_text,
+                    'font': font,
+                    'size': size,
+                    'color': color,
+                    'flags': flags,
+                    'page': pno + 1,
+                    'y_position': l["y"]
+                }
+                final_lines.append(line_with_format)
+        
+        # Result
+        for line_data in final_lines:
+            line_text = line_data['text']
+            print(line_text)
+            
+            if line_text:
+                # Create a formatted string with text properties
+                format_info = f"Font: {line_data['font']}, Size: {line_data['size']}, Color: {line_data['color']}"
+                lines_for_prompt.append(f"PAGE {pno+1}: {line_text} [{format_info}]")
+                lines_on_page += 1
+        
         if lines_on_page > 0:
             logger.debug(f"Page {pno}: collected {lines_on_page} lines")
         total_lines += lines_on_page
-    
+
     logger.info(f"Total lines collected for LLM: {total_lines}")
+
     
     if not lines_for_prompt:
         logger.warning("No lines collected for prompt")
@@ -1599,7 +1739,521 @@ def extract_section_under_header_tobebilledMultiplePDFS(multiplePDF_Paths,model,
 
    
     return jsons,identified_headers
- 
+
+
+
+
+def testFunction(pdf_path, model,LLM_prompt):
+    Alltexttobebilled=''
+    alltextWithoutNotbilled=''
+    # keywordstoSkip=["installation", "execution", "miscellaneous items", "workmanship", "testing", "labeling"]
+    
+    headertoContinue1 = False
+    headertoContinue2=False
+    
+    parsed_url = urlparse(pdf_path)
+    filename = os.path.basename(parsed_url.path)
+    filename = unquote(filename)  # decode URL-encoded characters
+
+    # Optimized URL handling
+    if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
+        pdf_path = pdf_path.replace('dl=0', 'dl=1')
+
+    # Cache frequently used values
+    response = requests.get(pdf_path)
+    pdf_content = BytesIO(response.content)
+    if not pdf_content:
+        raise ValueError("No valid PDF content found.")
+
+    doc = fitz.open(stream=pdf_content, filetype="pdf")
+    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")   
+    parsed_url = urlparse(pdf_path)
+    filename = os.path.basename(parsed_url.path)
+    filename = unquote(filename)  # decode URL-encoded characters
+
+#### Get regular tex font size, style , color
+    most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
+
+    # Precompute regex patterns
+    dot_pattern = re.compile(r'\.{3,}')
+    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+    highlighted=[]
+    processed_subjects = set()  # Initialize at the top of testFunction
+    toc_pages = get_toc_page_numbers(doc)
+    identified_headers=process_document_in_chunks(len(doc), pdf_path,LLM_prompt, model)
+    # identified_headers = identify_headers_with_openrouterNEWW(doc, api_key='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')# ['text', fontsize, page number,y]
+
+    # with open("identified_headers.txt", "w", encoding="utf-8") as f:
+    #     json.dump(identified_headers, f, indent=4)
+    # with open("identified_headers.txt", "r", encoding="utf-8") as f:
+    #     identified_headers = json.load(f)
+    print(identified_headers)
+    allheaders_LLM=[]
+    for h in identified_headers:
+        if int(h["page"]) in toc_pages:
+            continue
+        if h['text']:
+            allheaders_LLM.append(h['text'])   
+    
+    headers_json=headers_with_location(doc,identified_headers)
+    headers=filter_headers_outside_toc(headers_json,toc_pages)
+    hierarchy=build_hierarchy_from_llm(headers)
+    # identify_headers_and_save_excel(hierarchy)
+    listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
+    allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
+    allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
+    # print('allchildrenheaders_set',allchildrenheaders_set)
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2",'BodyText'])
+    dictionaryNBS={}
+    data_list_JSON = []
+    for heading_to_searchDict,pathss in listofHeaderstoMarkup:
+        heading_to_search = heading_to_searchDict['text']
+        heading_to_searchPageNum = heading_to_searchDict['page']
+        paths=heading_to_searchDict['path']        
+        xloc=heading_to_searchDict['x']
+        yloc=heading_to_searchDict['y']
+        
+        # Initialize variables
+        headertoContinue1 = False
+        headertoContinue2 = False
+        matched_header_line = None
+        done = False
+        collecting = False
+        collected_lines = []
+        page_highlights = {}
+        current_bbox = {}
+        last_y1s = {}
+        mainHeader = ''
+        subHeader = ''
+        matched_header_line_norm = heading_to_search
+        break_collecting = False
+        heading_norm = normalize_text(heading_to_search)
+        paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
+        
+        for page_num in range(heading_to_searchPageNum,len(doc)):
+            if page_num in toc_pages:
+              continue
+            if break_collecting:
+                break
+            page=doc[page_num]
+            page_height = page.rect.height
+            blocks = page.get_text("dict")["blocks"]
+
+            for block in blocks:
+                if break_collecting:
+                    break
+
+                lines = block.get("lines", [])
+                i = 0
+                while i < len(lines):
+                    if break_collecting:
+                        break
+
+                    spans = lines[i].get("spans", [])
+                    if not spans:
+                        i += 1
+                        continue
+
+                    # y0 = spans[0]["bbox"][1]
+                    # y1 = spans[0]["bbox"][3]
+                    x0 = spans[0]["bbox"][0]  # left
+                    x1 = spans[0]["bbox"][2]  # right
+                    y0 = spans[0]["bbox"][1]  # top
+                    y1 = spans[0]["bbox"][3]  # bottom
+
+                    if y0 < top_margin or y1 > (page_height - bottom_margin):
+                        i += 1
+                        continue
+
+                    line_text = get_spaced_text_from_spans(spans).lower()
+                    line_text_norm = normalize_text(line_text)
+
+                    # Combine with next line if available
+                    if i + 1 < len(lines):
+                        next_spans = lines[i + 1].get("spans", [])
+                        next_line_text = get_spaced_text_from_spans(next_spans).lower()
+                        combined_line_norm = normalize_text(line_text + " " + next_line_text)
+                    else:
+                        combined_line_norm = line_text_norm
+   
+                    # Check if we should continue processing
+                    if combined_line_norm and combined_line_norm in paths[0]:
+                        
+                        headertoContinue1 = combined_line_norm
+                    if combined_line_norm and combined_line_norm in paths[-2]:
+                    
+                        headertoContinue2 = combined_line_norm
+                    # print('paths',paths)
+         
+                    # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+                    # if any(word in paths[-2].lower() for word in keywordstoSkip):
+                    #   stringtowrite='Not to be billed'
+                    # else:
+                    stringtowrite='To be billed'
+                    if stringtowrite!='To be billed':
+                        alltextWithoutNotbilled+= combined_line_norm #################################################
+                    # Optimized header matching
+                    existsfull = (
+                        ( combined_line_norm in allchildrenheaders_set or
+                        combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
+                    )
+                    # existsfull=False
+                    # if xloc==x0 and yloc ==y0:
+                    #     existsfull=True
+                    # New word-based matching
+                    current_line_words = set(combined_line_norm.split())
+                    heading_words = set(heading_norm.split())
+                    all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
+
+                    substring_match = (
+                        heading_norm in combined_line_norm or
+                        combined_line_norm in heading_norm or
+                        all_words_match  # Include the new word-based matching
+                    )
+                    # substring_match = (
+                    #     heading_norm in combined_line_norm or
+                    #     combined_line_norm in heading_norm
+                    # )
+
+                    if ( substring_match and existsfull and not collecting and
+                        len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
+
+                        # Check header conditions more efficiently
+                        # header_spans = [
+                        #     span for span in spans
+                        #     if (is_header(span, most_common_font_size, most_common_color, most_common_font) )
+                        #         # and span['size'] >= subsubheaderFontSize
+                        #         # and span['size'] < mainHeaderFontSize)
+                        # ]
+                        if stringtowrite.startswith('To'):
+                            collecting = True
+                            # matched_header_font_size = max(span["size"] for span in header_spans)
+                            Alltexttobebilled+= ' '+ combined_line_norm
+                            
+                            # collected_lines.append(line_text)
+                            valid_spans = [span for span in spans if span.get("bbox")]
+
+                            if valid_spans:
+                                x0s = [span["bbox"][0] for span in valid_spans]
+                                x1s = [span["bbox"][2] for span in valid_spans]
+                                y0s = [span["bbox"][1] for span in valid_spans]
+                                y1s = [span["bbox"][3] for span in valid_spans]
+
+                                header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                if page_num in current_bbox:
+                                    cb = current_bbox[page_num]
+                                    current_bbox[page_num] = [
+                                        min(cb[0], header_bbox[0]),
+                                        min(cb[1], header_bbox[1]),
+                                        max(cb[2], header_bbox[2]),
+                                        max(cb[3], header_bbox[3])
+                                    ]
+                                else:
+                                    current_bbox[page_num] = header_bbox
+                                last_y1s[page_num] = header_bbox[3]
+                                x0, y0, x1, y1 = header_bbox
+
+                                zoom = 200
+                                left = int(x0)
+                                top = int(y0)
+                                zoom_str = f"{zoom},{left},{top}"
+                                pageNumberFound = page_num + 1
+
+                              # Build the query parameters
+                                params = {
+                                    'pdfLink': pdf_path,  # Your PDF link
+                                    'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                }
+
+                                # URL encode each parameter
+                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                # Construct the final encoded link
+                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                # Correctly construct the final URL with page and zoom
+                                final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+                                # Get current date and time
+                                now = datetime.now()
+
+                                # Format the output
+                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                # Optionally, add the URL to a DataFrame
+
+
+                                # Create the data entry only if the subject is unique
+                                if heading_to_search not in processed_subjects:
+                                    data_entry = {
+                                        "NBSLink": zoom_str,
+                                        "Subject": heading_to_search,
+                                        "Page": str(pageNumberFound),
+                                        "Author": "ADR",
+                                        "Creation Date": formatted_time,
+                                        "Layer": "Initial",
+                                        "Code": stringtowrite,
+                                        "BodyText": collected_lines,
+                                        "MC Connnection": 'Go to ' + paths[0].strip().split()[0] + '/' + heading_to_search.strip().split()[0] + ' in ' + filename
+                                    }
+
+                                    # Dynamically add hierarchy paths
+                                    for i, path_text in enumerate(paths[:-1]):
+                                        data_entry[f"head above {i+1}"] = path_text
+
+                                    # Append to the list and mark this subject as processed
+                                    data_list_JSON.append(data_entry)
+                                    processed_subjects.add(heading_to_search)
+                                else:
+                                    print(f"Skipping duplicate data entry for Subject: {heading_to_search}")
+
+                                # Convert list to JSON
+                                json_output = json.dumps(data_list_JSON, indent=4)
+
+                                i += 1
+                                continue
+                    else:
+                        if (substring_match and not collecting and
+                            len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
+
+                            # Calculate word match percentage
+                            word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
+
+                            # Check if at least 70% of header words exist in this line
+                            meets_word_threshold = word_match_percent >= 100
+
+                            # Check header conditions (including word threshold)
+                            # header_spans = [
+                            #     span for span in spans
+                            #     if (is_header(span, most_common_font_size, most_common_color, most_common_font))
+                            #         # and span['size'] >= subsubheaderFontSize
+                            #         # and span['size'] < mainHeaderFontSize)
+                            # ]
+
+                            if  (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ) and stringtowrite.startswith('To'):
+                                collecting = True
+                                # matched_header_font_size = max(span["size"] for span in header_spans)
+                                Alltexttobebilled+= ' '+ combined_line_norm
+                                
+                                collected_lines.append(line_text)
+                                valid_spans = [span for span in spans if span.get("bbox")]
+
+                                if valid_spans:
+                                    x0s = [span["bbox"][0] for span in valid_spans]
+                                    x1s = [span["bbox"][2] for span in valid_spans]
+                                    y0s = [span["bbox"][1] for span in valid_spans]
+                                    y1s = [span["bbox"][3] for span in valid_spans]
+
+                                    header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                    if page_num in current_bbox:
+                                        cb = current_bbox[page_num]
+                                        current_bbox[page_num] = [
+                                            min(cb[0], header_bbox[0]),
+                                            min(cb[1], header_bbox[1]),
+                                            max(cb[2], header_bbox[2]),
+                                            max(cb[3], header_bbox[3])
+                                        ]
+                                    else:
+                                        current_bbox[page_num] = header_bbox
+
+                                    last_y1s[page_num] = header_bbox[3]
+                                    x0, y0, x1, y1 = header_bbox
+                                    zoom = 200
+                                    left = int(x0)
+                                    top = int(y0)
+                                    zoom_str = f"{zoom},{left},{top}"
+                                    pageNumberFound = page_num + 1
+
+                                  # Build the query parameters
+                                    params = {
+                                        'pdfLink': pdf_path,  # Your PDF link
+                                        'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                    }
+
+                                    # URL encode each parameter
+                                    encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                    # Construct the final encoded link
+                                    encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                    # Correctly construct the final URL with page and zoom
+                                    final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+                                    # Get current date and time
+                                    now = datetime.now()
+
+                                    # Format the output
+                                    formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                    # Optionally, add the URL to a DataFrame
+
+
+                                    # Create the data entry only if the subject is unique
+                                    if heading_to_search not in processed_subjects:
+                                        data_entry = {
+                                            "NBSLink": zoom_str,
+                                            "Subject": heading_to_search,
+                                            "Page": str(pageNumberFound),
+                                            "Author": "ADR",
+                                            "Creation Date": formatted_time,
+                                            "Layer": "Initial",
+                                            "Code": stringtowrite,
+                                            "BodyText": collected_lines,
+                                            "MC Connnection": 'Go to ' + paths[0].strip().split()[0] + '/' + heading_to_search.strip().split()[0] + ' in ' + filename
+                                        }
+
+                                        # Dynamically add hierarchy paths
+                                        for i, path_text in enumerate(paths[:-1]):
+                                            data_entry[f"head above {i+1}"] = path_text
+
+                                        # Append to the list and mark this subject as processed
+                                        data_list_JSON.append(data_entry)
+                                        processed_subjects.add(heading_to_search)
+                                    else:
+                                        print(f"Skipping duplicate data entry for Subject: {heading_to_search}")
+                                    # Convert list to JSON
+                                    json_output = json.dumps(data_list_JSON, indent=4)
+
+                
+                                    i += 2
+                                    continue
+                    if collecting:
+                        norm_line = normalize_text(line_text)
+                        def normalize(text):
+                            if isinstance(text, list):
+                                text = " ".join(text)
+                            return " ".join(text.lower().split())
+
+                        def is_similar(a, b, threshold=0.75):
+                            return SequenceMatcher(None, a, b).ratio() >= threshold
+                        # Optimized URL check
+                        if url_pattern.match(norm_line):
+                            line_is_header = False
+                        else:
+                            line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font,allheaders_LLM) for span in spans) 
+                            # def normalize(text): 
+                            #     return " ".join(text.lower().split()) 
+                            # line_text = " ".join(span["text"] for span in spans).strip() 
+                            # line_is_header = any( normalize(line_text) == normalize(header) for header in allheaders_LLM )
+
+
+                            # for line_text in lines:
+                            #     if collecting:
+                            #         # Join all spans into one line
+                            #         line_text = " ".join(span["text"] for span in spans).strip()
+                            #         norm_line = normalize(line_text)
+
+                            #         # Get max font size in this line
+                            #         max_font_size = max(span.get("size", 0) for span in spans)
+
+                            #         # Skip URLs
+                            #         if url_pattern.match(norm_line):
+                            #             line_is_header = False
+                            #         else:
+                            #             text_matches_header = any(
+                            #                 is_similar(norm_line, normalize(header))
+                            #                 if not isinstance(header, list)
+                            #                 else is_similar(norm_line, normalize(" ".join(header)))
+                            #                 for header in allheaders_LLM
+                            #             )
+
+                            #             # ✅ FINAL header condition
+                            #             line_is_header = text_matches_header and max_font_size > 11
+
+
+                        if line_is_header:
+                            header_font_size = max(span["size"] for span in spans)
+                            is_probably_real_header = (
+                                # header_font_size >= matched_header_font_size and
+                                # is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
+                                len(line_text.strip()) > 2
+                            )
+
+                            if (norm_line != matched_header_line_norm and
+                                norm_line != heading_norm and
+                                is_probably_real_header):
+                                if line_text not in heading_norm:
+                                  collecting = False
+                                  done = True
+                                  headertoContinue1 = False
+                                  headertoContinue2=False
+                                  for page_num, bbox in current_bbox.items():
+                                      bbox[3] = last_y1s.get(page_num, bbox[3])
+                                      page_highlights[page_num] = bbox
+                                      can_highlight=False
+                                      if [page_num,bbox] not in highlighted:
+                                        highlighted.append([page_num,bbox])
+                                        can_highlight=True
+                                  if can_highlight:
+                                    highlight_boxes(docHighlights, page_highlights,stringtowrite)
+                                  
+                                  break_collecting = True
+                                  
+                                  break
+
+                        if break_collecting:
+                            break
+
+
+                        collected_lines.append(line_text)
+                        
+                        valid_spans = [span for span in spans if span.get("bbox")]
+                        if valid_spans:
+                            x0s = [span["bbox"][0] for span in valid_spans]
+                            x1s = [span["bbox"][2] for span in valid_spans]
+                            y0s = [span["bbox"][1] for span in valid_spans]
+                            y1s = [span["bbox"][3] for span in valid_spans]
+
+                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                            if page_num in current_bbox:
+                                cb = current_bbox[page_num]
+                                current_bbox[page_num] = [
+                                    min(cb[0], line_bbox[0]),
+                                    min(cb[1], line_bbox[1]),
+                                    max(cb[2], line_bbox[2]),
+                                    max(cb[3], line_bbox[3])
+                                ]
+                            else:
+                                current_bbox[page_num] = line_bbox
+
+                            last_y1s[page_num] = line_bbox[3]
+                    i += 1
+
+        if not done:
+            for page_num, bbox in current_bbox.items():
+                bbox[3] = last_y1s.get(page_num, bbox[3])
+                page_highlights[page_num] = bbox
+            # if  'installation' in paths[-2].lower() or 'execution' in paths[-2].lower() or 'miscellaneous items' in paths[-2].lower() :
+            #     stringtowrite='Not to be billed'
+            # else:
+            stringtowrite='To be billed'
+            
+            highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+    print("Current working directory:", os.getcwd())
+    if data_list_JSON and not data_list_JSON[-1]["BodyText"] and collected_lines:
+        data_list_JSON[-1]["BodyText"] = collected_lines[1:] if len(collected_lines) > 0 else []
+# Final cleanup of the JSON data before returning
+    for entry in data_list_JSON:
+        # Check if BodyText exists and has content
+        if isinstance(entry.get("BodyText"), list) and len(entry["BodyText"]) > 0:
+            # Check if the first line of the body is essentially the same as the Subject
+            first_line = normalize_text(entry["BodyText"][0])
+            subject = normalize_text(entry["Subject"])
+            
+            # If they match or the subject is inside the first line, remove it
+            if subject in first_line or first_line in subject:
+                entry["BodyText"] = entry["BodyText"][1:]        
+    jsons.append(data_list_JSON)
+logger.info(f"Markups done! Uploading to dropbox")
+logger.info(f"Uploaded and Readyy!")
+
+
+return jsons,identified_headers
+
+
+
 def build_subject_body_map(jsons):
     subject_body = {}
 
@@ -1615,8 +2269,8 @@ def build_subject_body_map(jsons):
 
 def identify_headers_and_save_excel(pdf_path, model,LLM_prompt):
     try:
-        result = identify_headers_with_openrouterNEWW(pdf_path, model,LLM_prompt)
-        jsons = extract_section_under_header_tobebilledMultiplePDFS(pdf_path, model,result)
+        # result = identify_headers_with_openrouterNEWW(pdf_path, model,LLM_prompt)
+        jsons,result = testFunction(pdf_path, model,LLM_prompt)
         if not result:
             df = pd.DataFrame([{
                 "text": None,