Update InitialMarkups.py

InitialMarkups.py

@@ -8,8 +8,11 @@ Original file is located at
 """
 baselink='https://findconsole-initialmarkups.hf.space/view-pdf?'
 
-  
+newlink='https://findconsole-initialmarkups.hf.space/view-highlight?'
 
+    
+from urllib.parse import urlparse, unquote
+import os
 from io import BytesIO
 import re
 import requests
@@ -26,6 +29,13 @@ from collections import defaultdict, Counter
 import difflib
 from fuzzywuzzy import fuzz
 
+def filteredJsons(pdf_path,filteredjsonsfromrawan):
+    # for heading in subjects:
+    extract_section_under_headerRawan (pdf_path=pdf_path,listofheadingsfromrawan=filteredjsonsfromrawan)
+
+        
+
+    
 def get_regular_font_size_and_color(doc):
     font_sizes = []
     colors = []
@@ -203,6 +213,7 @@ def extract_headers(doc, toc_pages, most_common_font_size, most_common_color, mo
 
     # Get the smallest font size among valid ones
     smallest_font_size = min(valid_font_sizes) if valid_font_sizes else None
+    
     return headers, top_3_font_sizes, smallest_font_size, spans
 
 def is_numbered(text):
@@ -606,6 +617,10 @@ def extract_section_under_header(pdf_path):
     bottom_margin = 50
     headertoContinue1 = False
     headertoContinue2=False
+    
+    parsed_url = urlparse(pdf_path)
+    filename = os.path.basename(parsed_url.path)
+    filename = unquote(filename)  # decode URL-encoded characters
 
     # Optimized URL handling
     if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
@@ -845,7 +860,8 @@ def extract_section_under_header(pdf_path):
                                         "Layer": "Initial",
                                         "Code": stringtowrite,
                                         "head above 1":  paths[-2],
-                                        "head above 2":  paths[0]
+                                        "head above 2":  paths[0],
+                                        "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                     }
                                 data_list_JSON.append(data_entry)
 
@@ -941,7 +957,8 @@ def extract_section_under_header(pdf_path):
                                             "Layer": "Initial",
                                             "Code": stringtowrite,
                                             "head above 1":  paths[-2],
-                                            "head above 2":  paths[0]
+                                            "head above 2":  paths[0],
+                                            "MC Connnection": 'Go to ' +  paths[0].strip().split()[0] +'/'+ heading_to_search.strip().split()[0] + ' in '+ filename
                                         }
                                     data_list_JSON.append(data_entry)
 
@@ -1028,3 +1045,391 @@ def extract_section_under_header(pdf_path):
     docHighlights.save(pdf_bytes)
     print('JSONN',json_output)
     return pdf_bytes.getvalue(), docHighlights , json_output
+
+
+
+        
+def extract_section_under_headerRawan(pdf_path,headingjson,pagenum=0,incomingheader=0):
+    top_margin = 70
+    bottom_margin = 50
+    # Optimized URL handling
+    if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
+        pdf_path = pdf_path.replace('dl=0', 'dl=1')
+
+    # Cache frequently used values
+    response = requests.get(pdf_path)
+    pdf_content = BytesIO(response.content)
+    if not pdf_content:
+        raise ValueError("No valid PDF content found.")
+
+    doc = fitz.open(stream=pdf_content, filetype="pdf")
+    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
+    most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
+
+    # Precompute regex patterns
+    dot_pattern = re.compile(r'\.{3,}')
+    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+
+    def get_toc_page_numbers(doc, max_pages_to_check=15):
+        toc_pages = []
+        for page_num in range(min(len(doc), max_pages_to_check)):
+            page = doc.load_page(page_num)
+            blocks = page.get_text("dict")["blocks"]
+
+            dot_line_count = 0
+            for block in blocks:
+                for line in block.get("lines", []):
+                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
+                    if dot_pattern.search(line_text):
+                        dot_line_count += 1
+
+            if dot_line_count >= 3:
+                toc_pages.append(page_num)
+
+        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
+
+    toc_pages = get_toc_page_numbers(doc)
+
+    headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
+        doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
+    )
+    
+    listofheadingsfromrawan=[]
+    if type(headingjson) == str:
+        listofheadingsfromrawan.append(headingjson)
+        headingjson=[headingjson]
+    else:
+        for item in headingjson:
+            listofheadingsfromrawan.append(normalize_text(item['Subject']))
+    print('hereeeeeeeeeeeeeee0',listofheadingsfromrawan)
+    # Precompute all children headers once
+    allchildrenheaders = listofheadingsfromrawan
+    print('hereeeeeeeeeeeeeee00',allchildrenheaders)
+    allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
+    
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+    data_list_JSON = []
+
+    if len(top_3_font_sizes)==3:
+        mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
+    elif len(top_3_font_sizes)==2:
+        mainHeaderFontSize= top_3_font_sizes[0]
+        subHeaderFontSize= top_3_font_sizes[1]
+        subsubheaderFontSize= top_3_font_sizes[1]
+
+    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
+
+    # Preload all pages to avoid repeated loading
+    # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
+    newjsonList=[]
+    for heading_to_searchDict in headingjson:
+        if type(heading_to_searchDict) == str:
+            heading_to_search = heading_to_searchDict
+            heading_to_searchPageNum = pagenum
+        else:
+            heading_to_search = heading_to_searchDict['Subject']
+            heading_to_searchPageNum = int(heading_to_searchDict['Page'])-1
+            incomingheader = heading_to_searchDict['head above 1']
+            
+        print('hereeeeeeeeeeeeeee0',heading_to_searchPageNum)
+        done = False
+        collecting = False
+        collected_lines = []
+        page_highlights = {}
+        current_bbox = {}
+        last_y1s = {}
+        mainHeader = ''
+        subHeader = ''
+        matched_header_line_norm = heading_to_search
+        break_collecting = False
+        heading_norm = normalize_text(heading_to_search)
+
+        for page_num in range(heading_to_searchPageNum,len(doc)):
+            print('hereeeeeeeeeeeeeee1')
+            if page_num in toc_pages:
+              continue
+            if break_collecting:
+                break
+            page=doc[page_num]
+            page_height = page.rect.height
+            blocks = page.get_text("dict")["blocks"]
+
+            for block in blocks:
+                if break_collecting:
+                    break
+
+                lines = block.get("lines", [])
+                i = 0
+                while i < len(lines):
+                    if break_collecting:
+                        break
+
+                    spans = lines[i].get("spans", [])
+                    if not spans:
+                        i += 1
+                        continue
+
+                    y0 = spans[0]["bbox"][1]
+                    y1 = spans[0]["bbox"][3]
+                    if y0 < top_margin or y1 > (page_height - bottom_margin):
+                        i += 1
+                        continue
+
+                    line_text = get_spaced_text_from_spans(spans).lower()
+                    line_text_norm = normalize_text(line_text)
+
+                    # Combine with next line if available
+                    if i + 1 < len(lines):
+                        next_spans = lines[i + 1].get("spans", [])
+                        next_line_text = get_spaced_text_from_spans(next_spans).lower()
+                        combined_line_norm = normalize_text(line_text + " " + next_line_text)
+                    else:
+                        combined_line_norm = line_text_norm
+                    # Optimized header matching
+                    existsfull = (
+                        ( combined_line_norm in allchildrenheaders_set or
+                        combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
+                    )
+
+                    # New word-based matching
+                    current_line_words = set(combined_line_norm.split())
+                    heading_words = set(heading_norm.split())
+                    all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
+
+                    substring_match = (
+                        heading_norm in combined_line_norm or
+                        combined_line_norm in heading_norm or
+                        all_words_match  # Include the new word-based matching
+                    )
+
+                    if (substring_match and existsfull and not collecting and
+                        len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
+
+                        # Check header conditions more efficiently
+                        header_spans = [
+                            span for span in spans
+                            if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+                                # and span['size'] >= subsubheaderFontSize
+                                and span['size'] < mainHeaderFontSize)
+                        ]
+                        if header_spans:
+                            collecting = True
+                            matched_header_font_size = max(span["size"] for span in header_spans)
+                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
+
+                            collected_lines.append(line_text)
+                            valid_spans = [span for span in spans if span.get("bbox")]
+
+                            if valid_spans:
+                                x0s = [span["bbox"][0] for span in valid_spans]
+                                x1s = [span["bbox"][2] for span in valid_spans]
+                                y0s = [span["bbox"][1] for span in valid_spans]
+                                y1s = [span["bbox"][3] for span in valid_spans]
+
+                                header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                if page_num in current_bbox:
+                                    cb = current_bbox[page_num]
+                                    current_bbox[page_num] = [
+                                        min(cb[0], header_bbox[0]),
+                                        min(cb[1], header_bbox[1]),
+                                        max(cb[2], header_bbox[2]),
+                                        max(cb[3], header_bbox[3])
+                                    ]
+                                else:
+                                    current_bbox[page_num] = header_bbox
+                                last_y1s[page_num] = header_bbox[3]
+                                x0, y0, x1, y1 = header_bbox
+
+                                zoom = 200
+                                left = int(x0)
+                                top = int(y0)
+                                zoom_str = f"{zoom},{left},{top}"
+                                pageNumberFound = page_num + 1
+
+                              # Build the query parameters
+                                params = {
+                                    'pdfLink': pdf_path,  # Your PDF link
+                                    'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                }
+
+                                # URL encode each parameter
+                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                # Construct the final encoded link
+                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                # Correctly construct the final URL with page and zoom
+                                final_url = f"{newlink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+                                # Get current date and time
+                                now = datetime.now()
+
+                                # Format the output
+                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                # Optionally, add the URL to a DataFrame
+                                new_url= final_url
+                                if type(heading_to_searchDict) != str:
+                                    heading_to_searchDict['NBSLink']=new_url
+                                newjsonList.append(heading_to_searchDict)
+                                print("Final URL:", final_url)
+                                i += 2
+                                continue
+                    else:
+                        if (substring_match and not collecting and
+                            len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
+
+                            # Calculate word match percentage
+                            word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
+
+                            # Check if at least 70% of header words exist in this line
+                            meets_word_threshold = word_match_percent >= 100
+
+                            # Check header conditions (including word threshold)
+                            header_spans = [
+                                span for span in spans
+                                if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+                                    # and span['size'] >= subsubheaderFontSize
+                                    and span['size'] < mainHeaderFontSize)
+                            ]
+
+                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+                                collecting = True
+                                matched_header_font_size = max(span["size"] for span in header_spans)
+                                print(f"📥 Start collecting after header: {combined_line_norm} "
+                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
+
+                                collected_lines.append(line_text)
+                                valid_spans = [span for span in spans if span.get("bbox")]
+
+                                if valid_spans:
+                                    x0s = [span["bbox"][0] for span in valid_spans]
+                                    x1s = [span["bbox"][2] for span in valid_spans]
+                                    y0s = [span["bbox"][1] for span in valid_spans]
+                                    y1s = [span["bbox"][3] for span in valid_spans]
+
+                                    header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                    if page_num in current_bbox:
+                                        cb = current_bbox[page_num]
+                                        current_bbox[page_num] = [
+                                            min(cb[0], header_bbox[0]),
+                                            min(cb[1], header_bbox[1]),
+                                            max(cb[2], header_bbox[2]),
+                                            max(cb[3], header_bbox[3])
+                                        ]
+                                    else:
+                                        current_bbox[page_num] = header_bbox
+
+                                    last_y1s[page_num] = header_bbox[3]
+                                    x0, y0, x1, y1 = header_bbox
+                                    zoom = 200
+                                    left = int(x0)
+                                    top = int(y0)
+                                    zoom_str = f"{zoom},{left},{top}"
+                                    pageNumberFound = page_num + 1
+
+                                  # Build the query parameters
+                                    params = {
+                                        'pdfLink': pdf_path,  # Your PDF link
+                                        'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                    }
+
+                                    # URL encode each parameter
+                                    encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                    # Construct the final encoded link
+                                    encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                    # Correctly construct the final URL with page and zoom
+                                    final_url = f"{newlink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+                                    new_url= final_url
+                                    if type(heading_to_searchDict) != str:
+                                        heading_to_searchDict['NBSLink']=new_url
+                                    newjsonList.append(heading_to_searchDict)                                
+                                    print("Final URL:", final_url)
+                                    i += 2
+                                    continue
+                    if collecting:
+                        norm_line = normalize_text(line_text)
+
+                        # Optimized URL check
+                        if url_pattern.match(norm_line):
+                            line_is_header = False
+                        else:
+                            line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
+
+                        if line_is_header:
+                            header_font_size = max(span["size"] for span in spans)
+                            is_probably_real_header = (
+                                header_font_size >= matched_header_font_size and
+                                is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
+                                len(line_text.strip()) > 2
+                            )
+
+                            if (norm_line != matched_header_line_norm and
+                                norm_line != heading_norm and
+                                is_probably_real_header):
+                                if line_text not in heading_norm:
+                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
+                                  collecting = False
+                                  done = True
+                                  headertoContinue1 = False
+                                  headertoContinue2=False
+                                  for page_num, bbox in current_bbox.items():
+                                      bbox[3] = last_y1s.get(page_num, bbox[3])
+                                      page_highlights[page_num] = bbox
+                                  
+                                  if  'installation' in incomingheader or 'execution' in incomingheader or 'miscellaneous items' in incomingheader :
+                                    stringtowrite='Not to be billed'
+                                  else:
+                                    stringtowrite='To be billed'
+                                  highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+                                  break_collecting = True
+                                  break
+
+                        if break_collecting:
+                            break
+
+                        collected_lines.append(line_text)
+                        valid_spans = [span for span in spans if span.get("bbox")]
+                        if valid_spans:
+                            x0s = [span["bbox"][0] for span in valid_spans]
+                            x1s = [span["bbox"][2] for span in valid_spans]
+                            y0s = [span["bbox"][1] for span in valid_spans]
+                            y1s = [span["bbox"][3] for span in valid_spans]
+
+                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                            if page_num in current_bbox:
+                                cb = current_bbox[page_num]
+                                current_bbox[page_num] = [
+                                    min(cb[0], line_bbox[0]),
+                                    min(cb[1], line_bbox[1]),
+                                    max(cb[2], line_bbox[2]),
+                                    max(cb[3], line_bbox[3])
+                                ]
+                            else:
+                                current_bbox[page_num] = line_bbox
+
+                            last_y1s[page_num] = line_bbox[3]
+                    i += 1
+
+        if not done:
+            for page_num, bbox in current_bbox.items():
+                bbox[3] = last_y1s.get(page_num, bbox[3])
+                page_highlights[page_num] = bbox
+            if  'installation' in incomingheader or 'execution' in incomingheader or 'miscellaneous items' in incomingheader :
+                stringtowrite='Not to be billed'
+            else:
+                stringtowrite='To be billed'            
+            highlight_boxes(docHighlights, page_highlights,stringtowrite)
+
+    # docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
+
+    pdf_bytes = BytesIO()
+    docHighlights.save(pdf_bytes)
+    return pdf_bytes.getvalue(), docHighlights , newjsonList
+
+