Upload InitialMarkups.py

InitialMarkups.py

@@ -0,0 +1,978 @@
+# -*- coding: utf-8 -*-
+"""Copy of FindSpecsTrial(Retrieving+boundingBoxes)-InitialMarkups(ALL)_CleanedUp.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/12XfVkmKmN3oVjHhLVE0_GgkftgArFEK2
+"""
+
+pip install pymupdf
+
+pip install fuzzywuzzy
+
+from io import BytesIO
+import re
+import requests
+import pandas as pd
+import fitz  # PyMuPDF
+import re
+import urllib.parse
+import pandas as pd
+import math
+import random
+import json
+from datetime import datetime
+from collections import defaultdict, Counter
+import difflib
+from fuzzywuzzy import fuzz
+
+def get_regular_font_size_and_color(doc):
+    font_sizes = []
+    colors = []
+    fonts = []
+
+    # Loop through all pages
+    for page_num in range(len(doc)):
+        page = doc.load_page(page_num)
+        for span in page.get_text("dict")["blocks"]:
+            if "lines" in span:
+                for line in span["lines"]:
+                    for span in line["spans"]:
+                        font_sizes.append(span['size'])
+                        colors.append(span['color'])
+                        fonts.append(span['font'])
+
+    # Get the most common font size, color, and font
+    most_common_font_size = Counter(font_sizes).most_common(1)[0][0] if font_sizes else None
+    most_common_color = Counter(colors).most_common(1)[0][0] if colors else None
+    most_common_font = Counter(fonts).most_common(1)[0][0] if fonts else None
+
+    return most_common_font_size, most_common_color, most_common_font
+
+def normalize_text(text):
+    if text is None:
+        return ""
+    return re.sub(r'\s+', ' ', text.strip().lower())
+
+def get_spaced_text_from_spans(spans):
+    return normalize_text(" ".join(span["text"].strip() for span in spans))
+
+def is_header(span, most_common_font_size, most_common_color, most_common_font):
+    fontname = span.get("font", "").lower()
+    # is_italic = "italic" in fontname or "oblique" in fontname
+    is_bold = "bold" in fontname or span.get("bold", False)
+    return (
+        (
+            span["size"] > most_common_font_size or
+            span["font"].lower() != most_common_font.lower() or
+            is_bold
+        )
+    )
+
+def add_span_to_nearest_group(span_y, grouped_dict, pageNum=None, threshold=0.5):
+    for (p, y) in grouped_dict:
+        if pageNum is not None and p != pageNum:
+            continue
+        if abs(y - span_y) <= threshold:
+            return (p, y)
+    return (pageNum, span_y)
+
+def extract_headers(doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin):
+    print("Font baseline:", most_common_font_size, most_common_color, most_common_font)
+
+    grouped_headers = defaultdict(list)
+    spans = []
+    line_merge_threshold = 1.5  # Maximum vertical distance between lines to consider as part of same header
+
+    for pageNum in range(len(doc)):
+        if pageNum in toc_pages:
+            continue
+        page = doc.load_page(pageNum)
+        page_height = page.rect.height
+        text_instances = page.get_text("dict")
+
+        # First pass: collect all potential header spans
+        potential_header_spans = []
+        for block in text_instances['blocks']:
+            if block['type'] != 0:
+                continue
+
+            for line in block['lines']:
+                for span in line['spans']:
+                    span_y0 = span['bbox'][1]
+                    span_y1 = span['bbox'][3]
+
+                    if span_y0 < top_margin or span_y1 > (page_height - bottom_margin):
+                        continue
+
+                    span_text = normalize_text(span.get('text', ''))
+                    if not span_text:
+                        continue
+                    if span_text.startswith('http://www') or span_text.startswith('www'):
+                        continue
+                    if any((
+                        'page' in span_text,
+                        not re.search(r'[a-z0-9]', span_text),
+                        'end of section' in span_text,
+                        re.search(r'page\s+\d+\s+of\s+\d+', span_text),
+                        re.search(r'\b(?:\d{1,2}[/-])?\d{1,2}[/-]\d{2,4}\b', span_text),
+                        # re.search(r'\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)', span_text),
+                        'specification:' in span_text
+                    )):
+                        continue
+
+                    cleaned_text = re.sub(r'[.\-]{4,}.*$', '', span_text).strip()
+                    cleaned_text = normalize_text(cleaned_text)
+
+                    if is_header(span, most_common_font_size, most_common_color, most_common_font):
+                        potential_header_spans.append({
+                            'text': cleaned_text,
+                            'size': span['size'],
+                            'pageNum': pageNum,
+                            'y0': span_y0,
+                            'y1': span_y1,
+                            'x0': span['bbox'][0],
+                            'x1': span['bbox'][2],
+                            'span': span
+                        })
+
+        # Sort spans by vertical position (top to bottom)
+        potential_header_spans.sort(key=lambda s: (s['pageNum'], s['y0']))
+
+        # Second pass: group spans that are vertically close and likely part of same header
+        i = 0
+        while i < len(potential_header_spans):
+            current = potential_header_spans[i]
+            header_text = current['text']
+            header_size = current['size']
+            header_page = current['pageNum']
+            min_y = current['y0']
+            max_y = current['y1']
+            spans_group = [current['span']]
+
+            # Look ahead to find adjacent lines that might be part of same header
+            j = i + 1
+            while j < len(potential_header_spans):
+                next_span = potential_header_spans[j]
+                # Check if on same page and vertically close with similar styling
+                if (next_span['pageNum'] == header_page and
+                    next_span['y0'] - max_y < line_merge_threshold and
+                    abs(next_span['size'] - header_size) < 0.5):
+                    header_text += " " + next_span['text']
+                    max_y = next_span['y1']
+                    spans_group.append(next_span['span'])
+                    j += 1
+                else:
+                    break
+
+            # Add the merged header
+            grouped_headers[(header_page, min_y)].append({
+                "text": header_text.strip(),
+                "size": header_size,
+                "pageNum": header_page,
+                "spans": spans_group
+            })
+            spans.extend(spans_group)
+            i = j  # Skip the spans we've already processed
+
+    # Prepare final headers list
+    headers = []
+    for (pageNum, y), header_groups in sorted(grouped_headers.items()):
+        for group in header_groups:
+            headers.append([
+                group['text'],
+                group['size'],
+                group['pageNum'],
+                y
+            ])
+
+    font_sizes = [size for _, size, _, _ in headers]
+    font_size_counts = Counter(font_sizes)
+
+      # Filter font sizes that appear at least 3 times
+    valid_font_sizes = [size for size, count in font_size_counts.items() if count >= 3]
+
+    # Sort in descending order
+    valid_font_sizes_sorted = sorted(valid_font_sizes, reverse=True)
+
+    # If only 2 sizes, repeat the second one
+    if len(valid_font_sizes_sorted) == 2:
+        top_3_font_sizes = [valid_font_sizes_sorted[0], valid_font_sizes_sorted[1], valid_font_sizes_sorted[1]]
+    else:
+        top_3_font_sizes = valid_font_sizes_sorted[:3]
+
+    # Get the smallest font size among valid ones
+    smallest_font_size = min(valid_font_sizes) if valid_font_sizes else None
+    return headers, top_3_font_sizes, smallest_font_size, spans
+
+def is_numbered(text):
+    return bool(re.match(r'^\d', text.strip()))
+
+def is_similar(a, b, threshold=0.85):
+    return difflib.SequenceMatcher(None, a, b).ratio() > threshold
+
+def normalize(text):
+    text = text.lower()
+    text = re.sub(r'\.{2,}', '', text)  # remove long dots
+    text = re.sub(r'\s+', ' ', text)    # replace multiple spaces with one
+    return text.strip()
+
+def clean_toc_entry(toc_text):
+    """Remove page numbers and formatting from TOC entries"""
+    # Remove everything after last sequence of dots/whitespace followed by digits
+    return re.sub(r'[\.\s]+\d+.*$', '', toc_text).strip('. ')
+
+def build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin=70, bottom_margin=70):
+    # Extract headers with margin handling
+    headers_list, top_3_font_sizes, smallest_font_size, spans = extract_headers(
+        doc,
+        toc_pages=toc_pages,
+        most_common_font_size=most_common_font_size,
+        most_common_color=most_common_color,
+        most_common_font=most_common_font,
+        top_margin=top_margin,
+        bottom_margin=bottom_margin
+    )
+
+    # Step 1: Collect and filter potential headers
+    headers = []
+    seen_headers = set()
+
+    # First extract TOC entries to get exact level 0 header texts
+    toc_entries = {}
+    for pno in toc_pages:
+        page = doc.load_page(pno)
+        toc_text = page.get_text()
+        for line in toc_text.split('\n'):
+            clean_line = line.strip()
+            if clean_line:
+                norm_line = normalize(clean_line)
+                toc_entries[norm_line] = clean_line  # Store original text
+
+    for h in headers_list:
+        text, size, pageNum, y = h[:4]
+        page = doc.load_page(pageNum)
+        page_height = page.rect.height
+
+        # Skip margin areas
+        if y < top_margin or y > (page_height - bottom_margin):
+            continue
+
+        norm_text = normalize(text)
+        if len(norm_text) > 2 and size >= most_common_font_size:
+            headers.append({
+                "text": text,
+                "page": pageNum,
+                "y": y,
+                "size": size,
+                "bold": h[4] if len(h) > 4 else False,
+                # "italic": h[5] if len(h) > 5 else False,
+                "color": h[6] if len(h) > 6 else None,
+                "font": h[7] if len(h) > 7 else None,
+                "children": [],
+                "is_numbered": is_numbered(text),
+                "original_size": size,
+                "norm_text": norm_text,
+                "level": -1  # Initialize as unassigned
+            })
+
+    # Sort by page and vertical position
+    headers.sort(key=lambda h: (h['page'], h['y']))
+    # Step 2: Detect consecutive headers and assign levels
+    i = 0
+    while i < len(headers) - 1:
+        current = headers[i]
+        next_header = headers[i+1]
+
+        # Check if they are on the same page and very close vertically (likely consecutive lines)
+        if (current['page'] == next_header['page'] and
+            abs(current['y'] - next_header['y']) < 20):  # 20pt threshold for "same line"
+
+            # Case 1: Both unassigned - make current level 1 and next level 2
+            if current['level'] == -1 and next_header['level'] == -1:
+                current['level'] = 1
+                next_header['level'] = 2
+                i += 1  # Skip next header since we processed it
+
+            # Case 2: Current unassigned, next assigned - make current one level above
+            elif current['level'] == -1 and next_header['level'] != -1:
+                current['level'] = max(1, next_header['level'] - 1)
+
+            # Case 3: Current assigned, next unassigned - make next one level below
+            elif current['level'] != -1 and next_header['level'] == -1:
+                next_header['level'] = current['level'] + 1
+                i += 1  # Skip next header since we processed it
+        i += 1
+    # Step 2: Identify level 0 headers (largest and in TOC)
+    # max_size = max(h['size'] for h in headers) if headers else 0
+    max_size,subheaderSize,nbsheadersize=top_3_font_sizes
+    print(max_size)
+    toc_text_match=[]
+    # Improved TOC matching with exact and substring matching
+    toc_matches = []
+    for h in headers:
+        norm_text = h['norm_text']
+        matching_toc_texts = []
+
+        # Check both exact matches and substring matches
+        for toc_norm, toc_text in toc_entries.items():
+            # Exact match case
+            if norm_text == toc_norm and len(toc_text)>4 and h['size']==max_size:
+                matching_toc_texts.append(toc_text)
+            # Substring match case (header is substring of TOC entry)
+            elif norm_text in toc_norm and len(toc_text)>4 and h['size']==max_size:
+                matching_toc_texts.append(toc_text)
+            # Substring match case (TOC entry is substring of header)
+            elif toc_norm in norm_text and len(toc_text)>4 and h['size']==max_size:
+                matching_toc_texts.append(toc_text)
+
+        if matching_toc_texts and h['size'] >= max_size * 0.9:
+            best_match = max(matching_toc_texts,
+                          key=lambda x: (len(x), -len(x.replace(norm_text, ''))))
+            h['text'] = normalize_text(clean_toc_entry(best_match))
+            h['level'] = 0
+            if h['text'] not in  toc_text_match:
+              toc_matches.append(h)
+              toc_text_match.append(h['text'])
+        elif matching_toc_texts and h['size'] < max_size * 0.9 and h['size'] > nbsheadersize : # h['size'] < max_size * 0.9 and h['size'] > max_size*0.75:
+             print(h['text'],matching_toc_texts)
+             headers.remove(h)
+             continue
+
+
+    # Remove duplicates - keep only first occurrence of each level 0 header
+    unique_level0 = []
+    seen_level0 = set()
+    for h in toc_matches:
+        # Use the cleaned text for duplicate checking
+        cleaned_text = clean_toc_entry(h['text'])
+        norm_cleaned_text = normalize(cleaned_text)
+
+        if norm_cleaned_text not in seen_level0:
+            seen_level0.add(norm_cleaned_text)
+            # Update the header text with cleaned version
+            h['text'] = cleaned_text
+            unique_level0.append(h)
+            print(f"Added unique header: {cleaned_text} (normalized: {norm_cleaned_text})")
+
+    # Step 3: Process headers under each level 0 to identify level 1 format
+
+    # First, group headers by their level 0 parent
+    level0_headers = [h for h in headers if h['level'] == 0]
+    header_groups = []
+
+    for i, level0 in enumerate(level0_headers):
+        start_idx = headers.index(level0)
+        end_idx = headers.index(level0_headers[i+1]) if i+1 < len(level0_headers) else len(headers)
+        group = headers[start_idx:end_idx]
+        header_groups.append(group)
+
+    # Now process each group to identify level 1 format
+    for group in header_groups:
+        level0 = group[0]
+        level1_candidates = [h for h in group[1:] if h['level'] == -1]
+
+        if not level1_candidates:
+            continue
+
+        # The first candidate is our reference level 1
+        first_level1 = level1_candidates[0]
+        level1_format = {
+            'font': first_level1['font'],
+            'color': first_level1['color'],
+            'starts_with_number': is_numbered(first_level1['text']),
+            'size': first_level1['size'],
+            'bold': first_level1['bold']
+            # 'italic': first_level1['italic']
+        }
+
+        # Assign levels based on the reference format
+        for h in level1_candidates:
+            current_format = {
+                'font': h['font'],
+                'color': h['color'],
+                'starts_with_number': is_numbered(h['text']),
+                'size': h['size'],
+                'bold': h['bold']
+                # 'italic': h['italic']
+            }
+
+            # Compare with level1 format
+            if (current_format['font'] == level1_format['font'] and
+                current_format['color'] == level1_format['color'] and
+                current_format['starts_with_number'] == level1_format['starts_with_number'] and
+                abs(current_format['size'] - level1_format['size']) <= 0.1 and
+                current_format['bold'] == level1_format['bold'] ): #and
+                # current_format['italic'] == level1_format['italic']):
+                h['level'] = 1
+            else:
+                h['level'] = 2
+
+    # Step 4: Assign levels to remaining unassigned headers
+    unassigned = [h for h in headers if h['level'] == -1]
+    if unassigned:
+        # Cluster by size with tolerance
+        sizes = sorted({h['size'] for h in unassigned}, reverse=True)
+        clusters = []
+
+        for size in sizes:
+            found_cluster = False
+            for cluster in clusters:
+                if abs(size - cluster['size']) <= max(size, cluster['size']) * 0.1:
+                    cluster['headers'].extend([h for h in unassigned if abs(h['size'] - size) <= size * 0.1])
+                    found_cluster = True
+                    break
+            if not found_cluster:
+                clusters.append({
+                    'size': size,
+                    'headers': [h for h in unassigned if abs(h['size'] - size) <= size * 0.1]
+                })
+
+        # Assign levels starting from 1
+        clusters.sort(key=lambda x: -x['size'])
+        for i, cluster in enumerate(clusters):
+            for h in cluster['headers']:
+                base_level = i + 1
+                if h['bold']:
+                    base_level = max(1, base_level - 1)
+                h['level'] = base_level
+
+    # Step 5: Build hierarchy
+    root = []
+    stack = []
+
+    # Create a set of normalized texts from unique_level0 to avoid duplicates
+    unique_level0_texts = {h['norm_text'] for h in unique_level0}
+
+    # Filter out any headers from the original list that match unique_level0 headers
+    filtered_headers = []
+    for h in headers:
+        if h['norm_text'] in unique_level0_texts and h not in unique_level0:
+              h['level'] = 0
+        filtered_headers.append(h)
+
+    # Combine all headers - unique_level0 first, then the filtered headers
+    all_headers = unique_level0 + filtered_headers
+    all_headers.sort(key=lambda h: (h['page'], h['y']))
+
+    # Track which level 0 headers we've already added
+    added_level0 = set()
+
+    for header in all_headers:
+        if header['level'] < 0:
+            continue
+
+        if header['level'] == 0:
+            norm_text = header['norm_text']
+            if norm_text in added_level0:
+                continue
+            added_level0.add(norm_text)
+
+        # Pop stack until we find a parent
+        while stack and stack[-1]['level'] >= header['level']:
+            stack.pop()
+
+        current_parent = stack[-1] if stack else None
+
+        if current_parent:
+            current_parent['children'].append(header)
+        else:
+            root.append(header)
+
+        stack.append(header)
+
+    # Step 6: Enforce proper nesting
+    def enforce_nesting(node_list, parent_level=-1):
+        for node in node_list:
+            if node['level'] <= parent_level:
+                node['level'] = parent_level + 1
+            enforce_nesting(node['children'], node['level'])
+
+    enforce_nesting(root)
+    root = [h for h in root if not (h['level'] == 0 and not h['children'])]
+    return root
+
+def adjust_levels_if_level0_not_in_toc(doc, toc_pages, root):
+    def normalize(text):
+        return re.sub(r'\s+', ' ', text.strip().lower())
+
+    toc_text = ""
+    for pno in toc_pages:
+        page = doc.load_page(pno)
+        toc_text += page.get_text()
+    toc_text_normalized = normalize(toc_text)
+
+    def is_level0_in_toc_text(header):
+        return header['level'] == 0 and normalize(header['text']) in toc_text_normalized
+
+    if any(is_level0_in_toc_text(h) for h in root):
+        return  # No change needed
+
+    def increase_levels(node_list):
+        for node in node_list:
+            node['level'] += 1
+            increase_levels(node['children'])
+
+def assign_numbers_to_headers(headers, prefix=None):
+    for idx, header in enumerate(headers, 1):
+        current_number = f"{prefix}.{idx}" if prefix else str(idx)
+        header["number"] = current_number
+        assign_numbers_to_headers(header["children"], current_number)
+
+def print_tree_with_numbers(headers, indent=0):
+    for header in headers:
+        size_info = f"size:{header['original_size']:.1f}" if 'original_size' in header else ""
+        print("  " * indent +
+              f"{header.get('number', '?')} {header['text']} " +
+              f"(Level {header['level']}, p:{header['page']+1}, {size_info})")
+        print_tree_with_numbers(header["children"], indent + 1)
+
+def highlight_boxes(doc, highlights):
+    for page_num, bbox in highlights.items():
+
+        page = doc.load_page(page_num)
+        page_width = page.rect.width
+        rect = fitz.Rect(bbox)
+        # Get the original bounding box
+        orig_rect = fitz.Rect(bbox)
+        rect_width = orig_rect.width
+        rect_height = orig_rect.height
+        if rect_width>10:
+          annot = page.add_rect_annot(rect)
+
+          annot.set_colors(stroke=(1,1,0), fill=(1,1,0))
+          annot.set_opacity(0.3)
+          annot.update()
+
+
+          # Calculate new x coordinates so the rect is centered on the page width
+          center_x = page_width / 2
+          new_x0 = center_x - rect_width / 2
+          new_x1 = center_x + rect_width / 2
+
+          # Create new rect centered on the page's x-center, preserving y-coordinates
+          new_rect = fitz.Rect(new_x0, orig_rect.y0, new_x1, orig_rect.y1)
+
+          # Add centered text annotation
+          text = "[To be billed]"
+          annot1 = page.add_freetext_annot(
+              new_rect,
+              text,
+              fontsize=15,
+              fontname='helv',
+              text_color=(1, 0, 0),
+              rotate=page.rotation,
+              align=1  # centered alignment
+          )
+          annot1.update()
+
+def get_leaf_headers_with_paths(listtoloop, path=None, output=None):
+    if path is None:
+        path = []
+    if output is None:
+        output = []
+    for header in listtoloop:
+        current_path = path + [header['text']]
+        if not header['children']:
+            if header['level'] != 0 and header['level'] != 1:
+                output.append((header, current_path))
+        else:
+            get_leaf_headers_with_paths(header['children'], current_path, output)
+    return output
+
+# Add this helper function at the top of your code
+def words_match_ratio(text1, text2):
+    words1 = set(text1.split())
+    words2 = set(text2.split())
+    if not words1 or not words2:
+        return 0.0
+    common_words = words1 & words2
+    return len(common_words) / len(words1)
+
+def same_start_word(s1, s2):
+    # Split both strings into words
+    words1 = s1.strip().split()
+    words2 = s2.strip().split()
+
+    # Check if both have at least one word and compare the first ones
+    if words1 and words2:
+        return words1[0].lower() == words2[0].lower()
+    return False
+
+baselink='https://marthee-nbslink.hf.space/view-pdf?'
+def extract_section_under_header(pdf_path):
+    top_margin = 70
+    bottom_margin = 50
+    headertoContinue1 = False
+    headertoContinue2=False
+
+    # Optimized URL handling
+    if pdf_path and ('http' in pdf_path or 'dropbox' in pdf_path):
+        pdf_path = pdf_path.replace('dl=0', 'dl=1')
+
+    # Cache frequently used values
+    response = requests.get(pdf_path)
+    pdf_content = BytesIO(response.content)
+    if not pdf_content:
+        raise ValueError("No valid PDF content found.")
+
+    doc = fitz.open(stream=pdf_content, filetype="pdf")
+    docHighlights = fitz.open(stream=pdf_content, filetype="pdf")
+    most_common_font_size, most_common_color, most_common_font = get_regular_font_size_and_color(doc)
+
+    # Precompute regex patterns
+    dot_pattern = re.compile(r'\.{3,}')
+    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+
+    def get_toc_page_numbers(doc, max_pages_to_check=15):
+        toc_pages = []
+        for page_num in range(min(len(doc), max_pages_to_check)):
+            page = doc.load_page(page_num)
+            blocks = page.get_text("dict")["blocks"]
+
+            dot_line_count = 0
+            for block in blocks:
+                for line in block.get("lines", []):
+                    line_text = get_spaced_text_from_spans(line["spans"]).strip()
+                    if dot_pattern.search(line_text):
+                        dot_line_count += 1
+
+            if dot_line_count >= 3:
+                toc_pages.append(page_num)
+
+        return list(range(0, toc_pages[-1] +1)) if toc_pages else toc_pages
+
+    toc_pages = get_toc_page_numbers(doc)
+
+    headers, top_3_font_sizes, smallest_font_size, headersSpans = extract_headers(
+        doc, toc_pages, most_common_font_size, most_common_color, most_common_font, top_margin, bottom_margin
+    )
+
+    hierarchy = build_header_hierarchy(doc, toc_pages, most_common_font_size, most_common_color, most_common_font)
+    listofHeaderstoMarkup = get_leaf_headers_with_paths(hierarchy)
+    print('listofHeaderstoMarkup',listofHeaderstoMarkup)
+    # Precompute all children headers once
+    allchildrenheaders = [normalize_text(item['text']) for item, p in listofHeaderstoMarkup]
+    allchildrenheaders_set = set(allchildrenheaders)  # For faster lookups
+
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
+    dictionaryNBS={}
+    data_list_JSON = []
+
+    if len(top_3_font_sizes)==3:
+        mainHeaderFontSize, subHeaderFontSize, subsubheaderFontSize = top_3_font_sizes
+    elif len(top_3_font_sizes)==2:
+        mainHeaderFontSize= top_3_font_sizes[0]
+        subHeaderFontSize= top_3_font_sizes[1]
+        subsubheaderFontSize= top_3_font_sizes[1]
+
+    print("📌 Has TOC:", bool(toc_pages), " | Pages to skip:", toc_pages)
+
+    # Preload all pages to avoid repeated loading
+    # pages = [doc.load_page(page_num) for page_num in range(len(doc)) if page_num not in toc_pages]
+
+    for heading_to_searchDict, paths in listofHeaderstoMarkup:
+        heading_to_search = heading_to_searchDict['text']
+        heading_to_searchPageNum = heading_to_searchDict['page']
+
+        print('headertosearch', heading_to_search)
+
+        # Initialize variables
+        headertoContinue1 = False
+        headertoContinue2 = False
+        matched_header_line = None
+        done = False
+        collecting = False
+        collected_lines = []
+        page_highlights = {}
+        current_bbox = {}
+        last_y1s = {}
+        mainHeader = ''
+        subHeader = ''
+        matched_header_line_norm = heading_to_search
+        break_collecting = False
+        heading_norm = normalize_text(heading_to_search)
+        paths_norm = [normalize_text(p) for p in paths[0]] if paths and paths[0] else []
+
+        for page_num in range(heading_to_searchPageNum,len(doc)):
+            if page_num in toc_pages:
+              continue
+            if break_collecting:
+                break
+            page=doc[page_num]
+            page_height = page.rect.height
+            blocks = page.get_text("dict")["blocks"]
+
+            for block in blocks:
+                if break_collecting:
+                    break
+
+                lines = block.get("lines", [])
+                i = 0
+                while i < len(lines):
+                    if break_collecting:
+                        break
+
+                    spans = lines[i].get("spans", [])
+                    if not spans:
+                        i += 1
+                        continue
+
+                    y0 = spans[0]["bbox"][1]
+                    y1 = spans[0]["bbox"][3]
+                    if y0 < top_margin or y1 > (page_height - bottom_margin):
+                        i += 1
+                        continue
+
+                    line_text = get_spaced_text_from_spans(spans).lower()
+                    line_text_norm = normalize_text(line_text)
+
+                    # Combine with next line if available
+                    if i + 1 < len(lines):
+                        next_spans = lines[i + 1].get("spans", [])
+                        next_line_text = get_spaced_text_from_spans(next_spans).lower()
+                        combined_line_norm = normalize_text(line_text + " " + next_line_text)
+                    else:
+                        combined_line_norm = line_text_norm
+
+                    # Check if we should continue processing
+                    if combined_line_norm and combined_line_norm in paths[0]:
+                        print(combined_line_norm)
+                        headertoContinue1 = combined_line_norm
+                    if combined_line_norm and combined_line_norm in paths[-2]:
+                        print(combined_line_norm)
+                        headertoContinue2 = combined_line_norm
+
+                    # Optimized header matching
+                    existsfull = (
+                        ( combined_line_norm in allchildrenheaders_set or
+                        combined_line_norm in allchildrenheaders ) and heading_to_search in combined_line_norm
+                    )
+
+                    # New word-based matching
+                    current_line_words = set(combined_line_norm.split())
+                    heading_words = set(heading_norm.split())
+                    all_words_match = current_line_words.issubset(heading_words) and len(current_line_words) > 0
+
+                    substring_match = (
+                        heading_norm in combined_line_norm or
+                        combined_line_norm in heading_norm or
+                        all_words_match  # Include the new word-based matching
+                    )
+                    # substring_match = (
+                    #     heading_norm in combined_line_norm or
+                    #     combined_line_norm in heading_norm
+                    # )
+
+                    if (substring_match and existsfull and not collecting and
+                        len(combined_line_norm) > 0 ):#and (headertoContinue1 or headertoContinue2) ):
+
+                        # Check header conditions more efficiently
+                        header_spans = [
+                            span for span in spans
+                            if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+                                # and span['size'] >= subsubheaderFontSize
+                                and span['size'] < mainHeaderFontSize)
+                        ]
+                        if header_spans:
+                            collecting = True
+                            matched_header_font_size = max(span["size"] for span in header_spans)
+                            print(f"📥 Start collecting after header: {combined_line_norm} (Font size: {matched_header_font_size})")
+
+                            collected_lines.append(line_text)
+                            valid_spans = [span for span in spans if span.get("bbox")]
+
+                            if valid_spans:
+                                x0s = [span["bbox"][0] for span in valid_spans]
+                                x1s = [span["bbox"][2] for span in valid_spans]
+                                y0s = [span["bbox"][1] for span in valid_spans]
+                                y1s = [span["bbox"][3] for span in valid_spans]
+
+                                header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                if page_num in current_bbox:
+                                    cb = current_bbox[page_num]
+                                    current_bbox[page_num] = [
+                                        min(cb[0], header_bbox[0]),
+                                        min(cb[1], header_bbox[1]),
+                                        max(cb[2], header_bbox[2]),
+                                        max(cb[3], header_bbox[3])
+                                    ]
+                                else:
+                                    current_bbox[page_num] = header_bbox
+                                last_y1s[page_num] = header_bbox[3]
+                                x0, y0, x1, y1 = header_bbox
+
+                                zoom = 200
+                                left = int(x0)
+                                top = int(y0)
+                                zoom_str = f"{zoom},{left},{top}"
+                                pageNumberFound = page_num + 1
+
+                              # Build the query parameters
+                                params = {
+                                    'pdfLink': pdf_path,  # Your PDF link
+                                    'keyword': heading_to_search,  # Your keyword (could be a string or list)
+                                }
+
+                                # URL encode each parameter
+                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+
+                                # Construct the final encoded link
+                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+
+                                # Correctly construct the final URL with page and zoom
+                                final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+
+                                # Get current date and time
+                                now = datetime.now()
+
+                                # Format the output
+                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                # Optionally, add the URL to a DataFrame
+
+
+                                data_entry = {
+                                        "NBSLink": final_url,
+                                        "Subject": heading_to_search,
+                                        "Page": str(pageNumberFound),
+                                        "Author": "ADR",
+                                        "Creation Date": formatted_time,
+                                        "Layer": "Initial",
+                                        "Code": "to be added",
+                                        "head above 1":  paths[-2],
+                                        "head above 2":  paths[0]
+                                    }
+                                data_list_JSON.append(data_entry)
+
+                                # Convert list to JSON
+                                json_output = json.dumps(data_list_JSON, indent=4)
+
+                                print("Final URL:", final_url)
+                                i += 2
+                                continue
+                    else:
+                        if (substring_match and not collecting and
+                            len(combined_line_norm) > 0): # and (headertoContinue1 or headertoContinue2) ):
+
+                            # Calculate word match percentage
+                            word_match_percent = words_match_ratio(heading_norm, combined_line_norm) * 100
+
+                            # Check if at least 70% of header words exist in this line
+                            meets_word_threshold = word_match_percent >= 100
+
+                            # Check header conditions (including word threshold)
+                            header_spans = [
+                                span for span in spans
+                                if (is_header(span, most_common_font_size, most_common_color, most_common_font)
+                                    # and span['size'] >= subsubheaderFontSize
+                                    and span['size'] < mainHeaderFontSize)
+                            ]
+
+                            if header_spans and (meets_word_threshold or same_start_word(heading_to_search, combined_line_norm) ):
+                                collecting = True
+                                matched_header_font_size = max(span["size"] for span in header_spans)
+                                print(f"📥 Start collecting after header: {combined_line_norm} "
+                                      f"(Font: {matched_header_font_size}, Word match: {word_match_percent:.0f}%)")
+
+                                collected_lines.append(line_text)
+                                valid_spans = [span for span in spans if span.get("bbox")]
+
+                                if valid_spans:
+                                    x0s = [span["bbox"][0] for span in valid_spans]
+                                    x1s = [span["bbox"][2] for span in valid_spans]
+                                    y0s = [span["bbox"][1] for span in valid_spans]
+                                    y1s = [span["bbox"][3] for span in valid_spans]
+
+                                    header_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                                    if page_num in current_bbox:
+                                        cb = current_bbox[page_num]
+                                        current_bbox[page_num] = [
+                                            min(cb[0], header_bbox[0]),
+                                            min(cb[1], header_bbox[1]),
+                                            max(cb[2], header_bbox[2]),
+                                            max(cb[3], header_bbox[3])
+                                        ]
+                                    else:
+                                        current_bbox[page_num] = header_bbox
+
+                                    last_y1s[page_num] = header_bbox[3]
+                                    i += 2
+                                    continue
+                    if collecting:
+                        norm_line = normalize_text(line_text)
+
+                        # Optimized URL check
+                        if url_pattern.match(norm_line):
+                            line_is_header = False
+                        else:
+                            line_is_header = any(is_header(span, most_common_font_size, most_common_color, most_common_font) for span in spans)
+
+                        if line_is_header:
+                            header_font_size = max(span["size"] for span in spans)
+                            is_probably_real_header = (
+                                header_font_size >= matched_header_font_size and
+                                is_header(spans[0], most_common_font_size, most_common_color, most_common_font) and
+                                len(line_text.strip()) > 2
+                            )
+
+                            if (norm_line != matched_header_line_norm and
+                                norm_line != heading_norm and
+                                is_probably_real_header):
+                                if line_text not in heading_norm:
+                                  print(f"🛑 Stop at header with same or larger font: '{line_text}' ({header_font_size} ≥ {matched_header_font_size})")
+                                  collecting = False
+                                  done = True
+                                  headertoContinue1 = False
+                                  headertoContinue2=False
+                                  for page_num, bbox in current_bbox.items():
+                                      bbox[3] = last_y1s.get(page_num, bbox[3])
+                                      page_highlights[page_num] = bbox
+                                  highlight_boxes(docHighlights, page_highlights)
+
+                                  break_collecting = True
+                                  break
+
+                        if break_collecting:
+                            break
+
+                        collected_lines.append(line_text)
+                        valid_spans = [span for span in spans if span.get("bbox")]
+                        if valid_spans:
+                            x0s = [span["bbox"][0] for span in valid_spans]
+                            x1s = [span["bbox"][2] for span in valid_spans]
+                            y0s = [span["bbox"][1] for span in valid_spans]
+                            y1s = [span["bbox"][3] for span in valid_spans]
+
+                            line_bbox = [min(x0s), min(y0s), max(x1s), max(y1s)]
+
+                            if page_num in current_bbox:
+                                cb = current_bbox[page_num]
+                                current_bbox[page_num] = [
+                                    min(cb[0], line_bbox[0]),
+                                    min(cb[1], line_bbox[1]),
+                                    max(cb[2], line_bbox[2]),
+                                    max(cb[3], line_bbox[3])
+                                ]
+                            else:
+                                current_bbox[page_num] = line_bbox
+
+                            last_y1s[page_num] = line_bbox[3]
+                    i += 1
+
+        if not done:
+            for page_num, bbox in current_bbox.items():
+                bbox[3] = last_y1s.get(page_num, bbox[3])
+                page_highlights[page_num] = bbox
+            highlight_boxes(docHighlights, page_highlights)
+
+    docHighlights.save("highlighted_output.pdf", garbage=4, deflate=True)
+    return json_output
+
+pdflink='https://www.dropbox.com/scl/fi/jtffcxszwpcnc6wdo61p6/WH007-JAC-RP-XX-SP-AA-8501-Redoak-Pump-House-Specification.pdf?rlkey=unq4ag9eajezv2j6y6ewkkk5u&e=29&st=wu3vsd70&dl=0'
+
+jsonOutput=extract_section_under_header(pdflink)
+print(jsonOutput)
+