Update Find_Hyperlinking_text.py

Find_Hyperlinking_text.py

@@ -1,3 +1,4 @@
+
 import fitz  # PyMuPDF
 from io import BytesIO
 import re
@@ -10,7 +11,8 @@ import urllib.parse
 import pandas as pd
 import tempfile
 from fpdf import FPDF
-
+import json
+from datetime import datetime
 
 baselink='https://marthee-nbslink.hf.space/view-pdf?'
 class PDF(FPDF):
@@ -29,11 +31,15 @@ def save_df_to_pdf(df):
     pdf.set_right_margin(margin)
 
     pdf.add_page()
-    pdf.set_font("Arial", size=10)
+    pdf.set_font("Arial", size=8)  # Reduce font size to fit more text
+
+    # Table headers
+    headers = ["NBSLink", "Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"]
+    num_cols = len(headers)
 
-    # Set column widths and calculate total table width
-    col_width = 50
-    num_cols = 4
+    # Calculate column width dynamically
+    max_table_width = pdf.w - 2 * margin  # Total available width
+    col_width = max_table_width / num_cols  # Distribute evenly
     table_width = col_width * num_cols
 
     # Get page width and calculate left alignment
@@ -44,23 +50,20 @@ def save_df_to_pdf(df):
 
     # Table headers
     pdf.set_fill_color(200, 200, 200)  # Light gray background
-    pdf.set_font("Arial", "B", 10)
-    headers = ["NBS Link", "NBS", "Head Above 1", "Head Above 2"]
-
-    # Draw table headers
+    pdf.set_font("Arial", "B", 8)
+    
     for header in headers:
         pdf.cell(col_width, 8, header, border=1, fill=True, align="C")
     pdf.ln()
 
-    pdf.set_font("Arial", size=9)
+    pdf.set_font("Arial", size=7)  # Reduce font size for data rows
 
-    # Add rows
     for _, row in df.iterrows():
         x_start = start_x  # Ensure every row starts at the same position
         y_start = pdf.get_y()
 
         # Calculate max height needed for this row
-        text_lines = {col: pdf.multi_cell(col_width, 5, row[col], border=0, align="L", split_only=True) for col in ["NBS", "head above 1", "head above 2"]}
+        text_lines = {col: pdf.multi_cell(col_width, 5, row[col], border=0, align="L", split_only=True) for col in ["Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"]}
         max_lines = max(len(lines) for lines in text_lines.values())
         max_height = max_lines * 5
 
@@ -73,7 +76,7 @@ def save_df_to_pdf(df):
         pdf.set_xy(x_start + col_width, y_start)
 
         # Draw each cell manually, ensuring equal height
-        for i, col_name in enumerate(["NBS", "head above 1", "head above 2"]):
+        for i, col_name in enumerate(["Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"]):
             x_col = x_start + col_width * (i + 1)
             y_col = y_start
             pdf.multi_cell(col_width, 5, row[col_name], border=0, align="L")  # Draw text
@@ -82,11 +85,8 @@ def save_df_to_pdf(df):
 
         # Move to the next row
         pdf.ln(max_height)
-        # Save PDF to memory instead of a file
-    # pdf_output = BytesIO()
-    # pdf_output = 'output.pdf'
-    pdf_output = pdf.output(dest="S").encode("latin1")  # Returns the PDF as a byte string
-
+    # Save PDF to memory
+    pdf_output = pdf.output(dest="S").encode("latin1")  
     return pdf_output
 
 
@@ -96,7 +96,7 @@ def normalize_text(text):
     text = text.lower().strip()
     text = re.sub(r'\s+', ' ', text)  # Normalize multiple spaces
     return re.sub(r'[^\w\s]', '', text)  # Remove punctuation
-def get_repeated_texts(pdf_document, threshold=0.9):
+def get_repeated_texts(pdf_document, threshold=0.85):
     """
     Identify text that appears on most pages.
     :param pdf_document: The opened PDF document.
@@ -125,11 +125,9 @@ def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
     """
     Annotates text under a specific heading in a PDF, highlights it, 
     and constructs zoom coordinates for the first occurrence of the heading.
-
     Args:
         pdfshareablelinks (list): List of shareable links to PDFs.
         heading_to_search (str): The heading to search for in the PDF.
-
     Returns:
         Tuple: Annotated PDF bytes, count of heading occurrences, and zoom string.
     """
@@ -153,8 +151,9 @@ def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
     # Open the PDF using PyMuPDF
     pdf_document = fitz.open(stream=pdf_content, filetype="pdf")
     repeated_texts = get_repeated_texts(pdf_document)
-    df = pd.DataFrame(columns=["NBSLink","NBS", 'head above 1', "head above 2"])
+    df = pd.DataFrame(columns=["NBSLink","Subject","Page","Author","Creation Date","Layer",'Code', 'head above 1', "head above 2"])
     dictionaryNBS={}
+    data_list_JSON = []
     for NBSindex, heading_to_search in enumerate(LISTheading_to_search):
         if NBSindex == len(LISTheading_to_search) - 1:
             flagAllNBSvisited = True
@@ -164,6 +163,8 @@ def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
         f10_count = 0
         current_y = None
         highlight_rect = None
+        highlight_rectEnding=None
+        highlight_rectBegin=None
         zoom_str = None
         toc_flag = False
         span_font_goal = None
@@ -172,6 +173,7 @@ def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
         groupheadings = []
         merged_groupheadings = []
         collectheader2 = False
+        endingcontentFlag=True
         header2 = ''
         header2_first_span_size = 0
         previous_header = ''
@@ -213,177 +215,212 @@ def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
                             span_y = span['bbox'][1]
                             span_font = span['font']
                             span_size = span['size']
-
-                            if previous_y is None:
-                                previous_y = span_y  # Initialize on first span
-
-                            # If same Y coordinate as previous, append to the current line
-                            if abs(span_y - previous_y) < 5:  # Allow a small margin for OCR variations
-                                current_line_text += " " + span_text
-                                current_line_text = normalize_text(current_line_text)
-                                current_line_span_size = span_size
-                            else:
-                                # Store the complete line and reset for the new line
-                                if current_line_text.strip():
-                                    all_text.append(current_line_text.strip())
-
-                                current_line_text = span_text  # Start a new line
-                                previous_y = span_y  # Update the reference Y
-                            text = span_text
-                            if collecting_text and span_font == span_font_goal and span_size == span_size_goal and span_text[0].isdigit():
-                                print(f"Ending collection at heading: {span_text}")
-                                print("merged_groupheadings:", merged_groupheadings)
-                                collecting_text = False
-                                continue
-                            if collecting_text:
-                                annot = page.add_highlight_annot(highlight_rect)
-                                annot.update()
-
-                            if 'Content' in span_text:
-                                toc_flag = True
-                                TOC_start = span_text
-                                print('content', TOC_start, span_size)
-
-                            if toc_flag:
-                                if 'Content' not in span_text:
-                                    if current_y is None:
-                                        current_y = span_y
-                                        current_size = span_size  # Initialize the reference span size
-                                    # Check if the current span size deviates significantly
-                                    if abs(span_size - current_size) > 1:  # Threshold for size difference
-                                        toc_flag = False
-                                 
-                                    if abs(current_y - span_y) < 5:  # Allowing more flexibility for multi-line headings
-                                        current_line += " " + span_text  # Keep accumulating text
-                                    else:
-                                        if current_line.strip():  # Only process non-empty lines
-                                            pattern = r"^([A-Za-z0-9\s\/\-,]+)(?=\.+)"
-                                            match = re.match(pattern, current_line.strip())
-                                            
-                                            if match:
-                                                groupheadings.append(match.group(1).strip())
-                                        current_line = span_text
-                                        current_y = span_y
-                                        current_size = span_size  # Update reference span size
-                            if len(groupheadings) > 0:
-                                pattern = re.compile(r"^[A-Za-z]\d{2} ")  # Match headings starting with letter + 2 digits
-                                merged_groupheadings = []
-                                current_item = None  # Start as None to avoid an initial blank entry
-
-                                for item in groupheadings:
-                                    if pattern.match(item):  # If item starts with correct pattern, it's a new heading
-                                        if current_item:  # Append only if current_item is not empty
-                                            merged_groupheadings.append(current_item.strip())
-                                        current_item = item  # Start new heading
-                                    else:
-                                        if current_item:  
-                                            current_item += " " + item  # Merge with previous heading
-
-                                # Append last merged item after loop
-                                if current_item:
-                                    merged_groupheadings.append(current_item.strip())
-                            if span_text == first_word:
-                                print('First word found:', span_text)
-                                # Check if it's not the last span in the current line
-                                print(i + 1, len(spans))
-                                if i + 1 < len(spans):
-                                    next_span_text = (spans[i + 1]['text'].strip())
-                                # Check if the next span's text is in the heading list
-                                if next_span_text.replace(" ", "") in heading_to_search.replace(" ", ""):
-                                    text = (span_text + ' ' + next_span_text)
-                                            # After processing the current line, check if there's a next line
-                            if first_word == span_text:
-                                if line_index + 1 < len(block.get('lines', [])):
-                                    next_line = block['lines'][line_index + 1]
-                                    # You can process the spans of the next line here
-                                    for next_span in next_line.get('spans', []):
-                                        next_span_text = next_span['text'].strip()
-                                        text = span_text + ' ' + next_span_text
-                            if len(merged_groupheadings) > 0:
-                                if re.match(r"[A-Za-z]\d{2}", span_text) and span_size > 10:
-                                    previous_header = span_text  # Store last detected header
-                                    print('previous_header', span_text)
-                                groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
-
-                                if previous_header:
-                                    if not collectheader2:
-                                        if header2_first_span_size == 0:
-                                            spanSizeHeader = 10
+                            if normalize_text(span_text) not in repeated_texts and not (span_text.startswith('Page')):
+                                if previous_y is None:
+                                    previous_y = span_y  # Initialize on first span
+
+                                # If same Y coordinate as previous, append to the current line
+                                if abs(span_y - previous_y) < 5:  # Allow a small margin for OCR variations
+                                    current_line_text += " " + span_text
+                                    current_line_text = normalize_text(current_line_text)
+                                    current_line_span_size = span_size
+                                else:
+                                    # Store the complete line and reset for the new line
+                                    if current_line_text.strip():
+                                        all_text.append(current_line_text.strip())
+
+                                    current_line_text = span_text  # Start a new line
+                                    previous_y = span_y  # Update the reference Y
+                                text = span_text
+                                if collecting_text and span_font == span_font_goal and span_size == span_size_goal and span_text[0].isdigit():
+                                    print(f"Ending collection at heading: {span_text}")
+                                    highlight_rectEnding=highlight_rect
+                                    print("merged_groupheadings:", merged_groupheadings)
+                                    print('groupheadingss',groupheadings)
+                                    collecting_text = False
+                                    continue
+                                if collecting_text:
+                                    annot = page.add_rect_annot(highlight_rect)  # Create a rectangle annotation
+                                    annot.set_colors(stroke=(1, 0, 0))  # Set border color (Red)
+                                    annot.update()  # Apply changes
+
+                                if 'Content' in span_text:
+                                    toc_flag = True
+                                    TOC_start = span_text
+                                    print('content', TOC_start, span_size)
+
+                                if toc_flag or endingcontentFlag:
+                                    if 'Content' not in span_text:
+                                        if current_y is None:
+                                            current_y = span_y
+                                            current_size = span_size  # Initialize the reference span size
+                                        # Check if the current span size deviates significantly
+                                        if abs(span_size - current_size) > 1:  # Threshold for size difference
+                                            toc_flag = False
+                                    
+                                        if abs(current_y - span_y) < 5:  # Allowing more flexibility for multi-line headings
+                                            current_line += " " + span_text  # Keep accumulating text
+                                        else:
+                                            if current_line.strip():  # Only process non-empty lines
+                                                clean_text = re.sub(r'\.{5,}\d*$', '', current_line, flags=re.MULTILINE)  # Remove dots and trailing numbers
+
+                                                print(clean_text.strip())
+                                                if clean_text:
+                                                    groupheadings.append(clean_text)
+                                            # else:
+                                            #     toc_flag = False
+
+                                            current_line = span_text
+                                            current_y = span_y
+                                            current_size = span_size  # Update reference span size
+                                # print('outofcurrent')
+                                if len(groupheadings) > 0:
+                                    pattern = re.compile(r"^[A-Za-z]\d{2} ")  # Match headings starting with letter + 2 digits
+                                    merged_groupheadings = []
+                                    current_item = None  # Start as None to avoid an initial blank entry
+
+                                    for item in groupheadings:
+                                        if pattern.match(item):  # If item starts with correct pattern, it's a new heading
+                                            if current_item:  # Append only if current_item is not empty
+                                                if current_item not in merged_groupheadings:
+                                                    extracted_text = re.split(r"\.{3,}", current_item)[0].strip()
+                                                    merged_groupheadings.append(extracted_text.strip())
+                                            current_item = item  # Start new heading
                                         else:
-                                            spanSizeHeader = header2_first_span_size
-
-                                        for item in groupmainheadingFromArray:
-                                            if not any(normalize_text(current_line_text) in normalize_text(item) for item in groupmainheadingFromArray):
-                                                if span_size >= spanSizeHeader:
-                                                    if not re.match(r"^\d{2}", current_line_text) and current_line_text not in repeated_texts and "Bold" in span["font"] :
-                                                        if len(header2) > 0:
-                                                            header2_first_span_size = span_size
-                                                        header2 = current_line_text
-                                                        print('header2', header2, span_size, spanSizeHeader)
-
-                            trimmed_text = text.replace(" ", "")
-                            if len(text) > 0:
-                                if text.split()[0] in heading_words:
-                                    if len(trimmed_text) > 0 and (heading_to_search.replace(" ", "") in trimmed_text):
-                                        print(trimmed_text, heading_to_search)
-                                        f10_count += 1
-                                        # Start collecting text under the second occurrence of the heading
-                                        if f10_count == 1:
-                                            collecting_text = True
-                                            print(f"Starting collection under heading: {text}, {span_font}, {span_size}")
-                                            collectheader2 = True
-                                            NBS_heading = heading_to_searchNBS
-                                            x0, y0, x1, y1 = highlight_rect
-                                            
-                                            span_font_goal = span_font  # Capture the font at the first heading match
-                                            span_size_goal = span_size  # Capture the size at the first heading match
-                                            zoom = 200
-                                            left = int(x0)
-                                            top = int(y0)
-                                            zoom_str = f"{zoom},{left},{top}"
-                                            pageNumberFound = page_num + 1
-                                            dictionaryNBS[heading_to_searchNBS] = [pageNumberFound, zoom_str]
-
-                                            annot = page.add_highlight_annot(highlight_rect)
-                                            annot.update()
-                                            groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
-                                            
-                                                        # Build the query parameters
-                                            params = {
-                                                'pdfLink': link,  # Your PDF link
-                                                'keyword': NBS_heading,  # Your keyword (could be a string or list)                                              
-                                            }
-                                            
-                                            # URL encode each parameter
-                                            encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
-                                            
-                                            # Construct the final encoded link
-                                            encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
-                                            
-                                            # Correctly construct the final URL with page and zoom
-                                            final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
-                                            
-                                            # Optionally, add the URL to a DataFrame
-                                            if len(groupmainheadingFromArray) > 0:
-                                                df = pd.concat([df, pd.DataFrame([{
-                                                    "NBSLink": final_url,
-                                                    "NBS": NBS_heading,
-                                                    'head above 1': header2,
-                                                    "head above 2": groupmainheadingFromArray[0]
-                                                }])], ignore_index=True)
-                                            
-                                            print("Final URL:", final_url)
-
-                            if collecting_text:
-                                annot = page.add_highlight_annot(highlight_rect)
-                                annot.update()
+                                            if current_item: 
+                                                if item not in current_item: 
+                                                    current_item += " " + item  # Merge with previous heading
+
+                                    # Append last merged item after loop
+                                    if current_item:
+                                        if current_item not in merged_groupheadings:
+                                            extracted_text = re.split(r"\.{3,}", current_item)[0].strip()
+                                            merged_groupheadings.append(extracted_text.strip())
+                                if span_text == first_word:
+                                    print('First word found:', span_text)
+                                    # Check if it's not the last span in the current line
+                                    print(i + 1, len(spans))
+                                    if i + 1 < len(spans):
+                                        next_span_text = (spans[i + 1]['text'].strip())
+                                    # Check if the next span's text is in the heading list
+                                    if next_span_text.replace(" ", "") in heading_to_search.replace(" ", ""):
+                                        text = (span_text + ' ' + next_span_text)
+                                                # After processing the current line, check if there's a next line
+                                if first_word == span_text:
+                                    if line_index + 1 < len(block.get('lines', [])):
+                                        next_line = block['lines'][line_index + 1]
+                                        # You can process the spans of the next line here
+                                        for next_span in next_line.get('spans', []):
+                                            next_span_text = next_span['text'].strip()
+                                            text = span_text + ' ' + next_span_text
+                                if len(merged_groupheadings) > 0:
+                                    if re.match(r"[A-Za-z]\d{2}", span_text) and span_size > 10:
+                                        toc_flag = False
+                                        endingcontentFlag=False
+                                        previous_header = span_text  # Store last detected header
+                                        print('previous_header', span_text)
+                                        
+                                    groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
+
+                                    if previous_header:
+                                        if not collectheader2:
+                                            if header2_first_span_size == 0:
+                                                spanSizeHeader = 10
+                                            else:
+                                                spanSizeHeader = header2_first_span_size
+
+                                            for item in groupmainheadingFromArray:
+                                                if not any(normalize_text(current_line_text) in normalize_text(item) for item in groupmainheadingFromArray):
+                                                    if not current_line_text[0].isdigit() :
+                                                        if span_size >= spanSizeHeader:
+                                                            if not re.match(r"^\d{2}", current_line_text) and current_line_text not in repeated_texts and "Bold" in span["font"] :
+                                                                if len(header2) > 0 :
+                                                                    header2_first_span_size = span_size
+                                                                header2 = current_line_text
+                                                                print('header2', header2, span_size, spanSizeHeader)
+
+                                trimmed_text = text.replace(" ", "")
+                                if len(text) > 0:
+                                    if text.split()[0] in heading_words:
+                                        if len(trimmed_text) > 0 and (heading_to_search.replace(" ", "") in trimmed_text):
+                                            print(trimmed_text, heading_to_search)
+                                            f10_count += 1
+                                            # Start collecting text under the second occurrence of the heading
+                                            if f10_count == 1:
+                                                collecting_text = True
+                                                print(f"Starting collection under heading: {text}, {span_font}, {span_size}")
+                                                collectheader2 = True
+                                                NBS_heading = heading_to_searchNBS
+                                                highlight_rectBegin=highlight_rect
+                                                x0, y0, x1, y1 = highlight_rectBegin
+                                                
+                                                span_font_goal = span_font  # Capture the font at the first heading match
+                                                span_size_goal = span_size  # Capture the size at the first heading match
+                                                zoom = 200
+                                                left = int(x0)
+                                                top = int(y0)
+                                                zoom_str = f"{zoom},{left},{top}"
+                                                pageNumberFound = page_num + 1
+                                                dictionaryNBS[heading_to_searchNBS] = [pageNumberFound, zoom_str]
+
+                                                annot = page.add_rect_annot(highlight_rect)  # Create a rectangle annotation
+                                                annot.set_colors(stroke=(1, 0, 0))  # Set border color (Red)
+                                                annot.update()  # Apply changes
+                                                groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
+                                                
+                                                            # Build the query parameters
+                                                params = {
+                                                    'pdfLink': link,  # Your PDF link
+                                                    'keyword': NBS_heading,  # Your keyword (could be a string or list)                                              
+                                                }
+                                                
+                                                # URL encode each parameter
+                                                encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+                                                
+                                                # Construct the final encoded link
+                                                encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+                                                
+                                                # Correctly construct the final URL with page and zoom
+                                                final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+                                                
+                                                # Get current date and time
+                                                now = datetime.now()
+
+                                                # Format the output
+                                                formatted_time = now.strftime("%d/%m/%Y %I:%M:%S %p")
+                                                # Optionally, add the URL to a DataFrame
+
+
+                                                if len(groupmainheadingFromArray) > 0:
+                                                    data_entry = {
+                                                        "NBSLink": final_url,
+                                                        "Subject": NBS_heading,
+                                                        "Page": str(pageNumberFound),
+                                                        "Author": "ADR",
+                                                        "Creation Date": formatted_time,
+                                                        "Layer": "Initial",
+                                                        "Code": "to be added",
+                                                        "head above 1": header2,
+                                                        "head above 2": groupmainheadingFromArray[0]
+                                                    }
+                                                    data_list_JSON.append(data_entry)
+
+                                                # Convert list to JSON
+                                                json_output = json.dumps(data_list_JSON, indent=4)
+                                                
+                                                print("Final URL:", final_url)
+
+                                if collecting_text:
+                                    annot = page.add_rect_annot(highlight_rect)  # Create a rectangle annotation
+                                    annot.set_colors(stroke=(1, 0, 0))  # Set border color (Red)
+                                    annot.update()  # Apply changes
                 if current_line.strip():
                     all_text += current_line.strip() + '\n'  # Append the current line
     print(df)
     print(dictionaryNBS)
-    xx=save_df_to_pdf(df)
-    outputpdfFitz =fitz.open('pdf',xx)
+    # xx=save_df_to_pdf(df)
+    # outputpdfFitz =fitz.open('pdf',xx)
     pdf_bytes = BytesIO()
     pdf_document.save(pdf_bytes)
-    return pdf_bytes.getvalue(), pdf_document , df,outputpdfFitz
-    
+    print('JSONN',json_output)
+    return pdf_bytes.getvalue(), pdf_document , df, json_output