Upload Find_Hyperlinking_text.py

Find_Hyperlinking_text.py

@@ -0,0 +1,392 @@
+import fitz  # PyMuPDF
+from io import BytesIO
+import re
+import requests
+import pandas as pd
+from collections import Counter
+import fitz  # PyMuPDF
+import re
+import urllib.parse
+import pandas as pd
+import tempfile
+from fpdf import FPDF
+
+
+baselink='https://marthee-nbslink.hf.space/view-pdf?'
+class PDF(FPDF):
+    def header(self):
+        self.set_font("Arial", "B", 12)
+        self.cell(0, 10, "NBS Document Links", ln=True, align="C")
+        self.ln(5)  # Space after header
+
+def save_df_to_pdf(df):
+    pdf = PDF()
+    pdf.set_auto_page_break(auto=True, margin=15)
+
+    # Set equal margins
+    margin = 15
+    pdf.set_left_margin(margin)
+    pdf.set_right_margin(margin)
+
+    pdf.add_page()
+    pdf.set_font("Arial", size=10)
+
+    # Set column widths and calculate total table width
+    col_width = 50
+    num_cols = 4
+    table_width = col_width * num_cols
+
+    # Get page width and calculate left alignment
+    page_width = pdf.w
+    start_x = (page_width - table_width) / 2  # Centering the table
+
+    pdf.set_x(start_x)  # Move to calculated start position
+
+    # Table headers
+    pdf.set_fill_color(200, 200, 200)  # Light gray background
+    pdf.set_font("Arial", "B", 10)
+    headers = ["NBS Link", "NBS", "Head Above 1", "Head Above 2"]
+
+    # Draw table headers
+    for header in headers:
+        pdf.cell(col_width, 8, header, border=1, fill=True, align="C")
+    pdf.ln()
+
+    pdf.set_font("Arial", size=9)
+
+    # Add rows
+    for _, row in df.iterrows():
+        x_start = start_x  # Ensure every row starts at the same position
+        y_start = pdf.get_y()
+
+        # Calculate max height needed for this row
+        text_lines = {col: pdf.multi_cell(col_width, 5, row[col], border=0, align="L", split_only=True) for col in ["NBS", "head above 1", "head above 2"]}
+        max_lines = max(len(lines) for lines in text_lines.values())
+        max_height = max_lines * 5
+
+        pdf.set_x(x_start)  # Ensure correct alignment for each row
+
+        # Clickable link cell (keeps same height as others)
+        pdf.cell(col_width, max_height, "Click Here", border=1, link=row["NBSLink"], align="C")
+
+        # Move to next column
+        pdf.set_xy(x_start + col_width, y_start)
+
+        # Draw each cell manually, ensuring equal height
+        for i, col_name in enumerate(["NBS", "head above 1", "head above 2"]):
+            x_col = x_start + col_width * (i + 1)
+            y_col = y_start
+            pdf.multi_cell(col_width, 5, row[col_name], border=0, align="L")  # Draw text
+            pdf.rect(x_col, y_col, col_width, max_height)  # Draw border
+            pdf.set_xy(x_col + col_width, y_start)  # Move to next column
+
+        # Move to the next row
+        pdf.ln(max_height)
+        # Save PDF to memory instead of a file
+    # pdf_output = BytesIO()
+    # pdf_output = 'output.pdf'
+    pdf_output = pdf.output(dest="S").encode("latin1")  # Returns the PDF as a byte string
+
+    return pdf_output
+
+
+    
+def normalize_text(text):
+    """Lowercase, remove extra spaces, and strip special characters."""
+    text = text.lower().strip()
+    text = re.sub(r'\s+', ' ', text)  # Normalize multiple spaces
+    return re.sub(r'[^\w\s]', '', text)  # Remove punctuation
+def get_repeated_texts(pdf_document, threshold=0.9):
+    """
+    Identify text that appears on most pages.
+    :param pdf_document: The opened PDF document.
+    :param threshold: The percentage of pages a text must appear on to be considered "repeated".
+    """
+    text_counts = Counter()
+    total_pages = pdf_document.page_count
+
+    for page_num in range(total_pages):
+        page = pdf_document.load_page(page_num)
+        page_text = page.get_text("text")
+        normalized_lines = {normalize_text(line) for line in page_text.splitlines() if line.strip()}
+
+        text_counts.update(normalized_lines)
+
+    # Find texts that appear in at least `threshold * total_pages` pages
+    min_occurrence = max(1, int(threshold * total_pages))
+    repeated_texts = {text for text, count in text_counts.items() if count >= min_occurrence}
+    return repeated_texts
+
+
+def split_links(links_string):
+    """Split a comma-separated string of links into an array of trimmed links."""
+    return [link.strip() for link in links_string.split(',')]
+def annotate_text_from_pdf(pdfshareablelinks, LISTheading_to_search):
+    """
+    Annotates text under a specific heading in a PDF, highlights it, 
+    and constructs zoom coordinates for the first occurrence of the heading.
+    Args:
+        pdfshareablelinks (list): List of shareable links to PDFs.
+        heading_to_search (str): The heading to search for in the PDF.
+    Returns:
+        Tuple: Annotated PDF bytes, count of heading occurrences, and zoom string.
+    """
+    print("Input links:", pdfshareablelinks)
+    print(LISTheading_to_search)
+    
+    link = pdfshareablelinks[0]
+    pdf_content = None
+    headings_TOC = []
+    # Modify Dropbox shareable link for direct download
+    if link and ('http' in link or 'dropbox' in link):
+        if 'dl=0' in link:
+            link = link.replace('dl=0', 'dl=1')
+
+    # Download the PDF content from the shareable link
+    response = requests.get(link)
+    pdf_content = BytesIO(response.content)  # Store the content in memory
+    if pdf_content is None:
+        raise ValueError("No valid PDF content found.")
+
+    # Open the PDF using PyMuPDF
+    pdf_document = fitz.open(stream=pdf_content, filetype="pdf")
+    repeated_texts = get_repeated_texts(pdf_document)
+    df = pd.DataFrame(columns=["NBSLink","NBS", 'head above 1', "head above 2"])
+    dictionaryNBS={}
+    for NBSindex, heading_to_search in enumerate(LISTheading_to_search):
+        if NBSindex == len(LISTheading_to_search) - 1:
+            flagAllNBSvisited = True
+        all_text = []
+        current_line = ""
+        collecting_text = False
+        f10_count = 0
+        current_y = None
+        highlight_rect = None
+        zoom_str = None
+        toc_flag = False
+        span_font_goal = None
+        span_size_goal = None
+        pageNumberFound = None
+        groupheadings = []
+        merged_groupheadings = []
+        collectheader2 = False
+        header2 = ''
+        header2_first_span_size = 0
+        previous_header = ''
+        next_span_text = ''
+        current_line_span_size = 0
+        flagAllNBSvisited = False
+        
+        text = ''
+        heading_to_searchNBS = heading_to_search
+        heading_words = heading_to_search.split()  # Split heading into words
+        first_word = heading_words[0]  # First word to search for
+        remaining_words = heading_words[1:]  # Remaining words to verify
+        print(heading_words)
+        heading_to_search = heading_to_search.replace(" ", "")
+
+        # Process each page in the PDF
+        for page_num in range(pdf_document.page_count):
+            page = pdf_document.load_page(page_num)
+            # Get page dimensions
+            page_height = page.rect.height
+            header_threshold = page_height * 0.1  # Top 10% of the page height
+            footer_threshold = page_height * 0.9  # Bottom 10% of the page height
+
+            # Extract text in dictionary format
+            text_dict = page.get_text("dict")
+
+            # Collect header y-coordinates to detect header area
+            header_threshold = 0  # Header area: top 10% of the page height
+            current_line_text = ""
+            previous_y = None
+            # Process text blocks
+            for block in text_dict['blocks']:
+                for line_index, line in enumerate(block.get('lines', [])):
+                    spans = line.get('spans', [])
+                    if spans and any(span['text'].strip() for span in spans):
+                        for i, span in enumerate(spans):
+                            span_text = span['text'].strip()
+                            highlight_rect = span['bbox']
+                            span_y = span['bbox'][1]
+                            span_font = span['font']
+                            span_size = span['size']
+
+                            if previous_y is None:
+                                previous_y = span_y  # Initialize on first span
+
+                            # If same Y coordinate as previous, append to the current line
+                            if abs(span_y - previous_y) < 5:  # Allow a small margin for OCR variations
+                                current_line_text += " " + span_text
+                                current_line_text = normalize_text(current_line_text)
+                                current_line_span_size = span_size
+                            else:
+                                # Store the complete line and reset for the new line
+                                if current_line_text.strip():
+                                    all_text.append(current_line_text.strip())
+
+                                current_line_text = span_text  # Start a new line
+                                previous_y = span_y  # Update the reference Y
+                            text = span_text
+                            if collecting_text and span_font == span_font_goal and span_size == span_size_goal and span_text[0].isdigit():
+                                print(f"Ending collection at heading: {span_text}")
+                                print("merged_groupheadings:", merged_groupheadings)
+                                print('groupheadingss',groupheadings)
+                                collecting_text = False
+                                continue
+                            if collecting_text:
+                                annot = page.add_highlight_annot(highlight_rect)
+                                annot.update()
+
+                            if 'Content' in span_text:
+                                toc_flag = True
+                                TOC_start = span_text
+                                print('content', TOC_start, span_size)
+
+                            if toc_flag:
+                                if 'Content' not in span_text:
+                                    if current_y is None:
+                                        current_y = span_y
+                                        current_size = span_size  # Initialize the reference span size
+                                    # Check if the current span size deviates significantly
+                                    if abs(span_size - current_size) > 1:  # Threshold for size difference
+                                        toc_flag = False
+                                 
+                                    if abs(current_y - span_y) < 5:  # Allowing more flexibility for multi-line headings
+                                        current_line += " " + span_text  # Keep accumulating text
+                                    else:
+                                        if current_line.strip():  # Only process non-empty lines
+                                            print('current_line',current_line)
+                                            pattern = r"^([A-Za-z0-9\s\/\-,]+)(?=\.+)"
+                                            match = re.match(pattern, current_line.strip())
+                                            
+                                            if match:
+                                                groupheadings.append(match.group(1).strip())
+                                        # else:
+                                        #     toc_flag = False
+
+                                        current_line = span_text
+                                        current_y = span_y
+                                        current_size = span_size  # Update reference span size
+                            # print('outofcurrent')
+                            if len(groupheadings) > 0:
+                                pattern = re.compile(r"^[A-Za-z]\d{2} ")  # Match headings starting with letter + 2 digits
+                                merged_groupheadings = []
+                                current_item = None  # Start as None to avoid an initial blank entry
+
+                                for item in groupheadings:
+                                    if pattern.match(item):  # If item starts with correct pattern, it's a new heading
+                                        if current_item:  # Append only if current_item is not empty
+                                            merged_groupheadings.append(current_item.strip())
+                                        current_item = item  # Start new heading
+                                    else:
+                                        if current_item:  
+                                            current_item += " " + item  # Merge with previous heading
+
+                                # Append last merged item after loop
+                                if current_item:
+                                    merged_groupheadings.append(current_item.strip())
+                            if span_text == first_word:
+                                print('First word found:', span_text)
+                                # Check if it's not the last span in the current line
+                                print(i + 1, len(spans))
+                                if i + 1 < len(spans):
+                                    next_span_text = (spans[i + 1]['text'].strip())
+                                # Check if the next span's text is in the heading list
+                                if next_span_text.replace(" ", "") in heading_to_search.replace(" ", ""):
+                                    text = (span_text + ' ' + next_span_text)
+                                            # After processing the current line, check if there's a next line
+                            if first_word == span_text:
+                                if line_index + 1 < len(block.get('lines', [])):
+                                    next_line = block['lines'][line_index + 1]
+                                    # You can process the spans of the next line here
+                                    for next_span in next_line.get('spans', []):
+                                        next_span_text = next_span['text'].strip()
+                                        text = span_text + ' ' + next_span_text
+                            if len(merged_groupheadings) > 0:
+                                if re.match(r"[A-Za-z]\d{2}", span_text) and span_size > 10:
+                                    previous_header = span_text  # Store last detected header
+                                    print('previous_header', span_text)
+                                groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
+
+                                if previous_header:
+                                    if not collectheader2:
+                                        if header2_first_span_size == 0:
+                                            spanSizeHeader = 10
+                                        else:
+                                            spanSizeHeader = header2_first_span_size
+
+                                        for item in groupmainheadingFromArray:
+                                            if not any(normalize_text(current_line_text) in normalize_text(item) for item in groupmainheadingFromArray):
+                                                if span_size >= spanSizeHeader:
+                                                    if not re.match(r"^\d{2}", current_line_text) and current_line_text not in repeated_texts and "Bold" in span["font"] :
+                                                        if len(header2) > 0:
+                                                            header2_first_span_size = span_size
+                                                        header2 = current_line_text
+                                                        print('header2', header2, span_size, spanSizeHeader)
+
+                            trimmed_text = text.replace(" ", "")
+                            if len(text) > 0:
+                                if text.split()[0] in heading_words:
+                                    if len(trimmed_text) > 0 and (heading_to_search.replace(" ", "") in trimmed_text):
+                                        print(trimmed_text, heading_to_search)
+                                        f10_count += 1
+                                        # Start collecting text under the second occurrence of the heading
+                                        if f10_count == 1:
+                                            collecting_text = True
+                                            print(f"Starting collection under heading: {text}, {span_font}, {span_size}")
+                                            collectheader2 = True
+                                            NBS_heading = heading_to_searchNBS
+                                            x0, y0, x1, y1 = highlight_rect
+                                            
+                                            span_font_goal = span_font  # Capture the font at the first heading match
+                                            span_size_goal = span_size  # Capture the size at the first heading match
+                                            zoom = 200
+                                            left = int(x0)
+                                            top = int(y0)
+                                            zoom_str = f"{zoom},{left},{top}"
+                                            pageNumberFound = page_num + 1
+                                            dictionaryNBS[heading_to_searchNBS] = [pageNumberFound, zoom_str]
+
+                                            annot = page.add_highlight_annot(highlight_rect)
+                                            annot.update()
+                                            groupmainheadingFromArray = [item for item in merged_groupheadings if previous_header in item]
+                                            
+                                                        # Build the query parameters
+                                            params = {
+                                                'pdfLink': link,  # Your PDF link
+                                                'keyword': NBS_heading,  # Your keyword (could be a string or list)                                              
+                                            }
+                                            
+                                            # URL encode each parameter
+                                            encoded_params = {key: urllib.parse.quote(value, safe='') for key, value in params.items()}
+                                            
+                                            # Construct the final encoded link
+                                            encoded_link = '&'.join([f"{key}={value}" for key, value in encoded_params.items()])
+                                            
+                                            # Correctly construct the final URL with page and zoom
+                                            final_url = f"{baselink}{encoded_link}#page={str(pageNumberFound)}&zoom={zoom_str}"
+                                            
+                                            # Optionally, add the URL to a DataFrame
+                                            if len(groupmainheadingFromArray) > 0:
+                                                df = pd.concat([df, pd.DataFrame([{
+                                                    "NBSLink": final_url,
+                                                    "NBS": NBS_heading,
+                                                    'head above 1': header2,
+                                                    "head above 2": groupmainheadingFromArray[0]
+                                                }])], ignore_index=True)
+                                            
+                                            print("Final URL:", final_url)
+
+                            if collecting_text:
+                                annot = page.add_highlight_annot(highlight_rect)
+                                annot.update()
+                if current_line.strip():
+                    all_text += current_line.strip() + '\n'  # Append the current line
+    print(df)
+    print(dictionaryNBS)
+    xx=save_df_to_pdf(df)
+    outputpdfFitz =fitz.open('pdf',xx)
+    pdf_bytes = BytesIO()
+    pdf_document.save(pdf_bytes)
+    return pdf_bytes.getvalue(), pdf_document , df,outputpdfFitz