Delete src/pdf_counter.py

src/pdf_counter.py

@@ -1,326 +0,0 @@
-# pdf_counter.py
-import re
-from collections import Counter
-import fitz
-
-
-# ============================================================
-# REGEX PATTERNS
-# ============================================================
-# These patterns are used to identify page numbers and
-# running headers that should not be counted as content.
-
-PAGE_NUMBER_RE = re.compile(
-    r"^\s*(side\s*)?\d+\s*(/|af|-)?\s*\d*\s*$",
-    re.IGNORECASE,
-)
-
-RUNNING_HEADER_RE = re.compile(
-    r"^\d+(\.\d+)+\.?\s+.+\s+([ivxlcdm]+|\d+)$",
-    re.IGNORECASE,
-)
-
-
-# ============================================================
-# TEXT NORMALIZATION
-# ============================================================
-# Cleans extracted text by replacing multiple whitespace
-# characters (spaces, tabs, line breaks) with a single space.
-# This ensures consistent comparison and character counting.
-
-def normalize(text: str) -> str:
-    return re.sub(r"\s+", " ", text).strip()
-
-
-# ============================================================
-# PDF EXTRACTION
-# ============================================================
-# Reads the PDF and extracts all text blocks from each page.
-#
-# For every block we store:
-# - Page number
-# - Original text
-# - Lowercase version for comparisons
-# - Vertical coordinates on the page
-# - Page height
-#
-# The position data is later used to detect headers/footers.
-
-def extract_pages(pdf_bytes: bytes):
-    doc = fitz.open(stream=pdf_bytes, filetype="pdf")
-    pages = []
-
-    for page_no, page in enumerate(doc, start=1):
-        blocks = []
-
-        for block in page.get_text("blocks", sort=True):
-            x0, y0, x1, y1, text, *_ = block
-
-            text = normalize(text)
-
-            if text:
-                blocks.append({
-                    "page": page_no,
-                    "text": text,
-                    "text_key": text.lower(),
-                    "y0": y0,
-                    "y1": y1,
-                    "height": page.rect.height,
-                })
-
-        pages.append(blocks)
-
-    return pages
-
-
-# ============================================================
-# PAGE NUMBER DETECTION
-# ============================================================
-# Checks whether a text block looks like a page number.
-
-def is_page_number(text: str) -> bool:
-    return PAGE_NUMBER_RE.match(text) is not None
-
-
-# ============================================================
-# PAGE POSITION HELPERS
-# ============================================================
-# Determines whether a text block is located near the top
-# or bottom of the page.
-#
-# Top area  = top 15%
-# Bottom area = bottom 15%
-#
-# These areas are where headers and footers are expected.
-
-def is_top_area(block: dict) -> bool:
-    return block["y1"] <= block["height"] * 0.15
-
-
-def is_bottom_area(block: dict) -> bool:
-    return block["y0"] >= block["height"] * 0.85
-
-
-# ============================================================
-# RUNNING HEADER DETECTION
-# ============================================================
-# Identifies chapter-style running headers such as:
-#
-#   2.1 Methods 12
-#   4.3 Results iv
-#
-# They typically appear near the top of each page and
-# follow a numbering pattern.
-#
-# "Chapter X" headings are excluded because they are often
-# actual content rather than page headers.
-
-def is_running_header(block: dict) -> bool:
-    text = block["text"]
-
-    if text.lower().startswith("chapter "):
-        return False
-
-    return is_top_area(block) and RUNNING_HEADER_RE.match(text) is not None
-
-
-# ============================================================
-# HEADER / FOOTER DETECTION
-# ============================================================
-# Finds text that appears repeatedly in the top or bottom
-# regions of many pages.
-#
-# Repeated top text   -> header candidate
-# Repeated bottom text -> footer candidate
-#
-# A text must appear on at least min_ratio of pages before
-# it is classified as a header/footer.
-#
-# Default: 50% of pages.
-
-def detect_headers_and_footers(pages, min_ratio=0.5):
-    header_counter = Counter()
-    footer_counter = Counter()
-
-    running_headers = set()
-    page_numbers = set()
-
-    for blocks in pages:
-        headers_seen = set()
-        footers_seen = set()
-
-        for block in blocks:
-            text = block["text"]
-            text_key = block["text_key"]
-
-            # Collect page numbers separately
-            if is_page_number(text):
-                page_numbers.add(text)
-                continue
-
-            # Collect running headers separately
-            if is_running_header(block):
-                running_headers.add(text)
-                continue
-
-            # Potential header candidate
-            if is_top_area(block):
-                headers_seen.add(text_key)
-
-            # Potential footer candidate
-            if is_bottom_area(block):
-                footers_seen.add(text_key)
-
-        # Count once per page
-        header_counter.update(headers_seen)
-        footer_counter.update(footers_seen)
-
-    min_count = max(2, int(len(pages) * min_ratio))
-
-    detected_headers = {
-        text for text, count in header_counter.items()
-        if count >= min_count
-    }
-
-    detected_footers = {
-        text for text, count in footer_counter.items()
-        if count >= min_count
-    }
-
-    return (
-        detected_headers,
-        detected_footers,
-        running_headers,
-        page_numbers,
-    )
-
-
-# ============================================================
-# CHARACTER COUNTING ENGINE
-# ============================================================
-# Main workflow:
-#
-# 1. Extract all text blocks from the PDF.
-# 2. Detect repeated headers and footers.
-# 3. Detect page numbers.
-# 4. Remove unwanted elements.
-# 5. Count characters in remaining content.
-# 6. Return detailed results and diagnostics.
-
-def count_characters(
-    pdf_bytes: bytes,
-    excluded_pages: set[int] | None = None,
-    remove_headers: bool = True,
-    remove_footers: bool = True,
-    remove_page_numbers: bool = True,
-):
-    excluded_pages = excluded_pages or set()
-
-    # Extract all page data
-    pages = extract_pages(pdf_bytes)
-
-    # Detect recurring elements
-    (
-        detected_headers,
-        detected_footers,
-        running_headers,
-        detected_page_numbers,
-    ) = detect_headers_and_footers(pages)
-
-    included_text_parts = []
-    page_results = []
-    removed_items = []
-
-    # Process each page individually
-    for page_no, blocks in enumerate(pages, start=1):
-
-        # Skip pages excluded by the user
-        if page_no in excluded_pages:
-            page_results.append({
-                "Side": page_no,
-                "Tegn": 0,
-                "Status": "Fravalgt",
-            })
-            continue
-
-        kept_text = []
-
-        # Evaluate every text block
-        for block in blocks:
-            text = block["text"]
-            text_key = block["text_key"]
-
-            # Remove page numbers
-            if remove_page_numbers and is_page_number(text):
-                removed_items.append({
-                    "Side": page_no,
-                    "Type": "Sidetal",
-                    "Tekst": text,
-                })
-                continue
-
-            # Remove repeated headers
-            if remove_headers and text_key in detected_headers:
-                removed_items.append({
-                    "Side": page_no,
-                    "Type": "Sidehoved",
-                    "Tekst": text,
-                })
-                continue
-
-            # Remove running chapter headers
-            if remove_headers and is_running_header(block):
-                removed_items.append({
-                    "Side": page_no,
-                    "Type": "Løbende sidehoved",
-                    "Tekst": text,
-                })
-                continue
-
-            # Remove repeated footers
-            if remove_footers and text_key in detected_footers:
-                removed_items.append({
-                    "Side": page_no,
-                    "Type": "Sidefod",
-                    "Tekst": text,
-                })
-                continue
-
-            # Keep everything else
-            kept_text.append(text)
-
-        # Combine all remaining text on the page
-        page_text = " ".join(kept_text)
-
-        included_text_parts.append(page_text)
-
-        # Store page statistics
-        page_results.append({
-            "Side": page_no,
-            "Tegn": len(page_text),
-            "Status": "Talt med",
-        })
-
-    # Combine text from all included pages
-    full_text = " ".join(
-        t for t in included_text_parts if t
-    )
-
-    # Return complete result package
-    return {
-        "total_characters": len(full_text),
-        "page_results": page_results,
-        "included_text": full_text,
-
-        # Diagnostic information
-        "detected_headers": sorted(detected_headers),
-        "detected_footers": sorted(detected_footers),
-        "detected_running_headers": sorted(running_headers),
-        "detected_page_numbers": sorted(detected_page_numbers),
-
-        # Log of removed items
-        "removed_items": removed_items,
-
-        # Total pages in document
-        "page_count": len(pages),
-    }