modules/doc_reader.py

"""
Document Reader Module
Supports: PDF (pymupdf), DOCX (python-docx), TXT
Returns: plain text + metadata (page count, word count, source type)
"""

import fitz          # PyMuPDF
import docx
import os


def read_pdf(path: str) -> tuple[str, dict]:
    doc   = fitz.open(path)
    pages = []
    for page in doc:
        pages.append(page.get_text("text"))
    full_text = "\n".join(pages)
    meta = {
        "type":       "PDF",
        "pages":      doc.page_count,
        "word_count": len(full_text.split()),
        "filename":   os.path.basename(path),
    }
    doc.close()
    return full_text, meta


def read_docx(path: str) -> tuple[str, dict]:
    document   = docx.Document(path)
    paragraphs = [p.text for p in document.paragraphs if p.text.strip()]
    full_text  = "\n".join(paragraphs)
    meta = {
        "type":       "DOCX",
        "pages":      "N/A",
        "word_count": len(full_text.split()),
        "filename":   os.path.basename(path),
    }
    return full_text, meta


def read_txt(path: str) -> tuple[str, dict]:
    with open(path, "r", encoding="utf-8", errors="ignore") as f:
        full_text = f.read()
    meta = {
        "type":       "TXT",
        "pages":      "N/A",
        "word_count": len(full_text.split()),
        "filename":   os.path.basename(path),
    }
    return full_text, meta


def read_document(path: str) -> tuple[str, dict]:
    """Auto-detect file type and extract text."""
    ext = os.path.splitext(path)[-1].lower()
    if ext == ".pdf":
        return read_pdf(path)
    elif ext in (".docx", ".doc"):
        return read_docx(path)
    elif ext == ".txt":
        return read_txt(path)
    else:
        raise ValueError(f"Unsupported file type: {ext}. Supported: PDF, DOCX, TXT")


def chunk_text(text: str, max_chars: int = 400) -> list[str]:
    """
    Split long text into sentence-aware chunks for NER processing.
    BERT has a 512-token limit — chunking prevents truncation errors.
    """
    import re
    # Split on sentence boundaries
    sentences = re.split(r'(?<=[.!?])\s+', text)
    chunks    = []
    current   = ""

    for sent in sentences:
        if len(current) + len(sent) <= max_chars:
            current += " " + sent
        else:
            if current.strip():
                chunks.append(current.strip())
            current = sent

    if current.strip():
        chunks.append(current.strip())

    return chunks if chunks else [text[:max_chars]]