""" Document Reader Module Supports: PDF (pymupdf), DOCX (python-docx), TXT Returns: plain text + metadata (page count, word count, source type) """ import fitz # PyMuPDF import docx import os def read_pdf(path: str) -> tuple[str, dict]: doc = fitz.open(path) pages = [] for page in doc: pages.append(page.get_text("text")) full_text = "\n".join(pages) meta = { "type": "PDF", "pages": doc.page_count, "word_count": len(full_text.split()), "filename": os.path.basename(path), } doc.close() return full_text, meta def read_docx(path: str) -> tuple[str, dict]: document = docx.Document(path) paragraphs = [p.text for p in document.paragraphs if p.text.strip()] full_text = "\n".join(paragraphs) meta = { "type": "DOCX", "pages": "N/A", "word_count": len(full_text.split()), "filename": os.path.basename(path), } return full_text, meta def read_txt(path: str) -> tuple[str, dict]: with open(path, "r", encoding="utf-8", errors="ignore") as f: full_text = f.read() meta = { "type": "TXT", "pages": "N/A", "word_count": len(full_text.split()), "filename": os.path.basename(path), } return full_text, meta def read_document(path: str) -> tuple[str, dict]: """Auto-detect file type and extract text.""" ext = os.path.splitext(path)[-1].lower() if ext == ".pdf": return read_pdf(path) elif ext in (".docx", ".doc"): return read_docx(path) elif ext == ".txt": return read_txt(path) else: raise ValueError(f"Unsupported file type: {ext}. Supported: PDF, DOCX, TXT") def chunk_text(text: str, max_chars: int = 400) -> list[str]: """ Split long text into sentence-aware chunks for NER processing. BERT has a 512-token limit — chunking prevents truncation errors. """ import re # Split on sentence boundaries sentences = re.split(r'(?<=[.!?])\s+', text) chunks = [] current = "" for sent in sentences: if len(current) + len(sent) <= max_chars: current += " " + sent else: if current.strip(): chunks.append(current.strip()) current = sent if current.strip(): chunks.append(current.strip()) return chunks if chunks else [text[:max_chars]]