Spaces:
Sleeping
Sleeping
File size: 2,462 Bytes
fbd78fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """
Document Reader Module
Supports: PDF (pymupdf), DOCX (python-docx), TXT
Returns: plain text + metadata (page count, word count, source type)
"""
import fitz # PyMuPDF
import docx
import os
def read_pdf(path: str) -> tuple[str, dict]:
doc = fitz.open(path)
pages = []
for page in doc:
pages.append(page.get_text("text"))
full_text = "\n".join(pages)
meta = {
"type": "PDF",
"pages": doc.page_count,
"word_count": len(full_text.split()),
"filename": os.path.basename(path),
}
doc.close()
return full_text, meta
def read_docx(path: str) -> tuple[str, dict]:
document = docx.Document(path)
paragraphs = [p.text for p in document.paragraphs if p.text.strip()]
full_text = "\n".join(paragraphs)
meta = {
"type": "DOCX",
"pages": "N/A",
"word_count": len(full_text.split()),
"filename": os.path.basename(path),
}
return full_text, meta
def read_txt(path: str) -> tuple[str, dict]:
with open(path, "r", encoding="utf-8", errors="ignore") as f:
full_text = f.read()
meta = {
"type": "TXT",
"pages": "N/A",
"word_count": len(full_text.split()),
"filename": os.path.basename(path),
}
return full_text, meta
def read_document(path: str) -> tuple[str, dict]:
"""Auto-detect file type and extract text."""
ext = os.path.splitext(path)[-1].lower()
if ext == ".pdf":
return read_pdf(path)
elif ext in (".docx", ".doc"):
return read_docx(path)
elif ext == ".txt":
return read_txt(path)
else:
raise ValueError(f"Unsupported file type: {ext}. Supported: PDF, DOCX, TXT")
def chunk_text(text: str, max_chars: int = 400) -> list[str]:
"""
Split long text into sentence-aware chunks for NER processing.
BERT has a 512-token limit — chunking prevents truncation errors.
"""
import re
# Split on sentence boundaries
sentences = re.split(r'(?<=[.!?])\s+', text)
chunks = []
current = ""
for sent in sentences:
if len(current) + len(sent) <= max_chars:
current += " " + sent
else:
if current.strip():
chunks.append(current.strip())
current = sent
if current.strip():
chunks.append(current.strip())
return chunks if chunks else [text[:max_chars]]
|