File size: 2,862 Bytes
492ec2c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | """Extract plain text from uploaded documents."""
from __future__ import annotations
import io
from pathlib import Path
SUPPORTED_EXTENSIONS = {".txt", ".md", ".docx", ".pdf"}
def extract_text_from_path(path: str | Path) -> str:
path = Path(path)
suffix = path.suffix.lower()
if suffix not in SUPPORTED_EXTENSIONS:
raise ValueError(
f"Unsupported file type '{suffix}'. Use .txt, .md, .docx, or .pdf."
)
data = path.read_bytes()
return extract_text_from_bytes(data, suffix, path.name)
def extract_text_from_bytes(data: bytes, suffix: str, filename: str = "") -> str:
suffix = suffix.lower()
if not suffix.startswith("."):
suffix = f".{suffix}"
if suffix in {".txt", ".md"}:
return _decode_text(data)
if suffix == ".docx":
return _extract_docx(data)
if suffix == ".pdf":
return _extract_pdf(data)
raise ValueError(f"Unsupported file type for '{filename or suffix}'.")
def _decode_text(data: bytes) -> str:
for encoding in ("utf-8", "utf-8-sig", "cp1252", "latin-1"):
try:
text = data.decode(encoding)
break
except UnicodeDecodeError:
continue
else:
text = data.decode("utf-8", errors="replace")
text = text.strip()
if not text:
raise ValueError("The file is empty.")
return text
def _extract_docx(data: bytes) -> str:
try:
from docx import Document
except ImportError as exc:
raise RuntimeError("python-docx is required for .docx files.") from exc
doc = Document(io.BytesIO(data))
parts: list[str] = []
for para in doc.paragraphs:
line = para.text.strip()
if line:
parts.append(line)
for table in doc.tables:
for row in table.rows:
cells = [c.text.strip() for c in row.cells if c.text.strip()]
if cells:
parts.append(" | ".join(cells))
text = "\n\n".join(parts).strip()
if not text:
raise ValueError("No readable text found in the Word document.")
return text
def _extract_pdf(data: bytes) -> str:
try:
from pypdf import PdfReader
except ImportError as exc:
raise RuntimeError("pypdf is required for .pdf files.") from exc
reader = PdfReader(io.BytesIO(data))
parts: list[str] = []
for page in reader.pages:
page_text = (page.extract_text() or "").strip()
if page_text:
parts.append(page_text)
text = "\n\n".join(parts).strip()
if not text:
raise ValueError(
"No readable text found in the PDF. Scanned/image PDFs are not supported."
)
return text
def word_count(text: str) -> int:
return len(text.split())
|