| import hashlib |
| from pathlib import Path |
| from .config import MAX_PDF_BYTES |
| from .schema import Document, Page |
|
|
|
|
| def read_document(name, data, models): |
| if Path(name).suffix.lower() != '.pdf': |
| raise ValueError('Upload a PDF. Save other document formats as PDF first.') |
| if not data or len(data) > MAX_PDF_BYTES: |
| raise ValueError('Upload a non-empty PDF smaller than 40 MB.') |
| if b'%PDF-' not in data[:1024]: |
| raise ValueError('This file does not appear to be a PDF.') |
| extracted = models.extract_pdf(name, data) |
| pages = [Page(number=p.number, original=p.original, english=p.english, |
| language=p.language, method='Gemini PDF extraction', reviewed=False, |
| warnings=p.warnings) for p in extracted] |
| if not pages or not any(p.english.strip() for p in pages): |
| raise ValueError('No readable text was found. Upload a clearer, unlocked PDF.') |
| return Document(id=hashlib.sha256(data + b'.pdf').hexdigest()[:12], |
| name=Path(name).name, pages=pages) |
|
|
|
|
| def split_text(text, limit=1000): |
| """Retained for Sarvam translation; never discards characters.""" |
| while text: |
| end = min(len(text), limit) |
| if end < len(text): |
| cut = max(text.rfind('\n', limit // 2, end), text.rfind('. ', limit // 2, end), |
| text.rfind(' ', limit // 2, end)) |
| if cut > 0: |
| end = cut + 1 |
| yield text[:end] |
| text = text[end:] |
|
|
