import hashlib from pathlib import Path from .config import MAX_PDF_BYTES from .schema import Document, Page def read_document(name, data, models): if Path(name).suffix.lower() != '.pdf': raise ValueError('Upload a PDF. Save other document formats as PDF first.') if not data or len(data) > MAX_PDF_BYTES: raise ValueError('Upload a non-empty PDF smaller than 40 MB.') if b'%PDF-' not in data[:1024]: raise ValueError('This file does not appear to be a PDF.') extracted = models.extract_pdf(name, data) pages = [Page(number=p.number, original=p.original, english=p.english, language=p.language, method='Gemini PDF extraction', reviewed=False, warnings=p.warnings) for p in extracted] if not pages or not any(p.english.strip() for p in pages): raise ValueError('No readable text was found. Upload a clearer, unlocked PDF.') return Document(id=hashlib.sha256(data + b'.pdf').hexdigest()[:12], name=Path(name).name, pages=pages) def split_text(text, limit=1000): """Retained for Sarvam translation; never discards characters.""" while text: end = min(len(text), limit) if end < len(text): cut = max(text.rfind('\n', limit // 2, end), text.rfind('. ', limit // 2, end), text.rfind(' ', limit // 2, end)) if cut > 0: end = cut + 1 yield text[:end] text = text[end:]