"""Paragraph / sentence segmentation with batch streaming.""" from __future__ import annotations import re from collections.abc import Iterator from app.engine.models import DocumentBlock from app.pipeline.nlp import get_nlp def split_sentences(text: str) -> list[str]: """Split a paragraph into sentences (spaCy preferred).""" raw = (text or "").strip() if not raw: return [] nlp = get_nlp() if nlp is not None: try: doc = nlp(raw) sents = [s.text.strip() for s in doc.sents if s.text.strip()] if sents: return sents except Exception: pass return [ part.strip() for part in re.split(r"(?<=[.!?])\s+", raw) if part.strip() ] def iter_paragraph_batches( blocks: list[DocumentBlock], *, batch_paras: int = 20, ) -> Iterator[list[DocumentBlock]]: """Yield batches of rewriteable paragraph blocks interleaved with passthrough. Each yielded list preserves document order for that slice of blocks. Non-rewriteable blocks are included so stitch can keep structure. Batching is driven by count of rewriteable paragraphs. """ batch_paras = max(1, min(int(batch_paras), 100)) batch: list[DocumentBlock] = [] rewriteable_count = 0 for block in blocks: batch.append(block) if block.rewriteable and block.kind == "paragraph": rewriteable_count += 1 if rewriteable_count >= batch_paras: yield batch batch = [] rewriteable_count = 0 if batch: yield batch def word_count(text: str) -> int: return len((text or "").split()) if (text or "").strip() else 0