| """Paragraph / sentence segmentation with batch streaming.""" | |
| from __future__ import annotations | |
| import re | |
| from collections.abc import Iterator | |
| from app.engine.models import DocumentBlock | |
| from app.pipeline.nlp import get_nlp | |
| def split_sentences(text: str) -> list[str]: | |
| """Split a paragraph into sentences (spaCy preferred).""" | |
| raw = (text or "").strip() | |
| if not raw: | |
| return [] | |
| nlp = get_nlp() | |
| if nlp is not None: | |
| try: | |
| doc = nlp(raw) | |
| sents = [s.text.strip() for s in doc.sents if s.text.strip()] | |
| if sents: | |
| return sents | |
| except Exception: | |
| pass | |
| return [ | |
| part.strip() | |
| for part in re.split(r"(?<=[.!?])\s+", raw) | |
| if part.strip() | |
| ] | |
| def iter_paragraph_batches( | |
| blocks: list[DocumentBlock], | |
| *, | |
| batch_paras: int = 20, | |
| ) -> Iterator[list[DocumentBlock]]: | |
| """Yield batches of rewriteable paragraph blocks interleaved with passthrough. | |
| Each yielded list preserves document order for that slice of blocks. | |
| Non-rewriteable blocks are included so stitch can keep structure. | |
| Batching is driven by count of rewriteable paragraphs. | |
| """ | |
| batch_paras = max(1, min(int(batch_paras), 100)) | |
| batch: list[DocumentBlock] = [] | |
| rewriteable_count = 0 | |
| for block in blocks: | |
| batch.append(block) | |
| if block.rewriteable and block.kind == "paragraph": | |
| rewriteable_count += 1 | |
| if rewriteable_count >= batch_paras: | |
| yield batch | |
| batch = [] | |
| rewriteable_count = 0 | |
| if batch: | |
| yield batch | |
| def word_count(text: str) -> int: | |
| return len((text or "").split()) if (text or "").strip() else 0 | |