File size: 1,788 Bytes
8f6d79d 39cfcd1 8f6d79d 39cfcd1 8f6d79d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """Paragraph / sentence segmentation with batch streaming."""
from __future__ import annotations
import re
from collections.abc import Iterator
from app.engine.models import DocumentBlock
from app.pipeline.nlp import get_nlp
def split_sentences(text: str) -> list[str]:
"""Split a paragraph into sentences (spaCy preferred)."""
raw = (text or "").strip()
if not raw:
return []
nlp = get_nlp()
if nlp is not None:
try:
doc = nlp(raw)
sents = [s.text.strip() for s in doc.sents if s.text.strip()]
if sents:
return sents
except Exception:
pass
return [
part.strip()
for part in re.split(r"(?<=[.!?])\s+", raw)
if part.strip()
]
def iter_paragraph_batches(
blocks: list[DocumentBlock],
*,
batch_paras: int = 20,
) -> Iterator[list[DocumentBlock]]:
"""Yield batches of rewriteable paragraph blocks interleaved with passthrough.
Each yielded list preserves document order for that slice of blocks.
Non-rewriteable blocks are included so stitch can keep structure.
Batching is driven by count of rewriteable paragraphs.
"""
batch_paras = max(1, min(int(batch_paras), 100))
batch: list[DocumentBlock] = []
rewriteable_count = 0
for block in blocks:
batch.append(block)
if block.rewriteable and block.kind == "paragraph":
rewriteable_count += 1
if rewriteable_count >= batch_paras:
yield batch
batch = []
rewriteable_count = 0
if batch:
yield batch
def word_count(text: str) -> int:
return len((text or "").split()) if (text or "").strip() else 0
|