| """Normalize and segment text without neural generation.""" | |
| from __future__ import annotations | |
| import re | |
| def normalize_whitespace(text: str) -> str: | |
| text = text.replace("\r\n", "\n").replace("\r", "\n") | |
| text = re.sub(r"[ \t]+", " ", text) | |
| text = re.sub(r"\n{3,}", "\n\n", text) | |
| return text.strip() | |
| def split_paragraphs(text: str) -> list[str]: | |
| parts = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] | |
| return parts or ([text.strip()] if text.strip() else []) | |
| def split_sentences_regex(text: str) -> list[str]: | |
| """Fallback sentence splitter when spaCy is unavailable.""" | |
| text = text.strip() | |
| if not text: | |
| return [] | |
| parts = re.split(r"(?<=[.!?])\s+", text) | |
| return [p.strip() for p in parts if p.strip()] | |
| def word_count(text: str) -> int: | |
| return len(text.split()) if text.strip() else 0 | |