File size: 890 Bytes
155974e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | """Normalize and segment text without neural generation."""
from __future__ import annotations
import re
def normalize_whitespace(text: str) -> str:
text = text.replace("\r\n", "\n").replace("\r", "\n")
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def split_paragraphs(text: str) -> list[str]:
parts = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
return parts or ([text.strip()] if text.strip() else [])
def split_sentences_regex(text: str) -> list[str]:
"""Fallback sentence splitter when spaCy is unavailable."""
text = text.strip()
if not text:
return []
parts = re.split(r"(?<=[.!?])\s+", text)
return [p.strip() for p in parts if p.strip()]
def word_count(text: str) -> int:
return len(text.split()) if text.strip() else 0
|