| """ |
| Text utility functions: sentence segmentation and confidence level classification. |
| """ |
|
|
| import re |
|
|
| |
| ABBREVIATIONS = ( |
| r"\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|vs|etc|i\.e|e\.g|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec|No|Vol|pp|al|Fig|Eq)\." |
| ) |
|
|
|
|
| def split_into_sentences(text: str) -> list[str]: |
| """ |
| Split text into individual sentences while preserving abbreviations, |
| decimals, and proper boundary punctuation. |
| """ |
| if not text or not text.strip(): |
| return [] |
|
|
| |
| normalized = re.sub(r"[\r\n]+", " ", text) |
| normalized = re.sub(r"\s+", " ", normalized).strip() |
|
|
| |
| pattern = r'(?<=[.?!])\s+(?=[A-Z0-9"\'\(\[\{])' |
| raw_segments = re.split(pattern, normalized) |
|
|
| sentences: list[str] = [] |
| buffer = "" |
|
|
| for segment in raw_segments: |
| seg = segment.strip() |
| if not seg: |
| continue |
| if buffer: |
| candidate = f"{buffer} {seg}" |
| else: |
| candidate = seg |
|
|
| |
| if re.search(ABBREVIATIONS + r"$", candidate, flags=re.IGNORECASE): |
| buffer = candidate |
| else: |
| sentences.append(candidate) |
| buffer = "" |
|
|
| if buffer: |
| sentences.append(buffer) |
|
|
| return sentences |
|
|
|
|
| def get_confidence_band(prob: float) -> str: |
| """ |
| Compute confidence band for a given AI probability (0.0 to 1.0). |
| - high: prob >= 0.8 OR prob <= 0.2 |
| - medium: prob >= 0.65 OR prob <= 0.35 |
| - low: otherwise (between 0.35 and 0.65) |
| """ |
| if prob >= 0.8 or prob <= 0.2: |
| return "high" |
| elif prob >= 0.65 or prob <= 0.35: |
| return "medium" |
| return "low" |
|
|
