"""Versioned BFB input contract. Immutable originals; only CRLF/CR -> LF. Offsets are Python Unicode character boundaries, not UTF-16 or byte indices. The interior boundary of CRLF is deliberately unrepresentable after normalization. """ from dataclasses import dataclass import re VERSION = 'bfb111-lf-v1' SENT = re.compile(r'''[.!?。!?…]+["'”’」』》〉】〕)>]*(?=\s|$)|[”’」』](?=\s|$)''') ABBREV = re.compile(r'\b(?:Mr|Mrs|Ms|Dr|St|Prof|vs|Sr|Jr)\.$', re.I) @dataclass class InputText: original: str text: str to_original: list to_canonical: list def original_span(self, start, end): return self.to_original[start], self.to_original[end] def position(self, original_position): p = self.to_canonical[original_position] if self.to_original[p] != original_position: raise ValueError('Boundary lies inside CRLF') return p def normalize(original): out, back, forward = [], [0], [0] * (len(original) + 1) i = 0 while i < len(original): start = i if original[i] == '\r': i += 2 if original[i:i+2] == '\r\n' else 1 ch = '\n' else: ch = original[i] i += 1 out.append(ch) for j in range(start+1, i+1): forward[j] = len(out) back.append(i) return InputText(original, ''.join(out), back, forward) def candidates(text, language): """Candidate locations only. Punctuation never supplies semantic labels.""" last_content = len(text.rstrip()) result = [] for match in SENT.finditer(text): p = match.end() if p >= last_content: continue if language == 'en' and ABBREV.search(text[max(0, p-25):p].strip()): continue result.append(p) return sorted(set(result)) def original_candidates(inp, language): return [inp.to_original[p] for p in candidates(inp.text, language)]