| """Versioned BFB input contract. Immutable originals; only CRLF/CR -> LF. |
| |
| Offsets are Python Unicode character boundaries, not UTF-16 or byte indices. |
| The interior boundary of CRLF is deliberately unrepresentable after normalization. |
| """ |
| from dataclasses import dataclass |
| import re |
|
|
| VERSION = 'bfb111-lf-v1' |
| SENT = re.compile(r'''[.!?γοΌοΌβ¦]+["'ββγγγγγγ)>]*(?=\s|$)|[ββγγ](?=\s|$)''') |
| ABBREV = re.compile(r'\b(?:Mr|Mrs|Ms|Dr|St|Prof|vs|Sr|Jr)\.$', re.I) |
|
|
|
|
| @dataclass |
| class InputText: |
| original: str |
| text: str |
| to_original: list |
| to_canonical: list |
|
|
| def original_span(self, start, end): |
| return self.to_original[start], self.to_original[end] |
|
|
| def position(self, original_position): |
| p = self.to_canonical[original_position] |
| if self.to_original[p] != original_position: |
| raise ValueError('Boundary lies inside CRLF') |
| return p |
|
|
|
|
| def normalize(original): |
| out, back, forward = [], [0], [0] * (len(original) + 1) |
| i = 0 |
| while i < len(original): |
| start = i |
| if original[i] == '\r': |
| i += 2 if original[i:i+2] == '\r\n' else 1 |
| ch = '\n' |
| else: |
| ch = original[i] |
| i += 1 |
| out.append(ch) |
| for j in range(start+1, i+1): |
| forward[j] = len(out) |
| back.append(i) |
| return InputText(original, ''.join(out), back, forward) |
|
|
|
|
| def candidates(text, language): |
| """Candidate locations only. Punctuation never supplies semantic labels.""" |
| last_content = len(text.rstrip()) |
| result = [] |
| for match in SENT.finditer(text): |
| p = match.end() |
| if p >= last_content: |
| continue |
| if language == 'en' and ABBREV.search(text[max(0, p-25):p].strip()): |
| continue |
| result.append(p) |
| return sorted(set(result)) |
|
|
|
|
| def original_candidates(inp, language): |
| return [inp.to_original[p] for p in candidates(inp.text, language)] |
|
|
