File size: 1,959 Bytes
af5c818 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Versioned BFB input contract. Immutable originals; only CRLF/CR -> LF.
Offsets are Python Unicode character boundaries, not UTF-16 or byte indices.
The interior boundary of CRLF is deliberately unrepresentable after normalization.
"""
from dataclasses import dataclass
import re
VERSION = 'bfb111-lf-v1'
SENT = re.compile(r'''[.!?γοΌοΌβ¦]+["'ββγγγγγγ)>]*(?=\s|$)|[ββγγ](?=\s|$)''')
ABBREV = re.compile(r'\b(?:Mr|Mrs|Ms|Dr|St|Prof|vs|Sr|Jr)\.$', re.I)
@dataclass
class InputText:
original: str
text: str
to_original: list
to_canonical: list
def original_span(self, start, end):
return self.to_original[start], self.to_original[end]
def position(self, original_position):
p = self.to_canonical[original_position]
if self.to_original[p] != original_position:
raise ValueError('Boundary lies inside CRLF')
return p
def normalize(original):
out, back, forward = [], [0], [0] * (len(original) + 1)
i = 0
while i < len(original):
start = i
if original[i] == '\r':
i += 2 if original[i:i+2] == '\r\n' else 1
ch = '\n'
else:
ch = original[i]
i += 1
out.append(ch)
for j in range(start+1, i+1):
forward[j] = len(out)
back.append(i)
return InputText(original, ''.join(out), back, forward)
def candidates(text, language):
"""Candidate locations only. Punctuation never supplies semantic labels."""
last_content = len(text.rstrip())
result = []
for match in SENT.finditer(text):
p = match.end()
if p >= last_content:
continue
if language == 'en' and ABBREV.search(text[max(0, p-25):p].strip()):
continue
result.append(p)
return sorted(set(result))
def original_candidates(inp, language):
return [inp.to_original[p] for p in candidates(inp.text, language)]
|