| """Text normalisation and tokenisation. |
| |
| Tokens keep their character offsets in the normalised text, so every value the copilot |
| fills can point back at the words that produced it (the UI highlights them). |
| """ |
| from __future__ import annotations |
|
|
| import re |
| import unicodedata |
| from dataclasses import dataclass |
|
|
| MAX_CHARS = 2000 |
|
|
| _QUOTES = {'‘': "'", '’': "'", '‚': "'", '‛': "'", '′': "'", |
| '“': '"', '”': '"', '„': '"', '″': '"', '«': '"', '»': '"', |
| '–': '-', '—': '-', '−': '-', ' ': ' '} |
|
|
| CONTRACTIONS = { |
| "don't": 'do not', 'dont': 'do not', "doesn't": 'does not', 'doesnt': 'does not', "didn't": 'did not', |
| "can't": 'can not', 'cant': 'can not', 'cannot': 'can not', "won't": 'will not', 'wont': 'will not', |
| "shouldn't": 'should not', 'shouldnt': 'should not', "mustn't": 'must not', 'mustnt': 'must not', |
| "isn't": 'is not', "aren't": 'are not', "wasn't": 'was not', "weren't": 'were not', |
| "wouldn't": 'would not', "couldn't": 'could not', "haven't": 'have not', "hasn't": 'has not', |
| "shan't": 'shall not', "needn't": 'need not', "mayn't": 'may not', "ain't": 'is not', |
| "let's": 'let us', "it's": 'it is', "i'm": 'i am', "they're": 'they are', "we're": 'we are', |
| "you're": 'you are', "that's": 'that is', "what's": 'what is', "there's": 'there is', |
| "i'd": 'i would', "we'd": 'we would', "i'll": 'i will', "we'll": 'we will', "i've": 'i have', "we've": 'we have', |
| } |
|
|
| NUMBER_WORDS = { |
| 'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, |
| 'nine': 9, 'ten': 10, 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14, 'fifteen': 15, |
| 'sixteen': 16, 'seventeen': 17, 'eighteen': 18, 'nineteen': 19, 'twenty': 20, 'thirty': 30, |
| 'forty': 40, 'fifty': 50, 'sixty': 60, 'seventy': 70, 'eighty': 80, 'ninety': 90, |
| 'hundred': 100, 'thousand': 1000, 'dozen': 12, |
| } |
|
|
| |
| _TOKEN_RE = re.compile(r""" |
| (?P<quoted>"[^"\n]{1,80}"|(?<![\w'])'[^'\n]{1,60}'(?![\w'])) |
| | (?P<email>[a-z0-9._%+-]+@[a-z0-9-]+(?:\.[a-z0-9-]+)+) |
| | (?P<url>https?://[^\s,;"'<>()]+) |
| | (?P<path>(?:\*\*/|\*|~)?[./\\]?[a-z0-9_*-]*\.(?:env|pem|key|pfx|p12|kdbx|ssh|aws|npmrc|pypirc|netrc|git-credentials)\b|\*\*?/[^\s,;]+|\*\.[a-z0-9]{1,8}\b) |
| | (?P<time>\d{1,2}(?::\d{2})?\s?(?:am|pm)\b|\d{1,2}:\d{2}\b) |
| | (?P<number>\d[\d,]*(?:\.\d+)?(?=(?:kb|mb|gb|tb)\b|[^a-z0-9.,]|[.,](?![0-9])|$)) # whole numbers only: "1234abcd" is a serial, "50mb" is 50 + mb |
| | (?P<word>[a-z0-9][a-z0-9_]*(?:['&.+-][a-z0-9_]+)*(?:'s)?) |
| | (?P<punct>[,.;:!?/()&|\n-]) |
| """, re.VERBOSE) |
|
|
| _TLD = re.compile(r'\.[a-z]{2,24}$') |
| |
| FILE_EXTS = {'md', 'json', 'txt', 'env', 'yaml', 'yml', 'py', 'js', 'ts', 'pem', 'key', 'csv', 'docx', 'xlsx', 'pdf', |
| 'sh', 'toml', 'ini', 'cfg', 'log', 'xml', 'doc', 'xls', 'ppt', 'pptx', 'zip', 'exe', 'dll', 'jar', 'sql', |
| 'bak', 'conf', 'pfx', 'p12', 'kdbx', 'rules', 'lock'} |
| _DOMAIN = re.compile(r'^(?:[a-z0-9-]+\.)+[a-z]{2,24}$') |
| _SERIAL = re.compile(r'^(?=.*\d)(?=.*[a-z])[a-z0-9]+(?:-[a-z0-9]+)+$|^(?=.*\d)[a-z]{2,}\d{4,}[a-z0-9]*$' |
| r'|^(?=.*\d)(?=.*[a-z])[a-z0-9]{8,}$') |
|
|
|
|
| @dataclass |
| class Token: |
| text: str |
| start: int |
| end: int |
| kind: str |
| value: object = None |
|
|
| @property |
| def is_word(self) -> bool: |
| return self.kind in ('word', 'domain') |
|
|
|
|
| _TAGS = re.compile(r'<\s*(script|style)\b.*?<\s*/\s*\1\s*>|<[^<>]{1,200}>', re.I | re.S) |
| _PLUS = re.compile(r'(?<=[a-z0-9])\s*\+\s*(?=[a-z])', re.I) |
|
|
|
|
| def clean(raw: object) -> tuple[str, list[str]]: |
| """NFKC, straight quotes, no control characters, length cap. Returns (text, notes).""" |
| notes: list[str] = [] |
| if not isinstance(raw, str): |
| raw = '' |
| text = unicodedata.normalize('NFKC', raw) |
| text = _TAGS.sub(' ', text) |
| text = _PLUS.sub(' and ', text) |
| text = ''.join(_QUOTES.get(c, c) for c in text) |
| text = ''.join(c if (c in '\n\t' or unicodedata.category(c)[0] != 'C') else ' ' for c in text) |
| text = re.sub(r'[ \t]+', ' ', text).strip() |
| if len(text) > MAX_CHARS: |
| text = text[:MAX_CHARS] |
| notes.append('INPUT_TRUNCATED') |
| return text, notes |
|
|
|
|
| def tokenize(text: str) -> list[Token]: |
| """Tokens of the (already cleaned) text. Offsets refer to `text`.""" |
| low = text.lower() |
| out: list[Token] = [] |
| for m in _TOKEN_RE.finditer(low): |
| kind = m.lastgroup |
| s, e = m.span() |
| t = m.group() |
| if kind == 'quoted': |
| inner = t[1:-1].strip() |
| if inner: |
| out.append(Token(t, s, e, 'quoted', inner)) |
| continue |
| if kind == 'email': |
| out.append(Token(t, s, e, 'email', t)) |
| continue |
| if kind == 'url': |
| host = re.sub(r'^https?://', '', t).split('/')[0].split(':')[0] |
| out.append(Token(host, s, e, 'domain', host)) |
| continue |
| if kind == 'path': |
| out.append(Token(t, s, e, 'path', t)) |
| continue |
| if kind == 'time': |
| out.append(Token(t.replace(' ', ''), s, e, 'time', _parse_time(t))) |
| continue |
| if kind == 'number': |
| try: |
| v = float(t.replace(',', '')) |
| v = int(v) if v.is_integer() else v |
| except ValueError: |
| v = None |
| out.append(Token(t, s, e, 'number', v)) |
| continue |
| if kind == 'punct': |
| out.append(Token(t, s, e, 'punct')) |
| continue |
| |
| exp = CONTRACTIONS.get(t) |
| if exp: |
| for part in exp.split(): |
| out.append(Token(part, s, e, 'word')) |
| continue |
| if t.endswith("'s") and len(t) > 3: |
| t_base = t[:-2] |
| else: |
| t_base = t |
| if t_base in NUMBER_WORDS: |
| out.append(Token(t_base, s, e, 'number', NUMBER_WORDS[t_base])) |
| continue |
| if '.' in t_base and _DOMAIN.match(t_base) and _TLD.search(t_base): |
| if t_base.rsplit('.', 1)[1] in FILE_EXTS: |
| out.append(Token(t_base, s, e, 'path', t_base)) |
| else: |
| out.append(Token(t_base, s, e, 'domain', t_base)) |
| continue |
| out.append(Token(t_base if t_base != t and len(t_base) > 1 else t, s, e, 'word')) |
| return _merge_number_words(out) |
|
|
|
|
| def _merge_number_words(tokens: list[Token]) -> list[Token]: |
| """'twenty five' -> 25, 'two hundred' -> 200, 'a hundred' -> 100.""" |
| out: list[Token] = [] |
| for t in tokens: |
| prev = out[-1] if out else None |
| if t.kind == 'number' and t.text in NUMBER_WORDS and prev is not None: |
| if prev.kind == 'number' and isinstance(prev.value, int): |
| if t.value in (100, 1000): |
| prev.value = prev.value * t.value |
| prev.end, prev.text = t.end, f'{prev.text} {t.text}' |
| continue |
| if prev.value >= 20 and prev.value % 10 == 0 and t.value < 10: |
| prev.value += t.value |
| prev.end, prev.text = t.end, f'{prev.text} {t.text}' |
| continue |
| if prev.kind == 'word' and prev.text == 'a' and t.value in (100, 1000, 12): |
| out[-1] = Token(f'a {t.text}', prev.start, t.end, 'number', t.value) |
| continue |
| out.append(t) |
| return out |
|
|
|
|
| def _parse_time(t: str) -> int | None: |
| """'6pm' -> 18, '9:30 am' -> 9, '18:00' -> 18 (hours only: the platform stores whole hours).""" |
| m = re.match(r'(\d{1,2})(?::(\d{2}))?\s?(am|pm)?', t) |
| if not m: |
| return None |
| h = int(m.group(1)) |
| ap = m.group(3) |
| if ap == 'pm' and h < 12: |
| h += 12 |
| if ap == 'am' and h == 12: |
| h = 0 |
| return h if 0 <= h <= 23 else None |
|
|
|
|
| def is_domain(s: str) -> bool: |
| return bool(_DOMAIN.match(s)) |
|
|
|
|
| def is_serial(s: str) -> bool: |
| return bool(_SERIAL.match(s)) |
|
|
|
|
| def stem(word: str) -> str: |
| """Plural to singular, conservatively. Only used when the exact word is not in the vocabulary.""" |
| if len(word) < 3 or not word.isalpha(): |
| return word |
| if len(word) == 3: |
| |
| return word[:-1] if word.endswith('s') and word not in ('bus', 'gas', 'his', 'has', 'was', 'yes', 'its', |
| 'ios', 'aws', 'sms', 'ssn', 'pls') else word |
| if word.endswith('ies') and len(word) > 4: |
| return word[:-3] + 'y' |
| if word.endswith(('ses', 'xes', 'ches', 'shes')): |
| return word[:-2] |
| if word.endswith('s') and not word.endswith(('ss', 'us', 'is')): |
| return word[:-1] |
| return word |
|
|
