ud_tokenize.py

#!/usr/bin/env python3
"""Lightweight tokenizer approximating UDPipe tokenization, for the webapp's
local (UDPipe-free) mode.

Rules were derived empirically from the training corpora's UD token
columns (results_cleaned_UD/words):

English (english-ewt):
- contractions split for straight and curly apostrophes
  (don't -> do + n't, we’re -> we + ’re, John's -> John + 's),
- hyphenated compounds kept whole (non-executive), en-dash ranges kept
  whole (6–24), ellipsis "..." kept as one token.

Czech (czech-pdtc):
- no word-internal splitting at all (abych, kdybys stay whole; Czech-PDT
  UD has no multiword tokens),
- punctuation runs split into single characters ("..." -> . + . + .,
  "?!" -> ? + !).

Known approximations: glued artifacts like "3.One" are split here but
kept whole by UDPipe; URLs/emails are not treated specially. Good enough
for the surface-only detector; for exact annotation use the UDPipe mode.
"""

import re

_TOKEN_EN = re.compile(
    r"\d+(?:[.,:–-]\d+)+"       # 3.14, 12:30, 6–24, 2020-21
    r"|\w+(?:['’–-]\w+)*"  # words incl. apostrophes and hyphens
    r"|\.{2,}|-{2,}|[!?]{2,}"        # ... -- ?!
    r"|\S", re.UNICODE)

_TOKEN_CS = re.compile(
    r"\d+(?:[.,:]\d+)+"              # 3,14, 12:30
    r"|\w+"                          # words, nothing glued
    r"|\S", re.UNICODE)              # punctuation char by char

_EN_CONTRACTION = re.compile(
    r"(?i)^(.+?)(n['’]t|['’](?:re|ll|ve|s|d|m))$")


def tokenize(text, lang):
    """Return a list of word tokens approximating UDPipe output."""
    if lang == "cs":
        return _TOKEN_CS.findall(text)
    out = []
    for tok in _TOKEN_EN.findall(text):
        if tok.lower() == "cannot":
            out += [tok[:3], tok[3:]]
            continue
        m = _EN_CONTRACTION.match(tok)
        if m and m.group(1):
            out += [m.group(1), m.group(2)]
        else:
            out.append(tok)
    return out


if __name__ == "__main__":
    print(tokenize("Don't worry — we’re fine (non-executive, 6–24)...", "en"))
    print(tokenize("Řekl, abychom šli domů. Kdybys věděl?! No...", "cs"))