"""Briques communes du chantier ponctuation/majuscules. Schema d'etiquettes, par mot NORMALISE (normalize() d'evaluate_asr.py) : - casse : L (minuscule), C (Capitale initiale), U (TOUT EN MAJUSCULES, >= 2 lettres), M (mixte, ex. "McDonald", "iPhone" : forme rendue via une table apprise) - ponct. : signe qui suit le mot : O (aucun) , . ? ! : ; ("..." est compte comme "." ; guillemets, parentheses, tirets d'incise ne sont pas predits : voir README). Alignement texte brut -> mots normalises : normalize() agit caractere par caractere a l'interieur de chaque jeton separe par des blancs (minuscules, puis tout caractere hors [a-z0-9'-] devient un blanc, puis les jetons "-" et "'" isoles sont supprimes). On rejoue exactement cette operation en gardant la position de chaque caractere dans le texte brut : chaque mot normalise est alors une plage du texte brut, d'ou on lit la casse ; le "trou" entre deux mots (blancs, ponctuation, lettres accentuees supprimees) donne la ponctuation. Si la reconstruction ne redonne pas normalize(texte).split(), l'exemple est ignore et compte. """ import os import re import sys import unicodedata try: from evaluate_asr import APOS, DASH, normalize # noqa: E402 except ImportError: # hors du depot (Space, paquet du modele) : copie conforme APOS = dict.fromkeys(map(ord, "’ʼʻ`´"), "'") DASH = dict.fromkeys(map(ord, "‐‑‒–—―"), "-") _KEEP_RE = re.compile(r"[^a-z0-9'\- ]+") def normalize(text): if not text: return "" text = unicodedata.normalize("NFC", text).translate(APOS).translate(DASH).lower() text = re.sub(r"\s+", " ", _KEEP_RE.sub(" ", text)).strip() return " ".join(w for w in text.split() if w not in {"-", "'"}) CASES = ["L", "C", "U", "M"] PUNCTS = ["O", ",", ".", "?", "!", ":", ";"] CASE2ID = {c: i for i, c in enumerate(CASES)} P2ID = {p: i for i, p in enumerate(PUNCTS)} ALLOWED = set("abcdefghijklmnopqrstuvwxyz0123456789'- ") # Priorite quand un trou contient plusieurs signes (ex. '".' ou ').'). PRIORITY = ["?", "!", ".", ":", ";", ","] def gap_punct(gap): for p in PRIORITY: if p in gap: return p return "O" def case_of(s): letters = [c for c in s if c.isalpha()] if not letters: return "L" if all(c.islower() for c in letters): return "L" if letters[0].isupper() and all(c.islower() for c in letters[1:]): return "C" if len(letters) >= 2 and all(c.isupper() for c in letters): return "U" if len(letters) == 1: return "C" return "M" def segments(raw): """Rend [(debut, fin)] des mots normalises dans le texte brut (NFC), et le texte NFC. Les positions renvoient au texte NFC + apostrophes/tirets unifies (meme longueur que le NFC, la traduction etant caractere pour caractere).""" t = unicodedata.normalize("NFC", raw).translate(APOS).translate(DASH) segs, start = [], None for i, ch in enumerate(t): low = ch.lower() keep = len(low) == 1 and low in ALLOWED and low != " " if ch.isspace(): keep = False if keep: if start is None: start = i else: if start is not None: segs.append((start, i)) start = None if start is not None: segs.append((start, len(t))) segs = [(a, b) for a, b in segs if t[a:b] not in ("-", "'")] return segs, t def label_text(raw): """Texte brut -> (mots normalises, casses, ponctuations, formes brutes) ou None si desaccord.""" segs, t = segments(raw) words = [t[a:b].lower() for a, b in segs] if words != normalize(raw).split(): return None cases, puncts, forms = [], [], [] for k, (a, b) in enumerate(segs): nxt = segs[k + 1][0] if k + 1 < len(segs) else len(t) cases.append(case_of(t[a:b])) puncts.append(gap_punct(t[b:nxt])) forms.append(t[a:b]) return words, cases, puncts, forms def apply_case(word, c, mixed_table=None): if c == "L": return word if c == "U": return word.upper() if c == "M" and mixed_table and word in mixed_table: return mixed_table[word] start = 0 if c == "M": # M hors table : le cas typique est un nom propre apres elision ou # trait d'union ("an'Antananarivo", "maha-Malagasy"). cut = max(word.rfind("'"), word.rfind("-")) if 0 <= cut < len(word) - 1: start = cut + 1 # C : premiere lettre en majuscule. for i, ch in enumerate(word): if i < start: continue if ch.isalpha(): return word[:i] + ch.upper() + word[i + 1 :] return word def render(words, cases, puncts, mixed_table=None, lex_table=None): """Assemble la sortie. lex_table (optionnelle) : n-gramme normalise -> forme brute minuscule accentuee (ex. ("tan","na") -> "tanàna"), appliquee avant la casse. La sortie verifie toujours normalize(sortie).split() == words.""" out, i, n = [], 0, len(words) while i < n: span = 1 form = None if lex_table: for k in (3, 2, 1): if i + k <= n: key = " ".join(words[i : i + k]) if key in lex_table: form, span = lex_table[key], k break if form is None: tok = apply_case(words[i], cases[i], mixed_table) else: tok = apply_case(form, cases[i], None) p = puncts[i + span - 1] out.append(tok + ("" if p == "O" else p)) i += span return " ".join(out) def check_words(inp_words, text): return normalize(text).split() == inp_words def trivial_baseline(norm_text): """Baseline triviale : majuscule en debut + point final.""" w = norm_text.split() if not w: return "" return render(w, ["C"] + ["L"] * (len(w) - 1), ["O"] * (len(w) - 1) + ["."]) WORK = os.path.dirname(os.path.abspath(__file__))