"""Restore the checkpoint's postfix casing and punctuation annotations. This is model-output formatting, not WER normalization: words are never expanded, corrected, or lowercased. Unsupported angle-bracket tokens, including ````, are preserved verbatim so uncertain model output is not discarded. """ from __future__ import annotations import re _TAG = re.compile(r"<[^<>]+>") _WORD = re.compile(r"\S+") _PUNCTUATION = {"<.>": ".", "<,>": ",", "": "?"} _STRUCTURAL = {"", ""} _OPENING_PUNCTUATION = "\"'“‘([{" def _case_previous_word(text: str, *, uppercase: bool) -> str: """Apply a postfix case marker without changing the rest of the text.""" words = list(_WORD.finditer(text)) if not words: # A hallucinated leading case marker has no preceding word to format. return text previous = words[-1] word = previous.group() if "<" in word or ">" in word: # Keep unsupported annotations intact, even beside a case marker. return text if uppercase: replacement = word.upper() else: replacement = word for index, char in enumerate(word): if char.isalpha(): replacement = word[:index] + char.upper() + word[index + 1 :] break return text[: previous.start()] + replacement + text[previous.end() :] def format_transcript(text: str) -> str: """Format already detokenized icefall output into a readable transcript. ```` capitalizes the first letter of the preceding word and ```` uppercases that word. These markers may be glued to a word or surrounded by whitespace. ``<.>``, ``<,>`` and ```` become ordinary punctuation. Blank and start/end decoder tokens are removed; other tags are preserved. BPE word boundaries must already have been converted from ``\u2581`` to spaces by the decoder. This function does not guess missing word boundaries or automatically capitalize sentences which have no casing annotation. """ if not isinstance(text, str): raise TypeError("The transcript must be a string.") output = "" cursor = 0 punctuation_boundary = False def append_plain(fragment: str) -> None: nonlocal output, punctuation_boundary if not fragment: return # Tags can be emitted without BPE whitespace, e.g. a<,>b. Ordinary # punctuation in the original text (decimals, URLs, etc.) is untouched. # A word can start with an apostrophe or opening quote/bracket. Keep # that word separate as well, so its postfix case tag cannot reach # across the rendered sentence punctuation. A lone/closing quote, # or a quote followed by whitespace, stays attached to the left. after_opening = fragment.lstrip(_OPENING_PUNCTUATION) quoted_word = ( after_opening != fragment and bool(after_opening) and after_opening[0].isalpha() ) if punctuation_boundary and (fragment[0].isalnum() or quoted_word): output += " " output += fragment if fragment.strip(): punctuation_boundary = False for match in _TAG.finditer(text): append_plain(text[cursor : match.start()]) tag = match.group() if tag == "": output = _case_previous_word(output, uppercase=False) elif tag == "": output = _case_previous_word(output, uppercase=True) elif tag in _PUNCTUATION: output = output.rstrip() + _PUNCTUATION[tag] punctuation_boundary = True elif tag not in _STRUCTURAL: output += tag punctuation_boundary = False cursor = match.end() append_plain(text[cursor:]) output = " ".join(output.split()) return re.sub(r"\s+([,.;:!?])", r"\1", output)