EdgeASR / text_format.py
frankyoujian's picture
Add offline EdgeASR decoder and FFASR runtime materials
496a112 verified
Raw History Blame Contribute Delete
3.88 kB
"""Restore the checkpoint's postfix casing and punctuation annotations.
This is model-output formatting, not WER normalization: words are never
expanded, corrected, or lowercased. Unsupported angle-bracket tokens, including
``<unk>``, are preserved verbatim so uncertain model output is not discarded.
"""
from __future__ import annotations
import re
_TAG = re.compile(r"<[^<>]+>")
_WORD = re.compile(r"\S+")
_PUNCTUATION = {"<.>": ".", "<,>": ",", "<?>": "?"}
_STRUCTURAL = {"<blk>", "<sos/eos>"}
_OPENING_PUNCTUATION = "\"'“‘([{"
def _case_previous_word(text: str, *, uppercase: bool) -> str:
"""Apply a postfix case marker without changing the rest of the text."""
words = list(_WORD.finditer(text))
if not words:
# A hallucinated leading case marker has no preceding word to format.
return text
previous = words[-1]
word = previous.group()
if "<" in word or ">" in word:
# Keep unsupported annotations intact, even beside a case marker.
return text
if uppercase:
replacement = word.upper()
else:
replacement = word
for index, char in enumerate(word):
if char.isalpha():
replacement = word[:index] + char.upper() + word[index + 1 :]
break
return text[: previous.start()] + replacement + text[previous.end() :]
def format_transcript(text: str) -> str:
"""Format already detokenized icefall output into a readable transcript.
``<cap>`` capitalizes the first letter of the preceding word and ``<upp>``
uppercases that word. These markers may be glued to a word or surrounded
by whitespace. ``<.>``, ``<,>`` and ``<?>`` become ordinary punctuation.
Blank and start/end decoder tokens are removed; other tags are preserved.
BPE word boundaries must already have been converted from ``\u2581`` to
spaces by the decoder. This function does not guess missing word boundaries
or automatically capitalize sentences which have no casing annotation.
"""
if not isinstance(text, str):
raise TypeError("The transcript must be a string.")
output = ""
cursor = 0
punctuation_boundary = False
def append_plain(fragment: str) -> None:
nonlocal output, punctuation_boundary
if not fragment:
return
# Tags can be emitted without BPE whitespace, e.g. a<,>b. Ordinary
# punctuation in the original text (decimals, URLs, etc.) is untouched.
# A word can start with an apostrophe or opening quote/bracket. Keep
# that word separate as well, so its postfix case tag cannot reach
# across the rendered sentence punctuation. A lone/closing quote,
# or a quote followed by whitespace, stays attached to the left.
after_opening = fragment.lstrip(_OPENING_PUNCTUATION)
quoted_word = (
after_opening != fragment
and bool(after_opening)
and after_opening[0].isalpha()
)
if punctuation_boundary and (fragment[0].isalnum() or quoted_word):
output += " "
output += fragment
if fragment.strip():
punctuation_boundary = False
for match in _TAG.finditer(text):
append_plain(text[cursor : match.start()])
tag = match.group()
if tag == "<cap>":
output = _case_previous_word(output, uppercase=False)
elif tag == "<upp>":
output = _case_previous_word(output, uppercase=True)
elif tag in _PUNCTUATION:
output = output.rstrip() + _PUNCTUATION[tag]
punctuation_boundary = True
elif tag not in _STRUCTURAL:
output += tag
punctuation_boundary = False
cursor = match.end()
append_plain(text[cursor:])
output = " ".join(output.split())
return re.sub(r"\s+([,.;:!?])", r"\1", output)