Instructions to use frankyoujian/EdgeASR with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- K2
How to use frankyoujian/EdgeASR with K2:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download text_format.py from frankyoujian/EdgeASR: direct link, hf CLI and curl.
- Browser
- Download file 3.88 kB
-
https://huggingface.co/frankyoujian/EdgeASR/resolve/main/text_format.py
- Command line
-
hf download hf://frankyoujian/EdgeASR/text_format.py
-
curl -L -o text_format.py https://huggingface.co/frankyoujian/EdgeASR/resolve/main/text_format.py
3.88 kB
| """Restore the checkpoint's postfix casing and punctuation annotations. | |
| This is model-output formatting, not WER normalization: words are never | |
| expanded, corrected, or lowercased. Unsupported angle-bracket tokens, including | |
| ``<unk>``, are preserved verbatim so uncertain model output is not discarded. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| _TAG = re.compile(r"<[^<>]+>") | |
| _WORD = re.compile(r"\S+") | |
| _PUNCTUATION = {"<.>": ".", "<,>": ",", "<?>": "?"} | |
| _STRUCTURAL = {"<blk>", "<sos/eos>"} | |
| _OPENING_PUNCTUATION = "\"'“‘([{" | |
| def _case_previous_word(text: str, *, uppercase: bool) -> str: | |
| """Apply a postfix case marker without changing the rest of the text.""" | |
| words = list(_WORD.finditer(text)) | |
| if not words: | |
| # A hallucinated leading case marker has no preceding word to format. | |
| return text | |
| previous = words[-1] | |
| word = previous.group() | |
| if "<" in word or ">" in word: | |
| # Keep unsupported annotations intact, even beside a case marker. | |
| return text | |
| if uppercase: | |
| replacement = word.upper() | |
| else: | |
| replacement = word | |
| for index, char in enumerate(word): | |
| if char.isalpha(): | |
| replacement = word[:index] + char.upper() + word[index + 1 :] | |
| break | |
| return text[: previous.start()] + replacement + text[previous.end() :] | |
| def format_transcript(text: str) -> str: | |
| """Format already detokenized icefall output into a readable transcript. | |
| ``<cap>`` capitalizes the first letter of the preceding word and ``<upp>`` | |
| uppercases that word. These markers may be glued to a word or surrounded | |
| by whitespace. ``<.>``, ``<,>`` and ``<?>`` become ordinary punctuation. | |
| Blank and start/end decoder tokens are removed; other tags are preserved. | |
| BPE word boundaries must already have been converted from ``\u2581`` to | |
| spaces by the decoder. This function does not guess missing word boundaries | |
| or automatically capitalize sentences which have no casing annotation. | |
| """ | |
| if not isinstance(text, str): | |
| raise TypeError("The transcript must be a string.") | |
| output = "" | |
| cursor = 0 | |
| punctuation_boundary = False | |
| def append_plain(fragment: str) -> None: | |
| nonlocal output, punctuation_boundary | |
| if not fragment: | |
| return | |
| # Tags can be emitted without BPE whitespace, e.g. a<,>b. Ordinary | |
| # punctuation in the original text (decimals, URLs, etc.) is untouched. | |
| # A word can start with an apostrophe or opening quote/bracket. Keep | |
| # that word separate as well, so its postfix case tag cannot reach | |
| # across the rendered sentence punctuation. A lone/closing quote, | |
| # or a quote followed by whitespace, stays attached to the left. | |
| after_opening = fragment.lstrip(_OPENING_PUNCTUATION) | |
| quoted_word = ( | |
| after_opening != fragment | |
| and bool(after_opening) | |
| and after_opening[0].isalpha() | |
| ) | |
| if punctuation_boundary and (fragment[0].isalnum() or quoted_word): | |
| output += " " | |
| output += fragment | |
| if fragment.strip(): | |
| punctuation_boundary = False | |
| for match in _TAG.finditer(text): | |
| append_plain(text[cursor : match.start()]) | |
| tag = match.group() | |
| if tag == "<cap>": | |
| output = _case_previous_word(output, uppercase=False) | |
| elif tag == "<upp>": | |
| output = _case_previous_word(output, uppercase=True) | |
| elif tag in _PUNCTUATION: | |
| output = output.rstrip() + _PUNCTUATION[tag] | |
| punctuation_boundary = True | |
| elif tag not in _STRUCTURAL: | |
| output += tag | |
| punctuation_boundary = False | |
| cursor = match.end() | |
| append_plain(text[cursor:]) | |
| output = " ".join(output.split()) | |
| return re.sub(r"\s+([,.;:!?])", r"\1", output) | |