Download cavaalign/vocab.py from cava-reader/cavaalign-v1: direct link, hf CLI and curl.
- Browser
- Download file 2.73 kB
-
https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/cavaalign/vocab.py
- Command line
-
hf download hf://cava-reader/cavaalign-v1/cavaalign/vocab.py
-
curl -L -o vocab.py https://huggingface.co/cava-reader/cavaalign-v1/resolve/main/cavaalign/vocab.py
2.73 kB
| # SPDX-License-Identifier: AGPL-3.0-or-later | |
| """Token set for CavaAlign: a 29-class romanized alphabet. | |
| Character-level, deliberately: characters align straight onto the book's own orthography, which | |
| is the text the reader is looking at, with no per-language lexicon or G2P. Every script reaches | |
| this alphabet through `cavaalign.romanize`, so one model serves every language. | |
| """ | |
| from __future__ import annotations | |
| BLANK = 0 | |
| # blank, 26 letters ordered by frequency, apostrophe, and `<unk>` (written `*` in romanized text), | |
| # a wildcard for anything with no romanization. There is no space class: the DP scores spaces from | |
| # the blank column (`align.Token.is_space`), so a space maps onto the blank id. | |
| ROMAN_LETTERS = list("aienoutsrmkldghybpwcvjzf") + ["'"] + list("qx") | |
| class VocabRoman: | |
| """The 29-class romanized vocab used by cavaalign.onnx.""" | |
| name = "roman29" | |
| def __init__(self): | |
| self.chars = list(ROMAN_LETTERS) | |
| self.specials = ["<unk>"] | |
| self.tokens = ["<blank>"] + self.chars + self.specials | |
| self.stoi = {t: i for i, t in enumerate(self.tokens)} | |
| self.itos = {i: t for t, i in self.stoi.items()} | |
| self.char_to_id = {c: self.stoi[c] for c in self.chars} | |
| self.char_to_id["*"] = self.stoi["<unk>"] | |
| self.unk = self.stoi["<unk>"] | |
| def id_for(self, ch: str) -> int: | |
| """Token id for one character, case-folded; a space is the blank id.""" | |
| if ch == " ": | |
| return BLANK | |
| return self.char_to_id.get(ch.lower(), self.unk) | |
| def __len__(self) -> int: | |
| return len(self.tokens) | |
| def vocab_size(self) -> int: | |
| return len(self.tokens) | |
| def encode(self, text: str) -> list[int]: | |
| return [self.id_for(c) for c in text] | |
| def decode(self, ids: list[int], collapse_repeats: bool = True) -> str: | |
| out: list[str] = [] | |
| prev = None | |
| for i in ids: | |
| if i == BLANK: | |
| prev = None | |
| continue | |
| if collapse_repeats and i == prev: | |
| continue | |
| prev = i | |
| tok = self.itos[i] | |
| if not tok.startswith("<"): | |
| out.append(tok) | |
| return "".join(out) | |
| def vocab_for_tokens(tokens: list[str] | None) -> VocabRoman: | |
| """The vocab matching a model's `vocab_tokens` metadata. | |
| Raises rather than guessing: a model whose output ids mean different characters from the ones | |
| the aligner asks for still runs and still produces timings, and they are wrong. | |
| """ | |
| v = VocabRoman() | |
| if tokens is not None and list(tokens) != v.tokens: | |
| raise RuntimeError(f"unknown vocabulary ({len(tokens)} tokens, e.g. {list(tokens)[1:5]!r})") | |
| return v | |