File size: 2,019 Bytes
8ce1de8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Deterministic character-level tokenizer.
Vocabulary policy
-----------------
* The vocabulary is the sorted set of unique characters in the training corpus
(65 characters for Tiny Shakespeare). Sorting makes it deterministic.
* There are NO special tokens (no <pad>, <bos>, <eos>, <unk>) during training: the corpus is a
single long stream of characters and we cut fixed-length windows from it.
* At inference time a character that is not in the vocabulary (e.g. a digit typed in a prompt)
is replaced by a fallback character (a space if available) so that the model never crashes.
"""
from __future__ import annotations
import json
from pathlib import Path
class CharTokenizer:
def __init__(self, chars: list[str] | str):
self.chars: list[str] = list(chars)
self.stoi = {c: i for i, c in enumerate(self.chars)}
self.itos = {i: c for i, c in enumerate(self.chars)}
self.fallback_id = self.stoi.get(" ", 0)
@classmethod
def from_text(cls, text: str) -> "CharTokenizer":
return cls(sorted(set(text)))
@property
def vocab_size(self) -> int:
return len(self.chars)
def encode(self, text: str, strict: bool = False) -> list[int]:
ids = []
for c in text:
if c in self.stoi:
ids.append(self.stoi[c])
elif strict:
raise KeyError(f"Character {c!r} is not in the vocabulary")
else:
ids.append(self.fallback_id)
return ids
def decode(self, ids) -> str:
return "".join(self.itos[int(i)] for i in ids)
def unknown_chars(self, text: str) -> list[str]:
return sorted({c for c in text if c not in self.stoi})
def save(self, path: str | Path) -> None:
Path(path).write_text(json.dumps({"chars": self.chars}, ensure_ascii=False), encoding="utf-8")
@classmethod
def load(cls, path: str | Path) -> "CharTokenizer":
return cls(json.loads(Path(path).read_text(encoding="utf-8"))["chars"])
|