File size: 1,648 Bytes
b3c2a26 5a98e33 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | from tokenizers import pre_tokenizers
from tokenizers.trainers import BpeTrainer
from korbyte.challenges import CHALLENGE_TEXTS
from korbyte.special_tokens import added_special_tokens, special_token_strings
from korbyte.train import build_tokenizer
from korbyte.validate import _random_challenges
def _small_tokenizer():
tokenizer = build_tokenizer()
trainer = BpeTrainer(
vocab_size=2_000,
min_frequency=1,
special_tokens=[],
initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
)
tokenizer.train_from_iterator(CHALLENGE_TEXTS * 4, trainer=trainer)
tokenizer.add_special_tokens(added_special_tokens())
return tokenizer
def test_byte_level_pipeline_is_exactly_reversible() -> None:
tokenizer = _small_tokenizer()
for text in [*CHALLENGE_TEXTS, *_random_challenges(count=50)]:
encoded = tokenizer.encode(text, add_special_tokens=False)
assert tokenizer.decode(encoded.ids, skip_special_tokens=False) == text
def test_special_tokens_are_added_in_declared_order() -> None:
tokenizer = _small_tokenizer()
vocab = tokenizer.get_vocab(with_added_tokens=True)
ids = [vocab[token] for token in special_token_strings()]
assert ids == list(range(ids[0], ids[0] + 256))
def test_pre_tokenizer_keeps_word_boundaries_and_six_digit_numbers() -> None:
segments = build_tokenizer().pre_tokenizer.pre_tokenize_str(
"가나다 라마바 123456 1234567, 테스트"
)
assert [offset for _, offset in segments] == [
(0, 3),
(3, 7),
(7, 14),
(14, 21),
(21, 22),
(22, 23),
(23, 27),
]
|