tests/test_tokenizer.py

def test_vocab_size(tokenizer):
    assert tokenizer.vocab_size == 65


def test_encode_decode_roundtrip(tokenizer):
    text = "ROMEO:"

    ids = tokenizer.encode(text)

    decoded = tokenizer.decode(ids)

    assert decoded == text


def test_vocabulary_is_stable(tokenizer):
    for token_id in range(65):
        token = tokenizer.id_to_token[token_id]
        assert tokenizer.token_to_id[token] == token_id