MANTA-20M / tests /test_decoder.py
average23's picture
Release MANTA 20M v0.1.0
63df241 verified
Raw History Blame Contribute Delete
7.29 kB
"""Generation contracts, including BPE boundaries and incomplete UTF-8."""
import pytest
import torch
from manta.decoder import GenerativeKeyboard, word_state
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
from transformers import Qwen2Config, Qwen2ForCausalLM
@pytest.fixture
def keyboard():
tokenizer = Tokenizer(models.BPE(unk_token='<unk>'))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.train_from_iterator(['я домой поеду поесть ёлка docker перезапущу'] * 8,
trainers.BpeTrainer(vocab_size=320, special_tokens=['<unk>', '<pad>', '<bos>', '<eos>'],
initial_alphabet=pre_tokenizers.ByteLevel.alphabet()))
torch.manual_seed(4)
model = Qwen2ForCausalLM(Qwen2Config(vocab_size=tokenizer.get_vocab_size(), hidden_size=16,
intermediate_size=32, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1,
bos_token_id=2, eos_token_id=3, pad_token_id=1, max_position_embeddings=128)).eval()
return GenerativeKeyboard(model, tokenizer, beam_size=4, max_steps=16, max_seconds=10, context_tokens=64)
@pytest.mark.parametrize('word', ['поеду', 'ёлка', 'docker', 'е\u0308лка', 'naïve', '变量'])
def test_every_unicode_prefix_can_reach_whole_word_without_tokenizing_partial_word(keyboard, word):
for n in range(1, len(word)):
base, required = keyboard.anchor('я домой ', word[:n])
sequence = keyboard.tokenizer.encode(' '+word, add_special_tokens=False).ids
raw = b''
for token in sequence:
raw += keyboard.token_bytes[token]
assert word_state(raw, required, 48)[0] != 'reject'
assert word_state(raw + b' ', required, 48) == ('done', word)
assert base[0] == 2
def test_utf8_fragment_is_not_replacement_character():
raw = ' ё'.encode()
assert word_state(raw[:-1], b' ', 48)[0] == 'open'
assert word_state(raw+b'!', b' ', 48) == ('done', 'ё')
assert word_state(b' \xff', b' ', 48)[0] == 'reject'
assert word_state(' поел '.encode(), ' пое'.encode(), 48) == ('done', 'поел')
assert word_state(' поел '.encode(), ' поё'.encode(), 48)[0] == 'reject'
def test_generation_is_bounded_prefix_preserving_and_dictionary_free(keyboard):
result = keyboard.complete('я домой ', 'пое')
assert all(r['text'].startswith('пое') and len(r['text']) <= 48 for r in result)
assert len({r['text'] for r in result}) == len(result) <= 3
assert keyboard.last_search['steps'] <= 16
assert not hasattr(keyboard, 'dictionary')
with pytest.raises(ValueError):
keyboard.complete('я ', 'два слова')
def test_incremental_logits_equal_uncached_logits_and_cache_is_not_mutated(keyboard):
a = keyboard.distribution([2, 15, 20])
keyboard.distribution([2, 15, 20, 30])
repeated = keyboard.distribution([2, 15, 20])
direct = keyboard.model(torch.tensor([[2, 15, 20]]), use_cache=False).logits[0, -1]
torch.testing.assert_close(a[0], direct)
torch.testing.assert_close(repeated[0], direct)
def oracle(keyboard, pieces, transitions):
keyboard.token_bytes = pieces
def distribution(ids):
probability = torch.full((keyboard.model.config.vocab_size,), 1e-35)
for token, value in transitions.get(tuple(ids[1:]), {3: 1.}).items():
probability[token] = value
return probability.log(), torch.zeros(16)
keyboard.distribution = distribution
return keyboard
def test_word_ranking_sums_alternative_token_paths_before_stopping(keyboard):
model = oracle(keyboard, {10:b'ab',11:b'a',12:b'b',13:b'x ',14:b' '},
{():{10:.30,11:.25,13:.35,3:.10},(10,):{14:1.},
(11,):{12:1.},(11,12):{14:1.}})
result = model.complete('', '', top_k=1)
assert result[0]['text'] == 'ab'
assert result[0]['log_probability'] == pytest.approx(torch.tensor(.55).log().item(),abs=1e-5)
def test_eos_cannot_finish_a_truncated_unicode_sequence(keyboard):
model = oracle(keyboard, {10:b'a\xc3',11:b'b '}, {():{10:.8,11:.2},(10,):{3:1.}})
assert [r['text'] for r in model.complete('', '',top_k=1)] == ['b']
def test_search_reports_actual_stop_reason_and_completed_word_count(keyboard):
keyboard.max_steps = 1
model = oracle(keyboard, {10: b'ab '}, {(): {10: 1.}})
assert model.complete('', '', top_k=1)[0]['text'] == 'ab'
assert model.last_search['stop_reason'] == 'exhausted'
assert model.last_search['finished_candidates'] == 1
assert model.last_search['step_limit'] is False
def test_canonical_equivalence_keeps_literal_typed_prefix(keyboard):
model = oracle(keyboard, {10:'ёлка '.encode()}, {():{10:1.}})
assert model.complete('', 'е\u0308',top_k=1)[0]['text'] == 'е\u0308лка'
assert model.complete('', 'е',top_k=1) == []
def test_byte_mapping_round_trips_every_byte_and_real_unicode(keyboard):
for word in ['ё','变量','👨🏽‍💻','я щас домой','naïve','e\u0308']:
ids = keyboard.tokenizer.encode(word,add_special_tokens=False).ids
assert b''.join(keyboard.token_bytes[t] for t in ids) == word.encode()
def test_canonical_aliases_are_merged_before_top_k(keyboard):
model=oracle(keyboard,{10:'ёлка '.encode(),11:'е\u0308лка '.encode(),12:'ёжик '.encode()},
{():{10:.4,11:.35,12:.25}})
result=model.complete('', 'е\u0308',top_k=2)
assert [r['text'] for r in result] == ['е\u0308лка','е\u0308жик']
assert result[0]['log_probability'] == pytest.approx(torch.tensor(.75).log().item(),abs=1e-5)
def test_impossible_utf8_fragment_cannot_displace_a_valid_prefix_path(keyboard):
keyboard.beam_size=1
model=oracle(keyboard,{10:b'\xd0',11:'щу'.encode(),12:'ка '.encode()},
{():{10:.9,11:.1},(10,):{3:1.},(11,):{12:1.}})
assert model.complete('', 'щ',top_k=1)[0]['text'] == 'щука'
assert word_state(b'\xd0','щ'.encode(),48)[0] == 'reject'
assert word_state(b'\xd1','е\u0308'.encode(),48)[0] == 'open'
def test_literal_combining_cluster_can_extend_even_when_nfd_reorders_marks(keyboard):
prefix='a\u0315'
word='a\u0315\u0300x'
assert word_state((word+' ').encode(),prefix.encode(),48) == ('done',word)
model=oracle(keyboard,{10:(word+' ').encode()},{():{10:1.}})
assert model.complete('',prefix,top_k=1)[0]['text'] == word
def test_incomplete_literal_prefix_with_reordering_marks_remains_reachable(keyboard):
prefix='a\u0315\u0300'
assert word_state('a\u0315'.encode(),prefix.encode(),48)[0] == 'open'
model=oracle(keyboard,{10:'a\u0315'.encode(),11:'\u0300x '.encode()},
{():{10:1.},(10,):{11:1.}})
assert model.complete('',prefix,top_k=1)[0]['text'] == prefix+'x'
def test_plain_e_prefix_cannot_be_satisfied_by_rewriting_yo_as_decomposed_e(keyboard):
model=oracle(keyboard,{10:'ёлка '.encode(),11:'ель '.encode()},{():{10:.9,11:.1}})
assert [row['text'] for row in model.complete('', 'е',top_k=1)] == ['ель']
assert word_state('ё'.encode(),'е'.encode(),48)[0] == 'reject'
assert word_state(b'\xd1','е'.encode(),48)[0] == 'reject'