"""Generation contracts, including BPE boundaries and incomplete UTF-8.""" import pytest import torch from manta.decoder import GenerativeKeyboard, word_state from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers from transformers import Qwen2Config, Qwen2ForCausalLM @pytest.fixture def keyboard(): tokenizer = Tokenizer(models.BPE(unk_token='')) tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tokenizer.decoder = decoders.ByteLevel() tokenizer.train_from_iterator(['я домой поеду поесть ёлка docker перезапущу'] * 8, trainers.BpeTrainer(vocab_size=320, special_tokens=['', '', '', ''], initial_alphabet=pre_tokenizers.ByteLevel.alphabet())) torch.manual_seed(4) model = Qwen2ForCausalLM(Qwen2Config(vocab_size=tokenizer.get_vocab_size(), hidden_size=16, intermediate_size=32, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1, bos_token_id=2, eos_token_id=3, pad_token_id=1, max_position_embeddings=128)).eval() return GenerativeKeyboard(model, tokenizer, beam_size=4, max_steps=16, max_seconds=10, context_tokens=64) @pytest.mark.parametrize('word', ['поеду', 'ёлка', 'docker', 'е\u0308лка', 'naïve', '变量']) def test_every_unicode_prefix_can_reach_whole_word_without_tokenizing_partial_word(keyboard, word): for n in range(1, len(word)): base, required = keyboard.anchor('я домой ', word[:n]) sequence = keyboard.tokenizer.encode(' '+word, add_special_tokens=False).ids raw = b'' for token in sequence: raw += keyboard.token_bytes[token] assert word_state(raw, required, 48)[0] != 'reject' assert word_state(raw + b' ', required, 48) == ('done', word) assert base[0] == 2 def test_utf8_fragment_is_not_replacement_character(): raw = ' ё'.encode() assert word_state(raw[:-1], b' ', 48)[0] == 'open' assert word_state(raw+b'!', b' ', 48) == ('done', 'ё') assert word_state(b' \xff', b' ', 48)[0] == 'reject' assert word_state(' поел '.encode(), ' пое'.encode(), 48) == ('done', 'поел') assert word_state(' поел '.encode(), ' поё'.encode(), 48)[0] == 'reject' def test_generation_is_bounded_prefix_preserving_and_dictionary_free(keyboard): result = keyboard.complete('я домой ', 'пое') assert all(r['text'].startswith('пое') and len(r['text']) <= 48 for r in result) assert len({r['text'] for r in result}) == len(result) <= 3 assert keyboard.last_search['steps'] <= 16 assert not hasattr(keyboard, 'dictionary') with pytest.raises(ValueError): keyboard.complete('я ', 'два слова') def test_incremental_logits_equal_uncached_logits_and_cache_is_not_mutated(keyboard): a = keyboard.distribution([2, 15, 20]) keyboard.distribution([2, 15, 20, 30]) repeated = keyboard.distribution([2, 15, 20]) direct = keyboard.model(torch.tensor([[2, 15, 20]]), use_cache=False).logits[0, -1] torch.testing.assert_close(a[0], direct) torch.testing.assert_close(repeated[0], direct) def oracle(keyboard, pieces, transitions): keyboard.token_bytes = pieces def distribution(ids): probability = torch.full((keyboard.model.config.vocab_size,), 1e-35) for token, value in transitions.get(tuple(ids[1:]), {3: 1.}).items(): probability[token] = value return probability.log(), torch.zeros(16) keyboard.distribution = distribution return keyboard def test_word_ranking_sums_alternative_token_paths_before_stopping(keyboard): model = oracle(keyboard, {10:b'ab',11:b'a',12:b'b',13:b'x ',14:b' '}, {():{10:.30,11:.25,13:.35,3:.10},(10,):{14:1.}, (11,):{12:1.},(11,12):{14:1.}}) result = model.complete('', '', top_k=1) assert result[0]['text'] == 'ab' assert result[0]['log_probability'] == pytest.approx(torch.tensor(.55).log().item(),abs=1e-5) def test_eos_cannot_finish_a_truncated_unicode_sequence(keyboard): model = oracle(keyboard, {10:b'a\xc3',11:b'b '}, {():{10:.8,11:.2},(10,):{3:1.}}) assert [r['text'] for r in model.complete('', '',top_k=1)] == ['b'] def test_search_reports_actual_stop_reason_and_completed_word_count(keyboard): keyboard.max_steps = 1 model = oracle(keyboard, {10: b'ab '}, {(): {10: 1.}}) assert model.complete('', '', top_k=1)[0]['text'] == 'ab' assert model.last_search['stop_reason'] == 'exhausted' assert model.last_search['finished_candidates'] == 1 assert model.last_search['step_limit'] is False def test_canonical_equivalence_keeps_literal_typed_prefix(keyboard): model = oracle(keyboard, {10:'ёлка '.encode()}, {():{10:1.}}) assert model.complete('', 'е\u0308',top_k=1)[0]['text'] == 'е\u0308лка' assert model.complete('', 'е',top_k=1) == [] def test_byte_mapping_round_trips_every_byte_and_real_unicode(keyboard): for word in ['ё','变量','👨🏽‍💻','я щас домой','naïve','e\u0308']: ids = keyboard.tokenizer.encode(word,add_special_tokens=False).ids assert b''.join(keyboard.token_bytes[t] for t in ids) == word.encode() def test_canonical_aliases_are_merged_before_top_k(keyboard): model=oracle(keyboard,{10:'ёлка '.encode(),11:'е\u0308лка '.encode(),12:'ёжик '.encode()}, {():{10:.4,11:.35,12:.25}}) result=model.complete('', 'е\u0308',top_k=2) assert [r['text'] for r in result] == ['е\u0308лка','е\u0308жик'] assert result[0]['log_probability'] == pytest.approx(torch.tensor(.75).log().item(),abs=1e-5) def test_impossible_utf8_fragment_cannot_displace_a_valid_prefix_path(keyboard): keyboard.beam_size=1 model=oracle(keyboard,{10:b'\xd0',11:'щу'.encode(),12:'ка '.encode()}, {():{10:.9,11:.1},(10,):{3:1.},(11,):{12:1.}}) assert model.complete('', 'щ',top_k=1)[0]['text'] == 'щука' assert word_state(b'\xd0','щ'.encode(),48)[0] == 'reject' assert word_state(b'\xd1','е\u0308'.encode(),48)[0] == 'open' def test_literal_combining_cluster_can_extend_even_when_nfd_reorders_marks(keyboard): prefix='a\u0315' word='a\u0315\u0300x' assert word_state((word+' ').encode(),prefix.encode(),48) == ('done',word) model=oracle(keyboard,{10:(word+' ').encode()},{():{10:1.}}) assert model.complete('',prefix,top_k=1)[0]['text'] == word def test_incomplete_literal_prefix_with_reordering_marks_remains_reachable(keyboard): prefix='a\u0315\u0300' assert word_state('a\u0315'.encode(),prefix.encode(),48)[0] == 'open' model=oracle(keyboard,{10:'a\u0315'.encode(),11:'\u0300x '.encode()}, {():{10:1.},(10,):{11:1.}}) assert model.complete('',prefix,top_k=1)[0]['text'] == prefix+'x' def test_plain_e_prefix_cannot_be_satisfied_by_rewriting_yo_as_decomposed_e(keyboard): model=oracle(keyboard,{10:'ёлка '.encode(),11:'ель '.encode()},{():{10:.9,11:.1}}) assert [row['text'] for row in model.complete('', 'е',top_k=1)] == ['ель'] assert word_state('ё'.encode(),'е'.encode(),48)[0] == 'reject' assert word_state(b'\xd1','е'.encode(),48)[0] == 'reject'