File size: 7,293 Bytes
63df241
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
"""Generation contracts, including BPE boundaries and incomplete UTF-8."""
import pytest
import torch
from manta.decoder import GenerativeKeyboard, word_state
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
from transformers import Qwen2Config, Qwen2ForCausalLM


@pytest.fixture
def keyboard():
    tokenizer = Tokenizer(models.BPE(unk_token='<unk>'))
    tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
    tokenizer.decoder = decoders.ByteLevel()
    tokenizer.train_from_iterator(['я домой поеду поесть ёлка docker перезапущу'] * 8,
        trainers.BpeTrainer(vocab_size=320, special_tokens=['<unk>', '<pad>', '<bos>', '<eos>'],
                            initial_alphabet=pre_tokenizers.ByteLevel.alphabet()))
    torch.manual_seed(4)
    model = Qwen2ForCausalLM(Qwen2Config(vocab_size=tokenizer.get_vocab_size(), hidden_size=16,
        intermediate_size=32, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1,
        bos_token_id=2, eos_token_id=3, pad_token_id=1, max_position_embeddings=128)).eval()
    return GenerativeKeyboard(model, tokenizer, beam_size=4, max_steps=16, max_seconds=10, context_tokens=64)


@pytest.mark.parametrize('word', ['поеду', 'ёлка', 'docker', 'е\u0308лка', 'naïve', '变量'])
def test_every_unicode_prefix_can_reach_whole_word_without_tokenizing_partial_word(keyboard, word):
    for n in range(1, len(word)):
        base, required = keyboard.anchor('я домой ', word[:n])
        sequence = keyboard.tokenizer.encode(' '+word, add_special_tokens=False).ids
        raw = b''
        for token in sequence:
            raw += keyboard.token_bytes[token]
            assert word_state(raw, required, 48)[0] != 'reject'
        assert word_state(raw + b' ', required, 48) == ('done', word)
        assert base[0] == 2


def test_utf8_fragment_is_not_replacement_character():
    raw = ' ё'.encode()
    assert word_state(raw[:-1], b' ', 48)[0] == 'open'
    assert word_state(raw+b'!', b' ', 48) == ('done', 'ё')
    assert word_state(b' \xff', b' ', 48)[0] == 'reject'
    assert word_state(' поел '.encode(), ' пое'.encode(), 48) == ('done', 'поел')
    assert word_state(' поел '.encode(), ' поё'.encode(), 48)[0] == 'reject'


def test_generation_is_bounded_prefix_preserving_and_dictionary_free(keyboard):
    result = keyboard.complete('я домой ', 'пое')
    assert all(r['text'].startswith('пое') and len(r['text']) <= 48 for r in result)
    assert len({r['text'] for r in result}) == len(result) <= 3
    assert keyboard.last_search['steps'] <= 16
    assert not hasattr(keyboard, 'dictionary')
    with pytest.raises(ValueError):
        keyboard.complete('я ', 'два слова')


def test_incremental_logits_equal_uncached_logits_and_cache_is_not_mutated(keyboard):
    a = keyboard.distribution([2, 15, 20])
    keyboard.distribution([2, 15, 20, 30])
    repeated = keyboard.distribution([2, 15, 20])
    direct = keyboard.model(torch.tensor([[2, 15, 20]]), use_cache=False).logits[0, -1]
    torch.testing.assert_close(a[0], direct)
    torch.testing.assert_close(repeated[0], direct)


def oracle(keyboard, pieces, transitions):
    keyboard.token_bytes = pieces
    def distribution(ids):
        probability = torch.full((keyboard.model.config.vocab_size,), 1e-35)
        for token, value in transitions.get(tuple(ids[1:]), {3: 1.}).items():
            probability[token] = value
        return probability.log(), torch.zeros(16)
    keyboard.distribution = distribution
    return keyboard


def test_word_ranking_sums_alternative_token_paths_before_stopping(keyboard):
    model = oracle(keyboard, {10:b'ab',11:b'a',12:b'b',13:b'x ',14:b' '},
                   {():{10:.30,11:.25,13:.35,3:.10},(10,):{14:1.},
                    (11,):{12:1.},(11,12):{14:1.}})
    result = model.complete('', '', top_k=1)
    assert result[0]['text'] == 'ab'
    assert result[0]['log_probability'] == pytest.approx(torch.tensor(.55).log().item(),abs=1e-5)


def test_eos_cannot_finish_a_truncated_unicode_sequence(keyboard):
    model = oracle(keyboard, {10:b'a\xc3',11:b'b '}, {():{10:.8,11:.2},(10,):{3:1.}})
    assert [r['text'] for r in model.complete('', '',top_k=1)] == ['b']


def test_search_reports_actual_stop_reason_and_completed_word_count(keyboard):
    keyboard.max_steps = 1
    model = oracle(keyboard, {10: b'ab '}, {(): {10: 1.}})
    assert model.complete('', '', top_k=1)[0]['text'] == 'ab'
    assert model.last_search['stop_reason'] == 'exhausted'
    assert model.last_search['finished_candidates'] == 1
    assert model.last_search['step_limit'] is False


def test_canonical_equivalence_keeps_literal_typed_prefix(keyboard):
    model = oracle(keyboard, {10:'ёлка '.encode()}, {():{10:1.}})
    assert model.complete('', 'е\u0308',top_k=1)[0]['text'] == 'е\u0308лка'
    assert model.complete('', 'е',top_k=1) == []


def test_byte_mapping_round_trips_every_byte_and_real_unicode(keyboard):
    for word in ['ё','变量','👨🏽‍💻','я щас домой','naïve','e\u0308']:
        ids = keyboard.tokenizer.encode(word,add_special_tokens=False).ids
        assert b''.join(keyboard.token_bytes[t] for t in ids) == word.encode()


def test_canonical_aliases_are_merged_before_top_k(keyboard):
    model=oracle(keyboard,{10:'ёлка '.encode(),11:'е\u0308лка '.encode(),12:'ёжик '.encode()},
                 {():{10:.4,11:.35,12:.25}})
    result=model.complete('', 'е\u0308',top_k=2)
    assert [r['text'] for r in result] == ['е\u0308лка','е\u0308жик']
    assert result[0]['log_probability'] == pytest.approx(torch.tensor(.75).log().item(),abs=1e-5)


def test_impossible_utf8_fragment_cannot_displace_a_valid_prefix_path(keyboard):
    keyboard.beam_size=1
    model=oracle(keyboard,{10:b'\xd0',11:'щу'.encode(),12:'ка '.encode()},
                 {():{10:.9,11:.1},(10,):{3:1.},(11,):{12:1.}})
    assert model.complete('', 'щ',top_k=1)[0]['text'] == 'щука'
    assert word_state(b'\xd0','щ'.encode(),48)[0] == 'reject'
    assert word_state(b'\xd1','е\u0308'.encode(),48)[0] == 'open'


def test_literal_combining_cluster_can_extend_even_when_nfd_reorders_marks(keyboard):
    prefix='a\u0315'
    word='a\u0315\u0300x'
    assert word_state((word+' ').encode(),prefix.encode(),48) == ('done',word)
    model=oracle(keyboard,{10:(word+' ').encode()},{():{10:1.}})
    assert model.complete('',prefix,top_k=1)[0]['text'] == word


def test_incomplete_literal_prefix_with_reordering_marks_remains_reachable(keyboard):
    prefix='a\u0315\u0300'
    assert word_state('a\u0315'.encode(),prefix.encode(),48)[0] == 'open'
    model=oracle(keyboard,{10:'a\u0315'.encode(),11:'\u0300x '.encode()},
                 {():{10:1.},(10,):{11:1.}})
    assert model.complete('',prefix,top_k=1)[0]['text'] == prefix+'x'



def test_plain_e_prefix_cannot_be_satisfied_by_rewriting_yo_as_decomposed_e(keyboard):
    model=oracle(keyboard,{10:'ёлка '.encode(),11:'ель '.encode()},{():{10:.9,11:.1}})
    assert [row['text'] for row in model.complete('', 'е',top_k=1)] == ['ель']
    assert word_state('ё'.encode(),'е'.encode(),48)[0] == 'reject'
    assert word_state(b'\xd1','е'.encode(),48)[0] == 'reject'