File size: 3,105 Bytes
63df241
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
"""Public API and README examples, exercised against a real local checkpoint."""
import os
import unicodedata

import pytest
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

from manta import MantaKeyboard


@pytest.fixture(scope='module')
def keyboard():
    torch.set_num_threads(2)
    return MantaKeyboard.from_pretrained(os.environ['MANTA_MODEL_DIR'])


def test_standard_transformers_loading(keyboard):
    folder = os.environ['MANTA_MODEL_DIR']
    tokenizer = AutoTokenizer.from_pretrained(folder, trust_remote_code=False)
    model = AutoModelForCausalLM.from_pretrained(folder, trust_remote_code=False)
    assert sum(p.numel() for p in model.parameters()) == 20_084_064
    for text in ['я щас домой ', 'док', 'ёлка е\u0308лка', 'docker перезапущу', '  \n']:
        assert tokenizer.encode(text, add_special_tokens=False) == keyboard.tokenizer.encode(
            text, add_special_tokens=False).ids
    assert not model.training


def test_hub_loader_uses_pinned_release_and_only_model_files(monkeypatch):
    calls = []

    def download(model_id, **kwargs):
        calls.append((model_id, kwargs))
        return os.environ['MANTA_MODEL_DIR']

    monkeypatch.setattr('manta.snapshot_download', download)
    model = MantaKeyboard.from_pretrained('average23/MANTA-20M', local_files_only=True)
    assert model.model.config.model_type == 'qwen2'
    assert calls == [('average23/MANTA-20M', {
        'revision': 'v0.1.0', 'local_files_only': True,
        'allow_patterns': ['config.json', 'model.safetensors', 'tokenizer.json',
                           'tokenizer_config.json', 'generation_config.json'],
    })]


@pytest.mark.parametrize(('context', 'prefix'), [
    ('я щас домой ', ''), ('надо перезапустить ', 'док'),
    ('', ''), ('ща docker ', 'перезапу'), ('ну это ', 'пиз'),
    ('  \n', 'ё'), ('я ', 'е\u0308'),
])
def test_words_prefixes_and_repeatability(keyboard, context, prefix):
    keyboard.reset_session()
    first = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context)
    assert first
    assert 0 < len(first) <= 3
    assert len({r['text'] for r in first}) == len(first)
    for row in first:
        assert row['text'].startswith(prefix)
        assert all(c == '_' or unicodedata.category(c)[0] in 'LMN' for c in row['text'])
    assert not keyboard.last_search['timed_out']
    keyboard.reset_session()
    second = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context)
    assert first == second


def test_whitespace_and_invalid_input(keyboard):
    assert keyboard.next_word('я щас домой') == keyboard.next_word('я щас домой ')
    with pytest.raises(ValueError, match='whitespace'):
        keyboard.complete('я домой', prefix='до')
    with pytest.raises(ValueError, match='single Unicode word'):
        keyboard.complete('я ', prefix='два слова')
    with pytest.raises(ValueError, match='beam budget'):
        keyboard.next_word('я ', top_k=11)