Text Generation
Transformers
Safetensors
Russian
English
qwen2
MANTA
autocomplete
word-completion
text-generation-inference
Instructions to use average23/MANTA-20M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use average23/MANTA-20M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="average23/MANTA-20M")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("average23/MANTA-20M") model = AutoModelForCausalLM.from_pretrained("average23/MANTA-20M", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use average23/MANTA-20M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "average23/MANTA-20M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/average23/MANTA-20M
- SGLang
How to use average23/MANTA-20M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use average23/MANTA-20M with Docker Model Runner:
docker model run hf.co/average23/MANTA-20M
File size: 7,293 Bytes
63df241 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """Generation contracts, including BPE boundaries and incomplete UTF-8."""
import pytest
import torch
from manta.decoder import GenerativeKeyboard, word_state
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
from transformers import Qwen2Config, Qwen2ForCausalLM
@pytest.fixture
def keyboard():
tokenizer = Tokenizer(models.BPE(unk_token='<unk>'))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.train_from_iterator(['я домой поеду поесть ёлка docker перезапущу'] * 8,
trainers.BpeTrainer(vocab_size=320, special_tokens=['<unk>', '<pad>', '<bos>', '<eos>'],
initial_alphabet=pre_tokenizers.ByteLevel.alphabet()))
torch.manual_seed(4)
model = Qwen2ForCausalLM(Qwen2Config(vocab_size=tokenizer.get_vocab_size(), hidden_size=16,
intermediate_size=32, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1,
bos_token_id=2, eos_token_id=3, pad_token_id=1, max_position_embeddings=128)).eval()
return GenerativeKeyboard(model, tokenizer, beam_size=4, max_steps=16, max_seconds=10, context_tokens=64)
@pytest.mark.parametrize('word', ['поеду', 'ёлка', 'docker', 'е\u0308лка', 'naïve', '变量'])
def test_every_unicode_prefix_can_reach_whole_word_without_tokenizing_partial_word(keyboard, word):
for n in range(1, len(word)):
base, required = keyboard.anchor('я домой ', word[:n])
sequence = keyboard.tokenizer.encode(' '+word, add_special_tokens=False).ids
raw = b''
for token in sequence:
raw += keyboard.token_bytes[token]
assert word_state(raw, required, 48)[0] != 'reject'
assert word_state(raw + b' ', required, 48) == ('done', word)
assert base[0] == 2
def test_utf8_fragment_is_not_replacement_character():
raw = ' ё'.encode()
assert word_state(raw[:-1], b' ', 48)[0] == 'open'
assert word_state(raw+b'!', b' ', 48) == ('done', 'ё')
assert word_state(b' \xff', b' ', 48)[0] == 'reject'
assert word_state(' поел '.encode(), ' пое'.encode(), 48) == ('done', 'поел')
assert word_state(' поел '.encode(), ' поё'.encode(), 48)[0] == 'reject'
def test_generation_is_bounded_prefix_preserving_and_dictionary_free(keyboard):
result = keyboard.complete('я домой ', 'пое')
assert all(r['text'].startswith('пое') and len(r['text']) <= 48 for r in result)
assert len({r['text'] for r in result}) == len(result) <= 3
assert keyboard.last_search['steps'] <= 16
assert not hasattr(keyboard, 'dictionary')
with pytest.raises(ValueError):
keyboard.complete('я ', 'два слова')
def test_incremental_logits_equal_uncached_logits_and_cache_is_not_mutated(keyboard):
a = keyboard.distribution([2, 15, 20])
keyboard.distribution([2, 15, 20, 30])
repeated = keyboard.distribution([2, 15, 20])
direct = keyboard.model(torch.tensor([[2, 15, 20]]), use_cache=False).logits[0, -1]
torch.testing.assert_close(a[0], direct)
torch.testing.assert_close(repeated[0], direct)
def oracle(keyboard, pieces, transitions):
keyboard.token_bytes = pieces
def distribution(ids):
probability = torch.full((keyboard.model.config.vocab_size,), 1e-35)
for token, value in transitions.get(tuple(ids[1:]), {3: 1.}).items():
probability[token] = value
return probability.log(), torch.zeros(16)
keyboard.distribution = distribution
return keyboard
def test_word_ranking_sums_alternative_token_paths_before_stopping(keyboard):
model = oracle(keyboard, {10:b'ab',11:b'a',12:b'b',13:b'x ',14:b' '},
{():{10:.30,11:.25,13:.35,3:.10},(10,):{14:1.},
(11,):{12:1.},(11,12):{14:1.}})
result = model.complete('', '', top_k=1)
assert result[0]['text'] == 'ab'
assert result[0]['log_probability'] == pytest.approx(torch.tensor(.55).log().item(),abs=1e-5)
def test_eos_cannot_finish_a_truncated_unicode_sequence(keyboard):
model = oracle(keyboard, {10:b'a\xc3',11:b'b '}, {():{10:.8,11:.2},(10,):{3:1.}})
assert [r['text'] for r in model.complete('', '',top_k=1)] == ['b']
def test_search_reports_actual_stop_reason_and_completed_word_count(keyboard):
keyboard.max_steps = 1
model = oracle(keyboard, {10: b'ab '}, {(): {10: 1.}})
assert model.complete('', '', top_k=1)[0]['text'] == 'ab'
assert model.last_search['stop_reason'] == 'exhausted'
assert model.last_search['finished_candidates'] == 1
assert model.last_search['step_limit'] is False
def test_canonical_equivalence_keeps_literal_typed_prefix(keyboard):
model = oracle(keyboard, {10:'ёлка '.encode()}, {():{10:1.}})
assert model.complete('', 'е\u0308',top_k=1)[0]['text'] == 'е\u0308лка'
assert model.complete('', 'е',top_k=1) == []
def test_byte_mapping_round_trips_every_byte_and_real_unicode(keyboard):
for word in ['ё','变量','👨🏽💻','я щас домой','naïve','e\u0308']:
ids = keyboard.tokenizer.encode(word,add_special_tokens=False).ids
assert b''.join(keyboard.token_bytes[t] for t in ids) == word.encode()
def test_canonical_aliases_are_merged_before_top_k(keyboard):
model=oracle(keyboard,{10:'ёлка '.encode(),11:'е\u0308лка '.encode(),12:'ёжик '.encode()},
{():{10:.4,11:.35,12:.25}})
result=model.complete('', 'е\u0308',top_k=2)
assert [r['text'] for r in result] == ['е\u0308лка','е\u0308жик']
assert result[0]['log_probability'] == pytest.approx(torch.tensor(.75).log().item(),abs=1e-5)
def test_impossible_utf8_fragment_cannot_displace_a_valid_prefix_path(keyboard):
keyboard.beam_size=1
model=oracle(keyboard,{10:b'\xd0',11:'щу'.encode(),12:'ка '.encode()},
{():{10:.9,11:.1},(10,):{3:1.},(11,):{12:1.}})
assert model.complete('', 'щ',top_k=1)[0]['text'] == 'щука'
assert word_state(b'\xd0','щ'.encode(),48)[0] == 'reject'
assert word_state(b'\xd1','е\u0308'.encode(),48)[0] == 'open'
def test_literal_combining_cluster_can_extend_even_when_nfd_reorders_marks(keyboard):
prefix='a\u0315'
word='a\u0315\u0300x'
assert word_state((word+' ').encode(),prefix.encode(),48) == ('done',word)
model=oracle(keyboard,{10:(word+' ').encode()},{():{10:1.}})
assert model.complete('',prefix,top_k=1)[0]['text'] == word
def test_incomplete_literal_prefix_with_reordering_marks_remains_reachable(keyboard):
prefix='a\u0315\u0300'
assert word_state('a\u0315'.encode(),prefix.encode(),48)[0] == 'open'
model=oracle(keyboard,{10:'a\u0315'.encode(),11:'\u0300x '.encode()},
{():{10:1.},(10,):{11:1.}})
assert model.complete('',prefix,top_k=1)[0]['text'] == prefix+'x'
def test_plain_e_prefix_cannot_be_satisfied_by_rewriting_yo_as_decomposed_e(keyboard):
model=oracle(keyboard,{10:'ёлка '.encode(),11:'ель '.encode()},{():{10:.9,11:.1}})
assert [row['text'] for row in model.complete('', 'е',top_k=1)] == ['ель']
assert word_state('ё'.encode(),'е'.encode(),48)[0] == 'reject'
assert word_state(b'\xd1','е'.encode(),48)[0] == 'reject'
|