Text Generation
Transformers
Safetensors
Russian
English
qwen2
MANTA
autocomplete
word-completion
text-generation-inference
Instructions to use average23/MANTA-20M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use average23/MANTA-20M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="average23/MANTA-20M")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("average23/MANTA-20M") model = AutoModelForCausalLM.from_pretrained("average23/MANTA-20M", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use average23/MANTA-20M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "average23/MANTA-20M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/average23/MANTA-20M
- SGLang
How to use average23/MANTA-20M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use average23/MANTA-20M with Docker Model Runner:
docker model run hf.co/average23/MANTA-20M
Download tests/test_decoder.py from average23/MANTA-20M: direct link, hf CLI and curl.
- Browser
- Download file 7.29 kB
-
https://huggingface.co/average23/MANTA-20M/resolve/main/tests/test_decoder.py
- Command line
-
hf download hf://average23/MANTA-20M/tests/test_decoder.py
-
curl -L -o test_decoder.py https://huggingface.co/average23/MANTA-20M/resolve/main/tests/test_decoder.py
7.29 kB
| """Generation contracts, including BPE boundaries and incomplete UTF-8.""" | |
| import pytest | |
| import torch | |
| from manta.decoder import GenerativeKeyboard, word_state | |
| from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers | |
| from transformers import Qwen2Config, Qwen2ForCausalLM | |
| def keyboard(): | |
| tokenizer = Tokenizer(models.BPE(unk_token='<unk>')) | |
| tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) | |
| tokenizer.decoder = decoders.ByteLevel() | |
| tokenizer.train_from_iterator(['я домой поеду поесть ёлка docker перезапущу'] * 8, | |
| trainers.BpeTrainer(vocab_size=320, special_tokens=['<unk>', '<pad>', '<bos>', '<eos>'], | |
| initial_alphabet=pre_tokenizers.ByteLevel.alphabet())) | |
| torch.manual_seed(4) | |
| model = Qwen2ForCausalLM(Qwen2Config(vocab_size=tokenizer.get_vocab_size(), hidden_size=16, | |
| intermediate_size=32, num_hidden_layers=1, num_attention_heads=2, num_key_value_heads=1, | |
| bos_token_id=2, eos_token_id=3, pad_token_id=1, max_position_embeddings=128)).eval() | |
| return GenerativeKeyboard(model, tokenizer, beam_size=4, max_steps=16, max_seconds=10, context_tokens=64) | |
| def test_every_unicode_prefix_can_reach_whole_word_without_tokenizing_partial_word(keyboard, word): | |
| for n in range(1, len(word)): | |
| base, required = keyboard.anchor('я домой ', word[:n]) | |
| sequence = keyboard.tokenizer.encode(' '+word, add_special_tokens=False).ids | |
| raw = b'' | |
| for token in sequence: | |
| raw += keyboard.token_bytes[token] | |
| assert word_state(raw, required, 48)[0] != 'reject' | |
| assert word_state(raw + b' ', required, 48) == ('done', word) | |
| assert base[0] == 2 | |
| def test_utf8_fragment_is_not_replacement_character(): | |
| raw = ' ё'.encode() | |
| assert word_state(raw[:-1], b' ', 48)[0] == 'open' | |
| assert word_state(raw+b'!', b' ', 48) == ('done', 'ё') | |
| assert word_state(b' \xff', b' ', 48)[0] == 'reject' | |
| assert word_state(' поел '.encode(), ' пое'.encode(), 48) == ('done', 'поел') | |
| assert word_state(' поел '.encode(), ' поё'.encode(), 48)[0] == 'reject' | |
| def test_generation_is_bounded_prefix_preserving_and_dictionary_free(keyboard): | |
| result = keyboard.complete('я домой ', 'пое') | |
| assert all(r['text'].startswith('пое') and len(r['text']) <= 48 for r in result) | |
| assert len({r['text'] for r in result}) == len(result) <= 3 | |
| assert keyboard.last_search['steps'] <= 16 | |
| assert not hasattr(keyboard, 'dictionary') | |
| with pytest.raises(ValueError): | |
| keyboard.complete('я ', 'два слова') | |
| def test_incremental_logits_equal_uncached_logits_and_cache_is_not_mutated(keyboard): | |
| a = keyboard.distribution([2, 15, 20]) | |
| keyboard.distribution([2, 15, 20, 30]) | |
| repeated = keyboard.distribution([2, 15, 20]) | |
| direct = keyboard.model(torch.tensor([[2, 15, 20]]), use_cache=False).logits[0, -1] | |
| torch.testing.assert_close(a[0], direct) | |
| torch.testing.assert_close(repeated[0], direct) | |
| def oracle(keyboard, pieces, transitions): | |
| keyboard.token_bytes = pieces | |
| def distribution(ids): | |
| probability = torch.full((keyboard.model.config.vocab_size,), 1e-35) | |
| for token, value in transitions.get(tuple(ids[1:]), {3: 1.}).items(): | |
| probability[token] = value | |
| return probability.log(), torch.zeros(16) | |
| keyboard.distribution = distribution | |
| return keyboard | |
| def test_word_ranking_sums_alternative_token_paths_before_stopping(keyboard): | |
| model = oracle(keyboard, {10:b'ab',11:b'a',12:b'b',13:b'x ',14:b' '}, | |
| {():{10:.30,11:.25,13:.35,3:.10},(10,):{14:1.}, | |
| (11,):{12:1.},(11,12):{14:1.}}) | |
| result = model.complete('', '', top_k=1) | |
| assert result[0]['text'] == 'ab' | |
| assert result[0]['log_probability'] == pytest.approx(torch.tensor(.55).log().item(),abs=1e-5) | |
| def test_eos_cannot_finish_a_truncated_unicode_sequence(keyboard): | |
| model = oracle(keyboard, {10:b'a\xc3',11:b'b '}, {():{10:.8,11:.2},(10,):{3:1.}}) | |
| assert [r['text'] for r in model.complete('', '',top_k=1)] == ['b'] | |
| def test_search_reports_actual_stop_reason_and_completed_word_count(keyboard): | |
| keyboard.max_steps = 1 | |
| model = oracle(keyboard, {10: b'ab '}, {(): {10: 1.}}) | |
| assert model.complete('', '', top_k=1)[0]['text'] == 'ab' | |
| assert model.last_search['stop_reason'] == 'exhausted' | |
| assert model.last_search['finished_candidates'] == 1 | |
| assert model.last_search['step_limit'] is False | |
| def test_canonical_equivalence_keeps_literal_typed_prefix(keyboard): | |
| model = oracle(keyboard, {10:'ёлка '.encode()}, {():{10:1.}}) | |
| assert model.complete('', 'е\u0308',top_k=1)[0]['text'] == 'е\u0308лка' | |
| assert model.complete('', 'е',top_k=1) == [] | |
| def test_byte_mapping_round_trips_every_byte_and_real_unicode(keyboard): | |
| for word in ['ё','变量','👨🏽💻','я щас домой','naïve','e\u0308']: | |
| ids = keyboard.tokenizer.encode(word,add_special_tokens=False).ids | |
| assert b''.join(keyboard.token_bytes[t] for t in ids) == word.encode() | |
| def test_canonical_aliases_are_merged_before_top_k(keyboard): | |
| model=oracle(keyboard,{10:'ёлка '.encode(),11:'е\u0308лка '.encode(),12:'ёжик '.encode()}, | |
| {():{10:.4,11:.35,12:.25}}) | |
| result=model.complete('', 'е\u0308',top_k=2) | |
| assert [r['text'] for r in result] == ['е\u0308лка','е\u0308жик'] | |
| assert result[0]['log_probability'] == pytest.approx(torch.tensor(.75).log().item(),abs=1e-5) | |
| def test_impossible_utf8_fragment_cannot_displace_a_valid_prefix_path(keyboard): | |
| keyboard.beam_size=1 | |
| model=oracle(keyboard,{10:b'\xd0',11:'щу'.encode(),12:'ка '.encode()}, | |
| {():{10:.9,11:.1},(10,):{3:1.},(11,):{12:1.}}) | |
| assert model.complete('', 'щ',top_k=1)[0]['text'] == 'щука' | |
| assert word_state(b'\xd0','щ'.encode(),48)[0] == 'reject' | |
| assert word_state(b'\xd1','е\u0308'.encode(),48)[0] == 'open' | |
| def test_literal_combining_cluster_can_extend_even_when_nfd_reorders_marks(keyboard): | |
| prefix='a\u0315' | |
| word='a\u0315\u0300x' | |
| assert word_state((word+' ').encode(),prefix.encode(),48) == ('done',word) | |
| model=oracle(keyboard,{10:(word+' ').encode()},{():{10:1.}}) | |
| assert model.complete('',prefix,top_k=1)[0]['text'] == word | |
| def test_incomplete_literal_prefix_with_reordering_marks_remains_reachable(keyboard): | |
| prefix='a\u0315\u0300' | |
| assert word_state('a\u0315'.encode(),prefix.encode(),48)[0] == 'open' | |
| model=oracle(keyboard,{10:'a\u0315'.encode(),11:'\u0300x '.encode()}, | |
| {():{10:1.},(10,):{11:1.}}) | |
| assert model.complete('',prefix,top_k=1)[0]['text'] == prefix+'x' | |
| def test_plain_e_prefix_cannot_be_satisfied_by_rewriting_yo_as_decomposed_e(keyboard): | |
| model=oracle(keyboard,{10:'ёлка '.encode(),11:'ель '.encode()},{():{10:.9,11:.1}}) | |
| assert [row['text'] for row in model.complete('', 'е',top_k=1)] == ['ель'] | |
| assert word_state('ё'.encode(),'е'.encode(),48)[0] == 'reject' | |
| assert word_state(b'\xd1','е'.encode(),48)[0] == 'reject' | |