Text Generation
Transformers
Safetensors
Russian
English
qwen2
MANTA
autocomplete
word-completion
text-generation-inference
Instructions to use average23/MANTA-20M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use average23/MANTA-20M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="average23/MANTA-20M")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("average23/MANTA-20M") model = AutoModelForCausalLM.from_pretrained("average23/MANTA-20M", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use average23/MANTA-20M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "average23/MANTA-20M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/average23/MANTA-20M
- SGLang
How to use average23/MANTA-20M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use average23/MANTA-20M with Docker Model Runner:
docker model run hf.co/average23/MANTA-20M
File size: 3,105 Bytes
63df241 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | """Public API and README examples, exercised against a real local checkpoint."""
import os
import unicodedata
import pytest
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from manta import MantaKeyboard
@pytest.fixture(scope='module')
def keyboard():
torch.set_num_threads(2)
return MantaKeyboard.from_pretrained(os.environ['MANTA_MODEL_DIR'])
def test_standard_transformers_loading(keyboard):
folder = os.environ['MANTA_MODEL_DIR']
tokenizer = AutoTokenizer.from_pretrained(folder, trust_remote_code=False)
model = AutoModelForCausalLM.from_pretrained(folder, trust_remote_code=False)
assert sum(p.numel() for p in model.parameters()) == 20_084_064
for text in ['я щас домой ', 'док', 'ёлка е\u0308лка', 'docker перезапущу', ' \n']:
assert tokenizer.encode(text, add_special_tokens=False) == keyboard.tokenizer.encode(
text, add_special_tokens=False).ids
assert not model.training
def test_hub_loader_uses_pinned_release_and_only_model_files(monkeypatch):
calls = []
def download(model_id, **kwargs):
calls.append((model_id, kwargs))
return os.environ['MANTA_MODEL_DIR']
monkeypatch.setattr('manta.snapshot_download', download)
model = MantaKeyboard.from_pretrained('average23/MANTA-20M', local_files_only=True)
assert model.model.config.model_type == 'qwen2'
assert calls == [('average23/MANTA-20M', {
'revision': 'v0.1.0', 'local_files_only': True,
'allow_patterns': ['config.json', 'model.safetensors', 'tokenizer.json',
'tokenizer_config.json', 'generation_config.json'],
})]
@pytest.mark.parametrize(('context', 'prefix'), [
('я щас домой ', ''), ('надо перезапустить ', 'док'),
('', ''), ('ща docker ', 'перезапу'), ('ну это ', 'пиз'),
(' \n', 'ё'), ('я ', 'е\u0308'),
])
def test_words_prefixes_and_repeatability(keyboard, context, prefix):
keyboard.reset_session()
first = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context)
assert first
assert 0 < len(first) <= 3
assert len({r['text'] for r in first}) == len(first)
for row in first:
assert row['text'].startswith(prefix)
assert all(c == '_' or unicodedata.category(c)[0] in 'LMN' for c in row['text'])
assert not keyboard.last_search['timed_out']
keyboard.reset_session()
second = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context)
assert first == second
def test_whitespace_and_invalid_input(keyboard):
assert keyboard.next_word('я щас домой') == keyboard.next_word('я щас домой ')
with pytest.raises(ValueError, match='whitespace'):
keyboard.complete('я домой', prefix='до')
with pytest.raises(ValueError, match='single Unicode word'):
keyboard.complete('я ', prefix='два слова')
with pytest.raises(ValueError, match='beam budget'):
keyboard.next_word('я ', top_k=11)
|