Text Generation
Transformers
Safetensors
Russian
English
qwen2
MANTA
autocomplete
word-completion
text-generation-inference
Instructions to use average23/MANTA-20M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use average23/MANTA-20M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="average23/MANTA-20M")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("average23/MANTA-20M") model = AutoModelForCausalLM.from_pretrained("average23/MANTA-20M", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use average23/MANTA-20M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "average23/MANTA-20M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/average23/MANTA-20M
- SGLang
How to use average23/MANTA-20M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use average23/MANTA-20M with Docker Model Runner:
docker model run hf.co/average23/MANTA-20M
Download tests/test_runtime.py from average23/MANTA-20M: direct link, hf CLI and curl.
- Browser
- Download file 3.11 kB
-
https://huggingface.co/average23/MANTA-20M/resolve/main/tests/test_runtime.py
- Command line
-
hf download hf://average23/MANTA-20M/tests/test_runtime.py
-
curl -L -o test_runtime.py https://huggingface.co/average23/MANTA-20M/resolve/main/tests/test_runtime.py
3.11 kB
| """Public API and README examples, exercised against a real local checkpoint.""" | |
| import os | |
| import unicodedata | |
| import pytest | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from manta import MantaKeyboard | |
| def keyboard(): | |
| torch.set_num_threads(2) | |
| return MantaKeyboard.from_pretrained(os.environ['MANTA_MODEL_DIR']) | |
| def test_standard_transformers_loading(keyboard): | |
| folder = os.environ['MANTA_MODEL_DIR'] | |
| tokenizer = AutoTokenizer.from_pretrained(folder, trust_remote_code=False) | |
| model = AutoModelForCausalLM.from_pretrained(folder, trust_remote_code=False) | |
| assert sum(p.numel() for p in model.parameters()) == 20_084_064 | |
| for text in ['я щас домой ', 'док', 'ёлка е\u0308лка', 'docker перезапущу', ' \n']: | |
| assert tokenizer.encode(text, add_special_tokens=False) == keyboard.tokenizer.encode( | |
| text, add_special_tokens=False).ids | |
| assert not model.training | |
| def test_hub_loader_uses_pinned_release_and_only_model_files(monkeypatch): | |
| calls = [] | |
| def download(model_id, **kwargs): | |
| calls.append((model_id, kwargs)) | |
| return os.environ['MANTA_MODEL_DIR'] | |
| monkeypatch.setattr('manta.snapshot_download', download) | |
| model = MantaKeyboard.from_pretrained('average23/MANTA-20M', local_files_only=True) | |
| assert model.model.config.model_type == 'qwen2' | |
| assert calls == [('average23/MANTA-20M', { | |
| 'revision': 'v0.1.0', 'local_files_only': True, | |
| 'allow_patterns': ['config.json', 'model.safetensors', 'tokenizer.json', | |
| 'tokenizer_config.json', 'generation_config.json'], | |
| })] | |
| def test_words_prefixes_and_repeatability(keyboard, context, prefix): | |
| keyboard.reset_session() | |
| first = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context) | |
| assert first | |
| assert 0 < len(first) <= 3 | |
| assert len({r['text'] for r in first}) == len(first) | |
| for row in first: | |
| assert row['text'].startswith(prefix) | |
| assert all(c == '_' or unicodedata.category(c)[0] in 'LMN' for c in row['text']) | |
| assert not keyboard.last_search['timed_out'] | |
| keyboard.reset_session() | |
| second = keyboard.complete(context, prefix=prefix) if prefix else keyboard.next_word(context) | |
| assert first == second | |
| def test_whitespace_and_invalid_input(keyboard): | |
| assert keyboard.next_word('я щас домой') == keyboard.next_word('я щас домой ') | |
| with pytest.raises(ValueError, match='whitespace'): | |
| keyboard.complete('я домой', prefix='до') | |
| with pytest.raises(ValueError, match='single Unicode word'): | |
| keyboard.complete('я ', prefix='два слова') | |
| with pytest.raises(ValueError, match='beam budget'): | |
| keyboard.next_word('я ', top_k=11) | |