Text Generation
Transformers
Safetensors
Russian
English
qwen2
MANTA
autocomplete
word-completion
text-generation-inference
Instructions to use average23/MANTA-20M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use average23/MANTA-20M with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="average23/MANTA-20M")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("average23/MANTA-20M") model = AutoModelForCausalLM.from_pretrained("average23/MANTA-20M", device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use average23/MANTA-20M with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "average23/MANTA-20M" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/average23/MANTA-20M
- SGLang
How to use average23/MANTA-20M with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "average23/MANTA-20M" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "average23/MANTA-20M", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use average23/MANTA-20M with Docker Model Runner:
docker model run hf.co/average23/MANTA-20M
Download manta/decoder.py from average23/MANTA-20M: direct link, hf CLI and curl.
- Browser
- Download file 16.3 kB
-
https://huggingface.co/average23/MANTA-20M/resolve/main/manta/decoder.py
- Command line
-
hf download hf://average23/MANTA-20M/manta/decoder.py
-
curl -L -o decoder.py https://huggingface.co/average23/MANTA-20M/resolve/main/manta/decoder.py
16.3 kB
| """Dictionary-free, byte-prefix constrained word generation for local causal language models.""" | |
| import copy | |
| import json | |
| import math | |
| import re | |
| import time | |
| import unicodedata | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import numpy as np | |
| import torch | |
| def word_character(char): | |
| return char == '_' or unicodedata.category(char)[0] in 'LMN' | |
| def canonical_variants(): | |
| variants = {} | |
| for codepoint in range(0x110000): | |
| char = chr(codepoint) | |
| decomposition = unicodedata.decomposition(char) | |
| if (decomposition and not decomposition.startswith('<')) or 0xAC00 <= codepoint <= 0xD7A3: | |
| normalized = unicodedata.normalize('NFD', char) | |
| variants.setdefault(normalized[0],[]).append((normalized,char.encode('utf-8'))) | |
| return variants | |
| def word_state(raw, required, max_chars): | |
| """Inspect bytes before UTF-8 decoding; never replace incomplete code points.""" | |
| try: | |
| text = raw.decode('utf-8') | |
| except UnicodeDecodeError as error: | |
| if error.reason != 'unexpected end of data': | |
| return 'reject', '' | |
| text = raw[:error.start].decode('utf-8') | |
| literal_match = raw.startswith(required) or required.startswith(raw) | |
| normalized = unicodedata.normalize('NFD', text).encode('utf-8') | |
| required_nfc = unicodedata.normalize('NFC', required.decode('utf-8')) | |
| required = unicodedata.normalize('NFD', required.decode('utf-8')).encode('utf-8') | |
| common = min(len(normalized), len(required)) | |
| if normalized[:common] != required[:common] and not literal_match: | |
| return 'reject', '' | |
| if len(normalized) < len(required): | |
| pending = raw[len(text.encode('utf-8')):] | |
| if pending and not literal_match: | |
| remaining = required[len(normalized):].decode('utf-8') | |
| variants = [(remaining[0],remaining[0].encode('utf-8'))]+canonical_variants().get(remaining[0],[]) | |
| compatible = any(encoded.startswith(pending) and remaining.startswith(chars) | |
| for chars,encoded in variants) | |
| if not compatible: | |
| return 'reject', '' | |
| return 'open', '' | |
| if not literal_match and not unicodedata.normalize('NFC', text).startswith(required_nfc): | |
| return 'reject', '' | |
| required_text = required.decode('utf-8') | |
| context = re.match(r'(?s)(.*\s)\S*$', required_text) | |
| if context: | |
| length = len(context[1]) | |
| consumed, cut = 0, 0 | |
| while cut < len(text) and consumed < length: | |
| consumed += len(unicodedata.normalize('NFD', text[cut])) | |
| cut += 1 | |
| text = text[cut:] | |
| text = text.lstrip() | |
| end = next((i for i, char in enumerate(text) if not word_character(char)), len(text)) | |
| word = text[:end] | |
| if len(word) > max_chars or (end < len(text) and not word): | |
| return 'reject', '' | |
| return ('done' if end < len(text) else 'open'), word | |
| def byte_vocabulary(tokenizer): | |
| config = json.loads(tokenizer.to_str()) | |
| pre = config['pre_tokenizer'] | |
| bytelevel = pre if pre['type'] == 'ByteLevel' else ( | |
| pre['pretokenizers'][-1] if pre['type'] == 'Sequence' else {}) | |
| if (config['normalizer'] not in (None, {'type': 'NFC'}) or bytelevel.get('type') != 'ByteLevel' | |
| or config['decoder']['type'] != 'ByteLevel' or bytelevel.get('add_prefix_space')): | |
| raise ValueError('requires ByteLevel BPE with no normalization or NFC, without automatic prefix space') | |
| alphabet = list(range(33, 127)) + list(range(161, 173)) + list(range(174, 256)) | |
| chars = alphabet[:] | |
| extra = 0 | |
| for value in range(256): | |
| if value not in alphabet: | |
| alphabet.append(value) | |
| chars.append(256 + extra) | |
| extra += 1 | |
| inverse = dict(zip(map(chr, chars), alphabet)) | |
| special = {item['id'] for item in config['added_tokens'] if item['special']} | |
| return {index: bytes(inverse[c] for c in token) for token, index in tokenizer.get_vocab().items() | |
| if index not in special} | |
| class GenerativeKeyboard: | |
| """Python research API. Bounded beam search; scores are word-boundary path mass. | |
| The unfinished prefix is constrained in bytes, never encoded as a fixed token | |
| prefix. Finished context excludes trailing whitespace, which can merge into | |
| the first word token. No word list participates in search. | |
| """ | |
| def __init__(self, model, tokenizer, *, beam_size=8, max_steps=24, max_chars=48, | |
| max_seconds=2., context_tokens=128, memory=None, memory_alpha=.15, token_bytes=None): | |
| if min(beam_size, max_steps, max_chars, max_seconds, context_tokens) <= 0: | |
| raise ValueError('search budgets must be positive') | |
| if not 0 <= memory_alpha <= 1: | |
| raise ValueError('memory alpha must be in [0,1]') | |
| self.model, self.tokenizer = model.eval(), tokenizer | |
| self.beam_size, self.max_steps = beam_size, max_steps | |
| self.max_chars, self.max_seconds, self.context_tokens = max_chars, max_seconds, context_tokens | |
| self.memory, self.memory_alpha = memory, memory_alpha | |
| self.token_bytes = byte_vocabulary(tokenizer) if token_bytes is None else token_bytes | |
| self.device = model.device | |
| self.prepend_bos = model.config.model_type == 'qwen2' | |
| self._classes = None | |
| self._piece_trie = {} | |
| self._fragment_ids = [] | |
| self.bos, self.eos = model.config.bos_token_id, model.config.eos_token_id | |
| if tokenizer.get_vocab_size() > model.config.vocab_size or None in (self.bos, self.eos): | |
| raise ValueError('model/tokenizer contract mismatch') | |
| if context_tokens + max_steps + 1 > model.config.max_position_embeddings: | |
| raise ValueError('search exceeds model context capacity') | |
| self.reset_session() | |
| def load(cls, folder, device='cpu', **kwargs): | |
| from tokenizers import Tokenizer | |
| from transformers import AutoModelForCausalLM | |
| folder = Path(folder) | |
| if json.loads((folder/'config.json').read_text())['model_type'] not in {'qwen2', 'qwen3'}: | |
| raise ValueError('only local Qwen2/Qwen3 weights supported') | |
| model = AutoModelForCausalLM.from_pretrained(folder, local_files_only=True, | |
| trust_remote_code=False, attn_implementation='eager', | |
| torch_dtype=torch.bfloat16 if device == 'cuda' else torch.float32).to(device) | |
| return cls(model, Tokenizer.from_file(str(folder/'tokenizer.json')), **kwargs) | |
| def reset_session(self): | |
| self._ids, self._cache, self._result = [], None, None | |
| self.last_search = {} | |
| def distribution(self, ids): | |
| if ids == self._ids and self._result is not None: | |
| return self._result | |
| shared = self._cache is not None and len(ids) > len(self._ids) and ids[:len(self._ids)] == self._ids | |
| tail = ids[len(self._ids):] if shared else ids | |
| cache = copy.deepcopy(self._cache) if shared else None | |
| result = self.model(torch.tensor([tail], device=self.device), | |
| past_key_values=cache, use_cache=True, output_hidden_states=True) | |
| self._cache, self._ids = result.past_key_values, list(ids) | |
| self._result = result.logits[0, -1].float(), result.hidden_states[-1][0, -1].float() | |
| return self._result | |
| def anchor(self, context, prefix): | |
| if any(not word_character(char) for char in prefix) or len(prefix) > self.max_chars: | |
| raise ValueError('prefix must be a single Unicode word within the length budget') | |
| if context and not context[-1].isspace(): | |
| raise ValueError('completed context must end in whitespace') | |
| stable = context.rstrip() | |
| ids = self.tokenizer.encode(stable, add_special_tokens=False).ids | |
| ids = ids[-self.context_tokens:] | |
| base = ([self.bos]+ids) if self.prepend_bos or not ids else ids | |
| return base, (context[len(stable):]+prefix).encode('utf-8') | |
| def _word_extensions(self, raw, required, logp): | |
| """Vectorize the common case: whole UTF-8 word followed by another token.""" | |
| if self._classes is None: | |
| pure, boundaries, other = [], [], [] | |
| for token, piece in self.token_bytes.items(): | |
| try: | |
| text = piece.decode('utf-8') | |
| except UnicodeDecodeError: | |
| other.append(token) | |
| self._fragment_ids.append(token) | |
| continue | |
| node = self._piece_trie | |
| for byte in unicodedata.normalize('NFD', text).encode('utf-8'): | |
| node = node.setdefault(byte,{}) | |
| node.setdefault(None,[]).append(token) | |
| if text and all(word_character(c) for c in text): | |
| pure.append(token) | |
| elif text and not word_character(text[0]): | |
| boundaries.append(token) | |
| else: | |
| other.append(token) | |
| self._classes = tuple(np.asarray(group, dtype=np.int64) for group in (pure, boundaries, other)) | |
| status, word = word_state(raw, required, self.max_chars) | |
| try: | |
| decoded = raw.decode('utf-8') | |
| except UnicodeDecodeError: | |
| decoded = '' | |
| normalized = unicodedata.normalize('NFD', decoded).encode() | |
| normalized_required = unicodedata.normalize('NFD', required.decode('utf-8')).encode() | |
| combining = any(unicodedata.combining(c) for c in required.decode('utf-8')) | |
| if status != 'open' or not word or not decoded or len(normalized) < len(normalized_required): | |
| candidates = self.token_bytes | |
| if not combining and (decoded or not raw) and normalized_required.startswith(normalized) and len(normalized) < len(normalized_required): | |
| node, candidates = self._piece_trie, [] | |
| for byte in normalized_required[len(normalized):]: | |
| candidates.extend(node.get(None,[])) | |
| if byte not in node: | |
| node = {} | |
| break | |
| node = node[byte] | |
| stack = [node] | |
| while stack: | |
| node = stack.pop() | |
| candidates.extend(node.get(None,[])) | |
| stack.extend(child for key,child in node.items() if key is not None) | |
| candidates.extend(self._fragment_ids) | |
| for token in candidates: | |
| token = int(token) | |
| piece = self.token_bytes[token] | |
| state, text = word_state(raw+piece, required, self.max_chars) | |
| if state != 'reject': | |
| yield token, raw+piece, state, text, logp[token] | |
| return | |
| pure, boundaries, other = self._classes | |
| if len(boundaries): | |
| yield None, raw, 'done', word, float(np.logaddexp.reduce(logp[boundaries])) | |
| # At most beam_size pure-word extensions can survive this parent's beam. | |
| if len(pure): | |
| order = pure[np.argsort(-logp[pure])] | |
| accepted = 0 | |
| for token in order: | |
| data = raw+self.token_bytes[int(token)] | |
| state, text = word_state(data, required, self.max_chars) | |
| if state == 'open': | |
| yield int(token), data, state, text, logp[token] | |
| accepted += 1 | |
| if accepted == self.beam_size: | |
| break | |
| for token in other: | |
| data = raw+self.token_bytes[int(token)] | |
| state, text = word_state(data, required, self.max_chars) | |
| if state != 'reject': | |
| yield int(token), data, state, text, logp[token] | |
| def render_word(word, prefix): | |
| if prefix and not word.startswith(prefix): | |
| tail = unicodedata.normalize('NFD', word)[len(unicodedata.normalize('NFD', prefix)):] | |
| return prefix+tail | |
| return word | |
| def complete(self, context, prefix='', top_k=3): | |
| if not 1 <= top_k <= self.beam_size: | |
| raise ValueError('top_k must be within the beam budget') | |
| base, required = self.anchor(context, prefix) | |
| started = time.perf_counter() | |
| live, finished, steps = [(0., [], b'')], {}, 0 | |
| timed_out = False | |
| stop_reason = 'max_steps' | |
| for steps in range(1, self.max_steps+1): | |
| expanded = [] | |
| for score, tokens, raw in live: | |
| if time.perf_counter()-started > self.max_seconds: | |
| timed_out = True | |
| break | |
| logits, hidden = self.distribution(base+tokens) | |
| probability = logits.softmax(-1) | |
| if self.memory is not None: | |
| if hidden is None: | |
| raise ValueError('runtime does not expose contextual embeddings for memory') | |
| probability = self.memory.mix(probability, hidden, alpha=self.memory_alpha) | |
| logp = probability.clamp_min(1e-38).log().cpu().numpy() | |
| # Aggregate all boundary tokens, including tokens containing a following word. | |
| for token, data, status, word, delta in self._word_extensions(raw, required, logp): | |
| value = score+float(delta) | |
| if status == 'done' and word: | |
| word = self.render_word(word, prefix) | |
| finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value)) | |
| elif status == 'open': | |
| expanded.append((value, tokens+[token], data)) | |
| status, word = word_state(raw, required, self.max_chars) | |
| try: | |
| raw.decode('utf-8') | |
| valid_utf8 = True | |
| except UnicodeDecodeError: | |
| valid_utf8 = False | |
| if status == 'open' and word and valid_utf8: | |
| word = self.render_word(word, prefix) | |
| value = score+logp[self.eos] | |
| finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value)) | |
| if timed_out: | |
| stop_reason = 'timeout' | |
| break | |
| live = sorted(expanded, key=lambda row: -row[0])[:self.beam_size] | |
| if not live: | |
| stop_reason = 'exhausted' | |
| break | |
| if len(finished) >= top_k: | |
| ranked = sorted(finished.values(), reverse=True) | |
| mass = sum(math.exp(row[0]) for row in live) | |
| gaps = [math.exp(ranked[i])-math.exp(ranked[i+1]) | |
| for i in range(min(top_k, len(ranked)-1))] | |
| gaps.append(math.exp(ranked[top_k-1])) | |
| if mass < min(gaps): | |
| stop_reason = 'mass_bound' | |
| break | |
| self.last_search = {'steps': steps, 'timed_out': timed_out, | |
| 'step_limit': stop_reason == 'max_steps', 'seconds': time.perf_counter()-started, | |
| 'beam_size': self.beam_size, 'stop_reason': stop_reason, | |
| 'finished_candidates': len(finished)} | |
| return [{'text':word,'log_probability':score} for word,score in | |
| sorted(finished.items(),key=lambda row:(-row[1],row[0]))[:top_k]] | |
| def next_word(self, context, top_k=3): | |
| return self.complete(context if not context or context[-1].isspace() else context+' ', '', top_k) | |
| def observe(self, context, continuation): | |
| """Explicit confirmation only. Updates token memory, without gradients or network.""" | |
| if self.memory is None: | |
| raise ValueError('no personal memory configured') | |
| stable = context.rstrip() | |
| full = self.tokenizer.encode(context+continuation, add_special_tokens=False).ids | |
| base = self.tokenizer.encode(stable, add_special_tokens=False).ids | |
| shared = 0 | |
| while shared < min(len(base), len(full)) and base[shared] == full[shared]: | |
| shared += 1 | |
| for position in range(shared, len(full)): | |
| ids = full[max(0, position-self.context_tokens):position] | |
| if self.prepend_bos or not ids: | |
| ids = [self.bos]+ids | |
| _, hidden = self.distribution(ids) | |
| self.memory.add(hidden, full[position]) | |