MANTA-20M / manta /decoder.py
average23's picture
Release MANTA 20M v0.1.0
63df241 verified
Raw History Blame Contribute Delete
16.3 kB
"""Dictionary-free, byte-prefix constrained word generation for local causal language models."""
import copy
import json
import math
import re
import time
import unicodedata
from functools import lru_cache
from pathlib import Path
import numpy as np
import torch
def word_character(char):
return char == '_' or unicodedata.category(char)[0] in 'LMN'
@lru_cache(maxsize=1)
def canonical_variants():
variants = {}
for codepoint in range(0x110000):
char = chr(codepoint)
decomposition = unicodedata.decomposition(char)
if (decomposition and not decomposition.startswith('<')) or 0xAC00 <= codepoint <= 0xD7A3:
normalized = unicodedata.normalize('NFD', char)
variants.setdefault(normalized[0],[]).append((normalized,char.encode('utf-8')))
return variants
def word_state(raw, required, max_chars):
"""Inspect bytes before UTF-8 decoding; never replace incomplete code points."""
try:
text = raw.decode('utf-8')
except UnicodeDecodeError as error:
if error.reason != 'unexpected end of data':
return 'reject', ''
text = raw[:error.start].decode('utf-8')
literal_match = raw.startswith(required) or required.startswith(raw)
normalized = unicodedata.normalize('NFD', text).encode('utf-8')
required_nfc = unicodedata.normalize('NFC', required.decode('utf-8'))
required = unicodedata.normalize('NFD', required.decode('utf-8')).encode('utf-8')
common = min(len(normalized), len(required))
if normalized[:common] != required[:common] and not literal_match:
return 'reject', ''
if len(normalized) < len(required):
pending = raw[len(text.encode('utf-8')):]
if pending and not literal_match:
remaining = required[len(normalized):].decode('utf-8')
variants = [(remaining[0],remaining[0].encode('utf-8'))]+canonical_variants().get(remaining[0],[])
compatible = any(encoded.startswith(pending) and remaining.startswith(chars)
for chars,encoded in variants)
if not compatible:
return 'reject', ''
return 'open', ''
if not literal_match and not unicodedata.normalize('NFC', text).startswith(required_nfc):
return 'reject', ''
required_text = required.decode('utf-8')
context = re.match(r'(?s)(.*\s)\S*$', required_text)
if context:
length = len(context[1])
consumed, cut = 0, 0
while cut < len(text) and consumed < length:
consumed += len(unicodedata.normalize('NFD', text[cut]))
cut += 1
text = text[cut:]
text = text.lstrip()
end = next((i for i, char in enumerate(text) if not word_character(char)), len(text))
word = text[:end]
if len(word) > max_chars or (end < len(text) and not word):
return 'reject', ''
return ('done' if end < len(text) else 'open'), word
def byte_vocabulary(tokenizer):
config = json.loads(tokenizer.to_str())
pre = config['pre_tokenizer']
bytelevel = pre if pre['type'] == 'ByteLevel' else (
pre['pretokenizers'][-1] if pre['type'] == 'Sequence' else {})
if (config['normalizer'] not in (None, {'type': 'NFC'}) or bytelevel.get('type') != 'ByteLevel'
or config['decoder']['type'] != 'ByteLevel' or bytelevel.get('add_prefix_space')):
raise ValueError('requires ByteLevel BPE with no normalization or NFC, without automatic prefix space')
alphabet = list(range(33, 127)) + list(range(161, 173)) + list(range(174, 256))
chars = alphabet[:]
extra = 0
for value in range(256):
if value not in alphabet:
alphabet.append(value)
chars.append(256 + extra)
extra += 1
inverse = dict(zip(map(chr, chars), alphabet))
special = {item['id'] for item in config['added_tokens'] if item['special']}
return {index: bytes(inverse[c] for c in token) for token, index in tokenizer.get_vocab().items()
if index not in special}
class GenerativeKeyboard:
"""Python research API. Bounded beam search; scores are word-boundary path mass.
The unfinished prefix is constrained in bytes, never encoded as a fixed token
prefix. Finished context excludes trailing whitespace, which can merge into
the first word token. No word list participates in search.
"""
def __init__(self, model, tokenizer, *, beam_size=8, max_steps=24, max_chars=48,
max_seconds=2., context_tokens=128, memory=None, memory_alpha=.15, token_bytes=None):
if min(beam_size, max_steps, max_chars, max_seconds, context_tokens) <= 0:
raise ValueError('search budgets must be positive')
if not 0 <= memory_alpha <= 1:
raise ValueError('memory alpha must be in [0,1]')
self.model, self.tokenizer = model.eval(), tokenizer
self.beam_size, self.max_steps = beam_size, max_steps
self.max_chars, self.max_seconds, self.context_tokens = max_chars, max_seconds, context_tokens
self.memory, self.memory_alpha = memory, memory_alpha
self.token_bytes = byte_vocabulary(tokenizer) if token_bytes is None else token_bytes
self.device = model.device
self.prepend_bos = model.config.model_type == 'qwen2'
self._classes = None
self._piece_trie = {}
self._fragment_ids = []
self.bos, self.eos = model.config.bos_token_id, model.config.eos_token_id
if tokenizer.get_vocab_size() > model.config.vocab_size or None in (self.bos, self.eos):
raise ValueError('model/tokenizer contract mismatch')
if context_tokens + max_steps + 1 > model.config.max_position_embeddings:
raise ValueError('search exceeds model context capacity')
self.reset_session()
@classmethod
def load(cls, folder, device='cpu', **kwargs):
from tokenizers import Tokenizer
from transformers import AutoModelForCausalLM
folder = Path(folder)
if json.loads((folder/'config.json').read_text())['model_type'] not in {'qwen2', 'qwen3'}:
raise ValueError('only local Qwen2/Qwen3 weights supported')
model = AutoModelForCausalLM.from_pretrained(folder, local_files_only=True,
trust_remote_code=False, attn_implementation='eager',
torch_dtype=torch.bfloat16 if device == 'cuda' else torch.float32).to(device)
return cls(model, Tokenizer.from_file(str(folder/'tokenizer.json')), **kwargs)
def reset_session(self):
self._ids, self._cache, self._result = [], None, None
self.last_search = {}
@torch.inference_mode()
def distribution(self, ids):
if ids == self._ids and self._result is not None:
return self._result
shared = self._cache is not None and len(ids) > len(self._ids) and ids[:len(self._ids)] == self._ids
tail = ids[len(self._ids):] if shared else ids
cache = copy.deepcopy(self._cache) if shared else None
result = self.model(torch.tensor([tail], device=self.device),
past_key_values=cache, use_cache=True, output_hidden_states=True)
self._cache, self._ids = result.past_key_values, list(ids)
self._result = result.logits[0, -1].float(), result.hidden_states[-1][0, -1].float()
return self._result
def anchor(self, context, prefix):
if any(not word_character(char) for char in prefix) or len(prefix) > self.max_chars:
raise ValueError('prefix must be a single Unicode word within the length budget')
if context and not context[-1].isspace():
raise ValueError('completed context must end in whitespace')
stable = context.rstrip()
ids = self.tokenizer.encode(stable, add_special_tokens=False).ids
ids = ids[-self.context_tokens:]
base = ([self.bos]+ids) if self.prepend_bos or not ids else ids
return base, (context[len(stable):]+prefix).encode('utf-8')
def _word_extensions(self, raw, required, logp):
"""Vectorize the common case: whole UTF-8 word followed by another token."""
if self._classes is None:
pure, boundaries, other = [], [], []
for token, piece in self.token_bytes.items():
try:
text = piece.decode('utf-8')
except UnicodeDecodeError:
other.append(token)
self._fragment_ids.append(token)
continue
node = self._piece_trie
for byte in unicodedata.normalize('NFD', text).encode('utf-8'):
node = node.setdefault(byte,{})
node.setdefault(None,[]).append(token)
if text and all(word_character(c) for c in text):
pure.append(token)
elif text and not word_character(text[0]):
boundaries.append(token)
else:
other.append(token)
self._classes = tuple(np.asarray(group, dtype=np.int64) for group in (pure, boundaries, other))
status, word = word_state(raw, required, self.max_chars)
try:
decoded = raw.decode('utf-8')
except UnicodeDecodeError:
decoded = ''
normalized = unicodedata.normalize('NFD', decoded).encode()
normalized_required = unicodedata.normalize('NFD', required.decode('utf-8')).encode()
combining = any(unicodedata.combining(c) for c in required.decode('utf-8'))
if status != 'open' or not word or not decoded or len(normalized) < len(normalized_required):
candidates = self.token_bytes
if not combining and (decoded or not raw) and normalized_required.startswith(normalized) and len(normalized) < len(normalized_required):
node, candidates = self._piece_trie, []
for byte in normalized_required[len(normalized):]:
candidates.extend(node.get(None,[]))
if byte not in node:
node = {}
break
node = node[byte]
stack = [node]
while stack:
node = stack.pop()
candidates.extend(node.get(None,[]))
stack.extend(child for key,child in node.items() if key is not None)
candidates.extend(self._fragment_ids)
for token in candidates:
token = int(token)
piece = self.token_bytes[token]
state, text = word_state(raw+piece, required, self.max_chars)
if state != 'reject':
yield token, raw+piece, state, text, logp[token]
return
pure, boundaries, other = self._classes
if len(boundaries):
yield None, raw, 'done', word, float(np.logaddexp.reduce(logp[boundaries]))
# At most beam_size pure-word extensions can survive this parent's beam.
if len(pure):
order = pure[np.argsort(-logp[pure])]
accepted = 0
for token in order:
data = raw+self.token_bytes[int(token)]
state, text = word_state(data, required, self.max_chars)
if state == 'open':
yield int(token), data, state, text, logp[token]
accepted += 1
if accepted == self.beam_size:
break
for token in other:
data = raw+self.token_bytes[int(token)]
state, text = word_state(data, required, self.max_chars)
if state != 'reject':
yield int(token), data, state, text, logp[token]
@staticmethod
def render_word(word, prefix):
if prefix and not word.startswith(prefix):
tail = unicodedata.normalize('NFD', word)[len(unicodedata.normalize('NFD', prefix)):]
return prefix+tail
return word
def complete(self, context, prefix='', top_k=3):
if not 1 <= top_k <= self.beam_size:
raise ValueError('top_k must be within the beam budget')
base, required = self.anchor(context, prefix)
started = time.perf_counter()
live, finished, steps = [(0., [], b'')], {}, 0
timed_out = False
stop_reason = 'max_steps'
for steps in range(1, self.max_steps+1):
expanded = []
for score, tokens, raw in live:
if time.perf_counter()-started > self.max_seconds:
timed_out = True
break
logits, hidden = self.distribution(base+tokens)
probability = logits.softmax(-1)
if self.memory is not None:
if hidden is None:
raise ValueError('runtime does not expose contextual embeddings for memory')
probability = self.memory.mix(probability, hidden, alpha=self.memory_alpha)
logp = probability.clamp_min(1e-38).log().cpu().numpy()
# Aggregate all boundary tokens, including tokens containing a following word.
for token, data, status, word, delta in self._word_extensions(raw, required, logp):
value = score+float(delta)
if status == 'done' and word:
word = self.render_word(word, prefix)
finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value))
elif status == 'open':
expanded.append((value, tokens+[token], data))
status, word = word_state(raw, required, self.max_chars)
try:
raw.decode('utf-8')
valid_utf8 = True
except UnicodeDecodeError:
valid_utf8 = False
if status == 'open' and word and valid_utf8:
word = self.render_word(word, prefix)
value = score+logp[self.eos]
finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value))
if timed_out:
stop_reason = 'timeout'
break
live = sorted(expanded, key=lambda row: -row[0])[:self.beam_size]
if not live:
stop_reason = 'exhausted'
break
if len(finished) >= top_k:
ranked = sorted(finished.values(), reverse=True)
mass = sum(math.exp(row[0]) for row in live)
gaps = [math.exp(ranked[i])-math.exp(ranked[i+1])
for i in range(min(top_k, len(ranked)-1))]
gaps.append(math.exp(ranked[top_k-1]))
if mass < min(gaps):
stop_reason = 'mass_bound'
break
self.last_search = {'steps': steps, 'timed_out': timed_out,
'step_limit': stop_reason == 'max_steps', 'seconds': time.perf_counter()-started,
'beam_size': self.beam_size, 'stop_reason': stop_reason,
'finished_candidates': len(finished)}
return [{'text':word,'log_probability':score} for word,score in
sorted(finished.items(),key=lambda row:(-row[1],row[0]))[:top_k]]
def next_word(self, context, top_k=3):
return self.complete(context if not context or context[-1].isspace() else context+' ', '', top_k)
@torch.inference_mode()
def observe(self, context, continuation):
"""Explicit confirmation only. Updates token memory, without gradients or network."""
if self.memory is None:
raise ValueError('no personal memory configured')
stable = context.rstrip()
full = self.tokenizer.encode(context+continuation, add_special_tokens=False).ids
base = self.tokenizer.encode(stable, add_special_tokens=False).ids
shared = 0
while shared < min(len(base), len(full)) and base[shared] == full[shared]:
shared += 1
for position in range(shared, len(full)):
ids = full[max(0, position-self.context_tokens):position]
if self.prepend_bos or not ids:
ids = [self.bos]+ids
_, hidden = self.distribution(ids)
self.memory.add(hidden, full[position])