"""Dictionary-free, byte-prefix constrained word generation for local causal language models.""" import copy import json import math import re import time import unicodedata from functools import lru_cache from pathlib import Path import numpy as np import torch def word_character(char): return char == '_' or unicodedata.category(char)[0] in 'LMN' @lru_cache(maxsize=1) def canonical_variants(): variants = {} for codepoint in range(0x110000): char = chr(codepoint) decomposition = unicodedata.decomposition(char) if (decomposition and not decomposition.startswith('<')) or 0xAC00 <= codepoint <= 0xD7A3: normalized = unicodedata.normalize('NFD', char) variants.setdefault(normalized[0],[]).append((normalized,char.encode('utf-8'))) return variants def word_state(raw, required, max_chars): """Inspect bytes before UTF-8 decoding; never replace incomplete code points.""" try: text = raw.decode('utf-8') except UnicodeDecodeError as error: if error.reason != 'unexpected end of data': return 'reject', '' text = raw[:error.start].decode('utf-8') literal_match = raw.startswith(required) or required.startswith(raw) normalized = unicodedata.normalize('NFD', text).encode('utf-8') required_nfc = unicodedata.normalize('NFC', required.decode('utf-8')) required = unicodedata.normalize('NFD', required.decode('utf-8')).encode('utf-8') common = min(len(normalized), len(required)) if normalized[:common] != required[:common] and not literal_match: return 'reject', '' if len(normalized) < len(required): pending = raw[len(text.encode('utf-8')):] if pending and not literal_match: remaining = required[len(normalized):].decode('utf-8') variants = [(remaining[0],remaining[0].encode('utf-8'))]+canonical_variants().get(remaining[0],[]) compatible = any(encoded.startswith(pending) and remaining.startswith(chars) for chars,encoded in variants) if not compatible: return 'reject', '' return 'open', '' if not literal_match and not unicodedata.normalize('NFC', text).startswith(required_nfc): return 'reject', '' required_text = required.decode('utf-8') context = re.match(r'(?s)(.*\s)\S*$', required_text) if context: length = len(context[1]) consumed, cut = 0, 0 while cut < len(text) and consumed < length: consumed += len(unicodedata.normalize('NFD', text[cut])) cut += 1 text = text[cut:] text = text.lstrip() end = next((i for i, char in enumerate(text) if not word_character(char)), len(text)) word = text[:end] if len(word) > max_chars or (end < len(text) and not word): return 'reject', '' return ('done' if end < len(text) else 'open'), word def byte_vocabulary(tokenizer): config = json.loads(tokenizer.to_str()) pre = config['pre_tokenizer'] bytelevel = pre if pre['type'] == 'ByteLevel' else ( pre['pretokenizers'][-1] if pre['type'] == 'Sequence' else {}) if (config['normalizer'] not in (None, {'type': 'NFC'}) or bytelevel.get('type') != 'ByteLevel' or config['decoder']['type'] != 'ByteLevel' or bytelevel.get('add_prefix_space')): raise ValueError('requires ByteLevel BPE with no normalization or NFC, without automatic prefix space') alphabet = list(range(33, 127)) + list(range(161, 173)) + list(range(174, 256)) chars = alphabet[:] extra = 0 for value in range(256): if value not in alphabet: alphabet.append(value) chars.append(256 + extra) extra += 1 inverse = dict(zip(map(chr, chars), alphabet)) special = {item['id'] for item in config['added_tokens'] if item['special']} return {index: bytes(inverse[c] for c in token) for token, index in tokenizer.get_vocab().items() if index not in special} class GenerativeKeyboard: """Python research API. Bounded beam search; scores are word-boundary path mass. The unfinished prefix is constrained in bytes, never encoded as a fixed token prefix. Finished context excludes trailing whitespace, which can merge into the first word token. No word list participates in search. """ def __init__(self, model, tokenizer, *, beam_size=8, max_steps=24, max_chars=48, max_seconds=2., context_tokens=128, memory=None, memory_alpha=.15, token_bytes=None): if min(beam_size, max_steps, max_chars, max_seconds, context_tokens) <= 0: raise ValueError('search budgets must be positive') if not 0 <= memory_alpha <= 1: raise ValueError('memory alpha must be in [0,1]') self.model, self.tokenizer = model.eval(), tokenizer self.beam_size, self.max_steps = beam_size, max_steps self.max_chars, self.max_seconds, self.context_tokens = max_chars, max_seconds, context_tokens self.memory, self.memory_alpha = memory, memory_alpha self.token_bytes = byte_vocabulary(tokenizer) if token_bytes is None else token_bytes self.device = model.device self.prepend_bos = model.config.model_type == 'qwen2' self._classes = None self._piece_trie = {} self._fragment_ids = [] self.bos, self.eos = model.config.bos_token_id, model.config.eos_token_id if tokenizer.get_vocab_size() > model.config.vocab_size or None in (self.bos, self.eos): raise ValueError('model/tokenizer contract mismatch') if context_tokens + max_steps + 1 > model.config.max_position_embeddings: raise ValueError('search exceeds model context capacity') self.reset_session() @classmethod def load(cls, folder, device='cpu', **kwargs): from tokenizers import Tokenizer from transformers import AutoModelForCausalLM folder = Path(folder) if json.loads((folder/'config.json').read_text())['model_type'] not in {'qwen2', 'qwen3'}: raise ValueError('only local Qwen2/Qwen3 weights supported') model = AutoModelForCausalLM.from_pretrained(folder, local_files_only=True, trust_remote_code=False, attn_implementation='eager', torch_dtype=torch.bfloat16 if device == 'cuda' else torch.float32).to(device) return cls(model, Tokenizer.from_file(str(folder/'tokenizer.json')), **kwargs) def reset_session(self): self._ids, self._cache, self._result = [], None, None self.last_search = {} @torch.inference_mode() def distribution(self, ids): if ids == self._ids and self._result is not None: return self._result shared = self._cache is not None and len(ids) > len(self._ids) and ids[:len(self._ids)] == self._ids tail = ids[len(self._ids):] if shared else ids cache = copy.deepcopy(self._cache) if shared else None result = self.model(torch.tensor([tail], device=self.device), past_key_values=cache, use_cache=True, output_hidden_states=True) self._cache, self._ids = result.past_key_values, list(ids) self._result = result.logits[0, -1].float(), result.hidden_states[-1][0, -1].float() return self._result def anchor(self, context, prefix): if any(not word_character(char) for char in prefix) or len(prefix) > self.max_chars: raise ValueError('prefix must be a single Unicode word within the length budget') if context and not context[-1].isspace(): raise ValueError('completed context must end in whitespace') stable = context.rstrip() ids = self.tokenizer.encode(stable, add_special_tokens=False).ids ids = ids[-self.context_tokens:] base = ([self.bos]+ids) if self.prepend_bos or not ids else ids return base, (context[len(stable):]+prefix).encode('utf-8') def _word_extensions(self, raw, required, logp): """Vectorize the common case: whole UTF-8 word followed by another token.""" if self._classes is None: pure, boundaries, other = [], [], [] for token, piece in self.token_bytes.items(): try: text = piece.decode('utf-8') except UnicodeDecodeError: other.append(token) self._fragment_ids.append(token) continue node = self._piece_trie for byte in unicodedata.normalize('NFD', text).encode('utf-8'): node = node.setdefault(byte,{}) node.setdefault(None,[]).append(token) if text and all(word_character(c) for c in text): pure.append(token) elif text and not word_character(text[0]): boundaries.append(token) else: other.append(token) self._classes = tuple(np.asarray(group, dtype=np.int64) for group in (pure, boundaries, other)) status, word = word_state(raw, required, self.max_chars) try: decoded = raw.decode('utf-8') except UnicodeDecodeError: decoded = '' normalized = unicodedata.normalize('NFD', decoded).encode() normalized_required = unicodedata.normalize('NFD', required.decode('utf-8')).encode() combining = any(unicodedata.combining(c) for c in required.decode('utf-8')) if status != 'open' or not word or not decoded or len(normalized) < len(normalized_required): candidates = self.token_bytes if not combining and (decoded or not raw) and normalized_required.startswith(normalized) and len(normalized) < len(normalized_required): node, candidates = self._piece_trie, [] for byte in normalized_required[len(normalized):]: candidates.extend(node.get(None,[])) if byte not in node: node = {} break node = node[byte] stack = [node] while stack: node = stack.pop() candidates.extend(node.get(None,[])) stack.extend(child for key,child in node.items() if key is not None) candidates.extend(self._fragment_ids) for token in candidates: token = int(token) piece = self.token_bytes[token] state, text = word_state(raw+piece, required, self.max_chars) if state != 'reject': yield token, raw+piece, state, text, logp[token] return pure, boundaries, other = self._classes if len(boundaries): yield None, raw, 'done', word, float(np.logaddexp.reduce(logp[boundaries])) # At most beam_size pure-word extensions can survive this parent's beam. if len(pure): order = pure[np.argsort(-logp[pure])] accepted = 0 for token in order: data = raw+self.token_bytes[int(token)] state, text = word_state(data, required, self.max_chars) if state == 'open': yield int(token), data, state, text, logp[token] accepted += 1 if accepted == self.beam_size: break for token in other: data = raw+self.token_bytes[int(token)] state, text = word_state(data, required, self.max_chars) if state != 'reject': yield int(token), data, state, text, logp[token] @staticmethod def render_word(word, prefix): if prefix and not word.startswith(prefix): tail = unicodedata.normalize('NFD', word)[len(unicodedata.normalize('NFD', prefix)):] return prefix+tail return word def complete(self, context, prefix='', top_k=3): if not 1 <= top_k <= self.beam_size: raise ValueError('top_k must be within the beam budget') base, required = self.anchor(context, prefix) started = time.perf_counter() live, finished, steps = [(0., [], b'')], {}, 0 timed_out = False stop_reason = 'max_steps' for steps in range(1, self.max_steps+1): expanded = [] for score, tokens, raw in live: if time.perf_counter()-started > self.max_seconds: timed_out = True break logits, hidden = self.distribution(base+tokens) probability = logits.softmax(-1) if self.memory is not None: if hidden is None: raise ValueError('runtime does not expose contextual embeddings for memory') probability = self.memory.mix(probability, hidden, alpha=self.memory_alpha) logp = probability.clamp_min(1e-38).log().cpu().numpy() # Aggregate all boundary tokens, including tokens containing a following word. for token, data, status, word, delta in self._word_extensions(raw, required, logp): value = score+float(delta) if status == 'done' and word: word = self.render_word(word, prefix) finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value)) elif status == 'open': expanded.append((value, tokens+[token], data)) status, word = word_state(raw, required, self.max_chars) try: raw.decode('utf-8') valid_utf8 = True except UnicodeDecodeError: valid_utf8 = False if status == 'open' and word and valid_utf8: word = self.render_word(word, prefix) value = score+logp[self.eos] finished[word] = float(np.logaddexp(finished.get(word, -math.inf), value)) if timed_out: stop_reason = 'timeout' break live = sorted(expanded, key=lambda row: -row[0])[:self.beam_size] if not live: stop_reason = 'exhausted' break if len(finished) >= top_k: ranked = sorted(finished.values(), reverse=True) mass = sum(math.exp(row[0]) for row in live) gaps = [math.exp(ranked[i])-math.exp(ranked[i+1]) for i in range(min(top_k, len(ranked)-1))] gaps.append(math.exp(ranked[top_k-1])) if mass < min(gaps): stop_reason = 'mass_bound' break self.last_search = {'steps': steps, 'timed_out': timed_out, 'step_limit': stop_reason == 'max_steps', 'seconds': time.perf_counter()-started, 'beam_size': self.beam_size, 'stop_reason': stop_reason, 'finished_candidates': len(finished)} return [{'text':word,'log_probability':score} for word,score in sorted(finished.items(),key=lambda row:(-row[1],row[0]))[:top_k]] def next_word(self, context, top_k=3): return self.complete(context if not context or context[-1].isspace() else context+' ', '', top_k) @torch.inference_mode() def observe(self, context, continuation): """Explicit confirmation only. Updates token memory, without gradients or network.""" if self.memory is None: raise ValueError('no personal memory configured') stable = context.rstrip() full = self.tokenizer.encode(context+continuation, add_special_tokens=False).ids base = self.tokenizer.encode(stable, add_special_tokens=False).ids shared = 0 while shared < min(len(base), len(full)) and base[shared] == full[shared]: shared += 1 for position in range(shared, len(full)): ids = full[max(0, position-self.context_tokens):position] if self.prepend_bos or not ids: ids = [self.bos]+ids _, hidden = self.distribution(ids) self.memory.add(hidden, full[position])