"""Constrained decoding. On the main line no structure token and no can come, and only between
characters. Inside an IDS every step keeps it well formed: after a structure token; a structure token opens as many
places as it has parts; a part fills one place and is one CJK character, radical or stroke (base-vocabulary tokens are
followed byte by byte through UTF-8); comes exactly when no place is open. told(): what the model is told of an
IDS so far -- a structure token and the unfinished bytes of a part stand for themselves, whatever finishes a part is
told as ."""
from __future__ import annotations
import numpy as np
import mlx.core as mx
NEG=-1e9
LEAVES=((0x4E00,0x9FFF),(0x3400,0x4DBF),(0x20000,0x3347F),(0x2E80,0x2EFF),(0x31C0,0x31E5))
BASE,SPECIAL,GLYPH,EXTCHAR,KOPEN,KCLOSE,STRUCT=range(7)
def leaf(cp):
cp=np.asarray(cp);ok=np.zeros(cp.shape,bool)
for a,b in LEAVES:ok|=(cp>=a)&(cp<=b)
return ok
def byte_decoder():
bs=list(range(ord('!'),ord('~')+1))+list(range(ord('¡'),ord('¬')+1))+list(range(ord('®'),ord('ÿ')+1));cs=bs[:];n=0
for b in range(256):
if b not in bs:bs.append(b);cs.append(256+n);n+=1
return {chr(c):b for b,c in zip(bs,cs)}
def char_length(first):
return 1 if first<0x80 else 2 if 0xC0<=first<=0xDF else 3 if 0xE0<=first<=0xEF else 4 if 0xF0<=first<=0xF7 else 0
def code(seq):
cp=seq[0]&{1:0x7F,2:0x1F,3:0x0F,4:0x07}[char_length(seq[0])]
for b in seq[1:]:cp=(cp<<6)|(b&0x3F)
return cp
def partial_can_be_leaf(tail):
n=char_length(tail[0])
if n<3 or any(not 0x80<=b<=0xBF for b in tail[1:]):return False
low=code(list(tail)+[0x80]*(n-len(tail)));high=code(list(tail)+[0xBF]*(n-len(tail)))
return any(a<=high and low<=b for a,b in LEAVES)
def shape(data):
i=0
while ilen(data):trail=n-(len(data)-i);ok=ok and partial_can_be_leaf(data[i:]);break
piece=data[i:i+n]
if any(not 0x80<=b<=0xBF for b in piece[1:]):return lead,count,0,False
ok=ok and bool(leaf(code(list(piece))));count+=1;i+=n
return lead,count,trail,ok
class Vocab:
def __init__(self,tokenizer,info):
ids=info['ids'];V=ids['vocabulary'];self.V=V;self.open=ids['open'];self.close=ids['close'];self.standin=info['first_gw_id'];self.first_ids=ids['first_ids']
self.kind=np.zeros(V,np.int8);self.lead=np.zeros(V,np.int8);self.chars=np.zeros(V,np.int16)
self.trail=np.zeros(V,np.int8);self.ok=np.zeros(V,bool);self.arity=np.zeros(V,np.int8);self.data=[b'']*V
self.first=np.full((V,3),-1,np.int16)
specials={i for i in tokenizer.added_tokens_decoder if i0)
&(self.lead==np.array([len(d) for d in self.data])))[0]
def inside(self,need,pending,partial):
m=np.zeros(self.V,bool)
if need==0:m[self.close]=True;return m
if pending==0:
k=self.chars+(self.trail>0)
m=(self.kind==STRUCT)|((self.kind==EXTCHAR)&self.ok)|((self.kind==BASE)&(self.lead==0)&self.ok&(k>=1)&(k<=need))
return m
finish=np.where((self.kind==BASE)&(self.lead==pending))[0]
if len(finish):
tail=self.first[finish,:pending].astype(np.int64);cp=code(list(partial))
for j in range(pending):cp=(cp<<6)|(tail[:,j]&0x3F)
k=1+self.chars[finish]+(self.trail[finish]>0)
m[finish]=leaf(cp)&self.ok[finish]&(k<=need)
for i in self.continuation_only:
if self.lead[i]len(data):pending=n-(len(data)-i);partial=data[i:];break
done+=1;i+=n
return done,pending,partial
class Constraint:
def __init__(self,vocab,may_open=True):
self.v=vocab;self.may_open=may_open;self.primed=False;self.inside=False;self.first=False;self.need=0;self.pending=0;self.partial=b'';self.dead_ends=0
def feed(self,t):
v=self.v;k=v.kind[t]
if not self.inside:
if k==KOPEN:self.inside=True;self.first=True;self.need=1;self.pending=0;self.partial=b''
elif k==BASE:_,self.pending,self.partial=advance(self.pending,self.partial,v.data[t])
else:self.pending=0;self.partial=b''
return
self.first=False
if k==KCLOSE:self.inside=False
elif k==STRUCT:self.need+=int(v.arity[t])-1
elif k==EXTCHAR:self.need-=1
elif k==BASE:
done,self.pending,self.partial=advance(self.pending,self.partial,v.data[t]);self.need-=done
def allowed(self):
if not self.inside:return self.v.outside_np[self.may_open and self.pending==0]
if self.first:return self.v.structures
return self.v.inside(self.need,self.pending,self.partial)
def __call__(self,tokens,logits):
if self.primed:self.feed(int(tokens[-1].item()))
self.primed=True
if not self.inside:return mx.where(self.v.outside[self.may_open and self.pending==0][None],logits,NEG)
m=self.allowed()
if not m.any():self.dead_ends+=1;return logits
return mx.where(mx.array(m)[None],logits,NEG)
def told(vocab,constraint,t):
return int(t) if vocab.kind[t]==STRUCT or constraint.pending else int(vocab.close)