"""Constrained decoding. On the main line no structure token and no can come, and only between characters. Inside an IDS every step keeps it well formed: after a structure token; a structure token opens as many places as it has parts; a part fills one place and is one CJK character, radical or stroke (base-vocabulary tokens are followed byte by byte through UTF-8); comes exactly when no place is open. told(): what the model is told of an IDS so far -- a structure token and the unfinished bytes of a part stand for themselves, whatever finishes a part is told as .""" from __future__ import annotations import numpy as np import mlx.core as mx NEG=-1e9 LEAVES=((0x4E00,0x9FFF),(0x3400,0x4DBF),(0x20000,0x3347F),(0x2E80,0x2EFF),(0x31C0,0x31E5)) BASE,SPECIAL,GLYPH,EXTCHAR,KOPEN,KCLOSE,STRUCT=range(7) def leaf(cp): cp=np.asarray(cp);ok=np.zeros(cp.shape,bool) for a,b in LEAVES:ok|=(cp>=a)&(cp<=b) return ok def byte_decoder(): bs=list(range(ord('!'),ord('~')+1))+list(range(ord('¡'),ord('¬')+1))+list(range(ord('®'),ord('ÿ')+1));cs=bs[:];n=0 for b in range(256): if b not in bs:bs.append(b);cs.append(256+n);n+=1 return {chr(c):b for b,c in zip(bs,cs)} def char_length(first): return 1 if first<0x80 else 2 if 0xC0<=first<=0xDF else 3 if 0xE0<=first<=0xEF else 4 if 0xF0<=first<=0xF7 else 0 def code(seq): cp=seq[0]&{1:0x7F,2:0x1F,3:0x0F,4:0x07}[char_length(seq[0])] for b in seq[1:]:cp=(cp<<6)|(b&0x3F) return cp def partial_can_be_leaf(tail): n=char_length(tail[0]) if n<3 or any(not 0x80<=b<=0xBF for b in tail[1:]):return False low=code(list(tail)+[0x80]*(n-len(tail)));high=code(list(tail)+[0xBF]*(n-len(tail))) return any(a<=high and low<=b for a,b in LEAVES) def shape(data): i=0 while ilen(data):trail=n-(len(data)-i);ok=ok and partial_can_be_leaf(data[i:]);break piece=data[i:i+n] if any(not 0x80<=b<=0xBF for b in piece[1:]):return lead,count,0,False ok=ok and bool(leaf(code(list(piece))));count+=1;i+=n return lead,count,trail,ok class Vocab: def __init__(self,tokenizer,info): ids=info['ids'];V=ids['vocabulary'];self.V=V;self.open=ids['open'];self.close=ids['close'];self.standin=info['first_gw_id'];self.first_ids=ids['first_ids'] self.kind=np.zeros(V,np.int8);self.lead=np.zeros(V,np.int8);self.chars=np.zeros(V,np.int16) self.trail=np.zeros(V,np.int8);self.ok=np.zeros(V,bool);self.arity=np.zeros(V,np.int8);self.data=[b'']*V self.first=np.full((V,3),-1,np.int16) specials={i for i in tokenizer.added_tokens_decoder if i0) &(self.lead==np.array([len(d) for d in self.data])))[0] def inside(self,need,pending,partial): m=np.zeros(self.V,bool) if need==0:m[self.close]=True;return m if pending==0: k=self.chars+(self.trail>0) m=(self.kind==STRUCT)|((self.kind==EXTCHAR)&self.ok)|((self.kind==BASE)&(self.lead==0)&self.ok&(k>=1)&(k<=need)) return m finish=np.where((self.kind==BASE)&(self.lead==pending))[0] if len(finish): tail=self.first[finish,:pending].astype(np.int64);cp=code(list(partial)) for j in range(pending):cp=(cp<<6)|(tail[:,j]&0x3F) k=1+self.chars[finish]+(self.trail[finish]>0) m[finish]=leaf(cp)&self.ok[finish]&(k<=need) for i in self.continuation_only: if self.lead[i]len(data):pending=n-(len(data)-i);partial=data[i:];break done+=1;i+=n return done,pending,partial class Constraint: def __init__(self,vocab,may_open=True): self.v=vocab;self.may_open=may_open;self.primed=False;self.inside=False;self.first=False;self.need=0;self.pending=0;self.partial=b'';self.dead_ends=0 def feed(self,t): v=self.v;k=v.kind[t] if not self.inside: if k==KOPEN:self.inside=True;self.first=True;self.need=1;self.pending=0;self.partial=b'' elif k==BASE:_,self.pending,self.partial=advance(self.pending,self.partial,v.data[t]) else:self.pending=0;self.partial=b'' return self.first=False if k==KCLOSE:self.inside=False elif k==STRUCT:self.need+=int(v.arity[t])-1 elif k==EXTCHAR:self.need-=1 elif k==BASE: done,self.pending,self.partial=advance(self.pending,self.partial,v.data[t]);self.need-=done def allowed(self): if not self.inside:return self.v.outside_np[self.may_open and self.pending==0] if self.first:return self.v.structures return self.v.inside(self.need,self.pending,self.partial) def __call__(self,tokens,logits): if self.primed:self.feed(int(tokens[-1].item())) self.primed=True if not self.inside:return mx.where(self.v.outside[self.may_open and self.pending==0][None],logits,NEG) m=self.allowed() if not m.any():self.dead_ends+=1;return logits return mx.where(mx.array(m)[None],logits,NEG) def told(vocab,constraint,t): return int(t) if vocab.kind[t]==STRUCT or constraint.pending else int(vocab.close)