File size: 2,769 Bytes
8406e31 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Vocab-only GGUF conversions of kos-v4 tokenizer with different forced pre-types,
then token-parity vs HF across several representative strings. Fast (no weights)."""
import os, sys, re, subprocess, tempfile
LL="/workspace/llm/llama.cpp"
SRC="/workspace/llm/kos-v4-conv" # conversion mirror (no tokenizer.model)
TOKREF="/workspace/llm/kos-v4-instruct" # pristine HF tokenizer
OUTDIR="/tmp/claude-0/-workspace/f1993a73-9699-445b-818e-eb56628e7541/scratchpad/vocabtest"
os.makedirs(OUTDIR, exist_ok=True)
TOKBIN=f"{LL}/build/bin/llama-tokenize"
from transformers import AutoTokenizer
at = AutoTokenizer.from_pretrained(TOKREF)
PROMPTS = [
"<|im_start|>system\nYou are a helpful medical assistant.<|im_end|>\n<|im_start|>user\nList three symptoms of anemia.<|im_end|>\n<|im_start|>assistant\n",
"Write exactly 3 bullet points.\n- one\n- two\n- three\n",
"The patient's BP was 148/92 (elevated). Multiple spaces and\ttabs.\n\nNew paragraph.",
"Answer in ALL CAPS: what is 2+2? Also use commas, semicolons; and dashes—like this.",
]
def convert_vocab_only(pre):
out=f"{OUTDIR}/vocab_{pre}.gguf"
code=f'''
import os,sys,re,gguf
sys.path.insert(0,"{LL}"); os.chdir("{LL}")
from conversion.base import TextModel
def _pre(self,tok): return "{pre}"
TextModel.get_vocab_base_pre=_pre
_byte=re.compile(r"^<0x[0-9A-Fa-f]{{2}}>$")
_ob=TextModel.get_vocab_base
def _bp(self):
t,tt,tp=_ob(self)
for i,x in enumerate(t):
if _byte.match(x) and tt[i]!=gguf.TokenType.BYTE: tt[i]=gguf.TokenType.BYTE
return t,tt,tp
TextModel.get_vocab_base=_bp
import convert_hf_to_gguf as c
sys.argv=["x","--outfile","{out}","--outtype","f16","--vocab-only","{SRC}"]
try: c.main()
except SystemExit as e:
if e.code not in (None,0): raise
'''
r=subprocess.run([sys.executable,"-c",code],capture_output=True,text=True)
if not os.path.exists(out):
return None, r.stderr[-300:]
return out, None
def gg_tokenize(gguf_path, text):
r=subprocess.run([TOKBIN,"-m",gguf_path,"-p",text,"--ids"],capture_output=True,text=True)
line=next((l for l in r.stdout.splitlines() if l.strip().startswith("[")),"")
return [int(x) for x in line.strip().strip("[]").replace(","," ").split()] if line else None
for pre in ["qwen2","gpt-2","default","llama-bpe","tekken"]:
out,err=convert_vocab_only(pre)
if out is None:
print(f"pre={pre:10} CONVERT FAILED: {err}"); continue
allok=True; details=[]
for p in PROMPTS:
hf=at(p,add_special_tokens=False)["input_ids"]
gg=gg_tokenize(out,p)
ok=(gg==hf)
allok&=ok
details.append(("OK" if ok else f"MISS(hf{len(hf)}/gg{len(gg) if gg else '?'})"))
print(f"pre={pre:10} {'ALL-PARITY ✅' if allok else 'mismatch'} {details}")
|