testing/pretok_probe.py

"""Vocab-only GGUF conversions of kos-v4 tokenizer with different forced pre-types,
then token-parity vs HF across several representative strings. Fast (no weights)."""
import os, sys, re, subprocess, tempfile
LL="/workspace/llm/llama.cpp"
SRC="/workspace/llm/kos-v4-conv"          # conversion mirror (no tokenizer.model)
TOKREF="/workspace/llm/kos-v4-instruct"   # pristine HF tokenizer
OUTDIR="/tmp/claude-0/-workspace/f1993a73-9699-445b-818e-eb56628e7541/scratchpad/vocabtest"
os.makedirs(OUTDIR, exist_ok=True)
TOKBIN=f"{LL}/build/bin/llama-tokenize"

from transformers import AutoTokenizer
at = AutoTokenizer.from_pretrained(TOKREF)

PROMPTS = [
  "<|im_start|>system\nYou are a helpful medical assistant.<|im_end|>\n<|im_start|>user\nList three symptoms of anemia.<|im_end|>\n<|im_start|>assistant\n",
  "Write exactly 3 bullet points.\n- one\n- two\n- three\n",
  "The patient's BP was 148/92 (elevated).  Multiple   spaces and\ttabs.\n\nNew paragraph.",
  "Answer in ALL CAPS: what is 2+2? Also use commas, semicolons; and dashes—like this.",
]

def convert_vocab_only(pre):
    out=f"{OUTDIR}/vocab_{pre}.gguf"
    code=f'''
import os,sys,re,gguf
sys.path.insert(0,"{LL}"); os.chdir("{LL}")
from conversion.base import TextModel
def _pre(self,tok): return "{pre}"
TextModel.get_vocab_base_pre=_pre
_byte=re.compile(r"^<0x[0-9A-Fa-f]{{2}}>$")
_ob=TextModel.get_vocab_base
def _bp(self):
    t,tt,tp=_ob(self)
    for i,x in enumerate(t):
        if _byte.match(x) and tt[i]!=gguf.TokenType.BYTE: tt[i]=gguf.TokenType.BYTE
    return t,tt,tp
TextModel.get_vocab_base=_bp
import convert_hf_to_gguf as c
sys.argv=["x","--outfile","{out}","--outtype","f16","--vocab-only","{SRC}"]
try: c.main()
except SystemExit as e:
    if e.code not in (None,0): raise
'''
    r=subprocess.run([sys.executable,"-c",code],capture_output=True,text=True)
    if not os.path.exists(out):
        return None, r.stderr[-300:]
    return out, None

def gg_tokenize(gguf_path, text):
    r=subprocess.run([TOKBIN,"-m",gguf_path,"-p",text,"--ids"],capture_output=True,text=True)
    line=next((l for l in r.stdout.splitlines() if l.strip().startswith("[")),"")
    return [int(x) for x in line.strip().strip("[]").replace(","," ").split()] if line else None

for pre in ["qwen2","gpt-2","default","llama-bpe","tekken"]:
    out,err=convert_vocab_only(pre)
    if out is None:
        print(f"pre={pre:10} CONVERT FAILED: {err}"); continue
    allok=True; details=[]
    for p in PROMPTS:
        hf=at(p,add_special_tokens=False)["input_ids"]
        gg=gg_tokenize(out,p)
        ok=(gg==hf)
        allok&=ok
        details.append(("OK" if ok else f"MISS(hf{len(hf)}/gg{len(gg) if gg else '?'})"))
    print(f"pre={pre:10} {'ALL-PARITY ✅' if allok else 'mismatch'}  {details}")