testing/convert_gpt2.py

"""Correct HF->GGUF f16 conversion for the rebuilt v4 ByteLevel tokenizer:
force pre='gpt-2' (verified full token parity) instead of the wrong 'sarvam-moe'.
Keeps the BYTE-retype patch for <0xXX> byte-fallback tokens."""
import os, sys, re, gguf
LL = "/workspace/llm/llama.cpp"
SRC = "/workspace/llm/kos-v4-conv"
OUT = "/workspace/llm/kos-v4-gguf/kosv4-f16.gguf"
sys.path.insert(0, LL); os.chdir(LL)
from conversion.base import TextModel

def _pre(self, tokenizer):
    return "gpt-2"
TextModel.get_vocab_base_pre = _pre

_byte = re.compile(r"^<0x[0-9A-Fa-f]{2}>$")
_orig_base = TextModel.get_vocab_base
def _base_patched(self):
    tokens, toktypes, tokpre = _orig_base(self)
    n = 0
    for i, t in enumerate(tokens):
        if _byte.match(t) and toktypes[i] != gguf.TokenType.BYTE:
            toktypes[i] = gguf.TokenType.BYTE; n += 1
    if n: print(f"[patch] retyped {n} byte-fallback tokens as BYTE", file=sys.stderr)
    return tokens, toktypes, tokpre
TextModel.get_vocab_base = _base_patched

import convert_hf_to_gguf as conv
sys.argv = ["convert_hf_to_gguf.py", "--outfile", OUT, "--outtype", "f16", SRC]
try:
    conv.main()
except SystemExit as e:
    if e.code not in (None, 0): raise
print("[done] wrote", OUT)