python_scripts/veryfy_tokenizer_orgin.py

# ---------- check_tokenizer_size.py ----------
# (주식회사엠케이디 | MKD)
# Display total vocab tokens and a few sample tokens

from tokenizers import Tokenizer
import os

BASE_DIR = "/home/mkd-s003/AI/finetune"
TOKENIZER_PATH = os.path.join(BASE_DIR, "prod_tokenizer_v2.json")

print("🔍 Checking tokenizer information...\n")

# Load tokenizer
tok = Tokenizer.from_file(TOKENIZER_PATH)
vocab = tok.get_vocab()
vocab_size = len(vocab)

print(f"✅ Tokenizer loaded: {TOKENIZER_PATH}")
print(f"📘 Total vocabulary size: {vocab_size:,} tokens\n")

# Show first few sample tokens
sample_tokens = list(vocab.keys())[:20]
print("🔹 Sample tokens:")
for t in sample_tokens:
    print(f"   {t}")

# Check if special tokens exist
specials = ["<|endoftext|>", "<|pad|>", "<|unk|>", "<|im_start|>", "<|im_end|>"]
print("\n🔹 Special tokens present:")
for s in specials:
    print(f"   {s}: {'✅ Yes' if s in vocab else '❌ No'}")