Md Najmul Hossain
Upload python_scripts/veryfy_tokenizer_orgin.py with huggingface_hub
8265028 verified | # ---------- check_tokenizer_size.py ---------- | |
| # (์ฃผ์ํ์ฌ์ ์ผ์ด๋ | MKD) | |
| # Display total vocab tokens and a few sample tokens | |
| from tokenizers import Tokenizer | |
| import os | |
| BASE_DIR = "/home/mkd-s003/AI/finetune" | |
| TOKENIZER_PATH = os.path.join(BASE_DIR, "prod_tokenizer_v2.json") | |
| print("๐ Checking tokenizer information...\n") | |
| # Load tokenizer | |
| tok = Tokenizer.from_file(TOKENIZER_PATH) | |
| vocab = tok.get_vocab() | |
| vocab_size = len(vocab) | |
| print(f"โ Tokenizer loaded: {TOKENIZER_PATH}") | |
| print(f"๐ Total vocabulary size: {vocab_size:,} tokens\n") | |
| # Show first few sample tokens | |
| sample_tokens = list(vocab.keys())[:20] | |
| print("๐น Sample tokens:") | |
| for t in sample_tokens: | |
| print(f" {t}") | |
| # Check if special tokens exist | |
| specials = ["<|endoftext|>", "<|pad|>", "<|unk|>", "<|im_start|>", "<|im_end|>"] | |
| print("\n๐น Special tokens present:") | |
| for s in specials: | |
| print(f" {s}: {'โ Yes' if s in vocab else 'โ No'}") | |