| from pathlib import Path | |
| import tiktoken | |
| from transformers.integrations.tiktoken import convert_tiktoken_to_fast | |
| # Load the tiktoken encoding | |
| encoding = tiktoken.get_encoding("cl100k_base") | |
| # Output directory | |
| out_dir = Path("hf_tokenizer") | |
| out_dir.mkdir(exist_ok=True) | |
| # Generate tokenizer.json | |
| convert_tiktoken_to_fast( | |
| encoding, | |
| str(out_dir), | |
| ) | |
| print(f"Saved {out_dir / 'tokenizer.json'}") | |