from pathlib import Path

import tiktoken
from transformers.integrations.tiktoken import convert_tiktoken_to_fast

# Load the tiktoken encoding
encoding = tiktoken.get_encoding("cl100k_base")

# Output directory
out_dir = Path("hf_tokenizer")
out_dir.mkdir(exist_ok=True)

# Generate tokenizer.json
convert_tiktoken_to_fast(
    encoding,
    str(out_dir),
)

print(f"Saved {out_dir / 'tokenizer.json'}")