Update tokenizer.py

tokenizer.py

@@ -8,7 +8,7 @@ from transformers import PreTrainedTokenizer
 class ChessTokenizer(PreTrainedTokenizer):
     """
     Chess move tokenizer compatible with HuggingFace transformers.
-    Can be loaded with: AutoTokenizer.from_pretrained("ankanmbz/gambit-tok")
+    Can be loaded with: AutoTokenizer.from_pretrained("ankanmbz/chess-tok")
     """
     
     vocab_files_names = {
@@ -160,7 +160,7 @@ class ChessTokenizer(PreTrainedTokenizer):
 # Builder script to create HuggingFace-compatible tokenizer
 # ============================================================================
 
-def build_hf_tokenizer(dataset_path, output_dir="gambit-tok-hf"):
+def build_hf_tokenizer(dataset_path, output_dir="chess-tok-hf"):
     """Build HuggingFace-compatible tokenizer from dataset"""
     import pandas as pd
     from collections import Counter
@@ -262,4 +262,4 @@ def build_hf_tokenizer(dataset_path, output_dir="gambit-tok-hf"):
 if __name__ == "__main__":
     # Build the tokenizer
     dataset_path = "/vast/users/ankan.deria/Document/TinyRecursiveModels/data/chees_data/dataset.parquet"
-    build_hf_tokenizer(dataset_path, output_dir="gambit-tok-hf")
+    build_hf_tokenizer(dataset_path, output_dir="chess-tok-hf")