Upload scripts/train_alizee_v2_stage1_sft.py with huggingface_hub

scripts/train_alizee_v2_stage1_sft.py

@@ -9,6 +9,8 @@
 #     "bitsandbytes>=0.45.0",
 #     "trackio",
 #     "datasets>=3.0.0",
+#     "protobuf>=3.20.0",
+#     "sentencepiece>=0.2.0",
 # ]
 # ///
 
@@ -66,7 +68,11 @@ print("=" * 60)
 
 # Load tokenizer
 print("\n📝 Loading tokenizer...")
-tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(
+    BASE_MODEL,
+    trust_remote_code=True,
+    use_fast=False,  # Use slow tokenizer to avoid conversion issues
+)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token
 tokenizer.padding_side = "right"