Upload train_ministral_n8n.py with huggingface_hub

train_ministral_n8n.py

@@ -2,7 +2,7 @@
 # requires-python = ">=3.10"
 # dependencies = [
 #     "torch>=2.0.0",
-#     "transformers>=4.48.0",
+#     "transformers>=4.51.0",
 #     "datasets>=2.14.0",
 #     "accelerate>=0.24.0",
 #     "peft>=0.7.0",
@@ -10,6 +10,8 @@
 #     "bitsandbytes>=0.41.0",
 #     "huggingface_hub>=0.20.0",
 #     "tokenizers>=0.21.0",
+#     "protobuf",
+#     "mistral-common>=1.5.0",
 #     "trackio",
 # ]
 # ///
@@ -37,9 +39,8 @@ MAX_SEQ_LENGTH = 4096  # n8n workflows can be long
 trackio.init(project="ministral-3b-n8n-sft")
 
 print(f"Loading tokenizer from {MODEL_NAME}...")
-# Ministral uses tekken tokenizer - use PreTrainedTokenizerFast
-from transformers import PreTrainedTokenizerFast
-tokenizer = PreTrainedTokenizerFast.from_pretrained(MODEL_NAME)
+# Ministral uses Tekken tokenizer - use AutoTokenizer with trust_remote_code
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
 if tokenizer.pad_token is None:
     tokenizer.pad_token = tokenizer.eos_token