Upload train_code_reasoning.py

train_code_reasoning.py

@@ -13,7 +13,6 @@
 import os
 import random
 from datasets import load_dataset, concatenate_datasets
-from transformers import AutoTokenizer
 from trl import SFTTrainer, SFTConfig
 import trackio
 
@@ -25,9 +24,6 @@ OUTPUT_DIR = "./code-reasoning-1.5b"
 # Initialize Trackio
 trackio.init(project="code-reasoning-ft", name="qwen2.5-coder-1.5b-code-reasoning")
 
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
-
 print("Loading and preparing datasets...")
 
 all_datasets = []
@@ -188,8 +184,6 @@ trainer = SFTTrainer(
     model=MODEL_ID,
     train_dataset=train_dataset,
     args=training_args,
-    processing_class=tokenizer,
-    max_seq_length=2048,
 )
 
 print("Starting training...")