cpt core 4

scripts/cpt_core_model_4.py

@@ -55,7 +55,7 @@ model = FastLanguageModel.get_peft_model(
 
 
 #
-#
+# dataset
 #
 from datasets import Dataset
 from litdata import TokensLoader, StreamingDataset
@@ -74,11 +74,10 @@ def unlsoth_generator():
         yield {'input_ids': batch}
 
 
-# train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
 train_dataset = Dataset.from_generator(unlsoth_generator)
 
 #
-#
+# trainer
 #
 from trl import SFTTrainer
 from transformers import TrainingArguments
@@ -97,18 +96,12 @@ trainer = UnslothTrainer(
     packing=False, # Can make training 5x faster for short sequences.
 
     args = UnslothTrainingArguments(
-        # per_device_train_batch_size=16,
-        # gradient_accumulation_steps=64,
-        # per_device_train_batch_size=16,
-        # gradient_accumulation_steps=16,
         per_device_train_batch_size=1,
         # gradient_accumulation_steps=8,
 
         warmup_ratio=0,
         num_train_epochs=1,
 
-        # learning_rate=5e-5,
-        # embedding_learning_rate=5e-6,
         learning_rate = 5e-5,
         embedding_learning_rate = 5e-5 / 10.0,
 
@@ -116,8 +109,7 @@ trainer = UnslothTrainer(
         bf16=is_bfloat16_supported(),
         logging_steps=1,
         # optim='adamw_8bit',
-        optim='adamw_torch',
-        # optim='adamw_torch_fused',
+        optim='adamw_torch_fused',
         weight_decay=0.01,
         lr_scheduler_type='cosine',
         seed=23,