cpt core 4

scripts/cpt_core_model_4.py

@@ -32,28 +32,25 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 
 model = FastLanguageModel.get_peft_model(
     model,
-    r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    # r = 8, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    target_modules = [
+    r=256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    target_modules=[
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
         "up_proj", "down_proj",
         "embed_tokens", "lm_head",
     ],
-    lora_alpha = 32,
-    # lora_alpha = 2,
-    lora_dropout = 0, # Supports any, but = 0 is optimized
-    bias = "none",    # Supports any, but = "none" is optimized
+    lora_alpha=32,
+    lora_dropout=0, # Supports any, but = 0 is optimized
+    bias="none",    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
-    # use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
-    use_gradient_checkpointing = False,
-    random_state = 3407,
-    use_rslora = True,  # We support rank stabilized LoRA
-    loftq_config = None, # And LoftQ
+    # use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context
+    use_gradient_checkpointing=False,
+    random_state=23,
+    use_rslora=True,    # We support rank stabilized LoRA
+    loftq_config=None,  # And LoftQ
 )
 # print(f'{model=}')
 
-
 #
 # dataset
 #
@@ -89,21 +86,20 @@ trainer = UnslothTrainer(
     model=model,
     tokenizer=tokenizer,
     train_dataset=train_dataset,
-    # dataset_text_field='text',
     max_seq_length=max_seq_length,
     dataset_num_proc=32,
     max_steps=len(litgpt_streaming_dataset),
     packing=False, # Can make training 5x faster for short sequences.
 
-    args = UnslothTrainingArguments(
+    args=UnslothTrainingArguments(
         per_device_train_batch_size=1,
         # gradient_accumulation_steps=8,
 
         warmup_ratio=0,
         num_train_epochs=1,
 
-        learning_rate = 5e-5,
-        embedding_learning_rate = 5e-5 / 10.0,
+        learning_rate=5e-5,
+        embedding_learning_rate=5e-5 / 10.0,
 
         fp16=not is_bfloat16_supported(),
         bf16=is_bfloat16_supported(),
@@ -115,6 +111,10 @@ trainer = UnslothTrainer(
         seed=23,
         output_dir=output_dir,
         report_to='wandb',
+
+        do_eval=True,
+        save_steps=100,
+        eval_steps=100,
     ),
 )