cpt core 4

scripts/cpt_core_model_4.py

@@ -29,7 +29,7 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 model = FastLanguageModel.get_peft_model(
     model,
     # r = 256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
-    r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    r = 8, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
     target_modules = [
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
@@ -49,6 +49,9 @@ model = FastLanguageModel.get_peft_model(
 # print(f'{model=}')
 
 
+#
+#
+#
 from datasets import Dataset
 from litdata import TokensLoader, StreamingDataset
 
@@ -58,6 +61,7 @@ litgpt_streaming_dataset = StreamingDataset(
     item_loader=TokensLoader(block_size=dataset_block_size),
 )
 
+
 def unlsoth_generator():
     global litgpt_streaming_dataset
 
@@ -68,7 +72,9 @@ def unlsoth_generator():
 # train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
 train_dataset = Dataset.from_generator(unlsoth_generator)
 
-
+#
+#
+#
 from trl import SFTTrainer
 from transformers import TrainingArguments
 from unsloth import is_bfloat16_supported
@@ -104,7 +110,8 @@ trainer = UnslothTrainer(
         fp16=not is_bfloat16_supported(),
         bf16=is_bfloat16_supported(),
         logging_steps=1,
-        optim='adamw_8bit',
+        # optim='adamw_8bit',
+        optim='adamw',
         weight_decay=0.01,
         lr_scheduler_type='cosine',
         seed=23,