Upload training_config.yml with huggingface_hub

training_config.yml

@@ -3,7 +3,7 @@ model:
   lora_attn_modules:
   - q_proj
   - v_proj
-  apply_lora_to_mlp: false
+  apply_lora_to_mlp: true
   apply_lora_to_output: false
   lora_rank: 32
   lora_alpha: 64
@@ -16,7 +16,7 @@ checkpointer:
   _component_: torchtune.utils.FullModelMetaCheckpointer
   checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
   checkpoint_files:
-  - consolidated.00.pth
+  - meta_model_5.pt
   adapter_checkpoint: null
   recipe_checkpoint: null
   output_dir: output_checkpoints/experiment_1
@@ -46,19 +46,19 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 16
+batch_size: 32
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 1.0e-05
+  weight_decay: 0.0
   lr: 5.0e-05
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 2000
+  num_warmup_steps: 1000
 loss:
   _component_: torch.nn.CrossEntropyLoss
 epochs: 7
-max_steps_per_epoch: 2000
-gradient_accumulation_steps: 16
+max_steps_per_epoch: 1000
+gradient_accumulation_steps: 8
 compile: false
 output_dir: /tmp/lora_finetune_output
 metric_logger: