Upload training_config.yml with huggingface_hub

training_config.yml

@@ -5,30 +5,31 @@ model:
   - v_proj
   apply_lora_to_mlp: false
   apply_lora_to_output: false
-  lora_rank: 8
-  lora_alpha: 16
+  lora_rank: 32
+  lora_alpha: 64
   perception_tokens: 2
   use_clip: false
 tokenizer:
   _component_: models.a2a_tokenizer
-  path: checkpoints/Meta-Llama-3-8B-Instruct/original/tokenizer.model
+  path: models/tokenizer.model
 checkpointer:
   _component_: torchtune.utils.FullModelMetaCheckpointer
-  checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
+  checkpoint_dir: crazyfrog2
   checkpoint_files:
-  - consolidated.01.pth
+  - meta_model_0.pt
   adapter_checkpoint: null
   recipe_checkpoint: null
   output_dir: output_checkpoints/experiment_1
   model_type: LLAMA3
 resume_from_checkpoint: false
-interim_checkpoint_steps: 5000
+interim_checkpoint_steps: 15000
 interim_gen_steps: null
-max_new_tokens: 100
+max_new_tokens: 77
 temperature: 0.6
-top_k: 300
+top_k: 231
 dataset:
   _component_: ds.EvenBatcher
+  buffer_size: 72
   dataset:
     _component_: ds.RoundRobinDataset
     datasets:
@@ -45,19 +46,19 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 8
+batch_size: 6
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 0.01
-  lr: 3.0e-05
+  weight_decay: 0.99
+  lr: 0.0002
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 100
+  num_warmup_steps: 4
 loss:
   _component_: torch.nn.CrossEntropyLoss
-epochs: 1
+epochs: 60
 max_steps_per_epoch: null
-gradient_accumulation_steps: 16
+gradient_accumulation_steps: 260
 compile: false
 output_dir: /tmp/lora_finetune_output
 metric_logger:
@@ -76,7 +77,8 @@ inference:
     {video}
 
     Caption the previous video.'
-  max_new_tokens: 300
-  temperature: 0.6
-  top_k: 300
+  max_new_tokens: 231
+  temperature: 0.8
+  top_k: 231
   quantizer: null
+gradient-accumulation-steps: 32