Upload training_config.yml with huggingface_hub

training_config.yml

@@ -14,22 +14,22 @@ tokenizer:
   path: models/tokenizer.model
 checkpointer:
   _component_: torchtune.utils.FullModelMetaCheckpointer
-  checkpoint_dir: /workspace/omega_a2a/training
+  checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
   checkpoint_files:
   - consolidated.00.pth
   adapter_checkpoint: null
   recipe_checkpoint: null
-  output_dir: /workspace/omega_a2a/checkpoints
+  output_dir: output_checkpoints/experiment_1
   model_type: LLAMA3
 resume_from_checkpoint: false
 interim_checkpoint_steps: 5000
 interim_gen_steps: null
-max_new_tokens: 170
+max_new_tokens: 200
 temperature: 0.8
 top_k: 200
 dataset:
   _component_: ds.EvenBatcher
-  buffer_size: 36
+  buffer_size: 72
   dataset:
     _component_: ds.RoundRobinDataset
     datasets:
@@ -46,19 +46,19 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 4
+batch_size: 16
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 0.0001
-  lr: 3.0e-05
+  weight_decay: 0.001
+  lr: 0.0003
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 100
+  num_warmup_steps: 150
 loss:
   _component_: torch.nn.CrossEntropyLoss
 epochs: 6
 max_steps_per_epoch: null
-gradient_accumulation_steps: 64
+gradient_accumulation_steps: 16
 compile: false
 output_dir: /tmp/lora_finetune_output
 metric_logger:
@@ -67,7 +67,7 @@ metric_logger:
 log_every_n_steps: null
 device: cuda
 dtype: bf16
-enable_activation_checkpointing: false
+enable_activation_checkpointing: true
 profiler:
   _component_: torchtune.utils.profiler
   enabled: false
@@ -78,6 +78,6 @@ inference:
 
     Caption the previous video.'
   max_new_tokens: 300
-  temperature: 0.7
-  top_k: 250
+  temperature: 0.6
+  top_k: 200
   quantizer: null