| | model: |
| | _component_: models.lora_mmllama3_8b |
| | lora_attn_modules: |
| | - q_proj |
| | - v_proj |
| | apply_lora_to_mlp: false |
| | apply_lora_to_output: false |
| | lora_rank: 32 |
| | lora_alpha: 64 |
| | perception_tokens: 2 |
| | use_clip: false |
| | tokenizer: |
| | _component_: models.a2a_tokenizer |
| | path: models/tokenizer.model |
| | checkpointer: |
| | _component_: torchtune.utils.FullModelMetaCheckpointer |
| | checkpoint_dir: smtst2/ |
| | checkpoint_files: |
| | - meta_model_1.pt |
| | adapter_checkpoint: null |
| | recipe_checkpoint: null |
| | output_dir: output_checkpoints/experiment_1 |
| | model_type: LLAMA3 |
| | resume_from_checkpoint: false |
| | interim_checkpoint_steps: 10000 |
| | interim_gen_steps: null |
| | max_new_tokens: 100 |
| | temperature: 0.6 |
| | top_k: 225 |
| | dataset: |
| | _component_: ds.EvenBatcher |
| | buffer_size: 1 |
| | dataset: |
| | _component_: ds.RoundRobinDataset |
| | datasets: |
| | - _component_: ds.OmegaVideoCaptionDataset |
| | length: 500000 |
| | - _component_: ds.LlavaInstructDataset |
| | dataset_path: ds/coco_llava_instruct/output.parquet |
| | train_on_input: false |
| | - _component_: ds.LlavaInstructDataset |
| | dataset_path: ds/vision_flan/output.parquet |
| | train_on_input: false |
| | - _component_: ds.CaptionInstructDataset |
| | dataset_path: ds/sam_llava/output.parquet |
| | train_on_input: false |
| | |
| | |
| | |
| | seed: null |
| | shuffle: true |
| | batch_size: 64 |
| | optimizer: |
| | _component_: torch.optim.AdamW |
| | weight_decay: 1.5 |
| | lr: 1.0 |
| | lr_scheduler: |
| | _component_: torchtune.modules.get_cosine_schedule_with_warmup |
| | num_warmup_steps: 1000 |
| | loss: |
| | _component_: torch.nn.CrossEntropyLoss |
| | epochs: 3 |
| | max_steps_per_epoch: null |
| | gradient_accumulation_steps: 192 |
| | compile: false |
| | output_dir: /tmp/lora_finetune_output |
| | metric_logger: |
| | _component_: torchtune.utils.metric_logging.DiskLogger |
| | log_dir: ${output_dir} |
| | log_every_n_steps: null |
| | device: cuda |
| | dtype: bf16 |
| | enable_activation_checkpointing: false |
| | profiler: |
| | _component_: torchtune.utils.profiler |
| | enabled: false |
| | inference: |
| | prompt_template: 'Video: |
| | |
| | {video} |
| | |
| | Caption the previous video.' |
| | max_new_tokens: 300 |
| | temperature: 0.6 |
| | top_k: 300 |
| | quantizer: null |
| | gradient-accumulation-steps: 32 |
| |
|
