| model_name: meta-llama/Meta-Llama-3-8B-Instruct |
| exp_name: Llama-3-8b-optimizers |
| run_name: Riemannion-WD-OI-new |
| base_dir: /home/vabogachev/RiemanianFinetune/runs/${exp_name}/${run_name}/final |
| cfg_no: 1 |
| tasks: |
| - FINETUNE |
| - VALIDATE |
| - INFERENCE |
| max_length: 768 |
| dataset_path: /home/vabogachev/RiemanianFinetune/datasets/common_reasoning |
| n_shots: 0 |
| fp16: false |
| bf16: true |
| num_ths: 3 |
| loader_config: |
| num_proc: ${num_ths} |
| tokenizer_config: |
| padding_side: left |
| report_to: comet_ml |
| detailed_lora_logs: false |
| detailed_riemannian_logs: true |
| adapter_config: |
| peft_pretrained: true |
| peft_is_trainable: true |
| merge_tuned: true |
| peft_pretrained_path: ${base_dir}/finetuned_model_cfg-${cfg_no} |
| ft_strategy: LoRA |
| peft_init_path: ${base_dir}/initialization_${cfg_no} |
| target_modules: |
| - q_proj |
| - k_proj |
| - v_proj |
| - o_proj |
| - up_proj |
| - down_proj |
| - gate_proj |
| LoRA_config: |
| r: 32 |
| lora_alpha: ${adapter_config.LoRA_config.r} |
| lora_dropout: 0.05 |
| target_modules: ${adapter_config.target_modules} |
| split2zero: true |
| init_strategy: riemannian |
| B_gain_npwr: 0.5 |
| B_gain_rpwr: -0.5 |
| B_gain_mult: -1.0 |
| evaluation_config: |
| num_splits: 10 |
| max_new_tokens: 4 |
| batch_size: 8 |
| empty_cache: true |
| dump_path: ${base_dir}/cfg-${cfg_no}_preds_CR_{0}.bin |
| optimizer_config: |
| optim: Riemannion |
| lr: 0.0001 |
| momentum: 0.9 |
| nesterov: false |
| weight_decay: 0.00316 |
| trainer_config: |
| run_name: ${exp_name}/${run_name}-final-cfg${cfg_no} |
| output_dir: bogachevv/${exp_name}-${run_name}-final-cfg${cfg_no} |
| max_seq_length: ${max_length} |
| dataset_text_field: text |
| fp16: ${fp16} |
| bf16: ${bf16} |
| full_determinism: false |
| per_device_train_batch_size: 4 |
| per_device_eval_batch_size: 8 |
| gradient_accumulation_steps: 16 |
| lr_scheduler_type: linear |
| warmup_ratio: 0.1 |
| num_train_epochs: 2 |
| dataloader_num_workers: ${num_ths} |
| dataset_num_proc: ${num_ths} |
| eval_strategy: steps |
| eval_steps: 128 |
| logging_steps: 16 |
| load_best_model_at_end: true |
| seed: 23654 |
| data_seed: 23654 |
| report_to: ${report_to} |
| disable_tqdm: true |
| save_strategy: steps |
| save_steps: 128 |
| save_total_limit: 4 |
|
|
|
|
