| { |
| "trainer": { |
| "evaluation_strategy": "steps", |
| "per_device_train_batch_size": 4, |
| "per_device_eval_batch_size": 4, |
| "gradient_accumulation_steps": 32, |
| "eval_steps": 50, |
| "save_steps": 50, |
| "logging_steps": 5, |
| "learning_rate": 0.0003, |
| "num_train_epochs": 5, |
| "lr_scheduler_type": "cosine", |
| "warmup_steps": 30, |
| "fp16": true, |
| "bf16": false, |
| "torch_compile": false, |
| "optim": "adamw_torch" |
| }, |
| "lora": { |
| "r": 16, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "bias": "none", |
| "target_modules": ["q_proj", "v_proj", "k_proj", "o_proj"], |
| "task_type": "CAUSAL_LM" |
| }, |
| "load_in_8bit": true, |
| "only_target_loss": true, |
| "mode": "chat", |
| "templates_path": "ru_saiga_template.json", |
| "model_name": "models/llama-13b", |
| "model_type": "causal", |
| "max_tokens_count": 1024 |
| } |
|
|
|
|
