| { |
| "run_name": "qwen-ctfer", |
| "seed": 42, |
| "model": { |
| "name_or_path": "Qwen/Qwen2.5-Coder-7B-Instruct", |
| "torch_dtype": "bfloat16", |
| "trust_remote_code": false, |
| "use_fast_tokenizer": true, |
| "padding_side": "right", |
| "attn_implementation": "sdpa", |
| "device_map": "auto", |
| "load_in_4bit": true, |
| "bnb_4bit_quant_type": "nf4", |
| "bnb_4bit_use_double_quant": true, |
| "bnb_4bit_compute_dtype": "bfloat16" |
| }, |
| "data": { |
| "train_file": "data/processed/train.jsonl", |
| "eval_file": "data/processed/eval.jsonl" |
| }, |
| "formatting": { |
| "assistant_only_loss": true |
| }, |
| "lora": { |
| "r": 16, |
| "alpha": 32, |
| "dropout": 0.05, |
| "bias": "none", |
| "target_modules": "all-linear", |
| "task_type": "CAUSAL_LM" |
| }, |
| "training": { |
| "output_dir": "outputs/checkpoints/qwen-ctfer", |
| "num_train_epochs": 2, |
| "per_device_train_batch_size": 1, |
| "per_device_eval_batch_size": 1, |
| "gradient_accumulation_steps": 8, |
| "learning_rate": 0.0002, |
| "lr_scheduler_type": "cosine", |
| "warmup_ratio": 0.03, |
| "weight_decay": 0.0, |
| "max_grad_norm": 1.0, |
| "logging_steps": 5, |
| "save_strategy": "steps", |
| "save_steps": 25, |
| "save_total_limit": 2, |
| "evaluation_strategy": "steps", |
| "eval_steps": 25, |
| "gradient_checkpointing": true, |
| "bf16": true, |
| "fp16": false, |
| "packing": false, |
| "max_seq_length": 2048, |
| "report_to": [ |
| "none" |
| ] |
| } |
| } |
|
|
