| { |
| "_doc": "Smoke for MATH: verify dataset loads, code paths run, OOM safe at full seq len.", |
| "base_model": "Qwen/Qwen2.5-Math-7B-Instruct", |
| "use_lora": true, "lora_r": 16, "lora_alpha": 32, "lora_dropout": 0.05, |
| "lora_target_modules": ["q_proj", "k_proj", "v_proj", "o_proj"], |
| "dtype": "bfloat16", "attn_impl": "eager", |
| "gradient_checkpointing": false, |
| "K_latents": 16, "K_curriculum": [[0, 16]], |
| "block_y_to_x": true, "block_z_to_x": false, |
| "proj_init_scale": 0.02, "proj_mlp": true, "proj_hidden_mult": 4, |
| "lambda_lm": 1.0, "lambda_id": 1.0, "lambda_kl": 0.0001, |
| "tau_infonce": 0.2, |
| "infonce_full_answer": true, "infonce_target_max_len": 256, |
| "lr_lora": 1e-4, "lr_proj": 5e-5, "lr_head": 1e-4, |
| "weight_decay": 0.01, "max_grad_norm": 1.0, "warmup_steps": 10, |
| "batch_size": 4, "grad_accum": 1, "max_steps": 5, |
| "max_prompt_len": 256, "max_answer_len": 256, |
| "log_every": 1, "eval_every": 0, "eval_size": 8, "save_every": 0, |
| "seed": 23, "dataset": "math", |
| "output_dir": "/home/ubuntu/work/blt_math_smoke", |
| "data_train_size": 64, "data_eval_size": 8 |
| } |
|
|
