| { | |
| "vocab_size": 50304, | |
| "d_model": 896, | |
| "n_layers": 18, | |
| "n_heads": 14, | |
| "n_kv_heads": 2, | |
| "max_len": 2048, | |
| "rope_theta": 500000.0, | |
| "window_size": 512, | |
| "pattern_mult": 2.5, | |
| "reason_mult": 0.75, | |
| "reason_depth": 2, | |
| "gate_hidden": 64, | |
| "gate_init": 0.0, | |
| "diff_lambda_init": 0.8, | |
| "tie_embeddings": true, | |
| "sft_lr": 1e-05, | |
| "sft_min_lr": 1e-06, | |
| "warmup": 50, | |
| "epochs": 3, | |
| "micro_batch": 4, | |
| "grad_accum": 4, | |
| "grad_clip": 1.0, | |
| "wd": 0.0, | |
| "log_every": 10, | |
| "sample_every": 50, | |
| "ckpt_every": 200, | |
| "max_hours": 11.0, | |
| "max_examples": 1096, | |
| "end_token_weight": 5.0, | |
| "tag_close_weight": 3.0, | |
| "step": 205, | |
| "loss": 4.547707557678223, | |
| "arch": "CodVa-2-Stop-Think" | |
| } |