{ "model_cfg": { "arch": "ngpt", "vocab_size": 4096, "d_model": 320, "n_heads": 8, "seq_len": 256, "n_input": 4, "n_core": 10, "n_output": 4, "n_experts": 4, "expert_dim": 120, "max_runs": 4, "target_mean": 2.5, "prefix": 32, "use_experts": true, "between_layers": 0, "between_mode": "shared", "between_merge": false, "boundary_norm": true, "active_experts": 1, "route_per_loop": true, "run_signal": false, "route_per_token": false, "run_mode": "joint", "neuron": "net", "neuron_train": "sampled" }, "train_cfg": { "batch_size": 64, "stage1_steps": 24000, "stage2_steps": 24000, "cycle_steps": 6000, "heavy_pieces": 2, "steps": 73180, "lr": 0.006, "weight_decay": 0.01, "grad_clip": 1.0, "warmup_lr_steps": 1000, "balance_penalty": 0.05, "router_sharpen": 0.05, "tau_start": 1.0, "tau_end": 0.3, "one_run_share": 0.1, "explore": 0.2, "neuron_weight": 5.0, "run_penalty": 0.2, "neuron_explore": 0.1, "eval_windows": 256, "log_every": 100, "save_every": 500, "keep_every": 20000, "seed": 0 }, "step": 73180 }