Download perloop_s0/config.json from Davidow/spidory: direct link, hf CLI and curl.
- Browser
- Download file 1.11 kB
-
https://huggingface.co/Davidow/spidory/resolve/main/perloop_s0/config.json
- Command line
-
hf download hf://Davidow/spidory/perloop_s0/config.json
-
curl -L -o config.json https://huggingface.co/Davidow/spidory/resolve/main/perloop_s0/config.json
1.11 kB
| { | |
| "model_cfg": { | |
| "arch": "ngpt", | |
| "vocab_size": 4096, | |
| "d_model": 320, | |
| "n_heads": 8, | |
| "seq_len": 256, | |
| "n_input": 4, | |
| "n_core": 10, | |
| "n_output": 4, | |
| "n_experts": 4, | |
| "expert_dim": 120, | |
| "max_runs": 4, | |
| "target_mean": 2.5, | |
| "prefix": 32, | |
| "use_experts": true, | |
| "between_layers": 0, | |
| "between_mode": "shared", | |
| "between_merge": false, | |
| "boundary_norm": true, | |
| "active_experts": 1, | |
| "route_per_loop": true, | |
| "run_signal": false, | |
| "route_per_token": false, | |
| "run_mode": "joint", | |
| "neuron": "net", | |
| "neuron_train": "sampled" | |
| }, | |
| "train_cfg": { | |
| "batch_size": 64, | |
| "stage1_steps": 24000, | |
| "stage2_steps": 24000, | |
| "cycle_steps": 6000, | |
| "heavy_pieces": 2, | |
| "steps": 73180, | |
| "lr": 0.006, | |
| "weight_decay": 0.01, | |
| "grad_clip": 1.0, | |
| "warmup_lr_steps": 1000, | |
| "balance_penalty": 0.05, | |
| "router_sharpen": 0.05, | |
| "tau_start": 1.0, | |
| "tau_end": 0.3, | |
| "one_run_share": 0.1, | |
| "explore": 0.2, | |
| "neuron_weight": 5.0, | |
| "run_penalty": 0.2, | |
| "neuron_explore": 0.1, | |
| "eval_windows": 256, | |
| "log_every": 100, | |
| "save_every": 500, | |
| "keep_every": 20000, | |
| "seed": 0 | |
| }, | |
| "step": 73180 | |
| } | |