Upload context-a, context-multi-bios, context-multi-bios-20k, context-multi-bios-20k-s1, dense-a, dense-a-k100, dense-a-to-b, latent-a, latent-a-all, latent-multi, latent-multi-nohop, lookup-a
790f3b4 verified | { | |
| "arm": "context", | |
| "data": "data/world-a", | |
| "out": "runs/context-a", | |
| "edits": 0, | |
| "mix": [ | |
| [ | |
| "qa", | |
| 1.0 | |
| ] | |
| ], | |
| "init": null, | |
| "init_checkpoint": "final", | |
| "size": "small", | |
| "dtype": "float32", | |
| "steps": 10000, | |
| "batch_size": 64, | |
| "seq_len": 256, | |
| "lr": 0.001, | |
| "min_lr_frac": 0.1, | |
| "warmup": 500, | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "log_every": 100, | |
| "eval_every": 2000, | |
| "eval_sets": "test_id,test_ood,test_1hop_ood", | |
| "eval_n": 200, | |
| "save_every": 2000, | |
| "seed": 0, | |
| "model": { | |
| "vocab_size": 576, | |
| "n_layer": 8, | |
| "n_head": 8, | |
| "d_model": 512, | |
| "mlp_ratio": 4, | |
| "rope_base": 10000.0 | |
| }, | |
| "params": 25469440, | |
| "vocab": 544 | |
| } |