| { |
| "format": "scholarly-ddp-v1", |
| "config": { |
| "world_size": 8, |
| "global_windows": 32, |
| "microbatch": 2, |
| "crop_padding": true, |
| "loss_chunk_size": 0, |
| "compile_model": true, |
| "cache_rope": true, |
| "context": 8192, |
| "target_tokens": 5000000000, |
| "warmup_tokens": 20000000, |
| "optimizer": "muon", |
| "muon_lr": 0.02, |
| "muon_momentum": 0.95, |
| "peak_lr": 0.0003, |
| "min_lr_ratio": 0.1, |
| "weight_decay": 0.1, |
| "seed": 42, |
| "checkpoint_every": 1000, |
| "evaluate_every": 500, |
| "pilot": false, |
| "experimental_data": true, |
| "prefetch_batches": 4, |
| "n_layer": 14, |
| "n_head": 12, |
| "n_embd": 768 |
| }, |
| "data_identity": { |
| "manifest_sha256": "70f63fa22fe19807bace17499f53a25c46145ad2c4e4966f61bef7e310528146", |
| "context": 8192, |
| "split": "train", |
| "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9" |
| }, |
| "source_sha256": { |
| "trainer": "e90356e9f0e7740c4aae2fbc257d7facef2afb4152731aeacfd8143a28c204a3", |
| "heldout_loader": "afa53c739b8c6b81540e4b9b6d85f90d5353802537ab564d97f1c29df196cda4", |
| "config": "b3a9a2438b5fac9f383d138bdc0bd9987d88497162e5260c2b96b2f9b7572a7b", |
| "optimizer": "de43032f664ab20a1471ffa9d5be987c8fa4634d7aafb43fa7f16a7ca4fe9759", |
| "model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326", |
| "loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2", |
| "raw_loader": "4312f800170e51f0a1d2be2b978c55fe9951f6df681c2bb225d83ecc0f33a6e7", |
| "prepared_loader": "c485580bd27298502a9ab4e9950f50e5680001bfe0e40cbb515906928ff76755", |
| "prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218", |
| "live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75", |
| "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da" |
| }, |
| "torch_version": "2.6.0+cu124", |
| "gpu": "NVIDIA H100 80GB HBM3", |
| "parameters": 124389632, |
| "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s019/step-000072999.pt", |
| "step": 77343, |
| "tokens": 4500012372, |
| "run_id": "python-124m-muon-5b-20261021-v1", |
| "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s022/step-000077343.pt", |
| "weights_sha256": "a55cd981a99fa31a4440f8f33792bec5db53273d5afb41e309e666cb194a080f", |
| "weight_roundtrip_exact": true, |
| "cpu_logits_roundtrip_exact": true, |
| "loss_token_exposures": { |
| "python_implementation": 2925006747, |
| "python_tests_examples": 675001115, |
| "python_documentation": 450000440, |
| "cs_math": 225001202, |
| "general_education": 135000835, |
| "books": 90002033 |
| } |
| } |
|
|
