Hugging Face's logo

jfrick
/
simple-llm-python-124m-5b

PyTorch
Safetensors
English
causal-lm
python
from-scratch
muon
experimental
Model card Files
xet
 Community
simple-llm-python-124m-5b
File size: 2,615 Bytes
4719692
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9beeb42
4719692
98cd910
 
 
4719692
98cd910
 
4719692
 
 
98cd910
 
 
 
 
 
4719692
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
{
  "format": "scholarly-ddp-v1",
  "config": {
    "world_size": 8,
    "global_windows": 32,
    "microbatch": 2,
    "crop_padding": true,
    "loss_chunk_size": 0,
    "compile_model": true,
    "cache_rope": true,
    "context": 8192,
    "target_tokens": 5000000000,
    "warmup_tokens": 20000000,
    "optimizer": "muon",
    "muon_lr": 0.02,
    "muon_momentum": 0.95,
    "peak_lr": 0.0003,
    "min_lr_ratio": 0.1,
    "weight_decay": 0.1,
    "seed": 42,
    "checkpoint_every": 1000,
    "evaluate_every": 500,
    "pilot": false,
    "experimental_data": true,
    "prefetch_batches": 4,
    "n_layer": 14,
    "n_head": 12,
    "n_embd": 768
  },
  "data_identity": {
    "manifest_sha256": "70f63fa22fe19807bace17499f53a25c46145ad2c4e4966f61bef7e310528146",
    "context": 8192,
    "split": "train",
    "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9"
  },
  "source_sha256": {
    "trainer": "e90356e9f0e7740c4aae2fbc257d7facef2afb4152731aeacfd8143a28c204a3",
    "heldout_loader": "afa53c739b8c6b81540e4b9b6d85f90d5353802537ab564d97f1c29df196cda4",
    "config": "b3a9a2438b5fac9f383d138bdc0bd9987d88497162e5260c2b96b2f9b7572a7b",
    "optimizer": "de43032f664ab20a1471ffa9d5be987c8fa4634d7aafb43fa7f16a7ca4fe9759",
    "model": "2c978432e84cad6b2f661c63e7e62bccfc393a2aeb9f0519f7ca3db6775ed326",
    "loader": "7821e13e765a9f2c0e0a94744bc7f2df8eea647f26d7aa0069f62556618a0ef2",
    "raw_loader": "4312f800170e51f0a1d2be2b978c55fe9951f6df681c2bb225d83ecc0f33a6e7",
    "prepared_loader": "c485580bd27298502a9ab4e9950f50e5680001bfe0e40cbb515906928ff76755",
    "prefetch": "1786b1423f5881b23223ff0db47284a522020237d77e3d7fa28d542deea5c218",
    "live_loader": "64457d0e60f1f1b664d3b174183c6e6f3d0c618973fe56712f9e72cd2a118a75",
    "tokenizer": "00e32adc13dab74f5a029c63304b4591bd48f7cd6b086b22d4624d244667b4da"
  },
  "torch_version": "2.6.0+cu124",
  "gpu": "NVIDIA H100 80GB HBM3",
  "parameters": 124389632,
  "resume_from": "/runs/python-124m-muon-5b-20261021-v1-s019/step-000072999.pt",
  "step": 77343,
  "tokens": 4500012372,
  "run_id": "python-124m-muon-5b-20261021-v1",
  "checkpoint": "/runs/python-124m-muon-5b-20261021-v1-s022/step-000077343.pt",
  "weights_sha256": "a55cd981a99fa31a4440f8f33792bec5db53273d5afb41e309e666cb194a080f",
  "weight_roundtrip_exact": true,
  "cpu_logits_roundtrip_exact": true,
  "loss_token_exposures": {
    "python_implementation": 2925006747,
    "python_tests_examples": 675001115,
    "python_documentation": 450000440,
    "cs_math": 225001202,
    "general_education": 135000835,
    "books": 90002033
  }
}