Hugging Face's logo

jfrick
/
simple-llm-python-124m-5b

PyTorch
Safetensors
English
causal-lm
python
from-scratch
muon
experimental
Model card Files
xet
 Community
simple-llm-python-124m-5b
File size: 2,684 Bytes
4719692
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
{
  "format": "scholarly-raw-v1",
  "training_ready": true,
  "max_passes": 2,
  "experiment": "python-124m-muon-5b",
  "prefer_collected": true,
  "target_tokens": 5000000000,
  "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9",
  "database": {
    "path": "raw.sqlite",
    "sha256": "3942c0e7edccc426cbf7f0c3b9d69144f0511960a36e6615151645c38da41565"
  },
  "sources": {
    "python_implementation": {
      "weight_percent": 65,
      "target_tokens": 3250000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 3344840434
        },
        "validation": {
          "loss_tokens": 202402128
        }
      }
    },
    "python_tests_examples": {
      "weight_percent": 15,
      "target_tokens": 750000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 1455959517
        },
        "validation": {
          "loss_tokens": 88631252
        }
      }
    },
    "python_documentation": {
      "weight_percent": 10,
      "target_tokens": 500000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 638756689
        },
        "validation": {
          "loss_tokens": 32309371
        }
      }
    },
    "cs_math": {
      "weight_percent": 5,
      "target_tokens": 250000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 255001393
        },
        "validation": {
          "loss_tokens": 31729746
        }
      }
    },
    "general_education": {
      "weight_percent": 3,
      "target_tokens": 150000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 153000047
        },
        "validation": {
          "loss_tokens": 59391033
        }
      }
    },
    "books": {
      "weight_percent": 2,
      "target_tokens": 100000000,
      "max_passes": 1,
      "splits": {
        "train": {
          "loss_tokens": 102061503
        },
        "validation": {
          "loss_tokens": 14413928
        }
      }
    }
  },
  "heldout": {
    "path": "heldout.jsonl.gz",
    "sha256": "58ed450084b99893f1e1b09d8f7556653ab36a66db1eb41644e8f92029490325",
    "windows": 192,
    "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9"
  },
  "limitations": [
    "Exact file/document deduplication; near clones can remain",
    "Path and content heuristics are imperfect",
    "Documentation includes explanatory Python technical Q&A",
    "Collected recent repositories preferred; supplements include older material",
    "No benchmark-contamination claim; detailed source provenance remains private"
  ]
}