Hugging Face's logo

jfrick
/
simple-llm-python-124m-5b

PyTorch
Safetensors
English
causal-lm
python
from-scratch
muon
experimental
Model card Files
xet
 Community
simple-llm-python-124m-5b
File size: 4,294 Bytes
4719692
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
{
  "experiment": "python-124m-muon-5b",
  "public_source_snapshots": [
    {
      "dataset": "codeparrot/codeparrot-clean-train",
      "revision": "3e6ab65f2864931e041f6a82db9b5a6ec2b71ab4",
      "representation": "complete Python files; dataset snapshot is pinned, individual upstream Git commits are not supplied"
    },
    {
      "dataset": "common-pile/stackexchange",
      "revision": "5ec3aa2e65b7c61b7c18f1ff18ccbc4d3279a23c",
      "representation": "complete explanatory Python Stack Overflow threads with per-record CC-BY-SA attribution"
    }
  ],
  "collected_repositories": "Complete pinned Git blobs from the user's collected snapshots; prefer recent snapshots and consume retained collected files first within each category. Repository identities, commits and notices remain in the private ledger.",
  "existing_source_pools": "Pinned prepared CodeParrot, FineWeb-Edu, FineMath, Gutenberg and programming-documentation pools. Original database hashes, upstream record metadata and license evidence remain in the private ledger.",
  "filters": [
    "Exclusive implementation, tests/examples, Python documentation and supporting-text categories",
    "Exclude vendored, generated, minified, boilerplate-heavy and malformed files; preserve accepted text unchanged",
    "Python 3.12 AST validation; newer syntax can be conservatively excluded",
    "Exact file/document deduplication across sources; near clones can remain",
    "Per-repository/category cap of 1 percent of that category's training budget",
    "Repository/thread holdouts before window construction; cross-source heldout repository and exact document-hash exclusions",
    "Exclude exact MBPP reference hashes; no exhaustive benchmark-contamination claim"
  ],
  "token_accounting": "Nonpadding next-token labels, including each document's EOS target. Existing frozen tokenizer; exact six-category budgets; prefer one pass, never exceed two.",
  "limitations": "Supplemental code and technical Q&A include older material. Heuristic filters and upstream licensing metadata are imperfect. Detailed provenance is retained privately; this file contains only aggregate and public-source information.",
  "corpus_database_sha256": "3942c0e7edccc426cbf7f0c3b9d69144f0511960a36e6615151645c38da41565",
  "tokenizer_sha256": "add5b2c7542e4efded329735825ecdcb94df1432617c3d3e4b7508aa459432f9",
  "capacity_by_origin": [
    {
      "category": "books",
      "origin": "existing_pinned_pool",
      "documents": 923,
      "unique_train_targets": 102061503
    },
    {
      "category": "cs_math",
      "origin": "existing_pinned_pool",
      "documents": 153044,
      "unique_train_targets": 255001393
    },
    {
      "category": "general_education",
      "origin": "existing_pinned_pool",
      "documents": 113805,
      "unique_train_targets": 153000047
    },
    {
      "category": "python_documentation",
      "origin": "collected_git",
      "documents": 8796,
      "unique_train_targets": 27901636
    },
    {
      "category": "python_documentation",
      "origin": "existing_pinned_pool",
      "documents": 55,
      "unique_train_targets": 1975348
    },
    {
      "category": "python_documentation",
      "origin": "python_explanatory_qa",
      "documents": 620563,
      "unique_train_targets": 608879705
    },
    {
      "category": "python_implementation",
      "origin": "collected_git",
      "documents": 10392,
      "unique_train_targets": 41406577
    },
    {
      "category": "python_implementation",
      "origin": "existing_pinned_pool",
      "documents": 708699,
      "unique_train_targets": 1422349105
    },
    {
      "category": "python_implementation",
      "origin": "supplemental_python_file",
      "documents": 870723,
      "unique_train_targets": 1881084752
    },
    {
      "category": "python_tests_examples",
      "origin": "collected_git",
      "documents": 11828,
      "unique_train_targets": 38755717
    },
    {
      "category": "python_tests_examples",
      "origin": "existing_pinned_pool",
      "documents": 274648,
      "unique_train_targets": 680803158
    },
    {
      "category": "python_tests_examples",
      "origin": "supplemental_python_file",
      "documents": 281242,
      "unique_train_targets": 736400642
    }
  ]
}