| { |
| "id": "scholarly-code-bpe-v1", |
| "library": "tokenizers==0.22.2", |
| "max_chunk_characters": 4096, |
| "max_chunks_per_document": 4, |
| "max_document_characters": 4000000, |
| "max_scanned_documents_per_source": 100000, |
| "model_target": "About150Mtotalparameters; embeddings included; model training not part of tokenizer job", |
| "sampling_weight_unit": "UTF-8 bytes before tokenizer exists; actual resulting token shares reported", |
| "seed": 20261016, |
| "selection_rule": "Prefer16K unless32K reduces total held-out tokens by at least10% and no individual source needs over2%moretokens; all roundtrip and bytecoverage checks must pass", |
| "sources": [ |
| { |
| "files": [ |
| "sample/10BT/000_00000.parquet", |
| "sample/10BT/001_00000.parquet", |
| "sample/10BT/002_00000.parquet", |
| "sample/10BT/003_00000.parquet", |
| "sample/10BT/004_00000.parquet", |
| "sample/10BT/005_00000.parquet", |
| "sample/10BT/006_00000.parquet", |
| "sample/10BT/007_00000.parquet", |
| "sample/10BT/008_00000.parquet", |
| "sample/10BT/009_00000.parquet", |
| "sample/10BT/010_00000.parquet", |
| "sample/10BT/011_00000.parquet", |
| "sample/10BT/012_00000.parquet", |
| "sample/10BT/013_00000.parquet" |
| ], |
| "format": "parquet", |
| "name": "fineweb_edu", |
| "repo": "HuggingFaceFW/fineweb-edu", |
| "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", |
| "source_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", |
| "synthetic": false, |
| "weight_percent": 10 |
| }, |
| { |
| "adapter": "codeparrot", |
| "allowed_spdx_licenses": [ |
| "MIT", |
| "Apache-2.0", |
| "BSD-2-Clause", |
| "BSD-3-Clause", |
| "ISC" |
| ], |
| "files": [ |
| "file-000000000001.json.gz", |
| "file-000000000002.json.gz", |
| "file-000000000003.json.gz", |
| "file-000000000004.json.gz", |
| "file-000000000005.json.gz", |
| "file-000000000006.json.gz", |
| "file-000000000007.json.gz", |
| "file-000000000008.json.gz", |
| "file-000000000009.json.gz", |
| "file-000000000010.json.gz", |
| "file-000000000011.json.gz", |
| "file-000000000012.json.gz", |
| "file-000000000013.json.gz", |
| "file-000000000014.json.gz", |
| "file-000000000015.json.gz", |
| "file-000000000016.json.gz", |
| "file-000000000017.json.gz", |
| "file-000000000018.json.gz", |
| "file-000000000019.json.gz", |
| "file-000000000020.json.gz", |
| "file-000000000021.json.gz", |
| "file-000000000022.json.gz", |
| "file-000000000023.json.gz", |
| "file-000000000024.json.gz", |
| "file-000000000025.json.gz", |
| "file-000000000026.json.gz", |
| "file-000000000027.json.gz", |
| "file-000000000028.json.gz", |
| "file-000000000029.json.gz", |
| "file-000000000030.json.gz", |
| "file-000000000031.json.gz", |
| "file-000000000032.json.gz", |
| "file-000000000033.json.gz", |
| "file-000000000034.json.gz", |
| "file-000000000035.json.gz", |
| "file-000000000036.json.gz", |
| "file-000000000037.json.gz", |
| "file-000000000038.json.gz", |
| "file-000000000039.json.gz", |
| "file-000000000040.json.gz", |
| "file-000000000041.json.gz", |
| "file-000000000042.json.gz", |
| "file-000000000043.json.gz", |
| "file-000000000044.json.gz", |
| "file-000000000045.json.gz", |
| "file-000000000046.json.gz", |
| "file-000000000047.json.gz", |
| "file-000000000048.json.gz", |
| "file-000000000049.json.gz", |
| "file-000000000050.json.gz", |
| "file-000000000051.json.gz", |
| "file-000000000052.json.gz", |
| "file-000000000053.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "code_python", |
| "repo": "codeparrot/codeparrot-clean-train", |
| "revision": "3e6ab65f2864931e041f6a82db9b5a6ec2b71ab4", |
| "source_url": "https://huggingface.co/datasets/codeparrot/codeparrot-clean-train", |
| "synthetic": false, |
| "weight_percent": 30 |
| }, |
| { |
| "files": [ |
| "finemath-4plus/train-00000-of-00064.parquet", |
| "finemath-4plus/train-00001-of-00064.parquet", |
| "finemath-4plus/train-00002-of-00064.parquet", |
| "finemath-4plus/train-00003-of-00064.parquet", |
| "finemath-4plus/train-00004-of-00064.parquet", |
| "finemath-4plus/train-00005-of-00064.parquet", |
| "finemath-4plus/train-00006-of-00064.parquet", |
| "finemath-4plus/train-00007-of-00064.parquet", |
| "finemath-4plus/train-00008-of-00064.parquet", |
| "finemath-4plus/train-00009-of-00064.parquet", |
| "finemath-4plus/train-00010-of-00064.parquet", |
| "finemath-4plus/train-00011-of-00064.parquet", |
| "finemath-4plus/train-00012-of-00064.parquet", |
| "finemath-4plus/train-00013-of-00064.parquet", |
| "finemath-4plus/train-00014-of-00064.parquet", |
| "finemath-4plus/train-00015-of-00064.parquet", |
| "finemath-4plus/train-00016-of-00064.parquet", |
| "finemath-4plus/train-00017-of-00064.parquet", |
| "finemath-4plus/train-00018-of-00064.parquet", |
| "finemath-4plus/train-00019-of-00064.parquet", |
| "finemath-4plus/train-00020-of-00064.parquet", |
| "finemath-4plus/train-00021-of-00064.parquet", |
| "finemath-4plus/train-00022-of-00064.parquet", |
| "finemath-4plus/train-00023-of-00064.parquet", |
| "finemath-4plus/train-00024-of-00064.parquet", |
| "finemath-4plus/train-00025-of-00064.parquet", |
| "finemath-4plus/train-00026-of-00064.parquet", |
| "finemath-4plus/train-00027-of-00064.parquet", |
| "finemath-4plus/train-00028-of-00064.parquet", |
| "finemath-4plus/train-00029-of-00064.parquet", |
| "finemath-4plus/train-00030-of-00064.parquet", |
| "finemath-4plus/train-00031-of-00064.parquet", |
| "finemath-4plus/train-00032-of-00064.parquet", |
| "finemath-4plus/train-00033-of-00064.parquet", |
| "finemath-4plus/train-00034-of-00064.parquet", |
| "finemath-4plus/train-00035-of-00064.parquet", |
| "finemath-4plus/train-00036-of-00064.parquet", |
| "finemath-4plus/train-00037-of-00064.parquet", |
| "finemath-4plus/train-00038-of-00064.parquet", |
| "finemath-4plus/train-00039-of-00064.parquet", |
| "finemath-4plus/train-00040-of-00064.parquet", |
| "finemath-4plus/train-00041-of-00064.parquet", |
| "finemath-4plus/train-00042-of-00064.parquet", |
| "finemath-4plus/train-00043-of-00064.parquet", |
| "finemath-4plus/train-00044-of-00064.parquet", |
| "finemath-4plus/train-00045-of-00064.parquet", |
| "finemath-4plus/train-00046-of-00064.parquet", |
| "finemath-4plus/train-00047-of-00064.parquet", |
| "finemath-4plus/train-00048-of-00064.parquet", |
| "finemath-4plus/train-00049-of-00064.parquet", |
| "finemath-4plus/train-00050-of-00064.parquet", |
| "finemath-4plus/train-00051-of-00064.parquet", |
| "finemath-4plus/train-00052-of-00064.parquet", |
| "finemath-4plus/train-00053-of-00064.parquet", |
| "finemath-4plus/train-00054-of-00064.parquet", |
| "finemath-4plus/train-00055-of-00064.parquet", |
| "finemath-4plus/train-00056-of-00064.parquet", |
| "finemath-4plus/train-00057-of-00064.parquet", |
| "finemath-4plus/train-00058-of-00064.parquet", |
| "finemath-4plus/train-00059-of-00064.parquet", |
| "finemath-4plus/train-00060-of-00064.parquet", |
| "finemath-4plus/train-00061-of-00064.parquet", |
| "finemath-4plus/train-00062-of-00064.parquet", |
| "finemath-4plus/train-00063-of-00064.parquet" |
| ], |
| "format": "parquet", |
| "name": "finemath_4plus", |
| "repo": "HuggingFaceTB/finemath", |
| "revision": "e92b25a616738fe95dc186b64dfb19f9c8525594", |
| "synthetic": false, |
| "weight_percent": 20 |
| }, |
| { |
| "files": [ |
| "peS2o-0000.json.gz", |
| "peS2o-0001.json.gz", |
| "peS2o-0002.json.gz", |
| "peS2o-0003.json.gz", |
| "peS2o-0004.json.gz", |
| "peS2o-0005.json.gz", |
| "peS2o-0006.json.gz", |
| "peS2o-0007.json.gz", |
| "peS2o-0008.json.gz", |
| "peS2o-0009.json.gz", |
| "peS2o-0010.json.gz", |
| "peS2o-0011.json.gz", |
| "peS2o-0012.json.gz", |
| "peS2o-0013.json.gz", |
| "peS2o-0014.json.gz", |
| "peS2o-0015.json.gz", |
| "peS2o-0016.json.gz", |
| "peS2o-0017.json.gz", |
| "peS2o-0018.json.gz", |
| "peS2o-0019.json.gz", |
| "peS2o-0020.json.gz", |
| "peS2o-0021.json.gz", |
| "peS2o-0022.json.gz", |
| "peS2o-0023.json.gz", |
| "peS2o-0024.json.gz", |
| "peS2o-0025.json.gz", |
| "peS2o-0026.json.gz", |
| "peS2o-0027.json.gz", |
| "peS2o-0028.json.gz", |
| "peS2o-0029.json.gz", |
| "peS2o-0030.json.gz", |
| "peS2o-0031.json.gz", |
| "peS2o-0032.json.gz", |
| "peS2o-0033.json.gz", |
| "peS2o-0034.json.gz", |
| "peS2o-0035.json.gz", |
| "peS2o-0036.json.gz", |
| "peS2o-0037.json.gz", |
| "peS2o-0038.json.gz", |
| "peS2o-0039.json.gz", |
| "peS2o-0040.json.gz", |
| "peS2o-0041.json.gz", |
| "peS2o-0042.json.gz", |
| "peS2o-0043.json.gz", |
| "peS2o-0044.json.gz", |
| "peS2o-0045.json.gz", |
| "peS2o-0046.json.gz", |
| "peS2o-0047.json.gz", |
| "peS2o-0048.json.gz", |
| "peS2o-0049.json.gz", |
| "peS2o-0050.json.gz", |
| "peS2o-0051.json.gz", |
| "peS2o-0052.json.gz", |
| "peS2o-0053.json.gz", |
| "peS2o-0054.json.gz", |
| "peS2o-0055.json.gz", |
| "peS2o-0056.json.gz", |
| "peS2o-0057.json.gz", |
| "peS2o-0058.json.gz", |
| "peS2o-0059.json.gz", |
| "peS2o-0060.json.gz", |
| "peS2o-0061.json.gz", |
| "peS2o-0062.json.gz", |
| "peS2o-0063.json.gz", |
| "peS2o-0064.json.gz", |
| "peS2o-0065.json.gz", |
| "peS2o-0066.json.gz", |
| "peS2o-0067.json.gz", |
| "peS2o-0068.json.gz", |
| "peS2o-0069.json.gz", |
| "peS2o-0070.json.gz", |
| "peS2o-0071.json.gz", |
| "peS2o-0072.json.gz", |
| "peS2o-0073.json.gz", |
| "peS2o-0074.json.gz", |
| "peS2o-0075.json.gz", |
| "peS2o-0076.json.gz", |
| "peS2o-0077.json.gz", |
| "peS2o-0078.json.gz", |
| "peS2o-0079.json.gz", |
| "peS2o-0080.json.gz", |
| "peS2o-0081.json.gz", |
| "peS2o-0082.json.gz", |
| "peS2o-0083.json.gz", |
| "peS2o-0084.json.gz", |
| "peS2o-0085.json.gz", |
| "peS2o-0086.json.gz", |
| "peS2o-0087.json.gz", |
| "peS2o-0088.json.gz", |
| "peS2o-0089.json.gz", |
| "peS2o-0090.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "pes2o", |
| "repo": "common-pile/peS2o_filtered", |
| "revision": "297747513bfb0ff1fbf61ddad3b03319d0f04597", |
| "source_url": "https://huggingface.co/datasets/common-pile/peS2o_filtered", |
| "synthetic": false, |
| "weight_percent": 10 |
| }, |
| { |
| "files": [ |
| "licensed_pubmed-0000.json.gz", |
| "licensed_pubmed-0001.json.gz", |
| "licensed_pubmed-0002.json.gz", |
| "licensed_pubmed-0003.json.gz", |
| "licensed_pubmed-0004.json.gz", |
| "licensed_pubmed-0005.json.gz", |
| "licensed_pubmed-0006.json.gz", |
| "licensed_pubmed-0007.json.gz", |
| "licensed_pubmed-0008.json.gz", |
| "licensed_pubmed-0009.json.gz", |
| "licensed_pubmed-0010.json.gz", |
| "licensed_pubmed-0011.json.gz", |
| "licensed_pubmed-0012.json.gz", |
| "licensed_pubmed-0013.json.gz", |
| "licensed_pubmed-0014.json.gz", |
| "licensed_pubmed-0015.json.gz", |
| "licensed_pubmed-0016.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "pubmed", |
| "repo": "common-pile/pubmed_filtered", |
| "revision": "c156f0569a92d8f2edc33cebe1f72f7d3e1cae84", |
| "source_url": "https://huggingface.co/datasets/common-pile/pubmed_filtered", |
| "synthetic": false, |
| "weight_percent": 3 |
| }, |
| { |
| "files": [ |
| "arxiv-papers-0000.json.gz", |
| "arxiv-papers-0001.json.gz", |
| "arxiv-papers-0002.json.gz", |
| "arxiv-papers-0003.json.gz", |
| "arxiv-papers-0004.json.gz", |
| "arxiv-papers-0005.json.gz", |
| "arxiv-papers-0006.json.gz", |
| "arxiv-papers-0007.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "arxiv", |
| "repo": "common-pile/arxiv_papers_filtered", |
| "revision": "033cf7f53f9b348deec868c1a5a48484f3ee9e52", |
| "source_url": "https://huggingface.co/datasets/common-pile/arxiv_papers_filtered", |
| "synthetic": false, |
| "weight_percent": 2 |
| }, |
| { |
| "files": [ |
| "pressbooks-0000.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "pressbooks", |
| "repo": "common-pile/pressbooks_filtered", |
| "revision": "1a1d3b50d77f834370f8eb4c0d174668dd1676bb", |
| "source_url": "https://huggingface.co/datasets/common-pile/pressbooks_filtered", |
| "synthetic": false, |
| "weight_percent": 3 |
| }, |
| { |
| "files": [ |
| "libretexts-0000.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "libretexts", |
| "repo": "common-pile/libretexts_filtered", |
| "revision": "70388bca52b4a93515e14b1d56618fd7944988fd", |
| "source_url": "https://huggingface.co/datasets/common-pile/libretexts_filtered", |
| "synthetic": false, |
| "weight_percent": 3 |
| }, |
| { |
| "files": [ |
| "doab-0000.json.gz", |
| "doab-0001.json.gz", |
| "doab-0002.json.gz", |
| "doab-0003.json.gz", |
| "doab-0004.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "doab", |
| "repo": "common-pile/doab_filtered", |
| "revision": "defb24ca72ef6aba6ce0228b669eec06dcfbffbc", |
| "source_url": "https://huggingface.co/datasets/common-pile/doab_filtered", |
| "synthetic": false, |
| "weight_percent": 4 |
| }, |
| { |
| "files": [ |
| "project_gutenberg-dolma-0000.json.gz", |
| "project_gutenberg-dolma-0001.json.gz", |
| "project_gutenberg-dolma-0002.json.gz", |
| "project_gutenberg-dolma-0003.json.gz", |
| "project_gutenberg-dolma-0004.json.gz", |
| "project_gutenberg-dolma-0005.json.gz", |
| "project_gutenberg-dolma-0006.json.gz", |
| "project_gutenberg-dolma-0007.json.gz", |
| "project_gutenberg-dolma-0008.json.gz", |
| "project_gutenberg-dolma-0009.json.gz", |
| "project_gutenberg-dolma-0010.json.gz", |
| "project_gutenberg-dolma-0011.json.gz", |
| "project_gutenberg-dolma-0012.json.gz", |
| "project_gutenberg-dolma-0013.json.gz", |
| "project_gutenberg-dolma-0014.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "gutenberg", |
| "repo": "common-pile/project_gutenberg_filtered", |
| "revision": "3cdf6879c807f4e4e063f2ceb23bc268d8c29ab7", |
| "source_url": "https://huggingface.co/datasets/common-pile/project_gutenberg_filtered", |
| "synthetic": false, |
| "weight_percent": 5 |
| }, |
| { |
| "files": [ |
| "public_library_1929_dolma-0000.json.gz", |
| "public_library_1929_dolma-0001.json.gz", |
| "public_library_1929_dolma-0002.json.gz", |
| "public_library_1929_dolma-0003.json.gz", |
| "public_library_1929_dolma-0004.json.gz", |
| "public_library_1929_dolma-0005.json.gz", |
| "public_library_1929_dolma-0006.json.gz", |
| "public_library_1929_dolma-0007.json.gz", |
| "public_library_1929_dolma-0008.json.gz", |
| "public_library_1929_dolma-0009.json.gz", |
| "public_library_1929_dolma-0010.json.gz", |
| "public_library_1929_dolma-0011.json.gz", |
| "public_library_1929_dolma-0012.json.gz", |
| "public_library_1929_dolma-0013.json.gz", |
| "public_library_1929_dolma-0014.json.gz", |
| "public_library_1929_dolma-0015.json.gz", |
| "public_library_1929_dolma-0016.json.gz", |
| "public_library_1929_dolma-0017.json.gz", |
| "public_library_1929_dolma-0018.json.gz", |
| "public_library_1929_dolma-0019.json.gz", |
| "public_library_1929_dolma-0020.json.gz", |
| "public_library_1929_dolma-0021.json.gz", |
| "public_library_1929_dolma-0022.json.gz", |
| "public_library_1929_dolma-0023.json.gz", |
| "public_library_1929_dolma-0024.json.gz", |
| "public_library_1929_dolma-0025.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "pre_1929_books", |
| "repo": "common-pile/pre_1929_books_filtered", |
| "revision": "23f9d96dbb1db3324bbc9fbfe1f8299cc799c4d1", |
| "synthetic": false, |
| "weight_percent": 7 |
| }, |
| { |
| "files": [ |
| "loc_books_dolma-0000.json.gz", |
| "loc_books_dolma-0001.json.gz", |
| "loc_books_dolma-0002.json.gz", |
| "loc_books_dolma-0003.json.gz", |
| "loc_books_dolma-0004.json.gz", |
| "loc_books_dolma-0005.json.gz", |
| "loc_books_dolma-0006.json.gz", |
| "loc_books_dolma-0007.json.gz", |
| "loc_books_dolma-0008.json.gz", |
| "loc_books_dolma-0009.json.gz", |
| "loc_books_dolma-0010.json.gz", |
| "loc_books_dolma-0011.json.gz", |
| "loc_books_dolma-0012.json.gz", |
| "loc_books_dolma-0013.json.gz", |
| "loc_books_dolma-0014.json.gz", |
| "loc_books_dolma-0015.json.gz", |
| "loc_books_dolma-0016.json.gz", |
| "loc_books_dolma-0017.json.gz", |
| "loc_books_dolma-0018.json.gz", |
| "loc_books_dolma-0019.json.gz", |
| "loc_books_dolma-0020.json.gz", |
| "loc_books_dolma-0021.json.gz", |
| "loc_books_dolma-0022.json.gz", |
| "loc_books_dolma-0023.json.gz", |
| "loc_books_dolma-0024.json.gz" |
| ], |
| "format": "json.gz", |
| "name": "library_of_congress", |
| "repo": "common-pile/library_of_congress_filtered", |
| "revision": "56725c7aa1bb320703e22eb5f42903173d5bac3d", |
| "synthetic": false, |
| "weight_percent": 3 |
| } |
| ], |
| "special_tokens": [ |
| "<|pad|>", |
| "<|bos|>", |
| "<|eos|>", |
| "<|unk|>" |
| ], |
| "split_policy": "Same mixture-split-v1 parent hash: buckets2..9 training, bucket0 validation, bucket1 excluded; no held-out fitting", |
| "tokenizer": "Byte-level BPE with complete256bytealphabet; no normalization; digit groups up to3; explicit whitespace; no automatic BOS/EOS", |
| "trainer": { |
| "max_token_length": 32, |
| "min_frequency": 2 |
| }, |
| "training_utf8_bytes": 67108864, |
| "validation_utf8_bytes": 8388608, |
| "vocab_sizes": [ |
| 16384, |
| 32768 |
| ] |
| } |
|
|
