{ "id": "scholarly-code-bpe-v1", "library": "tokenizers==0.22.2", "max_chunk_characters": 4096, "max_chunks_per_document": 4, "max_document_characters": 4000000, "max_scanned_documents_per_source": 100000, "model_target": "About150Mtotalparameters; embeddings included; model training not part of tokenizer job", "sampling_weight_unit": "UTF-8 bytes before tokenizer exists; actual resulting token shares reported", "seed": 20261016, "selection_rule": "Prefer16K unless32K reduces total held-out tokens by at least10% and no individual source needs over2%moretokens; all roundtrip and bytecoverage checks must pass", "sources": [ { "files": [ "sample/10BT/000_00000.parquet", "sample/10BT/001_00000.parquet", "sample/10BT/002_00000.parquet", "sample/10BT/003_00000.parquet", "sample/10BT/004_00000.parquet", "sample/10BT/005_00000.parquet", "sample/10BT/006_00000.parquet", "sample/10BT/007_00000.parquet", "sample/10BT/008_00000.parquet", "sample/10BT/009_00000.parquet", "sample/10BT/010_00000.parquet", "sample/10BT/011_00000.parquet", "sample/10BT/012_00000.parquet", "sample/10BT/013_00000.parquet" ], "format": "parquet", "name": "fineweb_edu", "repo": "HuggingFaceFW/fineweb-edu", "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", "source_url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", "synthetic": false, "weight_percent": 10 }, { "adapter": "codeparrot", "allowed_spdx_licenses": [ "MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC" ], "files": [ "file-000000000001.json.gz", "file-000000000002.json.gz", "file-000000000003.json.gz", "file-000000000004.json.gz", "file-000000000005.json.gz", "file-000000000006.json.gz", "file-000000000007.json.gz", "file-000000000008.json.gz", "file-000000000009.json.gz", "file-000000000010.json.gz", "file-000000000011.json.gz", "file-000000000012.json.gz", "file-000000000013.json.gz", "file-000000000014.json.gz", "file-000000000015.json.gz", "file-000000000016.json.gz", "file-000000000017.json.gz", "file-000000000018.json.gz", "file-000000000019.json.gz", "file-000000000020.json.gz", "file-000000000021.json.gz", "file-000000000022.json.gz", "file-000000000023.json.gz", "file-000000000024.json.gz", "file-000000000025.json.gz", "file-000000000026.json.gz", "file-000000000027.json.gz", "file-000000000028.json.gz", "file-000000000029.json.gz", "file-000000000030.json.gz", "file-000000000031.json.gz", "file-000000000032.json.gz", "file-000000000033.json.gz", "file-000000000034.json.gz", "file-000000000035.json.gz", "file-000000000036.json.gz", "file-000000000037.json.gz", "file-000000000038.json.gz", "file-000000000039.json.gz", "file-000000000040.json.gz", "file-000000000041.json.gz", "file-000000000042.json.gz", "file-000000000043.json.gz", "file-000000000044.json.gz", "file-000000000045.json.gz", "file-000000000046.json.gz", "file-000000000047.json.gz", "file-000000000048.json.gz", "file-000000000049.json.gz", "file-000000000050.json.gz", "file-000000000051.json.gz", "file-000000000052.json.gz", "file-000000000053.json.gz" ], "format": "json.gz", "name": "code_python", "repo": "codeparrot/codeparrot-clean-train", "revision": "3e6ab65f2864931e041f6a82db9b5a6ec2b71ab4", "source_url": "https://huggingface.co/datasets/codeparrot/codeparrot-clean-train", "synthetic": false, "weight_percent": 30 }, { "files": [ "finemath-4plus/train-00000-of-00064.parquet", "finemath-4plus/train-00001-of-00064.parquet", "finemath-4plus/train-00002-of-00064.parquet", "finemath-4plus/train-00003-of-00064.parquet", "finemath-4plus/train-00004-of-00064.parquet", "finemath-4plus/train-00005-of-00064.parquet", "finemath-4plus/train-00006-of-00064.parquet", "finemath-4plus/train-00007-of-00064.parquet", "finemath-4plus/train-00008-of-00064.parquet", "finemath-4plus/train-00009-of-00064.parquet", "finemath-4plus/train-00010-of-00064.parquet", "finemath-4plus/train-00011-of-00064.parquet", "finemath-4plus/train-00012-of-00064.parquet", "finemath-4plus/train-00013-of-00064.parquet", "finemath-4plus/train-00014-of-00064.parquet", "finemath-4plus/train-00015-of-00064.parquet", "finemath-4plus/train-00016-of-00064.parquet", "finemath-4plus/train-00017-of-00064.parquet", "finemath-4plus/train-00018-of-00064.parquet", "finemath-4plus/train-00019-of-00064.parquet", "finemath-4plus/train-00020-of-00064.parquet", "finemath-4plus/train-00021-of-00064.parquet", "finemath-4plus/train-00022-of-00064.parquet", "finemath-4plus/train-00023-of-00064.parquet", "finemath-4plus/train-00024-of-00064.parquet", "finemath-4plus/train-00025-of-00064.parquet", "finemath-4plus/train-00026-of-00064.parquet", "finemath-4plus/train-00027-of-00064.parquet", "finemath-4plus/train-00028-of-00064.parquet", "finemath-4plus/train-00029-of-00064.parquet", "finemath-4plus/train-00030-of-00064.parquet", "finemath-4plus/train-00031-of-00064.parquet", "finemath-4plus/train-00032-of-00064.parquet", "finemath-4plus/train-00033-of-00064.parquet", "finemath-4plus/train-00034-of-00064.parquet", "finemath-4plus/train-00035-of-00064.parquet", "finemath-4plus/train-00036-of-00064.parquet", "finemath-4plus/train-00037-of-00064.parquet", "finemath-4plus/train-00038-of-00064.parquet", "finemath-4plus/train-00039-of-00064.parquet", "finemath-4plus/train-00040-of-00064.parquet", "finemath-4plus/train-00041-of-00064.parquet", "finemath-4plus/train-00042-of-00064.parquet", "finemath-4plus/train-00043-of-00064.parquet", "finemath-4plus/train-00044-of-00064.parquet", "finemath-4plus/train-00045-of-00064.parquet", "finemath-4plus/train-00046-of-00064.parquet", "finemath-4plus/train-00047-of-00064.parquet", "finemath-4plus/train-00048-of-00064.parquet", "finemath-4plus/train-00049-of-00064.parquet", "finemath-4plus/train-00050-of-00064.parquet", "finemath-4plus/train-00051-of-00064.parquet", "finemath-4plus/train-00052-of-00064.parquet", "finemath-4plus/train-00053-of-00064.parquet", "finemath-4plus/train-00054-of-00064.parquet", "finemath-4plus/train-00055-of-00064.parquet", "finemath-4plus/train-00056-of-00064.parquet", "finemath-4plus/train-00057-of-00064.parquet", "finemath-4plus/train-00058-of-00064.parquet", "finemath-4plus/train-00059-of-00064.parquet", "finemath-4plus/train-00060-of-00064.parquet", "finemath-4plus/train-00061-of-00064.parquet", "finemath-4plus/train-00062-of-00064.parquet", "finemath-4plus/train-00063-of-00064.parquet" ], "format": "parquet", "name": "finemath_4plus", "repo": "HuggingFaceTB/finemath", "revision": "e92b25a616738fe95dc186b64dfb19f9c8525594", "synthetic": false, "weight_percent": 20 }, { "files": [ "peS2o-0000.json.gz", "peS2o-0001.json.gz", "peS2o-0002.json.gz", "peS2o-0003.json.gz", "peS2o-0004.json.gz", "peS2o-0005.json.gz", "peS2o-0006.json.gz", "peS2o-0007.json.gz", "peS2o-0008.json.gz", "peS2o-0009.json.gz", "peS2o-0010.json.gz", "peS2o-0011.json.gz", "peS2o-0012.json.gz", "peS2o-0013.json.gz", "peS2o-0014.json.gz", "peS2o-0015.json.gz", "peS2o-0016.json.gz", "peS2o-0017.json.gz", "peS2o-0018.json.gz", "peS2o-0019.json.gz", "peS2o-0020.json.gz", "peS2o-0021.json.gz", "peS2o-0022.json.gz", "peS2o-0023.json.gz", "peS2o-0024.json.gz", "peS2o-0025.json.gz", "peS2o-0026.json.gz", "peS2o-0027.json.gz", "peS2o-0028.json.gz", "peS2o-0029.json.gz", "peS2o-0030.json.gz", "peS2o-0031.json.gz", "peS2o-0032.json.gz", "peS2o-0033.json.gz", "peS2o-0034.json.gz", "peS2o-0035.json.gz", "peS2o-0036.json.gz", "peS2o-0037.json.gz", "peS2o-0038.json.gz", "peS2o-0039.json.gz", "peS2o-0040.json.gz", "peS2o-0041.json.gz", "peS2o-0042.json.gz", "peS2o-0043.json.gz", "peS2o-0044.json.gz", "peS2o-0045.json.gz", "peS2o-0046.json.gz", "peS2o-0047.json.gz", "peS2o-0048.json.gz", "peS2o-0049.json.gz", "peS2o-0050.json.gz", "peS2o-0051.json.gz", "peS2o-0052.json.gz", "peS2o-0053.json.gz", "peS2o-0054.json.gz", "peS2o-0055.json.gz", "peS2o-0056.json.gz", "peS2o-0057.json.gz", "peS2o-0058.json.gz", "peS2o-0059.json.gz", "peS2o-0060.json.gz", "peS2o-0061.json.gz", "peS2o-0062.json.gz", "peS2o-0063.json.gz", "peS2o-0064.json.gz", "peS2o-0065.json.gz", "peS2o-0066.json.gz", "peS2o-0067.json.gz", "peS2o-0068.json.gz", "peS2o-0069.json.gz", "peS2o-0070.json.gz", "peS2o-0071.json.gz", "peS2o-0072.json.gz", "peS2o-0073.json.gz", "peS2o-0074.json.gz", "peS2o-0075.json.gz", "peS2o-0076.json.gz", "peS2o-0077.json.gz", "peS2o-0078.json.gz", "peS2o-0079.json.gz", "peS2o-0080.json.gz", "peS2o-0081.json.gz", "peS2o-0082.json.gz", "peS2o-0083.json.gz", "peS2o-0084.json.gz", "peS2o-0085.json.gz", "peS2o-0086.json.gz", "peS2o-0087.json.gz", "peS2o-0088.json.gz", "peS2o-0089.json.gz", "peS2o-0090.json.gz" ], "format": "json.gz", "name": "pes2o", "repo": "common-pile/peS2o_filtered", "revision": "297747513bfb0ff1fbf61ddad3b03319d0f04597", "source_url": "https://huggingface.co/datasets/common-pile/peS2o_filtered", "synthetic": false, "weight_percent": 10 }, { "files": [ "licensed_pubmed-0000.json.gz", "licensed_pubmed-0001.json.gz", "licensed_pubmed-0002.json.gz", "licensed_pubmed-0003.json.gz", "licensed_pubmed-0004.json.gz", "licensed_pubmed-0005.json.gz", "licensed_pubmed-0006.json.gz", "licensed_pubmed-0007.json.gz", "licensed_pubmed-0008.json.gz", "licensed_pubmed-0009.json.gz", "licensed_pubmed-0010.json.gz", "licensed_pubmed-0011.json.gz", "licensed_pubmed-0012.json.gz", "licensed_pubmed-0013.json.gz", "licensed_pubmed-0014.json.gz", "licensed_pubmed-0015.json.gz", "licensed_pubmed-0016.json.gz" ], "format": "json.gz", "name": "pubmed", "repo": "common-pile/pubmed_filtered", "revision": "c156f0569a92d8f2edc33cebe1f72f7d3e1cae84", "source_url": "https://huggingface.co/datasets/common-pile/pubmed_filtered", "synthetic": false, "weight_percent": 3 }, { "files": [ "arxiv-papers-0000.json.gz", "arxiv-papers-0001.json.gz", "arxiv-papers-0002.json.gz", "arxiv-papers-0003.json.gz", "arxiv-papers-0004.json.gz", "arxiv-papers-0005.json.gz", "arxiv-papers-0006.json.gz", "arxiv-papers-0007.json.gz" ], "format": "json.gz", "name": "arxiv", "repo": "common-pile/arxiv_papers_filtered", "revision": "033cf7f53f9b348deec868c1a5a48484f3ee9e52", "source_url": "https://huggingface.co/datasets/common-pile/arxiv_papers_filtered", "synthetic": false, "weight_percent": 2 }, { "files": [ "pressbooks-0000.json.gz" ], "format": "json.gz", "name": "pressbooks", "repo": "common-pile/pressbooks_filtered", "revision": "1a1d3b50d77f834370f8eb4c0d174668dd1676bb", "source_url": "https://huggingface.co/datasets/common-pile/pressbooks_filtered", "synthetic": false, "weight_percent": 3 }, { "files": [ "libretexts-0000.json.gz" ], "format": "json.gz", "name": "libretexts", "repo": "common-pile/libretexts_filtered", "revision": "70388bca52b4a93515e14b1d56618fd7944988fd", "source_url": "https://huggingface.co/datasets/common-pile/libretexts_filtered", "synthetic": false, "weight_percent": 3 }, { "files": [ "doab-0000.json.gz", "doab-0001.json.gz", "doab-0002.json.gz", "doab-0003.json.gz", "doab-0004.json.gz" ], "format": "json.gz", "name": "doab", "repo": "common-pile/doab_filtered", "revision": "defb24ca72ef6aba6ce0228b669eec06dcfbffbc", "source_url": "https://huggingface.co/datasets/common-pile/doab_filtered", "synthetic": false, "weight_percent": 4 }, { "files": [ "project_gutenberg-dolma-0000.json.gz", "project_gutenberg-dolma-0001.json.gz", "project_gutenberg-dolma-0002.json.gz", "project_gutenberg-dolma-0003.json.gz", "project_gutenberg-dolma-0004.json.gz", "project_gutenberg-dolma-0005.json.gz", "project_gutenberg-dolma-0006.json.gz", "project_gutenberg-dolma-0007.json.gz", "project_gutenberg-dolma-0008.json.gz", "project_gutenberg-dolma-0009.json.gz", "project_gutenberg-dolma-0010.json.gz", "project_gutenberg-dolma-0011.json.gz", "project_gutenberg-dolma-0012.json.gz", "project_gutenberg-dolma-0013.json.gz", "project_gutenberg-dolma-0014.json.gz" ], "format": "json.gz", "name": "gutenberg", "repo": "common-pile/project_gutenberg_filtered", "revision": "3cdf6879c807f4e4e063f2ceb23bc268d8c29ab7", "source_url": "https://huggingface.co/datasets/common-pile/project_gutenberg_filtered", "synthetic": false, "weight_percent": 5 }, { "files": [ "public_library_1929_dolma-0000.json.gz", "public_library_1929_dolma-0001.json.gz", "public_library_1929_dolma-0002.json.gz", "public_library_1929_dolma-0003.json.gz", "public_library_1929_dolma-0004.json.gz", "public_library_1929_dolma-0005.json.gz", "public_library_1929_dolma-0006.json.gz", "public_library_1929_dolma-0007.json.gz", "public_library_1929_dolma-0008.json.gz", "public_library_1929_dolma-0009.json.gz", "public_library_1929_dolma-0010.json.gz", "public_library_1929_dolma-0011.json.gz", "public_library_1929_dolma-0012.json.gz", "public_library_1929_dolma-0013.json.gz", "public_library_1929_dolma-0014.json.gz", "public_library_1929_dolma-0015.json.gz", "public_library_1929_dolma-0016.json.gz", "public_library_1929_dolma-0017.json.gz", "public_library_1929_dolma-0018.json.gz", "public_library_1929_dolma-0019.json.gz", "public_library_1929_dolma-0020.json.gz", "public_library_1929_dolma-0021.json.gz", "public_library_1929_dolma-0022.json.gz", "public_library_1929_dolma-0023.json.gz", "public_library_1929_dolma-0024.json.gz", "public_library_1929_dolma-0025.json.gz" ], "format": "json.gz", "name": "pre_1929_books", "repo": "common-pile/pre_1929_books_filtered", "revision": "23f9d96dbb1db3324bbc9fbfe1f8299cc799c4d1", "synthetic": false, "weight_percent": 7 }, { "files": [ "loc_books_dolma-0000.json.gz", "loc_books_dolma-0001.json.gz", "loc_books_dolma-0002.json.gz", "loc_books_dolma-0003.json.gz", "loc_books_dolma-0004.json.gz", "loc_books_dolma-0005.json.gz", "loc_books_dolma-0006.json.gz", "loc_books_dolma-0007.json.gz", "loc_books_dolma-0008.json.gz", "loc_books_dolma-0009.json.gz", "loc_books_dolma-0010.json.gz", "loc_books_dolma-0011.json.gz", "loc_books_dolma-0012.json.gz", "loc_books_dolma-0013.json.gz", "loc_books_dolma-0014.json.gz", "loc_books_dolma-0015.json.gz", "loc_books_dolma-0016.json.gz", "loc_books_dolma-0017.json.gz", "loc_books_dolma-0018.json.gz", "loc_books_dolma-0019.json.gz", "loc_books_dolma-0020.json.gz", "loc_books_dolma-0021.json.gz", "loc_books_dolma-0022.json.gz", "loc_books_dolma-0023.json.gz", "loc_books_dolma-0024.json.gz" ], "format": "json.gz", "name": "library_of_congress", "repo": "common-pile/library_of_congress_filtered", "revision": "56725c7aa1bb320703e22eb5f42903173d5bac3d", "synthetic": false, "weight_percent": 3 } ], "special_tokens": [ "<|pad|>", "<|bos|>", "<|eos|>", "<|unk|>" ], "split_policy": "Same mixture-split-v1 parent hash: buckets2..9 training, bucket0 validation, bucket1 excluded; no held-out fitting", "tokenizer": "Byte-level BPE with complete256bytealphabet; no normalization; digit groups up to3; explicit whitespace; no automatic BOS/EOS", "trainer": { "max_token_length": 32, "min_frequency": 2 }, "training_utf8_bytes": 67108864, "validation_utf8_bytes": 8388608, "vocab_sizes": [ 16384, 32768 ] }