[ { "size_mib": 64, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 6710887, "code_python": 20132660, "finemath_4plus": 13421773, "pes2o": 6710887, "pubmed": 2013266, "arxiv": 1342178, "pressbooks": 2013266, "libretexts": 2013265, "doab": 2684354, "gutenberg": 3355443, "pre_1929_books": 4697620, "library_of_congress": 2013265 }, "training_seconds": 8.776563889999998, "peak_process_rss_mib": 409.15234375, "tokenizer_sha256": "97fa4177ad5a038cc18df87a2acb76c3ed3da89a9b7848517442bcbd7029d83d", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 809464, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.145266497336509, "chunk_tokens_p95": 2005 }, "code_python": { "bytes": 10066330, "tokens": 2750571, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.659723744633387, "chunk_tokens_p95": 2547 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1986189, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.3787756351485183, "chunk_tokens_p95": 2585 }, "pes2o": { "bytes": 3355444, "tokens": 810480, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.140070081926759, "chunk_tokens_p95": 2302 }, "pubmed": { "bytes": 1006633, "tokens": 270152, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.7261726731617757, "chunk_tokens_p95": 2723 }, "arxiv": { "bytes": 671089, "tokens": 192711, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.482359595456409, "chunk_tokens_p95": 2928 }, "pressbooks": { "bytes": 1006633, "tokens": 241100, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.175167980091248, "chunk_tokens_p95": 1871 }, "libretexts": { "bytes": 1006632, "tokens": 244293, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.1205928945978805, "chunk_tokens_p95": 2133 }, "doab": { "bytes": 1342177, "tokens": 333249, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.027549970142446, "chunk_tokens_p95": 2238 }, "gutenberg": { "bytes": 1677721, "tokens": 465276, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6058618970245617, "chunk_tokens_p95": 2807 }, "pre_1929_books": { "bytes": 2348810, "tokens": 594333, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9520100684296513, "chunk_tokens_p95": 2421 }, "library_of_congress": { "bytes": 1006632, "tokens": 258926, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8877208159860346, "chunk_tokens_p95": 2383 } }, "bytes": 33554432, "tokens": 8956744, "bytes_per_token": 3.7462756555283927, "evaluation_seconds": 14.238836671000001, "evaluation_mib_per_second": 2.2473746092736535, "vocabulary_used": 16248, "vocabulary_use_fraction": 0.99169921875, "vocabulary_unseen": 136, "vocabulary_used_at_most_10_times": 330, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 9, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 31, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 10, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 7, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 6, "123456789": 4, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "7a1bdf5cd3fbf82ee03ed76ea047edf9b155823a3470cec494b21e8f5fd23840", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.7253234702170097, 3.7711438880078716 ], "raw_result_sha256": "d09833d7252f1c57b21e523aacdd02633e2bf5eba311506133b5df942d4e8e34", "vs_64mib_same_vocab": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 64, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 6710887, "code_python": 20132660, "finemath_4plus": 13421773, "pes2o": 6710887, "pubmed": 2013266, "arxiv": 1342178, "pressbooks": 2013266, "libretexts": 2013265, "doab": 2684354, "gutenberg": 3355443, "pre_1929_books": 4697620, "library_of_congress": 2013265 }, "training_seconds": 9.810118514, "peak_process_rss_mib": 422.26171875, "tokenizer_sha256": "cd9109d658834401ef05b4de231cdfd789385bb0d9f294e343e19b447d94b52a", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 748726, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4815379725026245, "chunk_tokens_p95": 1858 }, "code_python": { "bytes": 10066330, "tokens": 2563077, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9274395580000134, "chunk_tokens_p95": 2356 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1885335, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.5595196609621103, "chunk_tokens_p95": 2441 }, "pes2o": { "bytes": 3355444, "tokens": 751239, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4665465983528545, "chunk_tokens_p95": 2143 }, "pubmed": { "bytes": 1006633, "tokens": 250154, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.024053183239125, "chunk_tokens_p95": 2566 }, "arxiv": { "bytes": 671089, "tokens": 180650, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.714857459175201, "chunk_tokens_p95": 2800 }, "pressbooks": { "bytes": 1006633, "tokens": 224760, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.478701726285816, "chunk_tokens_p95": 1751 }, "libretexts": { "bytes": 1006632, "tokens": 230366, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.369707335283852, "chunk_tokens_p95": 1999 }, "doab": { "bytes": 1342177, "tokens": 304345, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.410051093331581, "chunk_tokens_p95": 2026 }, "gutenberg": { "bytes": 1677721, "tokens": 430852, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8939612674421844, "chunk_tokens_p95": 2466 }, "pre_1929_books": { "bytes": 2348810, "tokens": 554432, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.236425747431606, "chunk_tokens_p95": 2263 }, "library_of_congress": { "bytes": 1006632, "tokens": 241709, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.164644262315429, "chunk_tokens_p95": 2228 } }, "bytes": 33554432, "tokens": 8365645, "bytes_per_token": 4.010979667437478, "evaluation_seconds": 18.018589408999997, "evaluation_mib_per_second": 1.7759436809196902, "vocabulary_used": 31820, "vocabulary_use_fraction": 0.9710693359375, "vocabulary_unseen": 948, "vocabulary_used_at_most_10_times": 2674, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 12, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 27, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 7, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 3, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "1b741432a4962272e45ebe2713a4b75ee6145f5afaefd8eec10e4e5fd64cde1c", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.988442103265165, 4.0371958036541455 ], "raw_result_sha256": "dac5ec5cc5c61a2fb5e3758320f1486c174ec7378d1b37b8628c7f642241dc7e", "vs_64mib_same_vocab": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] }, "vs_16k_same_size": { "token_saving_percent": 6.599485259375504, "paired_ci95": [ 6.532287848969547, 6.680290527180766 ] } }, { "size_mib": 64, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 6710887, "code_python": 20132660, "finemath_4plus": 13421773, "pes2o": 6710887, "pubmed": 2013266, "arxiv": 1342178, "pressbooks": 2013266, "libretexts": 2013265, "doab": 2684354, "gutenberg": 3355443, "pre_1929_books": 4697620, "library_of_congress": 2013265 }, "training_seconds": 10.408797823, "peak_process_rss_mib": 426.9296875, "tokenizer_sha256": "86de4d58e6eb5417264afe227857e98c0b0431eb7adbfaf6a9ebacac37d77726", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 722478, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.644354568582019, "chunk_tokens_p95": 1796 }, "code_python": { "bytes": 10066330, "tokens": 2480835, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.057637851771681, "chunk_tokens_p95": 2276 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1843677, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.639947235877, "chunk_tokens_p95": 2390 }, "pes2o": { "bytes": 3355444, "tokens": 726566, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.6182232584514, "chunk_tokens_p95": 2082 }, "pubmed": { "bytes": 1006633, "tokens": 241172, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.173921516593966, "chunk_tokens_p95": 2479 }, "arxiv": { "bytes": 671089, "tokens": 174905, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.836877161887882, "chunk_tokens_p95": 2743 }, "pressbooks": { "bytes": 1006633, "tokens": 217176, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.63510240542233, "chunk_tokens_p95": 1683 }, "libretexts": { "bytes": 1006632, "tokens": 224653, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4808304362728295, "chunk_tokens_p95": 1941 }, "doab": { "bytes": 1342177, "tokens": 291070, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.611182876971175, "chunk_tokens_p95": 1922 }, "gutenberg": { "bytes": 1677721, "tokens": 414289, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.049639261481719, "chunk_tokens_p95": 2338 }, "pre_1929_books": { "bytes": 2348810, "tokens": 536337, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.37935477134712, "chunk_tokens_p95": 2179 }, "library_of_congress": { "bytes": 1006632, "tokens": 233292, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.314901496836582, "chunk_tokens_p95": 2146 } }, "bytes": 33554432, "tokens": 8106450, "bytes_per_token": 4.139226418469244, "evaluation_seconds": 20.04189617, "evaluation_mib_per_second": 1.5966553128790109, "vocabulary_used": 47029, "vocabulary_use_fraction": 0.94058, "vocabulary_unseen": 2971, "vocabulary_used_at_most_10_times": 9496, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 26, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 7, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 2, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "8568773312de4a3c65765cd84df08d828ec4631c36ec91f878103783c7fae2c3", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.115876574322902, 4.165474937669827 ], "raw_result_sha256": "5ebb798a1720c5d288af67491d12e5daa8aae0c64e2bb3a4a2592cfa7c161fdb", "vs_64mib_same_vocab": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] }, "vs_16k_same_size": { "token_saving_percent": 9.49333820415098, "paired_ci95": [ 9.408400244317894, 9.605081462142396 ] } }, { "size_mib": 128, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 13421773, "code_python": 40265319, "finemath_4plus": 26843546, "pes2o": 13421773, "pubmed": 4026532, "arxiv": 2684355, "pressbooks": 4026532, "libretexts": 4026532, "doab": 5368709, "gutenberg": 6710886, "pre_1929_books": 9395240, "library_of_congress": 4026531 }, "training_seconds": 13.764946394000006, "peak_process_rss_mib": 618.10546875, "tokenizer_sha256": "64356327337221b75b445cc7c761d655a40c1af2bccd8f8ab5453fd1fa1035c0", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 805884, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.163681125323247, "chunk_tokens_p95": 2005 }, "code_python": { "bytes": 10066330, "tokens": 2740723, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.672873909548685, "chunk_tokens_p95": 2525 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1982520, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.3850286504045357, "chunk_tokens_p95": 2582 }, "pes2o": { "bytes": 3355444, "tokens": 808314, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.151164027840666, "chunk_tokens_p95": 2289 }, "pubmed": { "bytes": 1006633, "tokens": 269475, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.7355339085258374, "chunk_tokens_p95": 2714 }, "arxiv": { "bytes": 671089, "tokens": 192339, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.4890947753705697, "chunk_tokens_p95": 2915 }, "pressbooks": { "bytes": 1006633, "tokens": 240330, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.188544917405235, "chunk_tokens_p95": 1865 }, "libretexts": { "bytes": 1006632, "tokens": 243558, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.133027861946641, "chunk_tokens_p95": 2140 }, "doab": { "bytes": 1342177, "tokens": 332428, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.037496841421301, "chunk_tokens_p95": 2233 }, "gutenberg": { "bytes": 1677721, "tokens": 464540, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6115748912903087, "chunk_tokens_p95": 2821 }, "pre_1929_books": { "bytes": 2348810, "tokens": 592602, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9635539535809867, "chunk_tokens_p95": 2409 }, "library_of_congress": { "bytes": 1006632, "tokens": 258319, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8968562126672834, "chunk_tokens_p95": 2379 } }, "bytes": 33554432, "tokens": 8931032, "bytes_per_token": 3.757060998101899, "evaluation_seconds": 14.213449730000004, "evaluation_mib_per_second": 2.2513886922509974, "vocabulary_used": 16298, "vocabulary_use_fraction": 0.9947509765625, "vocabulary_unseen": 86, "vocabulary_used_at_most_10_times": 243, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 8, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 30, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 10, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 6, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 6, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "145aa86a48dce6b0ebb6dbe6629ce827e673e0482cb43deb17beade3bcec41a5", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.7362385971701717, 3.7822919781602184 ], "raw_result_sha256": "b41b8fb8fb102d965cecf80e38bc60a77013192d1ee549632d571e31d4b79a75", "vs_64mib_same_vocab": { "token_saving_percent": 0.2870686043946358, "paired_ci95": [ 0.2571423946282611, 0.31525380838660216 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 128, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 13421773, "code_python": 40265319, "finemath_4plus": 26843546, "pes2o": 13421773, "pubmed": 4026532, "arxiv": 2684355, "pressbooks": 4026532, "libretexts": 4026532, "doab": 5368709, "gutenberg": 6710886, "pre_1929_books": 9395240, "library_of_congress": 4026531 }, "training_seconds": 15.621106757000007, "peak_process_rss_mib": 655.171875, "tokenizer_sha256": "ade4e945319da6e9b010ad7f48f19d9489f7ac583448318a91cf302c21aaf321", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 745385, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.501625334558651, "chunk_tokens_p95": 1851 }, "code_python": { "bytes": 10066330, "tokens": 2551578, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9451390472876002, "chunk_tokens_p95": 2351 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1881693, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.56640907948321, "chunk_tokens_p95": 2424 }, "pes2o": { "bytes": 3355444, "tokens": 748062, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4855159064355625, "chunk_tokens_p95": 2134 }, "pubmed": { "bytes": 1006633, "tokens": 248357, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.053169429490612, "chunk_tokens_p95": 2542 }, "arxiv": { "bytes": 671089, "tokens": 180462, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.7187274883354946, "chunk_tokens_p95": 2800 }, "pressbooks": { "bytes": 1006633, "tokens": 223762, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.498677165917359, "chunk_tokens_p95": 1737 }, "libretexts": { "bytes": 1006632, "tokens": 229650, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.383331156107119, "chunk_tokens_p95": 1986 }, "doab": { "bytes": 1342177, "tokens": 302982, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4298902245017855, "chunk_tokens_p95": 2027 }, "gutenberg": { "bytes": 1677721, "tokens": 429461, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.906573588754276, "chunk_tokens_p95": 2460 }, "pre_1929_books": { "bytes": 2348810, "tokens": 552719, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.2495553798584815, "chunk_tokens_p95": 2259 }, "library_of_congress": { "bytes": 1006632, "tokens": 240838, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.179705860370872, "chunk_tokens_p95": 2213 } }, "bytes": 33554432, "tokens": 8334949, "bytes_per_token": 4.025751327332657, "evaluation_seconds": 17.542917185000004, "evaluation_mib_per_second": 1.8240979913740607, "vocabulary_used": 32155, "vocabulary_use_fraction": 0.981292724609375, "vocabulary_unseen": 613, "vocabulary_used_at_most_10_times": 2279, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 13, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 27, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 6, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 6, "roundtrip": true } ], "fragmentation": { "electromagnetism": 4, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "d2fa50e315c0349117a93e5f45fe45a411c7ab4fa9f5ff83af93173359bc367e", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.0024713775611165, 4.052006976084268 ], "raw_result_sha256": "2915a4d8951cafb394639013f01ed07bfa685dd6c9bf97179e416753a49b0e48", "vs_64mib_same_vocab": { "token_saving_percent": 0.3669292684544967, "paired_ci95": [ 0.33687187101464344, 0.4009128108909549 ] }, "vs_16k_same_size": { "token_saving_percent": 6.674290272389571, "paired_ci95": [ 6.604578111377216, 6.75937713765991 ] } }, { "size_mib": 128, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 13421773, "code_python": 40265319, "finemath_4plus": 26843546, "pes2o": 13421773, "pubmed": 4026532, "arxiv": 2684355, "pressbooks": 4026532, "libretexts": 4026532, "doab": 5368709, "gutenberg": 6710886, "pre_1929_books": 9395240, "library_of_congress": 4026531 }, "training_seconds": 14.651732101999997, "peak_process_rss_mib": 645.515625, "tokenizer_sha256": "db13f817c797466237e07a17ca3d62b65dafe334745d6c72714073fa47d36380", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 719232, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.665315225129026, "chunk_tokens_p95": 1787 }, "code_python": { "bytes": 10066330, "tokens": 2469423, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.076389504754754, "chunk_tokens_p95": 2265 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1839706, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.647804051299501, "chunk_tokens_p95": 2382 }, "pes2o": { "bytes": 3355444, "tokens": 723431, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.63823640402471, "chunk_tokens_p95": 2068 }, "pubmed": { "bytes": 1006633, "tokens": 239461, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.203745077486522, "chunk_tokens_p95": 2453 }, "arxiv": { "bytes": 671089, "tokens": 174403, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.847921194016158, "chunk_tokens_p95": 2742 }, "pressbooks": { "bytes": 1006633, "tokens": 216331, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.653207353546186, "chunk_tokens_p95": 1666 }, "libretexts": { "bytes": 1006632, "tokens": 223877, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.49636184154692, "chunk_tokens_p95": 1939 }, "doab": { "bytes": 1342177, "tokens": 290318, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.6231270537824045, "chunk_tokens_p95": 1929 }, "gutenberg": { "bytes": 1677721, "tokens": 412020, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.0719406824911415, "chunk_tokens_p95": 2297 }, "pre_1929_books": { "bytes": 2348810, "tokens": 534890, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.3912019293686555, "chunk_tokens_p95": 2172 }, "library_of_congress": { "bytes": 1006632, "tokens": 232714, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.325618570434095, "chunk_tokens_p95": 2140 } }, "bytes": 33554432, "tokens": 8075806, "bytes_per_token": 4.1549328946237685, "evaluation_seconds": 18.716767812, "evaluation_mib_per_second": 1.709696905011753, "vocabulary_used": 47778, "vocabulary_use_fraction": 0.95556, "vocabulary_unseen": 2222, "vocabulary_used_at_most_10_times": 9196, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 12, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 24, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 5, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 2, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 2, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "654d31ba0555fa7d894725c41b4969e986bd4992c145ab5cf107d2183e437600", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.131537559055759, 4.1815624765901855 ], "raw_result_sha256": "ae3b85d78322d9de54e8a94b59af2abf62cc6119a680ee32bd47d176455c4b5e", "vs_64mib_same_vocab": { "token_saving_percent": 0.37801997175087587, "paired_ci95": [ 0.3492808572410605, 0.40876432511092187 ] }, "vs_16k_same_size": { "token_saving_percent": 9.575892237313665, "paired_ci95": [ 9.4852345035617, 9.68628583930954 ] } }, { "size_mib": 256, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 26843546, "code_python": 80530637, "finemath_4plus": 53687092, "pes2o": 26843546, "pubmed": 8053064, "arxiv": 5368710, "pressbooks": 8053064, "libretexts": 8053063, "doab": 10737418, "gutenberg": 13421772, "pre_1929_books": 18790481, "library_of_congress": 8053063 }, "training_seconds": 22.75400429300001, "peak_process_rss_mib": 908.1640625, "tokenizer_sha256": "e76d8e8a71db29a3e1ed036d8e6813ce180ac568b0310d0cdc01a2c8974c92e6", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 804027, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.17329766288943, "chunk_tokens_p95": 2007 }, "code_python": { "bytes": 10066330, "tokens": 2736914, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6779854975348147, "chunk_tokens_p95": 2524 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1978378, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.3921156624264928, "chunk_tokens_p95": 2579 }, "pes2o": { "bytes": 3355444, "tokens": 807381, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.155961064231137, "chunk_tokens_p95": 2291 }, "pubmed": { "bytes": 1006633, "tokens": 268775, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.745262766254302, "chunk_tokens_p95": 2691 }, "arxiv": { "bytes": 671089, "tokens": 192437, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.487317927425599, "chunk_tokens_p95": 2930 }, "pressbooks": { "bytes": 1006633, "tokens": 240176, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.191230597561788, "chunk_tokens_p95": 1861 }, "libretexts": { "bytes": 1006632, "tokens": 243316, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.137138535895708, "chunk_tokens_p95": 2128 }, "doab": { "bytes": 1342177, "tokens": 332243, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.039745005914346, "chunk_tokens_p95": 2229 }, "gutenberg": { "bytes": 1677721, "tokens": 463963, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6160663673611904, "chunk_tokens_p95": 2817 }, "pre_1929_books": { "bytes": 2348810, "tokens": 592540, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9639686772201035, "chunk_tokens_p95": 2404 }, "library_of_congress": { "bytes": 1006632, "tokens": 258258, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8977766419626887, "chunk_tokens_p95": 2387 } }, "bytes": 33554432, "tokens": 8918408, "bytes_per_token": 3.7623791151963446, "evaluation_seconds": 14.001287165, "evaluation_mib_per_second": 2.285504155645964, "vocabulary_used": 16304, "vocabulary_use_fraction": 0.9951171875, "vocabulary_unseen": 80, "vocabulary_used_at_most_10_times": 215, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 8, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 29, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 9, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 7, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 6, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "16e9f72fd49ce84c4f68342e960bbf5e649c4a70ee0c10f5fbc178def20faa05", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.7410346836920847, 3.78745201861199 ], "raw_result_sha256": "90c567655c105853165ee9b9f8257245a7d38fc99e66fcb18501cfc406ee5e14", "vs_64mib_same_vocab": { "token_saving_percent": 0.4280126796076922, "paired_ci95": [ 0.3879827586257448, 0.4753906917193296 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 256, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 26843546, "code_python": 80530637, "finemath_4plus": 53687092, "pes2o": 26843546, "pubmed": 8053064, "arxiv": 5368710, "pressbooks": 8053064, "libretexts": 8053063, "doab": 10737418, "gutenberg": 13421772, "pre_1929_books": 18790481, "library_of_congress": 8053063 }, "training_seconds": 26.727447731000012, "peak_process_rss_mib": 974.83203125, "tokenizer_sha256": "3839415fcdd736bb9322b958a45ba82ba01310e2007a1b139f77e3486568c1b6", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 742828, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.517121056287593, "chunk_tokens_p95": 1848 }, "code_python": { "bytes": 10066330, "tokens": 2546847, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9524675019740094, "chunk_tokens_p95": 2357 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1877830, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.5737457597333093, "chunk_tokens_p95": 2439 }, "pes2o": { "bytes": 3355444, "tokens": 746903, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.492476265324949, "chunk_tokens_p95": 2131 }, "pubmed": { "bytes": 1006633, "tokens": 247495, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.067286207802177, "chunk_tokens_p95": 2540 }, "arxiv": { "bytes": 671089, "tokens": 179690, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.734704212810952, "chunk_tokens_p95": 2797 }, "pressbooks": { "bytes": 1006633, "tokens": 223451, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.5049384428800945, "chunk_tokens_p95": 1726 }, "libretexts": { "bytes": 1006632, "tokens": 229065, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.394525571344378, "chunk_tokens_p95": 1971 }, "doab": { "bytes": 1342177, "tokens": 302492, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.43706610422755, "chunk_tokens_p95": 2018 }, "gutenberg": { "bytes": 1677721, "tokens": 429061, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.910215563754338, "chunk_tokens_p95": 2451 }, "pre_1929_books": { "bytes": 2348810, "tokens": 552046, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.25473601837528, "chunk_tokens_p95": 2258 }, "library_of_congress": { "bytes": 1006632, "tokens": 240555, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.184623059175657, "chunk_tokens_p95": 2206 } }, "bytes": 33554432, "tokens": 8318263, "bytes_per_token": 4.033826773690613, "evaluation_seconds": 18.389478011999998, "evaluation_mib_per_second": 1.740125520643843, "vocabulary_used": 32356, "vocabulary_use_fraction": 0.9874267578125, "vocabulary_unseen": 412, "vocabulary_used_at_most_10_times": 2027, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 13, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 22, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 7, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 4, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "52408ec872ddc2d20e07fae1180f043b4b68fcfd01425ff095752562c41491c5", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.010703911098985, 4.059882958848434 ], "raw_result_sha256": "47255252cac983588d502289e862a109d82b7ba59e1c571f7183108c965c9355", "vs_64mib_same_vocab": { "token_saving_percent": 0.5663878876046136, "paired_ci95": [ 0.5291951126970479, 0.607556610571848 ] }, "vs_16k_same_size": { "token_saving_percent": 6.729283970861166, "paired_ci95": [ 6.662042324517174, 6.817499035195743 ] } }, { "size_mib": 256, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 26843546, "code_python": 80530637, "finemath_4plus": 53687092, "pes2o": 26843546, "pubmed": 8053064, "arxiv": 5368710, "pressbooks": 8053064, "libretexts": 8053063, "doab": 10737418, "gutenberg": 13421772, "pre_1929_books": 18790481, "library_of_congress": 8053063 }, "training_seconds": 28.223051318999993, "peak_process_rss_mib": 976.99609375, "tokenizer_sha256": "eb4b69846169b0aedd86c72eb2bc7e85b96b074519f7bba8563423cd200baf81", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 716688, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.681875516263702, "chunk_tokens_p95": 1786 }, "code_python": { "bytes": 10066330, "tokens": 2461951, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.088761311658924, "chunk_tokens_p95": 2264 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1836046, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6550756353598985, "chunk_tokens_p95": 2366 }, "pes2o": { "bytes": 3355444, "tokens": 720689, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.655883467071095, "chunk_tokens_p95": 2052 }, "pubmed": { "bytes": 1006633, "tokens": 238666, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.217747814938031, "chunk_tokens_p95": 2461 }, "arxiv": { "bytes": 671089, "tokens": 173666, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8642509184296294, "chunk_tokens_p95": 2713 }, "pressbooks": { "bytes": 1006633, "tokens": 216493, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.649725395278369, "chunk_tokens_p95": 1668 }, "libretexts": { "bytes": 1006632, "tokens": 223521, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.5035231588978215, "chunk_tokens_p95": 1914 }, "doab": { "bytes": 1342177, "tokens": 289529, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.635725609524434, "chunk_tokens_p95": 1911 }, "gutenberg": { "bytes": 1677721, "tokens": 411466, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.077423164975964, "chunk_tokens_p95": 2314 }, "pre_1929_books": { "bytes": 2348810, "tokens": 534248, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.396478788877076, "chunk_tokens_p95": 2170 }, "library_of_congress": { "bytes": 1006632, "tokens": 232261, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.334055222357606, "chunk_tokens_p95": 2145 } }, "bytes": 33554432, "tokens": 8055224, "bytes_per_token": 4.165549213777295, "evaluation_seconds": 14.800683191999994, "evaluation_mib_per_second": 2.1620623578576774, "vocabulary_used": 48420, "vocabulary_use_fraction": 0.9684, "vocabulary_unseen": 1580, "vocabulary_used_at_most_10_times": 9052, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 18, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 5, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 3, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 2, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "4b14d9a99be20c88aaab57029cd4634710350665995d74a357ab45a32f3b42f6", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.141883517407929, 4.1921418306820035 ], "raw_result_sha256": "d26837e3e6f7f669a0d3de13788bb1c9a52e99cce56d12d631f4c16c3a403331", "vs_64mib_same_vocab": { "token_saving_percent": 0.6319165602698851, "paired_ci95": [ 0.5918921376973765, 0.6721510169045986 ] }, "vs_16k_same_size": { "token_saving_percent": 9.67867807797086, "paired_ci95": [ 9.576043326252233, 9.790785308891813 ] } }, { "size_mib": 512, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 53687092, "code_python": 161061274, "finemath_4plus": 107374183, "pes2o": 53687092, "pubmed": 16106128, "arxiv": 10737418, "pressbooks": 16106127, "libretexts": 16106127, "doab": 21474836, "gutenberg": 26843545, "pre_1929_books": 37580963, "library_of_congress": 16106127 }, "training_seconds": 40.378172683, "peak_process_rss_mib": 1469.3984375, "tokenizer_sha256": "c3bae921b15a294391e45fe3bf2ce0a2f3ac1b6678c0f32541613106f322078e", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 803626, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.175380089743239, "chunk_tokens_p95": 2004 }, "code_python": { "bytes": 10066330, "tokens": 2734437, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6813172144759596, "chunk_tokens_p95": 2520 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1977334, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.393906643996411, "chunk_tokens_p95": 2565 }, "pes2o": { "bytes": 3355444, "tokens": 807061, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.157608904407473, "chunk_tokens_p95": 2284 }, "pubmed": { "bytes": 1006633, "tokens": 268706, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.746224498150395, "chunk_tokens_p95": 2689 }, "arxiv": { "bytes": 671089, "tokens": 192058, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.494199668850035, "chunk_tokens_p95": 2929 }, "pressbooks": { "bytes": 1006633, "tokens": 239977, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.19470615934027, "chunk_tokens_p95": 1861 }, "libretexts": { "bytes": 1006632, "tokens": 242822, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.145555180337861, "chunk_tokens_p95": 2123 }, "doab": { "bytes": 1342177, "tokens": 332709, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.034086844660048, "chunk_tokens_p95": 2240 }, "gutenberg": { "bytes": 1677721, "tokens": 464206, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6141734488567576, "chunk_tokens_p95": 2804 }, "pre_1929_books": { "bytes": 2348810, "tokens": 592079, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.967055072042751, "chunk_tokens_p95": 2402 }, "library_of_congress": { "bytes": 1006632, "tokens": 258039, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9010847197516654, "chunk_tokens_p95": 2396 } }, "bytes": 33554432, "tokens": 8913054, "bytes_per_token": 3.764639146133301, "evaluation_seconds": 18.321253727, "evaluation_mib_per_second": 1.7466053621014843, "vocabulary_used": 16307, "vocabulary_use_fraction": 0.99530029296875, "vocabulary_unseen": 77, "vocabulary_used_at_most_10_times": 195, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 8, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 30, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 9, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 7, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 5, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "989cc3f9196c5667734d4bf3d9aa9812a9e1f109f33c5ebaa67f6954a3f3fee9", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.743055062261041, 3.789654631203881 ], "raw_result_sha256": "75a63070c0c5a8f67c3043b7e5d57090e9f8e27c2c34181c1ede4c2b4535594f", "vs_64mib_same_vocab": { "token_saving_percent": 0.4877888661326102, "paired_ci95": [ 0.45379665743235653, 0.5309888029433135 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 512, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 53687092, "code_python": 161061274, "finemath_4plus": 107374183, "pes2o": 53687092, "pubmed": 16106128, "arxiv": 10737418, "pressbooks": 16106127, "libretexts": 16106127, "doab": 21474836, "gutenberg": 26843545, "pre_1929_books": 37580963, "library_of_congress": 16106127 }, "training_seconds": 46.02036403599999, "peak_process_rss_mib": 1471.63671875, "tokenizer_sha256": "ee276c0fba28e5d4f555f85971e9d99b5cee6b53aa5181bc1c4dd33417572829", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 742671, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.51807597172907, "chunk_tokens_p95": 1839 }, "code_python": { "bytes": 10066330, "tokens": 2541276, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9611321241769883, "chunk_tokens_p95": 2335 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1875724, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.577758241617637, "chunk_tokens_p95": 2437 }, "pes2o": { "bytes": 3355444, "tokens": 745660, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.499965131561301, "chunk_tokens_p95": 2130 }, "pubmed": { "bytes": 1006633, "tokens": 247341, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.069818590528865, "chunk_tokens_p95": 2545 }, "arxiv": { "bytes": 671089, "tokens": 179217, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.7445610628456008, "chunk_tokens_p95": 2799 }, "pressbooks": { "bytes": 1006633, "tokens": 223376, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.506451006374902, "chunk_tokens_p95": 1733 }, "libretexts": { "bytes": 1006632, "tokens": 228617, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.403137124535796, "chunk_tokens_p95": 1982 }, "doab": { "bytes": 1342177, "tokens": 302247, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.440662769192085, "chunk_tokens_p95": 2024 }, "gutenberg": { "bytes": 1677721, "tokens": 428638, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9140743471180808, "chunk_tokens_p95": 2461 }, "pre_1929_books": { "bytes": 2348810, "tokens": 551542, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.258624003249072, "chunk_tokens_p95": 2252 }, "library_of_congress": { "bytes": 1006632, "tokens": 240251, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.189918044045602, "chunk_tokens_p95": 2209 } }, "bytes": 33554432, "tokens": 8306560, "bytes_per_token": 4.039509977656214, "evaluation_seconds": 19.212221627999995, "evaluation_mib_per_second": 1.6656064363406586, "vocabulary_used": 32469, "vocabulary_use_fraction": 0.990875244140625, "vocabulary_unseen": 299, "vocabulary_used_at_most_10_times": 1812, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 12, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 24, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 6, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 4, "deoxyribonucleic": 6, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "7d09541cfba364ac660f4664e2cbe47e96fb7294701032766a872dfc0434d901", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.015809758793044, 4.065744954151812 ], "raw_result_sha256": "fe60794938bfefca360c98c7bf113d9c48089273a39d9f5e5d5e8d4d80b87d90", "vs_64mib_same_vocab": { "token_saving_percent": 0.7062814642505133, "paired_ci95": [ 0.6689432884540671, 0.7519069204712292 ] }, "vs_16k_same_size": { "token_saving_percent": 6.804558796569604, "paired_ci95": [ 6.735751471831152, 6.893860773834811 ] } }, { "size_mib": 512, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 53687092, "code_python": 161061274, "finemath_4plus": 107374183, "pes2o": 53687092, "pubmed": 16106128, "arxiv": 10737418, "pressbooks": 16106127, "libretexts": 16106127, "doab": 21474836, "gutenberg": 26843545, "pre_1929_books": 37580963, "library_of_congress": 16106127 }, "training_seconds": 41.69725704000001, "peak_process_rss_mib": 1463.0546875, "tokenizer_sha256": "1f4b70d34b608d97a3ad7a9bcd7b0094fa82c652281d39bcc12325c1f16865e2", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 715922, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.686884884107487, "chunk_tokens_p95": 1790 }, "code_python": { "bytes": 10066330, "tokens": 2455669, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.099221026938077, "chunk_tokens_p95": 2255 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1833707, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.659737897057709, "chunk_tokens_p95": 2373 }, "pes2o": { "bytes": 3355444, "tokens": 719660, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.662540644193091, "chunk_tokens_p95": 2054 }, "pubmed": { "bytes": 1006633, "tokens": 238078, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.228164719125664, "chunk_tokens_p95": 2444 }, "arxiv": { "bytes": 671089, "tokens": 173141, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.875968141572476, "chunk_tokens_p95": 2701 }, "pressbooks": { "bytes": 1006633, "tokens": 215986, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.660640041484171, "chunk_tokens_p95": 1673 }, "libretexts": { "bytes": 1006632, "tokens": 223088, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.512264218604318, "chunk_tokens_p95": 1909 }, "doab": { "bytes": 1342177, "tokens": 288858, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.646494125141073, "chunk_tokens_p95": 1912 }, "gutenberg": { "bytes": 1677721, "tokens": 411047, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.08157947874574, "chunk_tokens_p95": 2305 }, "pre_1929_books": { "bytes": 2348810, "tokens": 533677, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.401182737873283, "chunk_tokens_p95": 2165 }, "library_of_congress": { "bytes": 1006632, "tokens": 232106, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.336949497212481, "chunk_tokens_p95": 2150 } }, "bytes": 33554432, "tokens": 8040939, "bytes_per_token": 4.1729494527940085, "evaluation_seconds": 15.270844398000008, "evaluation_mib_per_second": 2.095496435298036, "vocabulary_used": 48885, "vocabulary_use_fraction": 0.9777, "vocabulary_unseen": 1115, "vocabulary_used_at_most_10_times": 8957, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 19, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 5, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 2, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 2, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "748f29ffa5967e98c916a1f29b1f1678a420ba6432d6f7e56d94cabfcbd27f62", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.14909233442202, 4.199119965141136 ], "raw_result_sha256": "e16aecacc179b080563cd8bdded280cb5607c3b231daba10a13937a90b7ff4ba", "vs_64mib_same_vocab": { "token_saving_percent": 0.8081342634568722, "paired_ci95": [ 0.7629974315430208, 0.8614506680344802 ] }, "vs_16k_same_size": { "token_saving_percent": 9.784693327337635, "paired_ci95": [ 9.686796230035283, 9.901877722230006 ] } }, { "size_mib": 1024, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 107374183, "code_python": 322122548, "finemath_4plus": 214748365, "pes2o": 107374183, "pubmed": 32212255, "arxiv": 21474837, "pressbooks": 32212255, "libretexts": 32212254, "doab": 42949672, "gutenberg": 53687091, "pre_1929_books": 75161927, "library_of_congress": 32212254 }, "training_seconds": 69.90681187199996, "peak_process_rss_mib": 2280.640625, "tokenizer_sha256": "020928562c47ea392514da50b9c4886b4e3e24fd01cfdc30501599042ffb1359", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 803728, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.174850198076961, "chunk_tokens_p95": 2008 }, "code_python": { "bytes": 10066330, "tokens": 2732020, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6845740514344696, "chunk_tokens_p95": 2520 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1976812, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.394802844175369, "chunk_tokens_p95": 2570 }, "pes2o": { "bytes": 3355444, "tokens": 806536, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.160315224614896, "chunk_tokens_p95": 2281 }, "pubmed": { "bytes": 1006633, "tokens": 268784, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.745137359366629, "chunk_tokens_p95": 2684 }, "arxiv": { "bytes": 671089, "tokens": 192191, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.4917816130828188, "chunk_tokens_p95": 2930 }, "pressbooks": { "bytes": 1006633, "tokens": 240161, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.191492373865865, "chunk_tokens_p95": 1860 }, "libretexts": { "bytes": 1006632, "tokens": 242804, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.14586250638375, "chunk_tokens_p95": 2126 }, "doab": { "bytes": 1342177, "tokens": 332589, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.035542366103509, "chunk_tokens_p95": 2245 }, "gutenberg": { "bytes": 1677721, "tokens": 464444, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6123213993506216, "chunk_tokens_p95": 2799 }, "pre_1929_books": { "bytes": 2348810, "tokens": 592152, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9665660168335157, "chunk_tokens_p95": 2410 }, "library_of_congress": { "bytes": 1006632, "tokens": 257923, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9028392194569697, "chunk_tokens_p95": 2397 } }, "bytes": 33554432, "tokens": 8910144, "bytes_per_token": 3.765868654872469, "evaluation_seconds": 17.980279679000006, "evaluation_mib_per_second": 1.7797275999757818, "vocabulary_used": 16311, "vocabulary_use_fraction": 0.99554443359375, "vocabulary_unseen": 73, "vocabulary_used_at_most_10_times": 191, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 9, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 29, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 8, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 7, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 6, "123456789": 4, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "e2dd656cce8fd693c83436fb691422442a04a5bf4dbf1cf613ec2dfb0ee42654", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.7443259960979614, 3.7909239993237693 ], "raw_result_sha256": "c24674878a3f53012b51d7ed72b6b9634399be97911e7aa0f5437995aa7c5460", "vs_64mib_same_vocab": { "token_saving_percent": 0.5202783511508269, "paired_ci95": [ 0.47699086385536854, 0.5633908637542873 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 1024, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 107374183, "code_python": 322122548, "finemath_4plus": 214748365, "pes2o": 107374183, "pubmed": 32212255, "arxiv": 21474837, "pressbooks": 32212255, "libretexts": 32212254, "doab": 42949672, "gutenberg": 53687091, "pre_1929_books": 75161927, "library_of_congress": 32212254 }, "training_seconds": 72.43962861700001, "peak_process_rss_mib": 2347.41796875, "tokenizer_sha256": "17ce952fb54679634b5a723756a217bb536ad1d2f6616917ac0172b7a6ef255c", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 742197, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.520961415904403, "chunk_tokens_p95": 1839 }, "code_python": { "bytes": 10066330, "tokens": 2537999, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.966246637607028, "chunk_tokens_p95": 2332 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1874070, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.5809158676036645, "chunk_tokens_p95": 2434 }, "pes2o": { "bytes": 3355444, "tokens": 745154, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.503020852065479, "chunk_tokens_p95": 2126 }, "pubmed": { "bytes": 1006633, "tokens": 247100, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.073787940105221, "chunk_tokens_p95": 2544 }, "arxiv": { "bytes": 671089, "tokens": 179080, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.747425731516641, "chunk_tokens_p95": 2774 }, "pressbooks": { "bytes": 1006633, "tokens": 223410, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.505765185085717, "chunk_tokens_p95": 1732 }, "libretexts": { "bytes": 1006632, "tokens": 228512, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.405160341688839, "chunk_tokens_p95": 1968 }, "doab": { "bytes": 1342177, "tokens": 301804, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.447180951876052, "chunk_tokens_p95": 2023 }, "gutenberg": { "bytes": 1677721, "tokens": 428498, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9153531638420715, "chunk_tokens_p95": 2451 }, "pre_1929_books": { "bytes": 2348810, "tokens": 551299, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.260501107384559, "chunk_tokens_p95": 2254 }, "library_of_congress": { "bytes": 1006632, "tokens": 240110, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.192378493190621, "chunk_tokens_p95": 2206 } }, "bytes": 33554432, "tokens": 8299233, "bytes_per_token": 4.043076269819151, "evaluation_seconds": 18.307522746999993, "evaluation_mib_per_second": 1.7479153483637626, "vocabulary_used": 32507, "vocabulary_use_fraction": 0.992034912109375, "vocabulary_unseen": 261, "vocabulary_used_at_most_10_times": 1685, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 12, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 23, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 6, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 6, "roundtrip": true } ], "fragmentation": { "electromagnetism": 4, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "1076cfae7a52016c78f2ce1e55fdb2658014bbb9cbf1661f5c2b6895a9291929", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.019696868302485, 4.069010372445005 ], "raw_result_sha256": "23a6072a121a13eb7e86099cd831b840f769d2a4b9f90c2200b430d5e64f2705", "vs_64mib_same_vocab": { "token_saving_percent": 0.793865864497012, "paired_ci95": [ 0.7530451926170176, 0.8462764487596336 ] }, "vs_16k_same_size": { "token_saving_percent": 6.856353836705676, "paired_ci95": [ 6.786942985526399, 6.941702923683285 ] } }, { "size_mib": 1024, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 107374183, "code_python": 322122548, "finemath_4plus": 214748365, "pes2o": 107374183, "pubmed": 32212255, "arxiv": 21474837, "pressbooks": 32212255, "libretexts": 32212254, "doab": 42949672, "gutenberg": 53687091, "pre_1929_books": 75161927, "library_of_congress": 32212254 }, "training_seconds": 85.27832251700002, "peak_process_rss_mib": 2405.546875, "tokenizer_sha256": "ce5e83a52cab742ea5edc2440c8defd26c946d05ca2f1d3f64ab3dedc26b744d", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 715090, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.692338027381169, "chunk_tokens_p95": 1788 }, "code_python": { "bytes": 10066330, "tokens": 2453113, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.103492175044525, "chunk_tokens_p95": 2256 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1831760, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.663627877014456, "chunk_tokens_p95": 2371 }, "pes2o": { "bytes": 3355444, "tokens": 719112, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.666093737832215, "chunk_tokens_p95": 2052 }, "pubmed": { "bytes": 1006633, "tokens": 237910, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.23115043503846, "chunk_tokens_p95": 2441 }, "arxiv": { "bytes": 671089, "tokens": 172908, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8811911536770998, "chunk_tokens_p95": 2694 }, "pressbooks": { "bytes": 1006633, "tokens": 215797, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.664721937747049, "chunk_tokens_p95": 1669 }, "libretexts": { "bytes": 1006632, "tokens": 222885, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.516373914799112, "chunk_tokens_p95": 1905 }, "doab": { "bytes": 1342177, "tokens": 288378, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.654228131133443, "chunk_tokens_p95": 1908 }, "gutenberg": { "bytes": 1677721, "tokens": 410567, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.086351314158225, "chunk_tokens_p95": 2291 }, "pre_1929_books": { "bytes": 2348810, "tokens": 533104, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.4059132927158675, "chunk_tokens_p95": 2159 }, "library_of_congress": { "bytes": 1006632, "tokens": 231822, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.342262598027797, "chunk_tokens_p95": 2145 } }, "bytes": 33554432, "tokens": 8032446, "bytes_per_token": 4.177361665425451, "evaluation_seconds": 16.72046845, "evaluation_mib_per_second": 1.9138219778764634, "vocabulary_used": 49103, "vocabulary_use_fraction": 0.98206, "vocabulary_unseen": 897, "vocabulary_used_at_most_10_times": 8880, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 19, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 5, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 3, "deoxyribonucleic": 5, "Schr\u00f6dinger": 3, "HTTPRequestHandler": 2, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "6f580c5480ac6649f55db977d2808c4b90af3b74a397876da9d2821d29c3b9f7", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.152758170068112, 4.203240076240967 ], "raw_result_sha256": "bb5308acb23d33b8db984c7837215346ce33284acc8ad0a9d53d34a17a6faa78", "vs_64mib_same_vocab": { "token_saving_percent": 0.9129026885998148, "paired_ci95": [ 0.8663809973552774, 0.9673188354975673 ] }, "vs_16k_same_size": { "token_saving_percent": 9.850547869933424, "paired_ci95": [ 9.750584481577206, 9.962689507195215 ] } }, { "size_mib": 2048, "requested_vocab_size": 16384, "actual_vocab_size": 16384, "actual_training_bytes": { "fineweb_edu": 214748365, "code_python": 644245095, "finemath_4plus": 429496730, "pes2o": 214748365, "pubmed": 64424510, "arxiv": 42949673, "pressbooks": 64424510, "libretexts": 64424509, "doab": 85899345, "gutenberg": 107374182, "pre_1929_books": 150323855, "library_of_congress": 64424509 }, "training_seconds": 125.62457658700004, "peak_process_rss_mib": 3352.40234375, "tokenizer_sha256": "f95ec3899180fd6aef876f92e8fa1e6ec9cd96c180e22b23ead4e88b3b002561", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 803594, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.175546357986745, "chunk_tokens_p95": 2010 }, "code_python": { "bytes": 10066330, "tokens": 2733636, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6823959005515, "chunk_tokens_p95": 2522 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1976064, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.3960878797447855, "chunk_tokens_p95": 2566 }, "pes2o": { "bytes": 3355444, "tokens": 806194, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.1620800948655035, "chunk_tokens_p95": 2292 }, "pubmed": { "bytes": 1006633, "tokens": 268693, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.7464057493124123, "chunk_tokens_p95": 2691 }, "arxiv": { "bytes": 671089, "tokens": 192144, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.4926357315346825, "chunk_tokens_p95": 2920 }, "pressbooks": { "bytes": 1006633, "tokens": 240230, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.190288473546185, "chunk_tokens_p95": 1857 }, "libretexts": { "bytes": 1006632, "tokens": 242602, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.149314515131779, "chunk_tokens_p95": 2115 }, "doab": { "bytes": 1342177, "tokens": 332651, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.034790215571275, "chunk_tokens_p95": 2248 }, "gutenberg": { "bytes": 1677721, "tokens": 464314, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.6133327877255477, "chunk_tokens_p95": 2717 }, "pre_1929_books": { "bytes": 2348810, "tokens": 591928, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.968067062210269, "chunk_tokens_p95": 2410 }, "library_of_congress": { "bytes": 1006632, "tokens": 257839, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9041107047421066, "chunk_tokens_p95": 2398 } }, "bytes": 33554432, "tokens": 8909889, "bytes_per_token": 3.7659764336009123, "evaluation_seconds": 18.35914522600001, "evaluation_mib_per_second": 1.7430005376656625, "vocabulary_used": 16310, "vocabulary_use_fraction": 0.9954833984375, "vocabulary_unseen": 74, "vocabulary_used_at_most_10_times": 193, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 14, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 8, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 29, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 4, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 10, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 7, "roundtrip": true } ], "fragmentation": { "electromagnetism": 5, "deoxyribonucleic": 6, "Schr\u00f6dinger": 6, "HTTPRequestHandler": 3, "get_user_configuration": 4, "np.linalg.eigh": 6, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "8db85e27082ec9c8e684fb1b7b9cdeab3e96f49942d5f6172f21cd4bb4afd418", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 3.7444001813488446, 3.7914213811223267 ], "raw_result_sha256": "c0c3439eda6da37e6afa45d895be966f1f2724ca26b449eb71c21a8cab92f717", "vs_64mib_same_vocab": { "token_saving_percent": 0.523125367879218, "paired_ci95": [ 0.48756702196798285, 0.5687806589113775 ] }, "vs_16k_same_size": { "token_saving_percent": 0.0, "paired_ci95": [ 0.0, 0.0 ] } }, { "size_mib": 2048, "requested_vocab_size": 32768, "actual_vocab_size": 32768, "actual_training_bytes": { "fineweb_edu": 214748365, "code_python": 644245095, "finemath_4plus": 429496730, "pes2o": 214748365, "pubmed": 64424510, "arxiv": 42949673, "pressbooks": 64424510, "libretexts": 64424509, "doab": 85899345, "gutenberg": 107374182, "pre_1929_books": 150323855, "library_of_congress": 64424509 }, "training_seconds": 136.48117239100003, "peak_process_rss_mib": 3509.57421875, "tokenizer_sha256": "6f8742023692929139e450a3a64802e355649742e7051f9d8a6e6db47fbc8e96", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 742149, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.521253818303332, "chunk_tokens_p95": 1847 }, "code_python": { "bytes": 10066330, "tokens": 2537953, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.9663185252051556, "chunk_tokens_p95": 2335 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1873768, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.581493013009081, "chunk_tokens_p95": 2436 }, "pes2o": { "bytes": 3355444, "tokens": 744976, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.504096776271988, "chunk_tokens_p95": 2130 }, "pubmed": { "bytes": 1006633, "tokens": 247017, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.07515677058664, "chunk_tokens_p95": 2542 }, "arxiv": { "bytes": 671089, "tokens": 178828, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.752706511284586, "chunk_tokens_p95": 2758 }, "pressbooks": { "bytes": 1006633, "tokens": 223411, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.505745017031391, "chunk_tokens_p95": 1726 }, "libretexts": { "bytes": 1006632, "tokens": 228434, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.406664507034855, "chunk_tokens_p95": 1963 }, "doab": { "bytes": 1342177, "tokens": 301431, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.452684030507811, "chunk_tokens_p95": 2014 }, "gutenberg": { "bytes": 1677721, "tokens": 428589, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.914521837937978, "chunk_tokens_p95": 2459 }, "pre_1929_books": { "bytes": 2348810, "tokens": 551192, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.261328176025777, "chunk_tokens_p95": 2256 }, "library_of_congress": { "bytes": 1006632, "tokens": 240038, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.193636007632125, "chunk_tokens_p95": 2197 } }, "bytes": 33554432, "tokens": 8297786, "bytes_per_token": 4.043781317088679, "evaluation_seconds": 17.973626337000013, "evaluation_mib_per_second": 1.7803864061714514, "vocabulary_used": 32532, "vocabulary_use_fraction": 0.9927978515625, "vocabulary_unseen": 236, "vocabulary_used_at_most_10_times": 1640, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 24, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 6, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 6, "roundtrip": true } ], "fragmentation": { "electromagnetism": 4, "deoxyribonucleic": 5, "Schr\u00f6dinger": 5, "HTTPRequestHandler": 3, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "372a33fb69da6f25775535e29a8b29f4e012672ea53b746d1ea2c78b547c14c7", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.020637191383395, 4.069643424675342 ], "raw_result_sha256": "439ceb897a3b169aea3e21aca388f703445bcea9da68faf11fca1aeff83349e4", "vs_64mib_same_vocab": { "token_saving_percent": 0.8111627973694913, "paired_ci95": [ 0.7650781112152172, 0.8572668574809261 ] }, "vs_16k_same_size": { "token_saving_percent": 6.869928458143537, "paired_ci95": [ 6.803539109129546, 6.949242854436372 ] } }, { "size_mib": 2048, "requested_vocab_size": 50000, "actual_vocab_size": 50000, "actual_training_bytes": { "fineweb_edu": 214748365, "code_python": 644245095, "finemath_4plus": 429496730, "pes2o": 214748365, "pubmed": 64424510, "arxiv": 42949673, "pressbooks": 64424510, "libretexts": 64424509, "doab": 85899345, "gutenberg": 107374182, "pre_1929_books": 150323855, "library_of_congress": 64424509 }, "training_seconds": 159.32488262100003, "peak_process_rss_mib": 3531.80859375, "tokenizer_sha256": "5e3de8f77cfa5a5820a96bf0205e7cceee7f27c43e16afdc16d452df7a37f724", "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", "tokenizers_version": "0.22.2", "execution": "modal-cpu", "metrics": { "sources": { "fineweb_edu": { "bytes": 3355444, "tokens": 714778, "chunks": 927, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.6943862290109655, "chunk_tokens_p95": 1785 }, "code_python": { "bytes": 10066330, "tokens": 2452093, "chunks": 2092, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.105199109495439, "chunk_tokens_p95": 2255 }, "finemath_4plus": { "bytes": 6710887, "tokens": 1831641, "chunks": 1789, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.663865899485762, "chunk_tokens_p95": 2362 }, "pes2o": { "bytes": 3355444, "tokens": 718383, "chunks": 422, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.670828791883995, "chunk_tokens_p95": 2049 }, "pubmed": { "bytes": 1006633, "tokens": 237852, "chunks": 128, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.232182197332795, "chunk_tokens_p95": 2459 }, "arxiv": { "bytes": 671089, "tokens": 172915, "chunks": 83, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 3.8810340340629788, "chunk_tokens_p95": 2695 }, "pressbooks": { "bytes": 1006633, "tokens": 215825, "chunks": 359, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.664116761264914, "chunk_tokens_p95": 1664 }, "libretexts": { "bytes": 1006632, "tokens": 222845, "chunks": 285, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.5171845901860035, "chunk_tokens_p95": 1902 }, "doab": { "bytes": 1342177, "tokens": 288273, "chunks": 315, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.65592337818665, "chunk_tokens_p95": 1905 }, "gutenberg": { "bytes": 1677721, "tokens": 410725, "chunks": 210, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.08477935358208, "chunk_tokens_p95": 2294 }, "pre_1929_books": { "bytes": 2348810, "tokens": 532881, "chunks": 291, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.407757078972604, "chunk_tokens_p95": 2157 }, "library_of_congress": { "bytes": 1006632, "tokens": 231766, "chunks": 127, "roundtrip_failures": 0, "unk_tokens": 0, "bytes_per_token": 4.343311788614378, "chunk_tokens_p95": 2121 } }, "bytes": 33554432, "tokens": 8029977, "bytes_per_token": 4.178646090767134, "evaluation_seconds": 14.937971026000014, "evaluation_mib_per_second": 2.1421918642299533, "vocabulary_used": 49193, "vocabulary_use_fraction": 0.98386, "vocabulary_unseen": 807, "vocabulary_used_at_most_10_times": 8918, "roundtrip_failures": 0, "unk_tokens": 0, "probes": [ { "text": "def square(x):\r\n\treturn x ** 2\r\n", "tokens": 10, "roundtrip": true }, { "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", "tokens": 11, "roundtrip": true }, { "text": "12345678901234567890", "tokens": 7, "roundtrip": true }, { "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", "tokens": 17, "roundtrip": true }, { "text": "\u0000\t \r\n", "tokens": 3, "roundtrip": true }, { "text": "HTTPRequestHandler snake_case_identifier", "tokens": 5, "roundtrip": true }, { "text": "e\u0301 \u2260 \u00e9", "tokens": 5, "roundtrip": true } ], "fragmentation": { "electromagnetism": 3, "deoxyribonucleic": 5, "Schr\u00f6dinger": 3, "HTTPRequestHandler": 2, "get_user_configuration": 3, "np.linalg.eigh": 4, "123456789": 3, "0.000001": 4, "\\frac{a}{b}": 5 } }, "correctness_passed": true, "requested_vocabulary_reached": true, "original_result_sha256": "896cdc975e9e70a05a843b6aeb54c356d697455617bfffc58c774909cc35dd87", "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", "evaluation_resource_requests": { "cpu": 2, "memory_mib": 8192 }, "evaluation_checks": { "byte_alphabet_complete": true, "special_ids_correct": true, "roundtrips_passed": true, "zero_unk": true }, "bytes_per_token_ci95": [ 4.154437229317744, 4.20486087426906 ], "raw_result_sha256": "cbcf97dbcf6b653f439c2822023c96459562673d96581ecffe6a602795e33dfc", "vs_64mib_same_vocab": { "token_saving_percent": 0.943359917103026, "paired_ci95": [ 0.8973001236409844, 0.9945821428537949 ] }, "vs_16k_same_size": { "token_saving_percent": 9.875678585894843, "paired_ci95": [ 9.771961349789661, 9.989392591793916 ] } } ]