| [ |
| { |
| "size_mib": 64, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 6710887, |
| "code_python": 20132660, |
| "finemath_4plus": 13421773, |
| "pes2o": 6710887, |
| "pubmed": 2013266, |
| "arxiv": 1342178, |
| "pressbooks": 2013266, |
| "libretexts": 2013265, |
| "doab": 2684354, |
| "gutenberg": 3355443, |
| "pre_1929_books": 4697620, |
| "library_of_congress": 2013265 |
| }, |
| "training_seconds": 8.776563889999998, |
| "peak_process_rss_mib": 409.15234375, |
| "tokenizer_sha256": "97fa4177ad5a038cc18df87a2acb76c3ed3da89a9b7848517442bcbd7029d83d", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 809464, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.145266497336509, |
| "chunk_tokens_p95": 2005 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2750571, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.659723744633387, |
| "chunk_tokens_p95": 2547 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1986189, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3787756351485183, |
| "chunk_tokens_p95": 2585 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 810480, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.140070081926759, |
| "chunk_tokens_p95": 2302 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 270152, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.7261726731617757, |
| "chunk_tokens_p95": 2723 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192711, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.482359595456409, |
| "chunk_tokens_p95": 2928 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 241100, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.175167980091248, |
| "chunk_tokens_p95": 1871 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 244293, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.1205928945978805, |
| "chunk_tokens_p95": 2133 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 333249, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.027549970142446, |
| "chunk_tokens_p95": 2238 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 465276, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6058618970245617, |
| "chunk_tokens_p95": 2807 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 594333, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9520100684296513, |
| "chunk_tokens_p95": 2421 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 258926, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8877208159860346, |
| "chunk_tokens_p95": 2383 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8956744, |
| "bytes_per_token": 3.7462756555283927, |
| "evaluation_seconds": 14.238836671000001, |
| "evaluation_mib_per_second": 2.2473746092736535, |
| "vocabulary_used": 16248, |
| "vocabulary_use_fraction": 0.99169921875, |
| "vocabulary_unseen": 136, |
| "vocabulary_used_at_most_10_times": 330, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 9, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 31, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 7, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 6, |
| "123456789": 4, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "7a1bdf5cd3fbf82ee03ed76ea047edf9b155823a3470cec494b21e8f5fd23840", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.7253234702170097, |
| 3.7711438880078716 |
| ], |
| "raw_result_sha256": "d09833d7252f1c57b21e523aacdd02633e2bf5eba311506133b5df942d4e8e34", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 64, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 6710887, |
| "code_python": 20132660, |
| "finemath_4plus": 13421773, |
| "pes2o": 6710887, |
| "pubmed": 2013266, |
| "arxiv": 1342178, |
| "pressbooks": 2013266, |
| "libretexts": 2013265, |
| "doab": 2684354, |
| "gutenberg": 3355443, |
| "pre_1929_books": 4697620, |
| "library_of_congress": 2013265 |
| }, |
| "training_seconds": 9.810118514, |
| "peak_process_rss_mib": 422.26171875, |
| "tokenizer_sha256": "cd9109d658834401ef05b4de231cdfd789385bb0d9f294e343e19b447d94b52a", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 748726, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4815379725026245, |
| "chunk_tokens_p95": 1858 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2563077, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9274395580000134, |
| "chunk_tokens_p95": 2356 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1885335, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5595196609621103, |
| "chunk_tokens_p95": 2441 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 751239, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4665465983528545, |
| "chunk_tokens_p95": 2143 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 250154, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.024053183239125, |
| "chunk_tokens_p95": 2566 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 180650, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.714857459175201, |
| "chunk_tokens_p95": 2800 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 224760, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.478701726285816, |
| "chunk_tokens_p95": 1751 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 230366, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.369707335283852, |
| "chunk_tokens_p95": 1999 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 304345, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.410051093331581, |
| "chunk_tokens_p95": 2026 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 430852, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8939612674421844, |
| "chunk_tokens_p95": 2466 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 554432, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.236425747431606, |
| "chunk_tokens_p95": 2263 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 241709, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.164644262315429, |
| "chunk_tokens_p95": 2228 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8365645, |
| "bytes_per_token": 4.010979667437478, |
| "evaluation_seconds": 18.018589408999997, |
| "evaluation_mib_per_second": 1.7759436809196902, |
| "vocabulary_used": 31820, |
| "vocabulary_use_fraction": 0.9710693359375, |
| "vocabulary_unseen": 948, |
| "vocabulary_used_at_most_10_times": 2674, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 12, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 27, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 3, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "1b741432a4962272e45ebe2713a4b75ee6145f5afaefd8eec10e4e5fd64cde1c", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.988442103265165, |
| 4.0371958036541455 |
| ], |
| "raw_result_sha256": "dac5ec5cc5c61a2fb5e3758320f1486c174ec7378d1b37b8628c7f642241dc7e", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.599485259375504, |
| "paired_ci95": [ |
| 6.532287848969547, |
| 6.680290527180766 |
| ] |
| } |
| }, |
| { |
| "size_mib": 64, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 6710887, |
| "code_python": 20132660, |
| "finemath_4plus": 13421773, |
| "pes2o": 6710887, |
| "pubmed": 2013266, |
| "arxiv": 1342178, |
| "pressbooks": 2013266, |
| "libretexts": 2013265, |
| "doab": 2684354, |
| "gutenberg": 3355443, |
| "pre_1929_books": 4697620, |
| "library_of_congress": 2013265 |
| }, |
| "training_seconds": 10.408797823, |
| "peak_process_rss_mib": 426.9296875, |
| "tokenizer_sha256": "86de4d58e6eb5417264afe227857e98c0b0431eb7adbfaf6a9ebacac37d77726", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 722478, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.644354568582019, |
| "chunk_tokens_p95": 1796 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2480835, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.057637851771681, |
| "chunk_tokens_p95": 2276 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1843677, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.639947235877, |
| "chunk_tokens_p95": 2390 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 726566, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.6182232584514, |
| "chunk_tokens_p95": 2082 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 241172, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.173921516593966, |
| "chunk_tokens_p95": 2479 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 174905, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.836877161887882, |
| "chunk_tokens_p95": 2743 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 217176, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.63510240542233, |
| "chunk_tokens_p95": 1683 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 224653, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4808304362728295, |
| "chunk_tokens_p95": 1941 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 291070, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.611182876971175, |
| "chunk_tokens_p95": 1922 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 414289, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.049639261481719, |
| "chunk_tokens_p95": 2338 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 536337, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.37935477134712, |
| "chunk_tokens_p95": 2179 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 233292, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.314901496836582, |
| "chunk_tokens_p95": 2146 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8106450, |
| "bytes_per_token": 4.139226418469244, |
| "evaluation_seconds": 20.04189617, |
| "evaluation_mib_per_second": 1.5966553128790109, |
| "vocabulary_used": 47029, |
| "vocabulary_use_fraction": 0.94058, |
| "vocabulary_unseen": 2971, |
| "vocabulary_used_at_most_10_times": 9496, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 26, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 2, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "8568773312de4a3c65765cd84df08d828ec4631c36ec91f878103783c7fae2c3", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.115876574322902, |
| 4.165474937669827 |
| ], |
| "raw_result_sha256": "5ebb798a1720c5d288af67491d12e5daa8aae0c64e2bb3a4a2592cfa7c161fdb", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.49333820415098, |
| "paired_ci95": [ |
| 9.408400244317894, |
| 9.605081462142396 |
| ] |
| } |
| }, |
| { |
| "size_mib": 128, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 13421773, |
| "code_python": 40265319, |
| "finemath_4plus": 26843546, |
| "pes2o": 13421773, |
| "pubmed": 4026532, |
| "arxiv": 2684355, |
| "pressbooks": 4026532, |
| "libretexts": 4026532, |
| "doab": 5368709, |
| "gutenberg": 6710886, |
| "pre_1929_books": 9395240, |
| "library_of_congress": 4026531 |
| }, |
| "training_seconds": 13.764946394000006, |
| "peak_process_rss_mib": 618.10546875, |
| "tokenizer_sha256": "64356327337221b75b445cc7c761d655a40c1af2bccd8f8ab5453fd1fa1035c0", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 805884, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.163681125323247, |
| "chunk_tokens_p95": 2005 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2740723, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.672873909548685, |
| "chunk_tokens_p95": 2525 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1982520, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3850286504045357, |
| "chunk_tokens_p95": 2582 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 808314, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.151164027840666, |
| "chunk_tokens_p95": 2289 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 269475, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.7355339085258374, |
| "chunk_tokens_p95": 2714 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192339, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4890947753705697, |
| "chunk_tokens_p95": 2915 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 240330, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.188544917405235, |
| "chunk_tokens_p95": 1865 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 243558, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.133027861946641, |
| "chunk_tokens_p95": 2140 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 332428, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.037496841421301, |
| "chunk_tokens_p95": 2233 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 464540, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6115748912903087, |
| "chunk_tokens_p95": 2821 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 592602, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9635539535809867, |
| "chunk_tokens_p95": 2409 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 258319, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8968562126672834, |
| "chunk_tokens_p95": 2379 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8931032, |
| "bytes_per_token": 3.757060998101899, |
| "evaluation_seconds": 14.213449730000004, |
| "evaluation_mib_per_second": 2.2513886922509974, |
| "vocabulary_used": 16298, |
| "vocabulary_use_fraction": 0.9947509765625, |
| "vocabulary_unseen": 86, |
| "vocabulary_used_at_most_10_times": 243, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 8, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 30, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 6, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 6, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "145aa86a48dce6b0ebb6dbe6629ce827e673e0482cb43deb17beade3bcec41a5", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.7362385971701717, |
| 3.7822919781602184 |
| ], |
| "raw_result_sha256": "b41b8fb8fb102d965cecf80e38bc60a77013192d1ee549632d571e31d4b79a75", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.2870686043946358, |
| "paired_ci95": [ |
| 0.2571423946282611, |
| 0.31525380838660216 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 128, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 13421773, |
| "code_python": 40265319, |
| "finemath_4plus": 26843546, |
| "pes2o": 13421773, |
| "pubmed": 4026532, |
| "arxiv": 2684355, |
| "pressbooks": 4026532, |
| "libretexts": 4026532, |
| "doab": 5368709, |
| "gutenberg": 6710886, |
| "pre_1929_books": 9395240, |
| "library_of_congress": 4026531 |
| }, |
| "training_seconds": 15.621106757000007, |
| "peak_process_rss_mib": 655.171875, |
| "tokenizer_sha256": "ade4e945319da6e9b010ad7f48f19d9489f7ac583448318a91cf302c21aaf321", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 745385, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.501625334558651, |
| "chunk_tokens_p95": 1851 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2551578, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9451390472876002, |
| "chunk_tokens_p95": 2351 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1881693, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.56640907948321, |
| "chunk_tokens_p95": 2424 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 748062, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4855159064355625, |
| "chunk_tokens_p95": 2134 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 248357, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.053169429490612, |
| "chunk_tokens_p95": 2542 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 180462, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.7187274883354946, |
| "chunk_tokens_p95": 2800 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 223762, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.498677165917359, |
| "chunk_tokens_p95": 1737 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 229650, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.383331156107119, |
| "chunk_tokens_p95": 1986 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 302982, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4298902245017855, |
| "chunk_tokens_p95": 2027 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 429461, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.906573588754276, |
| "chunk_tokens_p95": 2460 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 552719, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.2495553798584815, |
| "chunk_tokens_p95": 2259 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 240838, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.179705860370872, |
| "chunk_tokens_p95": 2213 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8334949, |
| "bytes_per_token": 4.025751327332657, |
| "evaluation_seconds": 17.542917185000004, |
| "evaluation_mib_per_second": 1.8240979913740607, |
| "vocabulary_used": 32155, |
| "vocabulary_use_fraction": 0.981292724609375, |
| "vocabulary_unseen": 613, |
| "vocabulary_used_at_most_10_times": 2279, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 13, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 27, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 6, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 6, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 4, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "d2fa50e315c0349117a93e5f45fe45a411c7ab4fa9f5ff83af93173359bc367e", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.0024713775611165, |
| 4.052006976084268 |
| ], |
| "raw_result_sha256": "2915a4d8951cafb394639013f01ed07bfa685dd6c9bf97179e416753a49b0e48", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.3669292684544967, |
| "paired_ci95": [ |
| 0.33687187101464344, |
| 0.4009128108909549 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.674290272389571, |
| "paired_ci95": [ |
| 6.604578111377216, |
| 6.75937713765991 |
| ] |
| } |
| }, |
| { |
| "size_mib": 128, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 13421773, |
| "code_python": 40265319, |
| "finemath_4plus": 26843546, |
| "pes2o": 13421773, |
| "pubmed": 4026532, |
| "arxiv": 2684355, |
| "pressbooks": 4026532, |
| "libretexts": 4026532, |
| "doab": 5368709, |
| "gutenberg": 6710886, |
| "pre_1929_books": 9395240, |
| "library_of_congress": 4026531 |
| }, |
| "training_seconds": 14.651732101999997, |
| "peak_process_rss_mib": 645.515625, |
| "tokenizer_sha256": "db13f817c797466237e07a17ca3d62b65dafe334745d6c72714073fa47d36380", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 719232, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.665315225129026, |
| "chunk_tokens_p95": 1787 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2469423, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.076389504754754, |
| "chunk_tokens_p95": 2265 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1839706, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.647804051299501, |
| "chunk_tokens_p95": 2382 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 723431, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.63823640402471, |
| "chunk_tokens_p95": 2068 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 239461, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.203745077486522, |
| "chunk_tokens_p95": 2453 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 174403, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.847921194016158, |
| "chunk_tokens_p95": 2742 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 216331, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.653207353546186, |
| "chunk_tokens_p95": 1666 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 223877, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.49636184154692, |
| "chunk_tokens_p95": 1939 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 290318, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.6231270537824045, |
| "chunk_tokens_p95": 1929 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 412020, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.0719406824911415, |
| "chunk_tokens_p95": 2297 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 534890, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.3912019293686555, |
| "chunk_tokens_p95": 2172 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 232714, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.325618570434095, |
| "chunk_tokens_p95": 2140 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8075806, |
| "bytes_per_token": 4.1549328946237685, |
| "evaluation_seconds": 18.716767812, |
| "evaluation_mib_per_second": 1.709696905011753, |
| "vocabulary_used": 47778, |
| "vocabulary_use_fraction": 0.95556, |
| "vocabulary_unseen": 2222, |
| "vocabulary_used_at_most_10_times": 9196, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 12, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 24, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 5, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 2, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 2, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "654d31ba0555fa7d894725c41b4969e986bd4992c145ab5cf107d2183e437600", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.131537559055759, |
| 4.1815624765901855 |
| ], |
| "raw_result_sha256": "ae3b85d78322d9de54e8a94b59af2abf62cc6119a680ee32bd47d176455c4b5e", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.37801997175087587, |
| "paired_ci95": [ |
| 0.3492808572410605, |
| 0.40876432511092187 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.575892237313665, |
| "paired_ci95": [ |
| 9.4852345035617, |
| 9.68628583930954 |
| ] |
| } |
| }, |
| { |
| "size_mib": 256, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 26843546, |
| "code_python": 80530637, |
| "finemath_4plus": 53687092, |
| "pes2o": 26843546, |
| "pubmed": 8053064, |
| "arxiv": 5368710, |
| "pressbooks": 8053064, |
| "libretexts": 8053063, |
| "doab": 10737418, |
| "gutenberg": 13421772, |
| "pre_1929_books": 18790481, |
| "library_of_congress": 8053063 |
| }, |
| "training_seconds": 22.75400429300001, |
| "peak_process_rss_mib": 908.1640625, |
| "tokenizer_sha256": "e76d8e8a71db29a3e1ed036d8e6813ce180ac568b0310d0cdc01a2c8974c92e6", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 804027, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.17329766288943, |
| "chunk_tokens_p95": 2007 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2736914, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6779854975348147, |
| "chunk_tokens_p95": 2524 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1978378, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3921156624264928, |
| "chunk_tokens_p95": 2579 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 807381, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.155961064231137, |
| "chunk_tokens_p95": 2291 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 268775, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.745262766254302, |
| "chunk_tokens_p95": 2691 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192437, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.487317927425599, |
| "chunk_tokens_p95": 2930 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 240176, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.191230597561788, |
| "chunk_tokens_p95": 1861 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 243316, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.137138535895708, |
| "chunk_tokens_p95": 2128 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 332243, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.039745005914346, |
| "chunk_tokens_p95": 2229 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 463963, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6160663673611904, |
| "chunk_tokens_p95": 2817 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 592540, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9639686772201035, |
| "chunk_tokens_p95": 2404 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 258258, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8977766419626887, |
| "chunk_tokens_p95": 2387 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8918408, |
| "bytes_per_token": 3.7623791151963446, |
| "evaluation_seconds": 14.001287165, |
| "evaluation_mib_per_second": 2.285504155645964, |
| "vocabulary_used": 16304, |
| "vocabulary_use_fraction": 0.9951171875, |
| "vocabulary_unseen": 80, |
| "vocabulary_used_at_most_10_times": 215, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 8, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 29, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 9, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 7, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 6, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "16e9f72fd49ce84c4f68342e960bbf5e649c4a70ee0c10f5fbc178def20faa05", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.7410346836920847, |
| 3.78745201861199 |
| ], |
| "raw_result_sha256": "90c567655c105853165ee9b9f8257245a7d38fc99e66fcb18501cfc406ee5e14", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.4280126796076922, |
| "paired_ci95": [ |
| 0.3879827586257448, |
| 0.4753906917193296 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 256, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 26843546, |
| "code_python": 80530637, |
| "finemath_4plus": 53687092, |
| "pes2o": 26843546, |
| "pubmed": 8053064, |
| "arxiv": 5368710, |
| "pressbooks": 8053064, |
| "libretexts": 8053063, |
| "doab": 10737418, |
| "gutenberg": 13421772, |
| "pre_1929_books": 18790481, |
| "library_of_congress": 8053063 |
| }, |
| "training_seconds": 26.727447731000012, |
| "peak_process_rss_mib": 974.83203125, |
| "tokenizer_sha256": "3839415fcdd736bb9322b958a45ba82ba01310e2007a1b139f77e3486568c1b6", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 742828, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.517121056287593, |
| "chunk_tokens_p95": 1848 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2546847, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9524675019740094, |
| "chunk_tokens_p95": 2357 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1877830, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5737457597333093, |
| "chunk_tokens_p95": 2439 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 746903, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.492476265324949, |
| "chunk_tokens_p95": 2131 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 247495, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.067286207802177, |
| "chunk_tokens_p95": 2540 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 179690, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.734704212810952, |
| "chunk_tokens_p95": 2797 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 223451, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.5049384428800945, |
| "chunk_tokens_p95": 1726 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 229065, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.394525571344378, |
| "chunk_tokens_p95": 1971 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 302492, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.43706610422755, |
| "chunk_tokens_p95": 2018 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 429061, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.910215563754338, |
| "chunk_tokens_p95": 2451 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 552046, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.25473601837528, |
| "chunk_tokens_p95": 2258 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 240555, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.184623059175657, |
| "chunk_tokens_p95": 2206 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8318263, |
| "bytes_per_token": 4.033826773690613, |
| "evaluation_seconds": 18.389478011999998, |
| "evaluation_mib_per_second": 1.740125520643843, |
| "vocabulary_used": 32356, |
| "vocabulary_use_fraction": 0.9874267578125, |
| "vocabulary_unseen": 412, |
| "vocabulary_used_at_most_10_times": 2027, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 13, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 22, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 4, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "52408ec872ddc2d20e07fae1180f043b4b68fcfd01425ff095752562c41491c5", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.010703911098985, |
| 4.059882958848434 |
| ], |
| "raw_result_sha256": "47255252cac983588d502289e862a109d82b7ba59e1c571f7183108c965c9355", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.5663878876046136, |
| "paired_ci95": [ |
| 0.5291951126970479, |
| 0.607556610571848 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.729283970861166, |
| "paired_ci95": [ |
| 6.662042324517174, |
| 6.817499035195743 |
| ] |
| } |
| }, |
| { |
| "size_mib": 256, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 26843546, |
| "code_python": 80530637, |
| "finemath_4plus": 53687092, |
| "pes2o": 26843546, |
| "pubmed": 8053064, |
| "arxiv": 5368710, |
| "pressbooks": 8053064, |
| "libretexts": 8053063, |
| "doab": 10737418, |
| "gutenberg": 13421772, |
| "pre_1929_books": 18790481, |
| "library_of_congress": 8053063 |
| }, |
| "training_seconds": 28.223051318999993, |
| "peak_process_rss_mib": 976.99609375, |
| "tokenizer_sha256": "eb4b69846169b0aedd86c72eb2bc7e85b96b074519f7bba8563423cd200baf81", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 716688, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.681875516263702, |
| "chunk_tokens_p95": 1786 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2461951, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.088761311658924, |
| "chunk_tokens_p95": 2264 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1836046, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6550756353598985, |
| "chunk_tokens_p95": 2366 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 720689, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.655883467071095, |
| "chunk_tokens_p95": 2052 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 238666, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.217747814938031, |
| "chunk_tokens_p95": 2461 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 173666, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8642509184296294, |
| "chunk_tokens_p95": 2713 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 216493, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.649725395278369, |
| "chunk_tokens_p95": 1668 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 223521, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.5035231588978215, |
| "chunk_tokens_p95": 1914 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 289529, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.635725609524434, |
| "chunk_tokens_p95": 1911 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 411466, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.077423164975964, |
| "chunk_tokens_p95": 2314 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 534248, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.396478788877076, |
| "chunk_tokens_p95": 2170 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 232261, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.334055222357606, |
| "chunk_tokens_p95": 2145 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8055224, |
| "bytes_per_token": 4.165549213777295, |
| "evaluation_seconds": 14.800683191999994, |
| "evaluation_mib_per_second": 2.1620623578576774, |
| "vocabulary_used": 48420, |
| "vocabulary_use_fraction": 0.9684, |
| "vocabulary_unseen": 1580, |
| "vocabulary_used_at_most_10_times": 9052, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 18, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 5, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 3, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 2, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "4b14d9a99be20c88aaab57029cd4634710350665995d74a357ab45a32f3b42f6", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.141883517407929, |
| 4.1921418306820035 |
| ], |
| "raw_result_sha256": "d26837e3e6f7f669a0d3de13788bb1c9a52e99cce56d12d631f4c16c3a403331", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.6319165602698851, |
| "paired_ci95": [ |
| 0.5918921376973765, |
| 0.6721510169045986 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.67867807797086, |
| "paired_ci95": [ |
| 9.576043326252233, |
| 9.790785308891813 |
| ] |
| } |
| }, |
| { |
| "size_mib": 512, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 53687092, |
| "code_python": 161061274, |
| "finemath_4plus": 107374183, |
| "pes2o": 53687092, |
| "pubmed": 16106128, |
| "arxiv": 10737418, |
| "pressbooks": 16106127, |
| "libretexts": 16106127, |
| "doab": 21474836, |
| "gutenberg": 26843545, |
| "pre_1929_books": 37580963, |
| "library_of_congress": 16106127 |
| }, |
| "training_seconds": 40.378172683, |
| "peak_process_rss_mib": 1469.3984375, |
| "tokenizer_sha256": "c3bae921b15a294391e45fe3bf2ce0a2f3ac1b6678c0f32541613106f322078e", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 803626, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.175380089743239, |
| "chunk_tokens_p95": 2004 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2734437, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6813172144759596, |
| "chunk_tokens_p95": 2520 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1977334, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.393906643996411, |
| "chunk_tokens_p95": 2565 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 807061, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.157608904407473, |
| "chunk_tokens_p95": 2284 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 268706, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.746224498150395, |
| "chunk_tokens_p95": 2689 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192058, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.494199668850035, |
| "chunk_tokens_p95": 2929 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 239977, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.19470615934027, |
| "chunk_tokens_p95": 1861 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 242822, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.145555180337861, |
| "chunk_tokens_p95": 2123 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 332709, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.034086844660048, |
| "chunk_tokens_p95": 2240 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 464206, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6141734488567576, |
| "chunk_tokens_p95": 2804 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 592079, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.967055072042751, |
| "chunk_tokens_p95": 2402 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 258039, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9010847197516654, |
| "chunk_tokens_p95": 2396 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8913054, |
| "bytes_per_token": 3.764639146133301, |
| "evaluation_seconds": 18.321253727, |
| "evaluation_mib_per_second": 1.7466053621014843, |
| "vocabulary_used": 16307, |
| "vocabulary_use_fraction": 0.99530029296875, |
| "vocabulary_unseen": 77, |
| "vocabulary_used_at_most_10_times": 195, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 8, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 30, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 9, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 7, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 5, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "989cc3f9196c5667734d4bf3d9aa9812a9e1f109f33c5ebaa67f6954a3f3fee9", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.743055062261041, |
| 3.789654631203881 |
| ], |
| "raw_result_sha256": "75a63070c0c5a8f67c3043b7e5d57090e9f8e27c2c34181c1ede4c2b4535594f", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.4877888661326102, |
| "paired_ci95": [ |
| 0.45379665743235653, |
| 0.5309888029433135 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 512, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 53687092, |
| "code_python": 161061274, |
| "finemath_4plus": 107374183, |
| "pes2o": 53687092, |
| "pubmed": 16106128, |
| "arxiv": 10737418, |
| "pressbooks": 16106127, |
| "libretexts": 16106127, |
| "doab": 21474836, |
| "gutenberg": 26843545, |
| "pre_1929_books": 37580963, |
| "library_of_congress": 16106127 |
| }, |
| "training_seconds": 46.02036403599999, |
| "peak_process_rss_mib": 1471.63671875, |
| "tokenizer_sha256": "ee276c0fba28e5d4f555f85971e9d99b5cee6b53aa5181bc1c4dd33417572829", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 742671, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.51807597172907, |
| "chunk_tokens_p95": 1839 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2541276, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9611321241769883, |
| "chunk_tokens_p95": 2335 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1875724, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.577758241617637, |
| "chunk_tokens_p95": 2437 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 745660, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.499965131561301, |
| "chunk_tokens_p95": 2130 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 247341, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.069818590528865, |
| "chunk_tokens_p95": 2545 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 179217, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.7445610628456008, |
| "chunk_tokens_p95": 2799 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 223376, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.506451006374902, |
| "chunk_tokens_p95": 1733 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 228617, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.403137124535796, |
| "chunk_tokens_p95": 1982 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 302247, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.440662769192085, |
| "chunk_tokens_p95": 2024 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 428638, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9140743471180808, |
| "chunk_tokens_p95": 2461 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 551542, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.258624003249072, |
| "chunk_tokens_p95": 2252 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 240251, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.189918044045602, |
| "chunk_tokens_p95": 2209 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8306560, |
| "bytes_per_token": 4.039509977656214, |
| "evaluation_seconds": 19.212221627999995, |
| "evaluation_mib_per_second": 1.6656064363406586, |
| "vocabulary_used": 32469, |
| "vocabulary_use_fraction": 0.990875244140625, |
| "vocabulary_unseen": 299, |
| "vocabulary_used_at_most_10_times": 1812, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 12, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 24, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 6, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 4, |
| "deoxyribonucleic": 6, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "7d09541cfba364ac660f4664e2cbe47e96fb7294701032766a872dfc0434d901", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.015809758793044, |
| 4.065744954151812 |
| ], |
| "raw_result_sha256": "fe60794938bfefca360c98c7bf113d9c48089273a39d9f5e5d5e8d4d80b87d90", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.7062814642505133, |
| "paired_ci95": [ |
| 0.6689432884540671, |
| 0.7519069204712292 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.804558796569604, |
| "paired_ci95": [ |
| 6.735751471831152, |
| 6.893860773834811 |
| ] |
| } |
| }, |
| { |
| "size_mib": 512, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 53687092, |
| "code_python": 161061274, |
| "finemath_4plus": 107374183, |
| "pes2o": 53687092, |
| "pubmed": 16106128, |
| "arxiv": 10737418, |
| "pressbooks": 16106127, |
| "libretexts": 16106127, |
| "doab": 21474836, |
| "gutenberg": 26843545, |
| "pre_1929_books": 37580963, |
| "library_of_congress": 16106127 |
| }, |
| "training_seconds": 41.69725704000001, |
| "peak_process_rss_mib": 1463.0546875, |
| "tokenizer_sha256": "1f4b70d34b608d97a3ad7a9bcd7b0094fa82c652281d39bcc12325c1f16865e2", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 715922, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.686884884107487, |
| "chunk_tokens_p95": 1790 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2455669, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.099221026938077, |
| "chunk_tokens_p95": 2255 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1833707, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.659737897057709, |
| "chunk_tokens_p95": 2373 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 719660, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.662540644193091, |
| "chunk_tokens_p95": 2054 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 238078, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.228164719125664, |
| "chunk_tokens_p95": 2444 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 173141, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.875968141572476, |
| "chunk_tokens_p95": 2701 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 215986, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.660640041484171, |
| "chunk_tokens_p95": 1673 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 223088, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.512264218604318, |
| "chunk_tokens_p95": 1909 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 288858, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.646494125141073, |
| "chunk_tokens_p95": 1912 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 411047, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.08157947874574, |
| "chunk_tokens_p95": 2305 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 533677, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.401182737873283, |
| "chunk_tokens_p95": 2165 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 232106, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.336949497212481, |
| "chunk_tokens_p95": 2150 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8040939, |
| "bytes_per_token": 4.1729494527940085, |
| "evaluation_seconds": 15.270844398000008, |
| "evaluation_mib_per_second": 2.095496435298036, |
| "vocabulary_used": 48885, |
| "vocabulary_use_fraction": 0.9777, |
| "vocabulary_unseen": 1115, |
| "vocabulary_used_at_most_10_times": 8957, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 19, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 5, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 2, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 2, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "748f29ffa5967e98c916a1f29b1f1678a420ba6432d6f7e56d94cabfcbd27f62", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.14909233442202, |
| 4.199119965141136 |
| ], |
| "raw_result_sha256": "e16aecacc179b080563cd8bdded280cb5607c3b231daba10a13937a90b7ff4ba", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.8081342634568722, |
| "paired_ci95": [ |
| 0.7629974315430208, |
| 0.8614506680344802 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.784693327337635, |
| "paired_ci95": [ |
| 9.686796230035283, |
| 9.901877722230006 |
| ] |
| } |
| }, |
| { |
| "size_mib": 1024, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 107374183, |
| "code_python": 322122548, |
| "finemath_4plus": 214748365, |
| "pes2o": 107374183, |
| "pubmed": 32212255, |
| "arxiv": 21474837, |
| "pressbooks": 32212255, |
| "libretexts": 32212254, |
| "doab": 42949672, |
| "gutenberg": 53687091, |
| "pre_1929_books": 75161927, |
| "library_of_congress": 32212254 |
| }, |
| "training_seconds": 69.90681187199996, |
| "peak_process_rss_mib": 2280.640625, |
| "tokenizer_sha256": "020928562c47ea392514da50b9c4886b4e3e24fd01cfdc30501599042ffb1359", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 803728, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.174850198076961, |
| "chunk_tokens_p95": 2008 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2732020, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6845740514344696, |
| "chunk_tokens_p95": 2520 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1976812, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.394802844175369, |
| "chunk_tokens_p95": 2570 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 806536, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.160315224614896, |
| "chunk_tokens_p95": 2281 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 268784, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.745137359366629, |
| "chunk_tokens_p95": 2684 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192191, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4917816130828188, |
| "chunk_tokens_p95": 2930 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 240161, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.191492373865865, |
| "chunk_tokens_p95": 1860 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 242804, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.14586250638375, |
| "chunk_tokens_p95": 2126 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 332589, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.035542366103509, |
| "chunk_tokens_p95": 2245 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 464444, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6123213993506216, |
| "chunk_tokens_p95": 2799 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 592152, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9665660168335157, |
| "chunk_tokens_p95": 2410 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 257923, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9028392194569697, |
| "chunk_tokens_p95": 2397 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8910144, |
| "bytes_per_token": 3.765868654872469, |
| "evaluation_seconds": 17.980279679000006, |
| "evaluation_mib_per_second": 1.7797275999757818, |
| "vocabulary_used": 16311, |
| "vocabulary_use_fraction": 0.99554443359375, |
| "vocabulary_unseen": 73, |
| "vocabulary_used_at_most_10_times": 191, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 9, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 29, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 8, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 7, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 6, |
| "123456789": 4, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "e2dd656cce8fd693c83436fb691422442a04a5bf4dbf1cf613ec2dfb0ee42654", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.7443259960979614, |
| 3.7909239993237693 |
| ], |
| "raw_result_sha256": "c24674878a3f53012b51d7ed72b6b9634399be97911e7aa0f5437995aa7c5460", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.5202783511508269, |
| "paired_ci95": [ |
| 0.47699086385536854, |
| 0.5633908637542873 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 1024, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 107374183, |
| "code_python": 322122548, |
| "finemath_4plus": 214748365, |
| "pes2o": 107374183, |
| "pubmed": 32212255, |
| "arxiv": 21474837, |
| "pressbooks": 32212255, |
| "libretexts": 32212254, |
| "doab": 42949672, |
| "gutenberg": 53687091, |
| "pre_1929_books": 75161927, |
| "library_of_congress": 32212254 |
| }, |
| "training_seconds": 72.43962861700001, |
| "peak_process_rss_mib": 2347.41796875, |
| "tokenizer_sha256": "17ce952fb54679634b5a723756a217bb536ad1d2f6616917ac0172b7a6ef255c", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 742197, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.520961415904403, |
| "chunk_tokens_p95": 1839 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2537999, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.966246637607028, |
| "chunk_tokens_p95": 2332 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1874070, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.5809158676036645, |
| "chunk_tokens_p95": 2434 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 745154, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.503020852065479, |
| "chunk_tokens_p95": 2126 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 247100, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.073787940105221, |
| "chunk_tokens_p95": 2544 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 179080, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.747425731516641, |
| "chunk_tokens_p95": 2774 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 223410, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.505765185085717, |
| "chunk_tokens_p95": 1732 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 228512, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.405160341688839, |
| "chunk_tokens_p95": 1968 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 301804, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.447180951876052, |
| "chunk_tokens_p95": 2023 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 428498, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9153531638420715, |
| "chunk_tokens_p95": 2451 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 551299, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.260501107384559, |
| "chunk_tokens_p95": 2254 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 240110, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.192378493190621, |
| "chunk_tokens_p95": 2206 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8299233, |
| "bytes_per_token": 4.043076269819151, |
| "evaluation_seconds": 18.307522746999993, |
| "evaluation_mib_per_second": 1.7479153483637626, |
| "vocabulary_used": 32507, |
| "vocabulary_use_fraction": 0.992034912109375, |
| "vocabulary_unseen": 261, |
| "vocabulary_used_at_most_10_times": 1685, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 12, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 23, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 6, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 6, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 4, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "1076cfae7a52016c78f2ce1e55fdb2658014bbb9cbf1661f5c2b6895a9291929", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.019696868302485, |
| 4.069010372445005 |
| ], |
| "raw_result_sha256": "23a6072a121a13eb7e86099cd831b840f769d2a4b9f90c2200b430d5e64f2705", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.793865864497012, |
| "paired_ci95": [ |
| 0.7530451926170176, |
| 0.8462764487596336 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.856353836705676, |
| "paired_ci95": [ |
| 6.786942985526399, |
| 6.941702923683285 |
| ] |
| } |
| }, |
| { |
| "size_mib": 1024, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 107374183, |
| "code_python": 322122548, |
| "finemath_4plus": 214748365, |
| "pes2o": 107374183, |
| "pubmed": 32212255, |
| "arxiv": 21474837, |
| "pressbooks": 32212255, |
| "libretexts": 32212254, |
| "doab": 42949672, |
| "gutenberg": 53687091, |
| "pre_1929_books": 75161927, |
| "library_of_congress": 32212254 |
| }, |
| "training_seconds": 85.27832251700002, |
| "peak_process_rss_mib": 2405.546875, |
| "tokenizer_sha256": "ce5e83a52cab742ea5edc2440c8defd26c946d05ca2f1d3f64ab3dedc26b744d", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 715090, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.692338027381169, |
| "chunk_tokens_p95": 1788 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2453113, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.103492175044525, |
| "chunk_tokens_p95": 2256 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1831760, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.663627877014456, |
| "chunk_tokens_p95": 2371 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 719112, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.666093737832215, |
| "chunk_tokens_p95": 2052 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 237910, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.23115043503846, |
| "chunk_tokens_p95": 2441 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 172908, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8811911536770998, |
| "chunk_tokens_p95": 2694 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 215797, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.664721937747049, |
| "chunk_tokens_p95": 1669 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 222885, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.516373914799112, |
| "chunk_tokens_p95": 1905 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 288378, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.654228131133443, |
| "chunk_tokens_p95": 1908 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 410567, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.086351314158225, |
| "chunk_tokens_p95": 2291 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 533104, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.4059132927158675, |
| "chunk_tokens_p95": 2159 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 231822, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.342262598027797, |
| "chunk_tokens_p95": 2145 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8032446, |
| "bytes_per_token": 4.177361665425451, |
| "evaluation_seconds": 16.72046845, |
| "evaluation_mib_per_second": 1.9138219778764634, |
| "vocabulary_used": 49103, |
| "vocabulary_use_fraction": 0.98206, |
| "vocabulary_unseen": 897, |
| "vocabulary_used_at_most_10_times": 8880, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 19, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 5, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 3, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 3, |
| "HTTPRequestHandler": 2, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "6f580c5480ac6649f55db977d2808c4b90af3b74a397876da9d2821d29c3b9f7", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.152758170068112, |
| 4.203240076240967 |
| ], |
| "raw_result_sha256": "bb5308acb23d33b8db984c7837215346ce33284acc8ad0a9d53d34a17a6faa78", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.9129026885998148, |
| "paired_ci95": [ |
| 0.8663809973552774, |
| 0.9673188354975673 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.850547869933424, |
| "paired_ci95": [ |
| 9.750584481577206, |
| 9.962689507195215 |
| ] |
| } |
| }, |
| { |
| "size_mib": 2048, |
| "requested_vocab_size": 16384, |
| "actual_vocab_size": 16384, |
| "actual_training_bytes": { |
| "fineweb_edu": 214748365, |
| "code_python": 644245095, |
| "finemath_4plus": 429496730, |
| "pes2o": 214748365, |
| "pubmed": 64424510, |
| "arxiv": 42949673, |
| "pressbooks": 64424510, |
| "libretexts": 64424509, |
| "doab": 85899345, |
| "gutenberg": 107374182, |
| "pre_1929_books": 150323855, |
| "library_of_congress": 64424509 |
| }, |
| "training_seconds": 125.62457658700004, |
| "peak_process_rss_mib": 3352.40234375, |
| "tokenizer_sha256": "f95ec3899180fd6aef876f92e8fa1e6ec9cd96c180e22b23ead4e88b3b002561", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 803594, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.175546357986745, |
| "chunk_tokens_p95": 2010 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2733636, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6823959005515, |
| "chunk_tokens_p95": 2522 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1976064, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.3960878797447855, |
| "chunk_tokens_p95": 2566 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 806194, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.1620800948655035, |
| "chunk_tokens_p95": 2292 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 268693, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.7464057493124123, |
| "chunk_tokens_p95": 2691 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 192144, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.4926357315346825, |
| "chunk_tokens_p95": 2920 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 240230, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.190288473546185, |
| "chunk_tokens_p95": 1857 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 242602, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.149314515131779, |
| "chunk_tokens_p95": 2115 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 332651, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.034790215571275, |
| "chunk_tokens_p95": 2248 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 464314, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.6133327877255477, |
| "chunk_tokens_p95": 2717 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 591928, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.968067062210269, |
| "chunk_tokens_p95": 2410 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 257839, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9041107047421066, |
| "chunk_tokens_p95": 2398 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8909889, |
| "bytes_per_token": 3.7659764336009123, |
| "evaluation_seconds": 18.35914522600001, |
| "evaluation_mib_per_second": 1.7430005376656625, |
| "vocabulary_used": 16310, |
| "vocabulary_use_fraction": 0.9954833984375, |
| "vocabulary_unseen": 74, |
| "vocabulary_used_at_most_10_times": 193, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 14, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 8, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 29, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 4, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 7, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 5, |
| "deoxyribonucleic": 6, |
| "Schr\u00f6dinger": 6, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 4, |
| "np.linalg.eigh": 6, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "8db85e27082ec9c8e684fb1b7b9cdeab3e96f49942d5f6172f21cd4bb4afd418", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 3.7444001813488446, |
| 3.7914213811223267 |
| ], |
| "raw_result_sha256": "c0c3439eda6da37e6afa45d895be966f1f2724ca26b449eb71c21a8cab92f717", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.523125367879218, |
| "paired_ci95": [ |
| 0.48756702196798285, |
| 0.5687806589113775 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 0.0, |
| "paired_ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| } |
| }, |
| { |
| "size_mib": 2048, |
| "requested_vocab_size": 32768, |
| "actual_vocab_size": 32768, |
| "actual_training_bytes": { |
| "fineweb_edu": 214748365, |
| "code_python": 644245095, |
| "finemath_4plus": 429496730, |
| "pes2o": 214748365, |
| "pubmed": 64424510, |
| "arxiv": 42949673, |
| "pressbooks": 64424510, |
| "libretexts": 64424509, |
| "doab": 85899345, |
| "gutenberg": 107374182, |
| "pre_1929_books": 150323855, |
| "library_of_congress": 64424509 |
| }, |
| "training_seconds": 136.48117239100003, |
| "peak_process_rss_mib": 3509.57421875, |
| "tokenizer_sha256": "6f8742023692929139e450a3a64802e355649742e7051f9d8a6e6db47fbc8e96", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 742149, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.521253818303332, |
| "chunk_tokens_p95": 1847 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2537953, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.9663185252051556, |
| "chunk_tokens_p95": 2335 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1873768, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.581493013009081, |
| "chunk_tokens_p95": 2436 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 744976, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.504096776271988, |
| "chunk_tokens_p95": 2130 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 247017, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.07515677058664, |
| "chunk_tokens_p95": 2542 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 178828, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.752706511284586, |
| "chunk_tokens_p95": 2758 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 223411, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.505745017031391, |
| "chunk_tokens_p95": 1726 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 228434, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.406664507034855, |
| "chunk_tokens_p95": 1963 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 301431, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.452684030507811, |
| "chunk_tokens_p95": 2014 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 428589, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.914521837937978, |
| "chunk_tokens_p95": 2459 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 551192, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.261328176025777, |
| "chunk_tokens_p95": 2256 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 240038, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.193636007632125, |
| "chunk_tokens_p95": 2197 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8297786, |
| "bytes_per_token": 4.043781317088679, |
| "evaluation_seconds": 17.973626337000013, |
| "evaluation_mib_per_second": 1.7803864061714514, |
| "vocabulary_used": 32532, |
| "vocabulary_use_fraction": 0.9927978515625, |
| "vocabulary_unseen": 236, |
| "vocabulary_used_at_most_10_times": 1640, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 24, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 6, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 6, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 4, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 5, |
| "HTTPRequestHandler": 3, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "372a33fb69da6f25775535e29a8b29f4e012672ea53b746d1ea2c78b547c14c7", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.020637191383395, |
| 4.069643424675342 |
| ], |
| "raw_result_sha256": "439ceb897a3b169aea3e21aca388f703445bcea9da68faf11fca1aeff83349e4", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.8111627973694913, |
| "paired_ci95": [ |
| 0.7650781112152172, |
| 0.8572668574809261 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 6.869928458143537, |
| "paired_ci95": [ |
| 6.803539109129546, |
| 6.949242854436372 |
| ] |
| } |
| }, |
| { |
| "size_mib": 2048, |
| "requested_vocab_size": 50000, |
| "actual_vocab_size": 50000, |
| "actual_training_bytes": { |
| "fineweb_edu": 214748365, |
| "code_python": 644245095, |
| "finemath_4plus": 429496730, |
| "pes2o": 214748365, |
| "pubmed": 64424510, |
| "arxiv": 42949673, |
| "pressbooks": 64424510, |
| "libretexts": 64424509, |
| "doab": 85899345, |
| "gutenberg": 107374182, |
| "pre_1929_books": 150323855, |
| "library_of_congress": 64424509 |
| }, |
| "training_seconds": 159.32488262100003, |
| "peak_process_rss_mib": 3531.80859375, |
| "tokenizer_sha256": "5e3de8f77cfa5a5820a96bf0205e7cceee7f27c43e16afdc16d452df7a37f724", |
| "corpus_manifest_sha256": "be5d52967413adaa08e74fc0cec028305c9d1297f9c07425efb3631b744fe39a", |
| "implementation_sha256": "5d992fec42941a33951b22ffb7468ae58811f6de79364cd9f5bcfd714ee772aa", |
| "config_sha256": "78620acfac5a23364592a2216d4048828b5d416e43baef5762f09930ac3f98f4", |
| "tokenizers_version": "0.22.2", |
| "execution": "modal-cpu", |
| "metrics": { |
| "sources": { |
| "fineweb_edu": { |
| "bytes": 3355444, |
| "tokens": 714778, |
| "chunks": 927, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.6943862290109655, |
| "chunk_tokens_p95": 1785 |
| }, |
| "code_python": { |
| "bytes": 10066330, |
| "tokens": 2452093, |
| "chunks": 2092, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.105199109495439, |
| "chunk_tokens_p95": 2255 |
| }, |
| "finemath_4plus": { |
| "bytes": 6710887, |
| "tokens": 1831641, |
| "chunks": 1789, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.663865899485762, |
| "chunk_tokens_p95": 2362 |
| }, |
| "pes2o": { |
| "bytes": 3355444, |
| "tokens": 718383, |
| "chunks": 422, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.670828791883995, |
| "chunk_tokens_p95": 2049 |
| }, |
| "pubmed": { |
| "bytes": 1006633, |
| "tokens": 237852, |
| "chunks": 128, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.232182197332795, |
| "chunk_tokens_p95": 2459 |
| }, |
| "arxiv": { |
| "bytes": 671089, |
| "tokens": 172915, |
| "chunks": 83, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 3.8810340340629788, |
| "chunk_tokens_p95": 2695 |
| }, |
| "pressbooks": { |
| "bytes": 1006633, |
| "tokens": 215825, |
| "chunks": 359, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.664116761264914, |
| "chunk_tokens_p95": 1664 |
| }, |
| "libretexts": { |
| "bytes": 1006632, |
| "tokens": 222845, |
| "chunks": 285, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.5171845901860035, |
| "chunk_tokens_p95": 1902 |
| }, |
| "doab": { |
| "bytes": 1342177, |
| "tokens": 288273, |
| "chunks": 315, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.65592337818665, |
| "chunk_tokens_p95": 1905 |
| }, |
| "gutenberg": { |
| "bytes": 1677721, |
| "tokens": 410725, |
| "chunks": 210, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.08477935358208, |
| "chunk_tokens_p95": 2294 |
| }, |
| "pre_1929_books": { |
| "bytes": 2348810, |
| "tokens": 532881, |
| "chunks": 291, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.407757078972604, |
| "chunk_tokens_p95": 2157 |
| }, |
| "library_of_congress": { |
| "bytes": 1006632, |
| "tokens": 231766, |
| "chunks": 127, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "bytes_per_token": 4.343311788614378, |
| "chunk_tokens_p95": 2121 |
| } |
| }, |
| "bytes": 33554432, |
| "tokens": 8029977, |
| "bytes_per_token": 4.178646090767134, |
| "evaluation_seconds": 14.937971026000014, |
| "evaluation_mib_per_second": 2.1421918642299533, |
| "vocabulary_used": 49193, |
| "vocabulary_use_fraction": 0.98386, |
| "vocabulary_unseen": 807, |
| "vocabulary_used_at_most_10_times": 8918, |
| "roundtrip_failures": 0, |
| "unk_tokens": 0, |
| "probes": [ |
| { |
| "text": "def square(x):\r\n\treturn x ** 2\r\n", |
| "tokens": 10, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u222b\u2080\u00b9 x\u00b2 dx = \u2153", |
| "tokens": 11, |
| "roundtrip": true |
| }, |
| { |
| "text": "12345678901234567890", |
| "tokens": 7, |
| "roundtrip": true |
| }, |
| { |
| "text": "na\u00efve caf\u00e9 \u4e2d\u6587 \u0627\u0644\u0639\u0631\u0628\u064a\u0629 \ud83d\ude00", |
| "tokens": 17, |
| "roundtrip": true |
| }, |
| { |
| "text": "\u0000\t \r\n", |
| "tokens": 3, |
| "roundtrip": true |
| }, |
| { |
| "text": "HTTPRequestHandler snake_case_identifier", |
| "tokens": 5, |
| "roundtrip": true |
| }, |
| { |
| "text": "e\u0301 \u2260 \u00e9", |
| "tokens": 5, |
| "roundtrip": true |
| } |
| ], |
| "fragmentation": { |
| "electromagnetism": 3, |
| "deoxyribonucleic": 5, |
| "Schr\u00f6dinger": 3, |
| "HTTPRequestHandler": 2, |
| "get_user_configuration": 3, |
| "np.linalg.eigh": 4, |
| "123456789": 3, |
| "0.000001": 4, |
| "\\frac{a}{b}": 5 |
| } |
| }, |
| "correctness_passed": true, |
| "requested_vocabulary_reached": true, |
| "original_result_sha256": "896cdc975e9e70a05a843b6aeb54c356d697455617bfffc58c774909cc35dd87", |
| "evaluation_manifest_sha256": "e4e66512ca51226f56844d61a4d0c065064900b6a6d0bd0243e9e64b8698024f", |
| "evaluation_implementation_sha256": "43b518effaffbde7f77ad293f2fafec2407b716bbc3c8b126896d93993f43e95", |
| "evaluation_resource_requests": { |
| "cpu": 2, |
| "memory_mib": 8192 |
| }, |
| "evaluation_checks": { |
| "byte_alphabet_complete": true, |
| "special_ids_correct": true, |
| "roundtrips_passed": true, |
| "zero_unk": true |
| }, |
| "bytes_per_token_ci95": [ |
| 4.154437229317744, |
| 4.20486087426906 |
| ], |
| "raw_result_sha256": "cbcf97dbcf6b653f439c2822023c96459562673d96581ecffe6a602795e33dfc", |
| "vs_64mib_same_vocab": { |
| "token_saving_percent": 0.943359917103026, |
| "paired_ci95": [ |
| 0.8973001236409844, |
| 0.9945821428537949 |
| ] |
| }, |
| "vs_16k_same_size": { |
| "token_saving_percent": 9.875678585894843, |
| "paired_ci95": [ |
| 9.771961349789661, |
| 9.989392591793916 |
| ] |
| } |
| } |
| ] |
|
|
