| { |
| "schema_version": 1, |
| "created_at": "2026-08-24T19:25:09+00:00", |
| "seed": 20260825, |
| "sampling": { |
| "method": "deterministic SHA-256 bottom-k, exact duplicates excluded from selected sample", |
| "requested_documents_per_corpus": 100000, |
| "min_chars": 250, |
| "max_chars": 20000, |
| "min_words": 20 |
| }, |
| "tokenizers": { |
| "new": "out/qwen35_2b_sft_lora_r64_grouped_identity_v1/merges/ckpt4749/merged", |
| "base": "/mnt/raid1/qwen3.5/pipelines/uzbek_tokenizer/artifacts/base_checkpoints/qwen35_2b_base", |
| "result_reuse_note": "The evaluated 2B tokenizer files are byte-identical to the evaluated 4B tokenizer files.", |
| "new_tokenizer_sha256": "e4731e5df2c95dbfd98c45e08feae7c5fcd0588172778ac35c3703921f5529f9", |
| "base_tokenizer_sha256": "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927", |
| "add_special_tokens": false, |
| "raw_text_no_preprocessing": true |
| }, |
| "corpora": { |
| "OSCAR Uzbek": { |
| "domain_status": "held-out web source", |
| "source": "xkas2001/uzbek-language-dataset, community-oscar-uzbek/all_metadata_text.txt", |
| "sampling": { |
| "records_scanned": 4762756, |
| "valid_records": 623128, |
| "sampled_records": 100000, |
| "selected_duplicate_occurrences_skipped": 316611, |
| "records_truncated_before_sampling": 67 |
| }, |
| "result": { |
| "documents": 100000, |
| "words": 7618770, |
| "characters": 62308307, |
| "new_tokens": 15469208, |
| "base_tokens": 25628881, |
| "new_tokens_per_word": 2.0304075329744826, |
| "base_tokens_per_word": 3.363913203837365, |
| "token_reduction": 0.3964150053995724, |
| "token_reduction_95ci": [ |
| 0.3956999048249437, |
| 0.3971071790721872 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.9714285714285715, |
| "base": 3.3536585365853657 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.99789, |
| "equal_fraction": 0.00171, |
| "new_more_fraction": 0.0004 |
| }, |
| "document_length_chars": { |
| "min": 250, |
| "median": 448, |
| "p90": 1105, |
| "p99": 2908, |
| "max": 19999 |
| }, |
| "by_script": { |
| "latin_dominant": { |
| "documents": 99876, |
| "words": 7609046, |
| "characters": 62243858, |
| "new_tokens": 15444043, |
| "base_tokens": 25601074, |
| "new_tokens_per_word": 2.029695049813078, |
| "base_tokens_per_word": 3.3645576593964606, |
| "token_reduction": 0.39674237885488706, |
| "token_reduction_95ci": [ |
| 0.3960664169063509, |
| 0.3974315537014118 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.9714285714285715, |
| "base": 3.3541666666666665 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9980375665825624, |
| "equal_fraction": 0.001571949217029116, |
| "new_more_fraction": 0.00039048420040850655 |
| } |
| }, |
| "cyrillic_dominant": { |
| "documents": 121, |
| "words": 8853, |
| "characters": 61058, |
| "new_tokens": 21987, |
| "base_tokens": 24636, |
| "new_tokens_per_word": 2.4835648932565233, |
| "base_tokens_per_word": 2.782785496441884, |
| "token_reduction": 0.10752557233317095, |
| "token_reduction_95ci": [ |
| 0.06903186700461168, |
| 0.1489608189386225 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.220779220779221, |
| "base": 2.5869565217391304 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9008264462809917, |
| "equal_fraction": 0.09917355371900827, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "other": { |
| "documents": 3, |
| "words": 871, |
| "characters": 3391, |
| "new_tokens": 3178, |
| "base_tokens": 3171, |
| "new_tokens_per_word": 3.6486796785304247, |
| "base_tokens_per_word": 3.640642939150402, |
| "token_reduction": -0.002207505518763808, |
| "token_reduction_95ci": [ |
| -0.06363636363636371, |
| 0.0 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 3.7318295739348373, |
| "base": 3.7318295739348373 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.0, |
| "equal_fraction": 0.6666666666666666, |
| "new_more_fraction": 0.3333333333333333 |
| } |
| } |
| }, |
| "by_document_length_quartile": { |
| "q1": { |
| "char_bounds": [ |
| 250, |
| 328 |
| ], |
| "documents": 25225, |
| "words": 886299, |
| "characters": 7229107, |
| "new_tokens": 1742025, |
| "base_tokens": 2981550, |
| "new_tokens_per_word": 1.9655048691243022, |
| "base_tokens_per_word": 3.3640453165353903, |
| "token_reduction": 0.41573175026412434, |
| "token_reduction_95ci": [ |
| 0.41438530829534354, |
| 0.4170515905402432 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.9142857142857144, |
| "base": 3.3636363636363638 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9958771060455897, |
| "equal_fraction": 0.004083250743310208, |
| "new_more_fraction": 3.9643211100099105e-05 |
| } |
| }, |
| "q2": { |
| "char_bounds": [ |
| 329, |
| 448 |
| ], |
| "documents": 24838, |
| "words": 1159735, |
| "characters": 9510849, |
| "new_tokens": 2254196, |
| "base_tokens": 3895455, |
| "new_tokens_per_word": 1.943716452465434, |
| "base_tokens_per_word": 3.358918201140778, |
| "token_reduction": 0.42132664861999436, |
| "token_reduction_95ci": [ |
| 0.4200747806331761, |
| 0.42257126807643025 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.9166666666666667, |
| "base": 3.3617021276595747 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9986311297205894, |
| "equal_fraction": 0.001328609388839681, |
| "new_more_fraction": 4.0260890570899426e-05 |
| } |
| }, |
| "q3": { |
| "char_bounds": [ |
| 449, |
| 694 |
| ], |
| "documents": 24973, |
| "words": 1683343, |
| "characters": 13783621, |
| "new_tokens": 3372637, |
| "base_tokens": 5654412, |
| "new_tokens_per_word": 2.0035352272234475, |
| "base_tokens_per_word": 3.359037344141984, |
| "token_reduction": 0.4035388648722449, |
| "token_reduction_95ci": [ |
| 0.4023960313587086, |
| 0.4046209067071377 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.9710144927536233, |
| "base": 3.3466666666666667 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9995194810395227, |
| "equal_fraction": 0.0004004324670644296, |
| "new_more_fraction": 8.008649341288591e-05 |
| } |
| }, |
| "q4": { |
| "char_bounds": [ |
| 695, |
| 19999 |
| ], |
| "documents": 24964, |
| "words": 3889393, |
| "characters": 31784730, |
| "new_tokens": 8100350, |
| "base_tokens": 13097464, |
| "new_tokens_per_word": 2.0826771683910574, |
| "base_tokens_per_word": 3.367482792302038, |
| "token_reduction": 0.3815329440874966, |
| "token_reduction_95ci": [ |
| 0.38033518553908646, |
| 0.38274511916670123 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.05603738317757, |
| "base": 3.347826086956522 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9975564813331197, |
| "equal_fraction": 0.00100144207659029, |
| "new_more_fraction": 0.0014420765902900176 |
| } |
| } |
| } |
| } |
| }, |
| "Uzbek legal corpus": { |
| "domain_status": "held-out legal source/domain", |
| "source": "sukhrobnurali/uzbek-legal-corpus-v1, articles", |
| "sampling": { |
| "records_scanned": 21021, |
| "valid_records": 18832, |
| "sampled_records": 18832, |
| "selected_duplicate_occurrences_skipped": 0, |
| "records_truncated_before_sampling": 3 |
| }, |
| "result": { |
| "documents": 18832, |
| "words": 2534566, |
| "characters": 21314804, |
| "new_tokens": 6018932, |
| "base_tokens": 7529043, |
| "new_tokens_per_word": 2.374738712663233, |
| "base_tokens_per_word": 2.970545253112367, |
| "token_reduction": 0.20057144048719078, |
| "token_reduction_95ci": [ |
| 0.19550471065059247, |
| 0.20567097138760268 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.8397191231160697, |
| "base": 3.3333333333333335 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.385089209855565, |
| "equal_fraction": 0.614910790144435, |
| "new_more_fraction": 0.0 |
| }, |
| "document_length_chars": { |
| "min": 250, |
| "median": 812, |
| "p90": 2183, |
| "p99": 5637, |
| "max": 19997 |
| }, |
| "by_script": { |
| "latin_dominant": { |
| "documents": 6354, |
| "words": 858053, |
| "characters": 7499938, |
| "new_tokens": 1410490, |
| "base_tokens": 2916524, |
| "new_tokens_per_word": 1.6438261972162558, |
| "base_tokens_per_word": 3.3990021595402613, |
| "token_reduction": 0.51637977263345, |
| "token_reduction_95ci": [ |
| 0.5147582025646525, |
| 0.5179880323158793 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.6153846153846154, |
| "base": 3.369890856442811 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 1.0, |
| "equal_fraction": 0.0, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "cyrillic_dominant": { |
| "documents": 12478, |
| "words": 1676513, |
| "characters": 13814866, |
| "new_tokens": 4608442, |
| "base_tokens": 4612519, |
| "new_tokens_per_word": 2.74882568760278, |
| "base_tokens_per_word": 2.7512575208185086, |
| "token_reduction": 0.0008838987980320789, |
| "token_reduction_95ci": [ |
| 0.000797593938754379, |
| 0.0009744198560997175 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 3.1314814814814813, |
| "base": 3.1314814814814813 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.07196666132393012, |
| "equal_fraction": 0.9280333386760699, |
| "new_more_fraction": 0.0 |
| } |
| } |
| }, |
| "by_document_length_quartile": { |
| "q1": { |
| "char_bounds": [ |
| 250, |
| 506 |
| ], |
| "documents": 4715, |
| "words": 213503, |
| "characters": 1769357, |
| "new_tokens": 505325, |
| "base_tokens": 619444, |
| "new_tokens_per_word": 2.366828569153595, |
| "base_tokens_per_word": 2.901336280989026, |
| "token_reduction": 0.1842281142443869, |
| "token_reduction_95ci": [ |
| 0.17673651196691142, |
| 0.19222917798968833 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.868421052631579, |
| "base": 3.2857142857142856 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.34612937433722163, |
| "equal_fraction": 0.6538706256627783, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "q2": { |
| "char_bounds": [ |
| 507, |
| 812 |
| ], |
| "documents": 4703, |
| "words": 369537, |
| "characters": 3066781, |
| "new_tokens": 873369, |
| "base_tokens": 1074262, |
| "new_tokens_per_word": 2.3634142183326703, |
| "base_tokens_per_word": 2.907048549942225, |
| "token_reduction": 0.18700559081490364, |
| "token_reduction_95ci": [ |
| 0.17978162099648665, |
| 0.19424970760537358 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.8421052631578947, |
| "base": 3.3076923076923075 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.35721879651286415, |
| "equal_fraction": 0.6427812034871359, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "q3": { |
| "char_bounds": [ |
| 813, |
| 1347 |
| ], |
| "documents": 4711, |
| "words": 594780, |
| "characters": 4943320, |
| "new_tokens": 1407482, |
| "base_tokens": 1750853, |
| "new_tokens_per_word": 2.366390934463163, |
| "base_tokens_per_word": 2.9436985103735833, |
| "token_reduction": 0.19611640725977564, |
| "token_reduction_95ci": [ |
| 0.18856405345722932, |
| 0.20422906152284168 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.8333333333333333, |
| "base": 3.3286713286713288 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.3922734026745914, |
| "equal_fraction": 0.6077265973254086, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "q4": { |
| "char_bounds": [ |
| 1348, |
| 19997 |
| ], |
| "documents": 4703, |
| "words": 1356746, |
| "characters": 11535346, |
| "new_tokens": 3232756, |
| "base_tokens": 4084484, |
| "new_tokens_per_word": 2.3827274965247733, |
| "base_tokens_per_word": 3.0105001230886255, |
| "token_reduction": 0.2085276867286051, |
| "token_reduction_95ci": [ |
| 0.1998185981685246, |
| 0.21714855506411254 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 1.819905213270142, |
| "base": 3.3981481481481484 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.4448224537529237, |
| "equal_fraction": 0.5551775462470764, |
| "new_more_fraction": 0.0 |
| } |
| } |
| } |
| } |
| }, |
| "uz-crawl control": { |
| "domain_status": "training-source control", |
| "source": "tahrirchi/uz-crawl local training source", |
| "sampling": { |
| "records_scanned": 1618035, |
| "valid_records": 1441908, |
| "sampled_records": 100000, |
| "selected_duplicate_occurrences_skipped": 174, |
| "records_truncated_before_sampling": 9216 |
| }, |
| "result": { |
| "documents": 100000, |
| "words": 20825680, |
| "characters": 174743319, |
| "new_tokens": 48327821, |
| "base_tokens": 69191245, |
| "new_tokens_per_word": 2.3205878991706395, |
| "base_tokens_per_word": 3.3224002769657464, |
| "token_reduction": 0.3015327155914018, |
| "token_reduction_95ci": [ |
| 0.3001693372220501, |
| 0.302962741125678 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.0866859344894024, |
| "base": 3.2613636363636362 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.86481, |
| "equal_fraction": 0.13306, |
| "new_more_fraction": 0.00213 |
| }, |
| "document_length_chars": { |
| "min": 250, |
| "median": 1036, |
| "p90": 3408, |
| "p99": 14367, |
| "max": 19999 |
| }, |
| "by_script": { |
| "latin_dominant": { |
| "documents": 82818, |
| "words": 17899459, |
| "characters": 152194135, |
| "new_tokens": 37942391, |
| "base_tokens": 58788459, |
| "new_tokens_per_word": 2.119750714253431, |
| "base_tokens_per_word": 3.284370717573084, |
| "token_reduction": 0.3545945642154015, |
| "token_reduction_95ci": [ |
| 0.35375779400897717, |
| 0.35539015324512163 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.0297619047619047, |
| "base": 3.2142857142857144 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 1.0, |
| "equal_fraction": 0.0, |
| "new_more_fraction": 0.0 |
| } |
| }, |
| "cyrillic_dominant": { |
| "documents": 17182, |
| "words": 2926221, |
| "characters": 22549184, |
| "new_tokens": 10385430, |
| "base_tokens": 10402786, |
| "new_tokens_per_word": 3.5490928402195188, |
| "base_tokens_per_word": 3.5550240395376838, |
| "token_reduction": 0.0016683992153639915, |
| "token_reduction_95ci": [ |
| 0.0013899888640842822, |
| 0.001964975103579841 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 3.542372881355932, |
| "base": 3.5457593062605754 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.21318822023047376, |
| "equal_fraction": 0.7744150855546502, |
| "new_more_fraction": 0.012396694214876033 |
| } |
| } |
| }, |
| "by_document_length_quartile": { |
| "q1": { |
| "char_bounds": [ |
| 250, |
| 642 |
| ], |
| "documents": 25014, |
| "words": 1443765, |
| "characters": 11560305, |
| "new_tokens": 3580625, |
| "base_tokens": 4717032, |
| "new_tokens_per_word": 2.4800608132209883, |
| "base_tokens_per_word": 3.2671743670195634, |
| "token_reduction": 0.2409156859652426, |
| "token_reduction_95ci": [ |
| 0.2388878125954489, |
| 0.243056097563355 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.2058823529411766, |
| "base": 3.2567567567567566 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.7540577276725035, |
| "equal_fraction": 0.24566242903973776, |
| "new_more_fraction": 0.00027984328775885504 |
| } |
| }, |
| "q2": { |
| "char_bounds": [ |
| 643, |
| 1036 |
| ], |
| "documents": 25032, |
| "words": 2534284, |
| "characters": 20698884, |
| "new_tokens": 5852869, |
| "base_tokens": 8233439, |
| "new_tokens_per_word": 2.3094763649220056, |
| "base_tokens_per_word": 3.248822547117845, |
| "token_reduction": 0.2891343459276251, |
| "token_reduction_95ci": [ |
| 0.2871033465654421, |
| 0.2910416333852472 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.0697674418604652, |
| "base": 3.2268041237113403 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.8623761585170981, |
| "equal_fraction": 0.13690476190476192, |
| "new_more_fraction": 0.0007190795781399808 |
| } |
| }, |
| "q3": { |
| "char_bounds": [ |
| 1037, |
| 1827 |
| ], |
| "documents": 24959, |
| "words": 4090354, |
| "characters": 34187499, |
| "new_tokens": 9144176, |
| "base_tokens": 13420837, |
| "new_tokens_per_word": 2.235546361024009, |
| "base_tokens_per_word": 3.281094252477903, |
| "token_reduction": 0.31865829232558296, |
| "token_reduction_95ci": [ |
| 0.3168107024780933, |
| 0.3205574932440705 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.051851851851852, |
| "base": 3.261904761904762 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.9184662847069194, |
| "equal_fraction": 0.08033174406025882, |
| "new_more_fraction": 0.0012019712328218278 |
| } |
| }, |
| "q4": { |
| "char_bounds": [ |
| 1828, |
| 19999 |
| ], |
| "documents": 24995, |
| "words": 12757277, |
| "characters": 108296631, |
| "new_tokens": 29750151, |
| "base_tokens": 42819937, |
| "new_tokens_per_word": 2.3320141908026297, |
| "base_tokens_per_word": 3.3565107193329737, |
| "token_reduction": 0.30522665178138864, |
| "token_reduction_95ci": [ |
| 0.3030310155219502, |
| 0.30739078599543024 |
| ], |
| "median_document_tokens_per_word": { |
| "new": 2.07344398340249, |
| "base": 3.2913907284768213 |
| }, |
| "document_token_count_comparison": { |
| "new_fewer_fraction": 0.924504900980196, |
| "equal_fraction": 0.0691738347669534, |
| "new_more_fraction": 0.00632126425285057 |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
|
|
