{
  "schema_version": 1,
  "created_at": "2026-08-24T19:25:09+00:00",
  "seed": 20260825,
  "sampling": {
    "method": "deterministic SHA-256 bottom-k, exact duplicates excluded from selected sample",
    "requested_documents_per_corpus": 100000,
    "min_chars": 250,
    "max_chars": 20000,
    "min_words": 20
  },
  "tokenizers": {
    "new": "out/qwen35_2b_sft_lora_r64_grouped_identity_v1/merges/ckpt4749/merged",
    "base": "/mnt/raid1/qwen3.5/pipelines/uzbek_tokenizer/artifacts/base_checkpoints/qwen35_2b_base",
    "result_reuse_note": "The evaluated 2B tokenizer files are byte-identical to the evaluated 4B tokenizer files.",
    "new_tokenizer_sha256": "e4731e5df2c95dbfd98c45e08feae7c5fcd0588172778ac35c3703921f5529f9",
    "base_tokenizer_sha256": "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927",
    "add_special_tokens": false,
    "raw_text_no_preprocessing": true
  },
  "corpora": {
    "OSCAR Uzbek": {
      "domain_status": "held-out web source",
      "source": "xkas2001/uzbek-language-dataset, community-oscar-uzbek/all_metadata_text.txt",
      "sampling": {
        "records_scanned": 4762756,
        "valid_records": 623128,
        "sampled_records": 100000,
        "selected_duplicate_occurrences_skipped": 316611,
        "records_truncated_before_sampling": 67
      },
      "result": {
        "documents": 100000,
        "words": 7618770,
        "characters": 62308307,
        "new_tokens": 15469208,
        "base_tokens": 25628881,
        "new_tokens_per_word": 2.0304075329744826,
        "base_tokens_per_word": 3.363913203837365,
        "token_reduction": 0.3964150053995724,
        "token_reduction_95ci": [
          0.3956999048249437,
          0.3971071790721872
        ],
        "median_document_tokens_per_word": {
          "new": 1.9714285714285715,
          "base": 3.3536585365853657
        },
        "document_token_count_comparison": {
          "new_fewer_fraction": 0.99789,
          "equal_fraction": 0.00171,
          "new_more_fraction": 0.0004
        },
        "document_length_chars": {
          "min": 250,
          "median": 448,
          "p90": 1105,
          "p99": 2908,
          "max": 19999
        },
        "by_script": {
          "latin_dominant": {
            "documents": 99876,
            "words": 7609046,
            "characters": 62243858,
            "new_tokens": 15444043,
            "base_tokens": 25601074,
            "new_tokens_per_word": 2.029695049813078,
            "base_tokens_per_word": 3.3645576593964606,
            "token_reduction": 0.39674237885488706,
            "token_reduction_95ci": [
              0.3960664169063509,
              0.3974315537014118
            ],
            "median_document_tokens_per_word": {
              "new": 1.9714285714285715,
              "base": 3.3541666666666665
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9980375665825624,
              "equal_fraction": 0.001571949217029116,
              "new_more_fraction": 0.00039048420040850655
            }
          },
          "cyrillic_dominant": {
            "documents": 121,
            "words": 8853,
            "characters": 61058,
            "new_tokens": 21987,
            "base_tokens": 24636,
            "new_tokens_per_word": 2.4835648932565233,
            "base_tokens_per_word": 2.782785496441884,
            "token_reduction": 0.10752557233317095,
            "token_reduction_95ci": [
              0.06903186700461168,
              0.1489608189386225
            ],
            "median_document_tokens_per_word": {
              "new": 2.220779220779221,
              "base": 2.5869565217391304
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9008264462809917,
              "equal_fraction": 0.09917355371900827,
              "new_more_fraction": 0.0
            }
          },
          "other": {
            "documents": 3,
            "words": 871,
            "characters": 3391,
            "new_tokens": 3178,
            "base_tokens": 3171,
            "new_tokens_per_word": 3.6486796785304247,
            "base_tokens_per_word": 3.640642939150402,
            "token_reduction": -0.002207505518763808,
            "token_reduction_95ci": [
              -0.06363636363636371,
              0.0
            ],
            "median_document_tokens_per_word": {
              "new": 3.7318295739348373,
              "base": 3.7318295739348373
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.0,
              "equal_fraction": 0.6666666666666666,
              "new_more_fraction": 0.3333333333333333
            }
          }
        },
        "by_document_length_quartile": {
          "q1": {
            "char_bounds": [
              250,
              328
            ],
            "documents": 25225,
            "words": 886299,
            "characters": 7229107,
            "new_tokens": 1742025,
            "base_tokens": 2981550,
            "new_tokens_per_word": 1.9655048691243022,
            "base_tokens_per_word": 3.3640453165353903,
            "token_reduction": 0.41573175026412434,
            "token_reduction_95ci": [
              0.41438530829534354,
              0.4170515905402432
            ],
            "median_document_tokens_per_word": {
              "new": 1.9142857142857144,
              "base": 3.3636363636363638
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9958771060455897,
              "equal_fraction": 0.004083250743310208,
              "new_more_fraction": 3.9643211100099105e-05
            }
          },
          "q2": {
            "char_bounds": [
              329,
              448
            ],
            "documents": 24838,
            "words": 1159735,
            "characters": 9510849,
            "new_tokens": 2254196,
            "base_tokens": 3895455,
            "new_tokens_per_word": 1.943716452465434,
            "base_tokens_per_word": 3.358918201140778,
            "token_reduction": 0.42132664861999436,
            "token_reduction_95ci": [
              0.4200747806331761,
              0.42257126807643025
            ],
            "median_document_tokens_per_word": {
              "new": 1.9166666666666667,
              "base": 3.3617021276595747
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9986311297205894,
              "equal_fraction": 0.001328609388839681,
              "new_more_fraction": 4.0260890570899426e-05
            }
          },
          "q3": {
            "char_bounds": [
              449,
              694
            ],
            "documents": 24973,
            "words": 1683343,
            "characters": 13783621,
            "new_tokens": 3372637,
            "base_tokens": 5654412,
            "new_tokens_per_word": 2.0035352272234475,
            "base_tokens_per_word": 3.359037344141984,
            "token_reduction": 0.4035388648722449,
            "token_reduction_95ci": [
              0.4023960313587086,
              0.4046209067071377
            ],
            "median_document_tokens_per_word": {
              "new": 1.9710144927536233,
              "base": 3.3466666666666667
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9995194810395227,
              "equal_fraction": 0.0004004324670644296,
              "new_more_fraction": 8.008649341288591e-05
            }
          },
          "q4": {
            "char_bounds": [
              695,
              19999
            ],
            "documents": 24964,
            "words": 3889393,
            "characters": 31784730,
            "new_tokens": 8100350,
            "base_tokens": 13097464,
            "new_tokens_per_word": 2.0826771683910574,
            "base_tokens_per_word": 3.367482792302038,
            "token_reduction": 0.3815329440874966,
            "token_reduction_95ci": [
              0.38033518553908646,
              0.38274511916670123
            ],
            "median_document_tokens_per_word": {
              "new": 2.05603738317757,
              "base": 3.347826086956522
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9975564813331197,
              "equal_fraction": 0.00100144207659029,
              "new_more_fraction": 0.0014420765902900176
            }
          }
        }
      }
    },
    "Uzbek legal corpus": {
      "domain_status": "held-out legal source/domain",
      "source": "sukhrobnurali/uzbek-legal-corpus-v1, articles",
      "sampling": {
        "records_scanned": 21021,
        "valid_records": 18832,
        "sampled_records": 18832,
        "selected_duplicate_occurrences_skipped": 0,
        "records_truncated_before_sampling": 3
      },
      "result": {
        "documents": 18832,
        "words": 2534566,
        "characters": 21314804,
        "new_tokens": 6018932,
        "base_tokens": 7529043,
        "new_tokens_per_word": 2.374738712663233,
        "base_tokens_per_word": 2.970545253112367,
        "token_reduction": 0.20057144048719078,
        "token_reduction_95ci": [
          0.19550471065059247,
          0.20567097138760268
        ],
        "median_document_tokens_per_word": {
          "new": 1.8397191231160697,
          "base": 3.3333333333333335
        },
        "document_token_count_comparison": {
          "new_fewer_fraction": 0.385089209855565,
          "equal_fraction": 0.614910790144435,
          "new_more_fraction": 0.0
        },
        "document_length_chars": {
          "min": 250,
          "median": 812,
          "p90": 2183,
          "p99": 5637,
          "max": 19997
        },
        "by_script": {
          "latin_dominant": {
            "documents": 6354,
            "words": 858053,
            "characters": 7499938,
            "new_tokens": 1410490,
            "base_tokens": 2916524,
            "new_tokens_per_word": 1.6438261972162558,
            "base_tokens_per_word": 3.3990021595402613,
            "token_reduction": 0.51637977263345,
            "token_reduction_95ci": [
              0.5147582025646525,
              0.5179880323158793
            ],
            "median_document_tokens_per_word": {
              "new": 1.6153846153846154,
              "base": 3.369890856442811
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 1.0,
              "equal_fraction": 0.0,
              "new_more_fraction": 0.0
            }
          },
          "cyrillic_dominant": {
            "documents": 12478,
            "words": 1676513,
            "characters": 13814866,
            "new_tokens": 4608442,
            "base_tokens": 4612519,
            "new_tokens_per_word": 2.74882568760278,
            "base_tokens_per_word": 2.7512575208185086,
            "token_reduction": 0.0008838987980320789,
            "token_reduction_95ci": [
              0.000797593938754379,
              0.0009744198560997175
            ],
            "median_document_tokens_per_word": {
              "new": 3.1314814814814813,
              "base": 3.1314814814814813
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.07196666132393012,
              "equal_fraction": 0.9280333386760699,
              "new_more_fraction": 0.0
            }
          }
        },
        "by_document_length_quartile": {
          "q1": {
            "char_bounds": [
              250,
              506
            ],
            "documents": 4715,
            "words": 213503,
            "characters": 1769357,
            "new_tokens": 505325,
            "base_tokens": 619444,
            "new_tokens_per_word": 2.366828569153595,
            "base_tokens_per_word": 2.901336280989026,
            "token_reduction": 0.1842281142443869,
            "token_reduction_95ci": [
              0.17673651196691142,
              0.19222917798968833
            ],
            "median_document_tokens_per_word": {
              "new": 1.868421052631579,
              "base": 3.2857142857142856
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.34612937433722163,
              "equal_fraction": 0.6538706256627783,
              "new_more_fraction": 0.0
            }
          },
          "q2": {
            "char_bounds": [
              507,
              812
            ],
            "documents": 4703,
            "words": 369537,
            "characters": 3066781,
            "new_tokens": 873369,
            "base_tokens": 1074262,
            "new_tokens_per_word": 2.3634142183326703,
            "base_tokens_per_word": 2.907048549942225,
            "token_reduction": 0.18700559081490364,
            "token_reduction_95ci": [
              0.17978162099648665,
              0.19424970760537358
            ],
            "median_document_tokens_per_word": {
              "new": 1.8421052631578947,
              "base": 3.3076923076923075
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.35721879651286415,
              "equal_fraction": 0.6427812034871359,
              "new_more_fraction": 0.0
            }
          },
          "q3": {
            "char_bounds": [
              813,
              1347
            ],
            "documents": 4711,
            "words": 594780,
            "characters": 4943320,
            "new_tokens": 1407482,
            "base_tokens": 1750853,
            "new_tokens_per_word": 2.366390934463163,
            "base_tokens_per_word": 2.9436985103735833,
            "token_reduction": 0.19611640725977564,
            "token_reduction_95ci": [
              0.18856405345722932,
              0.20422906152284168
            ],
            "median_document_tokens_per_word": {
              "new": 1.8333333333333333,
              "base": 3.3286713286713288
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.3922734026745914,
              "equal_fraction": 0.6077265973254086,
              "new_more_fraction": 0.0
            }
          },
          "q4": {
            "char_bounds": [
              1348,
              19997
            ],
            "documents": 4703,
            "words": 1356746,
            "characters": 11535346,
            "new_tokens": 3232756,
            "base_tokens": 4084484,
            "new_tokens_per_word": 2.3827274965247733,
            "base_tokens_per_word": 3.0105001230886255,
            "token_reduction": 0.2085276867286051,
            "token_reduction_95ci": [
              0.1998185981685246,
              0.21714855506411254
            ],
            "median_document_tokens_per_word": {
              "new": 1.819905213270142,
              "base": 3.3981481481481484
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.4448224537529237,
              "equal_fraction": 0.5551775462470764,
              "new_more_fraction": 0.0
            }
          }
        }
      }
    },
    "uz-crawl control": {
      "domain_status": "training-source control",
      "source": "tahrirchi/uz-crawl local training source",
      "sampling": {
        "records_scanned": 1618035,
        "valid_records": 1441908,
        "sampled_records": 100000,
        "selected_duplicate_occurrences_skipped": 174,
        "records_truncated_before_sampling": 9216
      },
      "result": {
        "documents": 100000,
        "words": 20825680,
        "characters": 174743319,
        "new_tokens": 48327821,
        "base_tokens": 69191245,
        "new_tokens_per_word": 2.3205878991706395,
        "base_tokens_per_word": 3.3224002769657464,
        "token_reduction": 0.3015327155914018,
        "token_reduction_95ci": [
          0.3001693372220501,
          0.302962741125678
        ],
        "median_document_tokens_per_word": {
          "new": 2.0866859344894024,
          "base": 3.2613636363636362
        },
        "document_token_count_comparison": {
          "new_fewer_fraction": 0.86481,
          "equal_fraction": 0.13306,
          "new_more_fraction": 0.00213
        },
        "document_length_chars": {
          "min": 250,
          "median": 1036,
          "p90": 3408,
          "p99": 14367,
          "max": 19999
        },
        "by_script": {
          "latin_dominant": {
            "documents": 82818,
            "words": 17899459,
            "characters": 152194135,
            "new_tokens": 37942391,
            "base_tokens": 58788459,
            "new_tokens_per_word": 2.119750714253431,
            "base_tokens_per_word": 3.284370717573084,
            "token_reduction": 0.3545945642154015,
            "token_reduction_95ci": [
              0.35375779400897717,
              0.35539015324512163
            ],
            "median_document_tokens_per_word": {
              "new": 2.0297619047619047,
              "base": 3.2142857142857144
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 1.0,
              "equal_fraction": 0.0,
              "new_more_fraction": 0.0
            }
          },
          "cyrillic_dominant": {
            "documents": 17182,
            "words": 2926221,
            "characters": 22549184,
            "new_tokens": 10385430,
            "base_tokens": 10402786,
            "new_tokens_per_word": 3.5490928402195188,
            "base_tokens_per_word": 3.5550240395376838,
            "token_reduction": 0.0016683992153639915,
            "token_reduction_95ci": [
              0.0013899888640842822,
              0.001964975103579841
            ],
            "median_document_tokens_per_word": {
              "new": 3.542372881355932,
              "base": 3.5457593062605754
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.21318822023047376,
              "equal_fraction": 0.7744150855546502,
              "new_more_fraction": 0.012396694214876033
            }
          }
        },
        "by_document_length_quartile": {
          "q1": {
            "char_bounds": [
              250,
              642
            ],
            "documents": 25014,
            "words": 1443765,
            "characters": 11560305,
            "new_tokens": 3580625,
            "base_tokens": 4717032,
            "new_tokens_per_word": 2.4800608132209883,
            "base_tokens_per_word": 3.2671743670195634,
            "token_reduction": 0.2409156859652426,
            "token_reduction_95ci": [
              0.2388878125954489,
              0.243056097563355
            ],
            "median_document_tokens_per_word": {
              "new": 2.2058823529411766,
              "base": 3.2567567567567566
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.7540577276725035,
              "equal_fraction": 0.24566242903973776,
              "new_more_fraction": 0.00027984328775885504
            }
          },
          "q2": {
            "char_bounds": [
              643,
              1036
            ],
            "documents": 25032,
            "words": 2534284,
            "characters": 20698884,
            "new_tokens": 5852869,
            "base_tokens": 8233439,
            "new_tokens_per_word": 2.3094763649220056,
            "base_tokens_per_word": 3.248822547117845,
            "token_reduction": 0.2891343459276251,
            "token_reduction_95ci": [
              0.2871033465654421,
              0.2910416333852472
            ],
            "median_document_tokens_per_word": {
              "new": 2.0697674418604652,
              "base": 3.2268041237113403
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.8623761585170981,
              "equal_fraction": 0.13690476190476192,
              "new_more_fraction": 0.0007190795781399808
            }
          },
          "q3": {
            "char_bounds": [
              1037,
              1827
            ],
            "documents": 24959,
            "words": 4090354,
            "characters": 34187499,
            "new_tokens": 9144176,
            "base_tokens": 13420837,
            "new_tokens_per_word": 2.235546361024009,
            "base_tokens_per_word": 3.281094252477903,
            "token_reduction": 0.31865829232558296,
            "token_reduction_95ci": [
              0.3168107024780933,
              0.3205574932440705
            ],
            "median_document_tokens_per_word": {
              "new": 2.051851851851852,
              "base": 3.261904761904762
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.9184662847069194,
              "equal_fraction": 0.08033174406025882,
              "new_more_fraction": 0.0012019712328218278
            }
          },
          "q4": {
            "char_bounds": [
              1828,
              19999
            ],
            "documents": 24995,
            "words": 12757277,
            "characters": 108296631,
            "new_tokens": 29750151,
            "base_tokens": 42819937,
            "new_tokens_per_word": 2.3320141908026297,
            "base_tokens_per_word": 3.3565107193329737,
            "token_reduction": 0.30522665178138864,
            "token_reduction_95ci": [
              0.3030310155219502,
              0.30739078599543024
            ],
            "median_document_tokens_per_word": {
              "new": 2.07344398340249,
              "base": 3.2913907284768213
            },
            "document_token_count_comparison": {
              "new_fewer_fraction": 0.924504900980196,
              "equal_fraction": 0.0691738347669534,
              "new_more_fraction": 0.00632126425285057
            }
          }
        }
      }
    }
  }
}