File size: 9,515 Bytes
f99a82c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 | {
"schema_version": 1,
"metric_note": "Accuracy for non-translation tasks; COMET for FLORES+; weighted sum normalized by 0.95.",
"weights": {
"uzlib_slice_full": 0.2,
"tumlu_uzbek_full": 0.2,
"flores_en_uz_full": 0.15,
"news_full": 0.1,
"mmlu_english_full": 0.1,
"mmlu_uz_full": 0.1,
"flores_uz_en_full": 0.05,
"sentiment_binary_full": 0.05
},
"labels": {
"uzlib_slice_full": "UzLiB",
"tumlu_uzbek_full": "TUMLU-Uzbek",
"flores_en_uz_full": "FLORES+ en to uz",
"news_full": "News",
"mmlu_english_full": "MMLU English",
"mmlu_uz_full": "MMLU Uzbek",
"flores_uz_en_full": "FLORES+ uz to en",
"sentiment_binary_full": "Sentiment"
},
"models": {
"NeuronAI-2B": {
"run_id": "qwen35_2b_lora_r64_grouped_identity_ckpt4749_public_v2",
"suite_id": "public_uzbek_full_comet_vllm_v2",
"tasks": {
"uzlib_slice_full": 0.4959699086512628,
"tumlu_uzbek_full": 0.32571428571428573,
"flores_en_uz_full": 0.8761537177251785,
"news_full": 0.7854903578426318,
"mmlu_english_full": 0.5406637231163652,
"mmlu_uz_full": 0.4685230024213075,
"flores_uz_en_full": 0.8534744450736248,
"sentiment_binary_full": 0.955
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0
},
"normalized_weighted_score": 0.5954224496037875
},
"Qwen3.5-2B": {
"run_id": "qwen35_2b_instruct_full_20260817_081135",
"suite_id": "public_uzbek_full_vllm",
"tasks": {
"uzlib_slice_full": 0.28694250403009136,
"tumlu_uzbek_full": 0.31285714285714283,
"flores_en_uz_full": 0.7009904753173863,
"news_full": 0.36753635144890173,
"mmlu_english_full": 0.5239282153539382,
"mmlu_uz_full": 0.3709585529126905,
"flores_uz_en_full": 0.8071526469372587,
"sentiment_binary_full": 0.7687
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0002
},
"normalized_weighted_score": 0.45278257367733765
},
"alloma-8B": {
"run_id": "alloma_8b_full_public_composite",
"suite_id": "public_uzbek_full_composite_apst",
"source_files": [
"reports/alloma_card_protocol_full/alloma-8B.json",
"reports/eval_suites/external_public/alloma_8b_uzlib_full_20260615/uzlib_full.json",
"reports/eval_suites/public_full_comet/alloma_8b_strict_missing_uz_metrics/tumlu_uzbek_full_apst.json",
"reports/eval_suites/public_full_comet/alloma_8b_strict_missing_uz_metrics/mmlu_uz_full_apst.json"
],
"tasks": {
"uzlib_slice_full": 0.4239656098871574,
"tumlu_uzbek_full": 0.20714285714285716,
"flores_en_uz_full": 0.8779,
"news_full": 0.5777,
"mmlu_english_full": 0.5347,
"mmlu_uz_full": 0.40037031761857284,
"flores_uz_en_full": 0.8713,
"sentiment_binary_full": 0.7994
},
"invalid_rate": {
"uzlib_slice_full": 0.0338527673293928,
"tumlu_uzbek_full": 0.25857142857142856,
"flores_en_uz_full": null,
"news_full": null,
"mmlu_english_full": null,
"mmlu_uz_full": 0.09051417177040308,
"flores_uz_en_full": null,
"sentiment_binary_full": null
},
"normalized_weighted_score": 0.5186512896503792
},
"alloma-3B": {
"run_id": "alloma_3B_apst_full_20260815_142714",
"suite_id": "public_uzbek_full_vllm_apst",
"tasks": {
"uzlib_slice_full": 0.32079527135948416,
"tumlu_uzbek_full": 0.27714285714285714,
"flores_en_uz_full": 0.8673252673692642,
"news_full": 0.1359802000618748,
"mmlu_english_full": 0.3872667711152257,
"mmlu_uz_full": 0.32744623273038026,
"flores_uz_en_full": 0.7954316840740269,
"sentiment_binary_full": 0.3885
},
"invalid_rate": {
"uzlib_slice_full": 0.0010746910263299302,
"tumlu_uzbek_full": 0.28714285714285714,
"flores_en_uz_full": null,
"news_full": 7.218727441476746e-05,
"mmlu_english_full": 0.06900726392251816,
"mmlu_uz_full": 0.07726819541375872,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.4405
},
"normalized_weighted_score": 0.41468665305295505
},
"alloma-1B": {
"run_id": "alloma_1B_apst_full_20260811_114037",
"suite_id": "public_uzbek_full_vllm_apst",
"tasks": {
"uzlib_slice_full": 0.23320795271359485,
"tumlu_uzbek_full": 0.22,
"flores_en_uz_full": 0.7383331350348018,
"news_full": 0.2541404558110756,
"mmlu_english_full": 0.21976926363765845,
"mmlu_uz_full": 0.21108104258652613,
"flores_uz_en_full": 0.7636193416348852,
"sentiment_binary_full": 0.7954
},
"invalid_rate": {
"uzlib_slice_full": 0.04567436861902203,
"tumlu_uzbek_full": 0.36714285714285716,
"flores_en_uz_full": null,
"news_full": 0.0003506239043002991,
"mmlu_english_full": 0.09393248824953711,
"mmlu_uz_full": 0.17882068081469876,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0063
},
"normalized_weighted_score": 0.3661490569296942
}
},
"checkpoints": {
"0.25 epoch": {
"run_id": "qwen35_2b_lora_r64_grouped_identity_ckpt1188_public_v2",
"suite_id": "public_uzbek_full_comet_vllm_v2",
"tasks": {
"uzlib_slice_full": 0.5432563138097797,
"tumlu_uzbek_full": 0.32,
"flores_en_uz_full": 0.8569777911726428,
"news_full": 0.7521810869341033,
"mmlu_english_full": 0.5305512035322604,
"mmlu_uz_full": 0.4598347813701752,
"flores_uz_en_full": 0.7069662087233046,
"sentiment_binary_full": 0.9087
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0
},
"normalized_weighted_score": 0.5855136305870227
},
"0.50 epoch": {
"run_id": "qwen35_2b_lora_r64_grouped_identity_ckpt2376_public_v2",
"suite_id": "public_uzbek_full_comet_vllm_v2",
"tasks": {
"uzlib_slice_full": 0.4954325631380978,
"tumlu_uzbek_full": 0.3057142857142857,
"flores_en_uz_full": 0.8641375761503245,
"news_full": 0.7519542126430855,
"mmlu_english_full": 0.5307648483122063,
"mmlu_uz_full": 0.46360917248255235,
"flores_uz_en_full": 0.8504324527211662,
"sentiment_binary_full": 0.9502
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0
},
"normalized_weighted_score": 0.5836994233398612
},
"0.75 epoch": {
"run_id": "qwen35_2b_lora_r64_grouped_identity_ckpt3564_public_v2",
"suite_id": "public_uzbek_full_comet_vllm_v2",
"tasks": {
"uzlib_slice_full": 0.4744760881246642,
"tumlu_uzbek_full": 0.30857142857142855,
"flores_en_uz_full": 0.8769590624563133,
"news_full": 0.7998040631123028,
"mmlu_english_full": 0.5434411052556616,
"mmlu_uz_full": 0.46802449793476714,
"flores_uz_en_full": 0.8535260226727719,
"sentiment_binary_full": 0.9529
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0
},
"normalized_weighted_score": 0.5890543478648181
},
"1.00 epoch": {
"run_id": "qwen35_2b_lora_r64_grouped_identity_ckpt4749_public_v2",
"suite_id": "public_uzbek_full_comet_vllm_v2",
"tasks": {
"uzlib_slice_full": 0.4959699086512628,
"tumlu_uzbek_full": 0.32571428571428573,
"flores_en_uz_full": 0.8761537177251785,
"news_full": 0.7854903578426318,
"mmlu_english_full": 0.5406637231163652,
"mmlu_uz_full": 0.4685230024213075,
"flores_uz_en_full": 0.8534744450736248,
"sentiment_binary_full": 0.955
},
"invalid_rate": {
"uzlib_slice_full": 0.0,
"tumlu_uzbek_full": 0.0,
"flores_en_uz_full": null,
"news_full": 0.0,
"mmlu_english_full": 0.0,
"mmlu_uz_full": 0.0,
"flores_uz_en_full": null,
"sentiment_binary_full": 0.0
},
"normalized_weighted_score": 0.5954224496037875
}
},
"tokenizer_fertility": {
"status": "superseded",
"result": "fertility_large_20260825.json",
"note": "Use the corpus-stratified 2026-08-25 evaluation; the old 20k uz-crawl-only result was in-domain."
}
}
|