Card update: results at updates 40,000 and 45,000
Browse files- EVALUATION.json +748 -8
- README.md +27 -21
EVALUATION.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
-
"status": "descriptive intermediate base checkpoint measurements
|
| 5 |
-
"source_aggregate_sha256": "
|
| 6 |
-
"source_generator_sha256":
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
| 9 |
"protocol": {
|
|
@@ -64,7 +64,7 @@
|
|
| 64 |
"method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
|
| 65 |
"replicates": 10000,
|
| 66 |
"base_seed": 20261110,
|
| 67 |
-
"seed_scope": "
|
| 68 |
"assumption": "items treated as independent; no subject, template or source clusters",
|
| 69 |
"interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
|
| 70 |
},
|
|
@@ -2354,6 +2354,514 @@
|
|
| 2354 |
"correct_primary": 3269
|
| 2355 |
}
|
| 2356 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2357 |
}
|
| 2358 |
},
|
| 2359 |
"additional_measurements": {
|
|
@@ -2553,9 +3061,9 @@
|
|
| 2553 |
}
|
| 2554 |
}
|
| 2555 |
},
|
| 2556 |
-
"source_generator_note": "
|
| 2557 |
"auxiliary_diagnostics": {
|
| 2558 |
-
"source_aggregate_sha256": "
|
| 2559 |
"method": "unweighted mean of class recalls (balanced accuracy); collected after the primary measurements; not a replacement for primary accuracy",
|
| 2560 |
"units": "percent",
|
| 2561 |
"uncertainty": "confidence intervals not supplied for these auxiliary values; no formal checkpoint-difference conclusion",
|
|
@@ -3021,7 +3529,239 @@
|
|
| 3021 |
]
|
| 3022 |
}
|
| 3023 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3024 |
}
|
| 3025 |
-
}
|
| 3026 |
-
|
|
|
|
|
|
|
| 3027 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
+
"status": "descriptive intermediate base checkpoint measurements through 1.09 update 45000; archive availability and execution state have separate receipts in the model card; this aggregate is not live telemetry; training-lineage contamination checks incomplete",
|
| 5 |
+
"source_aggregate_sha256": "d21e1d32ab57a4edb261fdc83ab898d47810f6c2444d084ff384920af282172f",
|
| 6 |
+
"source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
| 9 |
"protocol": {
|
|
|
|
| 64 |
"method": "percentile bootstrap via Binomial(n,p)/n for binary item scores",
|
| 65 |
"replicates": 10000,
|
| 66 |
"base_seed": 20261110,
|
| 67 |
+
"seed_scope": "One numpy default_rng per model: [base_seed, int(SHA256(model_label UTF-8).hexdigest()[:8],16)]. Tasks in source result order; acc then acc_norm; 10000 Binomial(n,p)/n draws per metric; numpy percentile 2.5/97.5, fractions rounded to six decimal places.",
|
| 68 |
"assumption": "items treated as independent; no subject, template or source clusters",
|
| 69 |
"interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance"
|
| 70 |
},
|
|
|
|
| 2354 |
"correct_primary": 3269
|
| 2355 |
}
|
| 2356 |
}
|
| 2357 |
+
},
|
| 2358 |
+
"1.09-k40000": {
|
| 2359 |
+
"step": 40000,
|
| 2360 |
+
"weights_sha256": "5d60d691f1ea87db4a900788ff253f2de55e75f9210c93e7e95f3b9f6c434591",
|
| 2361 |
+
"source_results_sha256": "b5823b87dd492662918d29893c78f2795c2b511b9085f7a5c8a862cdef2f91b7",
|
| 2362 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 2363 |
+
"weights_published": false,
|
| 2364 |
+
"tasks": {
|
| 2365 |
+
"arc_challenge": {
|
| 2366 |
+
"n": 1172,
|
| 2367 |
+
"n_pelne": 1172,
|
| 2368 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 2369 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 2370 |
+
"pliki_sha256": {
|
| 2371 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 2372 |
+
},
|
| 2373 |
+
"acc": 0.327645,
|
| 2374 |
+
"acc_ci95": [
|
| 2375 |
+
0.301195,
|
| 2376 |
+
0.354949
|
| 2377 |
+
],
|
| 2378 |
+
"acc_norm": 0.349829,
|
| 2379 |
+
"acc_norm_ci95": [
|
| 2380 |
+
0.322526,
|
| 2381 |
+
0.377986
|
| 2382 |
+
],
|
| 2383 |
+
"split": "test",
|
| 2384 |
+
"correct_primary": 410
|
| 2385 |
+
},
|
| 2386 |
+
"hellaswag": {
|
| 2387 |
+
"n": 10042,
|
| 2388 |
+
"n_pelne": 10042,
|
| 2389 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 2390 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 2391 |
+
"pliki_sha256": {
|
| 2392 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 2393 |
+
},
|
| 2394 |
+
"acc": 0.417347,
|
| 2395 |
+
"acc_ci95": [
|
| 2396 |
+
0.407787,
|
| 2397 |
+
0.427007
|
| 2398 |
+
],
|
| 2399 |
+
"acc_norm": 0.527982,
|
| 2400 |
+
"acc_norm_ci95": [
|
| 2401 |
+
0.518124,
|
| 2402 |
+
0.537841
|
| 2403 |
+
],
|
| 2404 |
+
"split": "validation",
|
| 2405 |
+
"correct_primary": 5302
|
| 2406 |
+
},
|
| 2407 |
+
"piqa": {
|
| 2408 |
+
"n": 1838,
|
| 2409 |
+
"n_pelne": 1838,
|
| 2410 |
+
"dataset_repo": "baber/piqa",
|
| 2411 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 2412 |
+
"pliki_sha256": {
|
| 2413 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 2414 |
+
},
|
| 2415 |
+
"acc": 0.70457,
|
| 2416 |
+
"acc_ci95": [
|
| 2417 |
+
0.683351,
|
| 2418 |
+
0.725245
|
| 2419 |
+
],
|
| 2420 |
+
"acc_norm": 0.712187,
|
| 2421 |
+
"acc_norm_ci95": [
|
| 2422 |
+
0.692043,
|
| 2423 |
+
0.732862
|
| 2424 |
+
],
|
| 2425 |
+
"split": "validation",
|
| 2426 |
+
"correct_primary": 1309
|
| 2427 |
+
},
|
| 2428 |
+
"winogrande": {
|
| 2429 |
+
"n": 1267,
|
| 2430 |
+
"n_pelne": 1267,
|
| 2431 |
+
"dataset_repo": "allenai/winogrande",
|
| 2432 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 2433 |
+
"pliki_sha256": {
|
| 2434 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 2435 |
+
},
|
| 2436 |
+
"acc": 0.576953,
|
| 2437 |
+
"acc_ci95": [
|
| 2438 |
+
0.550118,
|
| 2439 |
+
0.603788
|
| 2440 |
+
],
|
| 2441 |
+
"acc_norm": 0.576953,
|
| 2442 |
+
"acc_norm_ci95": [
|
| 2443 |
+
0.549329,
|
| 2444 |
+
0.604578
|
| 2445 |
+
],
|
| 2446 |
+
"split": "validation",
|
| 2447 |
+
"correct_primary": 731
|
| 2448 |
+
},
|
| 2449 |
+
"boolq": {
|
| 2450 |
+
"n": 3270,
|
| 2451 |
+
"n_pelne": 3270,
|
| 2452 |
+
"dataset_repo": "aps/super_glue",
|
| 2453 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 2454 |
+
"pliki_sha256": {
|
| 2455 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 2456 |
+
},
|
| 2457 |
+
"acc": 0.591743,
|
| 2458 |
+
"acc_ci95": [
|
| 2459 |
+
0.574618,
|
| 2460 |
+
0.608257
|
| 2461 |
+
],
|
| 2462 |
+
"acc_norm": 0.627217,
|
| 2463 |
+
"acc_norm_ci95": [
|
| 2464 |
+
0.611009,
|
| 2465 |
+
0.643425
|
| 2466 |
+
],
|
| 2467 |
+
"split": "validation",
|
| 2468 |
+
"correct_primary": 1935
|
| 2469 |
+
},
|
| 2470 |
+
"openbookqa": {
|
| 2471 |
+
"n": 500,
|
| 2472 |
+
"n_pelne": 500,
|
| 2473 |
+
"dataset_repo": "allenai/openbookqa",
|
| 2474 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 2475 |
+
"pliki_sha256": {
|
| 2476 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 2477 |
+
},
|
| 2478 |
+
"acc": 0.244,
|
| 2479 |
+
"acc_ci95": [
|
| 2480 |
+
0.208,
|
| 2481 |
+
0.282
|
| 2482 |
+
],
|
| 2483 |
+
"acc_norm": 0.346,
|
| 2484 |
+
"acc_norm_ci95": [
|
| 2485 |
+
0.304,
|
| 2486 |
+
0.388
|
| 2487 |
+
],
|
| 2488 |
+
"split": "test",
|
| 2489 |
+
"correct_primary": 173
|
| 2490 |
+
},
|
| 2491 |
+
"belebele_pol": {
|
| 2492 |
+
"n": 900,
|
| 2493 |
+
"n_pelne": 900,
|
| 2494 |
+
"dataset_repo": "facebook/belebele",
|
| 2495 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 2496 |
+
"pliki_sha256": {
|
| 2497 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 2498 |
+
},
|
| 2499 |
+
"acc": 0.255556,
|
| 2500 |
+
"acc_ci95": [
|
| 2501 |
+
0.227778,
|
| 2502 |
+
0.284444
|
| 2503 |
+
],
|
| 2504 |
+
"acc_norm": 0.255556,
|
| 2505 |
+
"acc_norm_ci95": [
|
| 2506 |
+
0.227778,
|
| 2507 |
+
0.284444
|
| 2508 |
+
],
|
| 2509 |
+
"split": "pol_Latn evaluation set",
|
| 2510 |
+
"correct_primary": 230
|
| 2511 |
+
},
|
| 2512 |
+
"hellaswag_pl": {
|
| 2513 |
+
"n": 10042,
|
| 2514 |
+
"n_pelne": 10042,
|
| 2515 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 2516 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 2517 |
+
"pliki_sha256": {
|
| 2518 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 2519 |
+
},
|
| 2520 |
+
"acc": 0.35949,
|
| 2521 |
+
"acc_ci95": [
|
| 2522 |
+
0.350129,
|
| 2523 |
+
0.36895
|
| 2524 |
+
],
|
| 2525 |
+
"acc_norm": 0.466142,
|
| 2526 |
+
"acc_norm_ci95": [
|
| 2527 |
+
0.456483,
|
| 2528 |
+
0.475702
|
| 2529 |
+
],
|
| 2530 |
+
"split": "validation",
|
| 2531 |
+
"correct_primary": 4681
|
| 2532 |
+
},
|
| 2533 |
+
"mmlu_pl": {
|
| 2534 |
+
"n": 14042,
|
| 2535 |
+
"n_pelne": 14042,
|
| 2536 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 2537 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 2538 |
+
"pliki_sha256": {
|
| 2539 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 2540 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 2541 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 2542 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 2543 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 2544 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 2545 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 2546 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 2547 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 2548 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 2549 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 2550 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 2551 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 2552 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 2553 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 2554 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 2555 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 2556 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 2557 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 2558 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 2559 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 2560 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 2561 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 2562 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 2563 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 2564 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 2565 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 2566 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 2567 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 2568 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 2569 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 2570 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 2571 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 2572 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 2573 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 2574 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 2575 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 2576 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 2577 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 2578 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 2579 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 2580 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 2581 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 2582 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 2583 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 2584 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 2585 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 2586 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 2587 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 2588 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 2589 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 2590 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 2591 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 2592 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 2593 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 2594 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 2595 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 2596 |
+
},
|
| 2597 |
+
"acc": 0.242416,
|
| 2598 |
+
"acc_ci95": [
|
| 2599 |
+
0.235437,
|
| 2600 |
+
0.24968
|
| 2601 |
+
],
|
| 2602 |
+
"acc_norm": 0.242416,
|
| 2603 |
+
"acc_norm_ci95": [
|
| 2604 |
+
0.235508,
|
| 2605 |
+
0.249396
|
| 2606 |
+
],
|
| 2607 |
+
"split": "test; micro-average over subjects",
|
| 2608 |
+
"correct_primary": 3404
|
| 2609 |
+
}
|
| 2610 |
+
}
|
| 2611 |
+
},
|
| 2612 |
+
"1.09-k45000": {
|
| 2613 |
+
"step": 45000,
|
| 2614 |
+
"weights_sha256": "7fbc475edb5a387170e1ef1a1de4ebdc3209f5f93c29fdfe48034fd13e99ad62",
|
| 2615 |
+
"source_results_sha256": "017ae81bdc8b13c932395e06f4643a0004093708f59e77eecac6f1d229b19531",
|
| 2616 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 2617 |
+
"weights_published": false,
|
| 2618 |
+
"tasks": {
|
| 2619 |
+
"arc_challenge": {
|
| 2620 |
+
"n": 1172,
|
| 2621 |
+
"n_pelne": 1172,
|
| 2622 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 2623 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 2624 |
+
"pliki_sha256": {
|
| 2625 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 2626 |
+
},
|
| 2627 |
+
"acc": 0.319113,
|
| 2628 |
+
"acc_ci95": [
|
| 2629 |
+
0.291809,
|
| 2630 |
+
0.346416
|
| 2631 |
+
],
|
| 2632 |
+
"acc_norm": 0.348123,
|
| 2633 |
+
"acc_norm_ci95": [
|
| 2634 |
+
0.320819,
|
| 2635 |
+
0.375427
|
| 2636 |
+
],
|
| 2637 |
+
"split": "test",
|
| 2638 |
+
"correct_primary": 408
|
| 2639 |
+
},
|
| 2640 |
+
"hellaswag": {
|
| 2641 |
+
"n": 10042,
|
| 2642 |
+
"n_pelne": 10042,
|
| 2643 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 2644 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 2645 |
+
"pliki_sha256": {
|
| 2646 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 2647 |
+
},
|
| 2648 |
+
"acc": 0.417845,
|
| 2649 |
+
"acc_ci95": [
|
| 2650 |
+
0.408186,
|
| 2651 |
+
0.427504
|
| 2652 |
+
],
|
| 2653 |
+
"acc_norm": 0.536547,
|
| 2654 |
+
"acc_norm_ci95": [
|
| 2655 |
+
0.526887,
|
| 2656 |
+
0.546206
|
| 2657 |
+
],
|
| 2658 |
+
"split": "validation",
|
| 2659 |
+
"correct_primary": 5388
|
| 2660 |
+
},
|
| 2661 |
+
"piqa": {
|
| 2662 |
+
"n": 1838,
|
| 2663 |
+
"n_pelne": 1838,
|
| 2664 |
+
"dataset_repo": "baber/piqa",
|
| 2665 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 2666 |
+
"pliki_sha256": {
|
| 2667 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 2668 |
+
},
|
| 2669 |
+
"acc": 0.713275,
|
| 2670 |
+
"acc_ci95": [
|
| 2671 |
+
0.692601,
|
| 2672 |
+
0.733406
|
| 2673 |
+
],
|
| 2674 |
+
"acc_norm": 0.702938,
|
| 2675 |
+
"acc_norm_ci95": [
|
| 2676 |
+
0.682263,
|
| 2677 |
+
0.723069
|
| 2678 |
+
],
|
| 2679 |
+
"split": "validation",
|
| 2680 |
+
"correct_primary": 1292
|
| 2681 |
+
},
|
| 2682 |
+
"winogrande": {
|
| 2683 |
+
"n": 1267,
|
| 2684 |
+
"n_pelne": 1267,
|
| 2685 |
+
"dataset_repo": "allenai/winogrande",
|
| 2686 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 2687 |
+
"pliki_sha256": {
|
| 2688 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 2689 |
+
},
|
| 2690 |
+
"acc": 0.590371,
|
| 2691 |
+
"acc_ci95": [
|
| 2692 |
+
0.562747,
|
| 2693 |
+
0.617206
|
| 2694 |
+
],
|
| 2695 |
+
"acc_norm": 0.590371,
|
| 2696 |
+
"acc_norm_ci95": [
|
| 2697 |
+
0.563536,
|
| 2698 |
+
0.617995
|
| 2699 |
+
],
|
| 2700 |
+
"split": "validation",
|
| 2701 |
+
"correct_primary": 748
|
| 2702 |
+
},
|
| 2703 |
+
"boolq": {
|
| 2704 |
+
"n": 3270,
|
| 2705 |
+
"n_pelne": 3270,
|
| 2706 |
+
"dataset_repo": "aps/super_glue",
|
| 2707 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 2708 |
+
"pliki_sha256": {
|
| 2709 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 2710 |
+
},
|
| 2711 |
+
"acc": 0.619266,
|
| 2712 |
+
"acc_ci95": [
|
| 2713 |
+
0.602446,
|
| 2714 |
+
0.636086
|
| 2715 |
+
],
|
| 2716 |
+
"acc_norm": 0.628135,
|
| 2717 |
+
"acc_norm_ci95": [
|
| 2718 |
+
0.611621,
|
| 2719 |
+
0.644343
|
| 2720 |
+
],
|
| 2721 |
+
"split": "validation",
|
| 2722 |
+
"correct_primary": 2025
|
| 2723 |
+
},
|
| 2724 |
+
"openbookqa": {
|
| 2725 |
+
"n": 500,
|
| 2726 |
+
"n_pelne": 500,
|
| 2727 |
+
"dataset_repo": "allenai/openbookqa",
|
| 2728 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 2729 |
+
"pliki_sha256": {
|
| 2730 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 2731 |
+
},
|
| 2732 |
+
"acc": 0.254,
|
| 2733 |
+
"acc_ci95": [
|
| 2734 |
+
0.216,
|
| 2735 |
+
0.292
|
| 2736 |
+
],
|
| 2737 |
+
"acc_norm": 0.36,
|
| 2738 |
+
"acc_norm_ci95": [
|
| 2739 |
+
0.318,
|
| 2740 |
+
0.402
|
| 2741 |
+
],
|
| 2742 |
+
"split": "test",
|
| 2743 |
+
"correct_primary": 180
|
| 2744 |
+
},
|
| 2745 |
+
"belebele_pol": {
|
| 2746 |
+
"n": 900,
|
| 2747 |
+
"n_pelne": 900,
|
| 2748 |
+
"dataset_repo": "facebook/belebele",
|
| 2749 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 2750 |
+
"pliki_sha256": {
|
| 2751 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 2752 |
+
},
|
| 2753 |
+
"acc": 0.253333,
|
| 2754 |
+
"acc_ci95": [
|
| 2755 |
+
0.225556,
|
| 2756 |
+
0.282222
|
| 2757 |
+
],
|
| 2758 |
+
"acc_norm": 0.253333,
|
| 2759 |
+
"acc_norm_ci95": [
|
| 2760 |
+
0.225556,
|
| 2761 |
+
0.282222
|
| 2762 |
+
],
|
| 2763 |
+
"split": "pol_Latn evaluation set",
|
| 2764 |
+
"correct_primary": 228
|
| 2765 |
+
},
|
| 2766 |
+
"hellaswag_pl": {
|
| 2767 |
+
"n": 10042,
|
| 2768 |
+
"n_pelne": 10042,
|
| 2769 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 2770 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 2771 |
+
"pliki_sha256": {
|
| 2772 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 2773 |
+
},
|
| 2774 |
+
"acc": 0.362079,
|
| 2775 |
+
"acc_ci95": [
|
| 2776 |
+
0.352519,
|
| 2777 |
+
0.37134
|
| 2778 |
+
],
|
| 2779 |
+
"acc_norm": 0.467835,
|
| 2780 |
+
"acc_norm_ci95": [
|
| 2781 |
+
0.457877,
|
| 2782 |
+
0.477495
|
| 2783 |
+
],
|
| 2784 |
+
"split": "validation",
|
| 2785 |
+
"correct_primary": 4698
|
| 2786 |
+
},
|
| 2787 |
+
"mmlu_pl": {
|
| 2788 |
+
"n": 14042,
|
| 2789 |
+
"n_pelne": 14042,
|
| 2790 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 2791 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 2792 |
+
"pliki_sha256": {
|
| 2793 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 2794 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 2795 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 2796 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 2797 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 2798 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 2799 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 2800 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 2801 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 2802 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 2803 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 2804 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 2805 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 2806 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 2807 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 2808 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 2809 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 2810 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 2811 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 2812 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 2813 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 2814 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 2815 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 2816 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 2817 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 2818 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 2819 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 2820 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 2821 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 2822 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 2823 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 2824 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 2825 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 2826 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 2827 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 2828 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 2829 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 2830 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 2831 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 2832 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 2833 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 2834 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 2835 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 2836 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 2837 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 2838 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 2839 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 2840 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 2841 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 2842 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 2843 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 2844 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 2845 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 2846 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 2847 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 2848 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 2849 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 2850 |
+
},
|
| 2851 |
+
"acc": 0.257371,
|
| 2852 |
+
"acc_ci95": [
|
| 2853 |
+
0.250247,
|
| 2854 |
+
0.264635
|
| 2855 |
+
],
|
| 2856 |
+
"acc_norm": 0.257371,
|
| 2857 |
+
"acc_norm_ci95": [
|
| 2858 |
+
0.25032,
|
| 2859 |
+
0.264492
|
| 2860 |
+
],
|
| 2861 |
+
"split": "test; micro-average over subjects",
|
| 2862 |
+
"correct_primary": 3614
|
| 2863 |
+
}
|
| 2864 |
+
}
|
| 2865 |
}
|
| 2866 |
},
|
| 2867 |
"additional_measurements": {
|
|
|
|
| 3061 |
}
|
| 3062 |
}
|
| 3063 |
},
|
| 3064 |
+
"source_generator_note": "Retained CI generator reproduces all 9 previous model records by value. Those published records were preserved unchanged; only the new checkpoint records were appended. The earlier missing stream/producer link is resolved by the retained generator and reproduction receipt.",
|
| 3065 |
"auxiliary_diagnostics": {
|
| 3066 |
+
"source_aggregate_sha256": "0c316a800deeb27c2f4acb8810d38f7accc4a310c016ebfb66d7941d78a97084",
|
| 3067 |
"method": "unweighted mean of class recalls (balanced accuracy); collected after the primary measurements; not a replacement for primary accuracy",
|
| 3068 |
"units": "percent",
|
| 3069 |
"uncertainty": "confidence intervals not supplied for these auxiliary values; no formal checkpoint-difference conclusion",
|
|
|
|
| 3529 |
]
|
| 3530 |
}
|
| 3531 |
}
|
| 3532 |
+
},
|
| 3533 |
+
"1.09-k40000": {
|
| 3534 |
+
"source_diagnostic_sha256": "f5775528d52415002414a054339ba382dddfe25626fbfdd814dfae2d048709c6",
|
| 3535 |
+
"source_diagnostic_kind": "derived balanced aggregate; no new model forward",
|
| 3536 |
+
"source_gold_diagnostic_sha256": "85cf33056bd30a85cf5411568ba6e2098eb31bbe9493d4156d800e98d6edcc1b",
|
| 3537 |
+
"source_track_results_sha256": "b5823b87dd492662918d29893c78f2795c2b511b9085f7a5c8a862cdef2f91b7",
|
| 3538 |
+
"tasks": {
|
| 3539 |
+
"boolq": {
|
| 3540 |
+
"n": 3270,
|
| 3541 |
+
"balanced_accuracy_percent": 60.36,
|
| 3542 |
+
"classes": [
|
| 3543 |
+
"no",
|
| 3544 |
+
"yes"
|
| 3545 |
+
],
|
| 3546 |
+
"gold_class_percent": [
|
| 3547 |
+
37.8,
|
| 3548 |
+
62.2
|
| 3549 |
+
],
|
| 3550 |
+
"class_totals": [
|
| 3551 |
+
1237,
|
| 3552 |
+
2033
|
| 3553 |
+
],
|
| 3554 |
+
"class_correct": [
|
| 3555 |
+
807,
|
| 3556 |
+
1128
|
| 3557 |
+
],
|
| 3558 |
+
"class_recalls": [
|
| 3559 |
+
0.6523848019401779,
|
| 3560 |
+
0.5548450565666503
|
| 3561 |
+
],
|
| 3562 |
+
"balanced_accuracy_fraction_exact": "3035967/5029642",
|
| 3563 |
+
"items_acc_sha256": "7ca8545f4e45c5ef4fc0024c31eb9ac0db17a25c48e4d557456119b229198aa5",
|
| 3564 |
+
"items_gold_sha256": "2404e078cb21f7e36be36272bdf4515a311d9104bf6c954eb091ffa20fd434a3",
|
| 3565 |
+
"predicted_class_percent": null,
|
| 3566 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3567 |
+
},
|
| 3568 |
+
"mmlu_pl": {
|
| 3569 |
+
"n": 14042,
|
| 3570 |
+
"balanced_accuracy_percent": 24.93,
|
| 3571 |
+
"classes": [
|
| 3572 |
+
"A",
|
| 3573 |
+
"B",
|
| 3574 |
+
"C",
|
| 3575 |
+
"D"
|
| 3576 |
+
],
|
| 3577 |
+
"gold_class_percent": [
|
| 3578 |
+
22.9,
|
| 3579 |
+
24.7,
|
| 3580 |
+
25.5,
|
| 3581 |
+
26.9
|
| 3582 |
+
],
|
| 3583 |
+
"class_totals": [
|
| 3584 |
+
3222,
|
| 3585 |
+
3462,
|
| 3586 |
+
3582,
|
| 3587 |
+
3776
|
| 3588 |
+
],
|
| 3589 |
+
"class_correct": [
|
| 3590 |
+
687,
|
| 3591 |
+
2634,
|
| 3592 |
+
69,
|
| 3593 |
+
14
|
| 3594 |
+
],
|
| 3595 |
+
"class_recalls": [
|
| 3596 |
+
0.2132216014897579,
|
| 3597 |
+
0.7608318890814558,
|
| 3598 |
+
0.019262981574539362,
|
| 3599 |
+
0.003707627118644068
|
| 3600 |
+
],
|
| 3601 |
+
"balanced_accuracy_fraction_exact": "116067551017/465655949952",
|
| 3602 |
+
"items_acc_sha256": "879c7e7788b17b0c657e0cd96a121aaa0d587ade71698aca432bf9190bc9e7f4",
|
| 3603 |
+
"items_gold_sha256": "d87e03f33f63c1efd17611ee976a17b25f70df9c1ed59f02b31d581437bd6701",
|
| 3604 |
+
"predicted_class_percent": null,
|
| 3605 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3606 |
+
},
|
| 3607 |
+
"belebele_pol": {
|
| 3608 |
+
"n": 900,
|
| 3609 |
+
"balanced_accuracy_percent": 26.29,
|
| 3610 |
+
"classes": [
|
| 3611 |
+
"A",
|
| 3612 |
+
"B",
|
| 3613 |
+
"C",
|
| 3614 |
+
"D"
|
| 3615 |
+
],
|
| 3616 |
+
"gold_class_percent": [
|
| 3617 |
+
22.9,
|
| 3618 |
+
27.9,
|
| 3619 |
+
27.3,
|
| 3620 |
+
21.9
|
| 3621 |
+
],
|
| 3622 |
+
"class_totals": [
|
| 3623 |
+
206,
|
| 3624 |
+
251,
|
| 3625 |
+
246,
|
| 3626 |
+
197
|
| 3627 |
+
],
|
| 3628 |
+
"class_correct": [
|
| 3629 |
+
138,
|
| 3630 |
+
31,
|
| 3631 |
+
51,
|
| 3632 |
+
10
|
| 3633 |
+
],
|
| 3634 |
+
"class_recalls": [
|
| 3635 |
+
0.6699029126213593,
|
| 3636 |
+
0.12350597609561753,
|
| 3637 |
+
0.2073170731707317,
|
| 3638 |
+
0.050761421319796954
|
| 3639 |
+
],
|
| 3640 |
+
"balanced_accuracy_fraction_exact": "439132005/1670517448",
|
| 3641 |
+
"items_acc_sha256": "89a83428b2a559eb15b18f63ce102100844533d15faab6e42cce32b261e173a2",
|
| 3642 |
+
"items_gold_sha256": "ae784c3f36b672f194365df64d33afeb4cef9377d4543b979580c743fe688dd4",
|
| 3643 |
+
"predicted_class_percent": null,
|
| 3644 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3645 |
+
}
|
| 3646 |
+
}
|
| 3647 |
+
},
|
| 3648 |
+
"1.09-k45000": {
|
| 3649 |
+
"source_diagnostic_sha256": "f5775528d52415002414a054339ba382dddfe25626fbfdd814dfae2d048709c6",
|
| 3650 |
+
"source_diagnostic_kind": "derived balanced aggregate; no new model forward",
|
| 3651 |
+
"source_gold_diagnostic_sha256": "85cf33056bd30a85cf5411568ba6e2098eb31bbe9493d4156d800e98d6edcc1b",
|
| 3652 |
+
"source_track_results_sha256": "017ae81bdc8b13c932395e06f4643a0004093708f59e77eecac6f1d229b19531",
|
| 3653 |
+
"tasks": {
|
| 3654 |
+
"boolq": {
|
| 3655 |
+
"n": 3270,
|
| 3656 |
+
"balanced_accuracy_percent": 58.92,
|
| 3657 |
+
"classes": [
|
| 3658 |
+
"no",
|
| 3659 |
+
"yes"
|
| 3660 |
+
],
|
| 3661 |
+
"gold_class_percent": [
|
| 3662 |
+
37.8,
|
| 3663 |
+
62.2
|
| 3664 |
+
],
|
| 3665 |
+
"class_totals": [
|
| 3666 |
+
1237,
|
| 3667 |
+
2033
|
| 3668 |
+
],
|
| 3669 |
+
"class_correct": [
|
| 3670 |
+
576,
|
| 3671 |
+
1449
|
| 3672 |
+
],
|
| 3673 |
+
"class_recalls": [
|
| 3674 |
+
0.465642683912692,
|
| 3675 |
+
0.7127397934087555
|
| 3676 |
+
],
|
| 3677 |
+
"balanced_accuracy_fraction_exact": "2963421/5029642",
|
| 3678 |
+
"items_acc_sha256": "0a9fafc3832b5e4e688997f4f0b63f5de3c366936b45252ec2a05ddadb326e1c",
|
| 3679 |
+
"items_gold_sha256": "2404e078cb21f7e36be36272bdf4515a311d9104bf6c954eb091ffa20fd434a3",
|
| 3680 |
+
"predicted_class_percent": null,
|
| 3681 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3682 |
+
},
|
| 3683 |
+
"mmlu_pl": {
|
| 3684 |
+
"n": 14042,
|
| 3685 |
+
"balanced_accuracy_percent": 24.91,
|
| 3686 |
+
"classes": [
|
| 3687 |
+
"A",
|
| 3688 |
+
"B",
|
| 3689 |
+
"C",
|
| 3690 |
+
"D"
|
| 3691 |
+
],
|
| 3692 |
+
"gold_class_percent": [
|
| 3693 |
+
22.9,
|
| 3694 |
+
24.7,
|
| 3695 |
+
25.5,
|
| 3696 |
+
26.9
|
| 3697 |
+
],
|
| 3698 |
+
"class_totals": [
|
| 3699 |
+
3222,
|
| 3700 |
+
3462,
|
| 3701 |
+
3582,
|
| 3702 |
+
3776
|
| 3703 |
+
],
|
| 3704 |
+
"class_correct": [
|
| 3705 |
+
445,
|
| 3706 |
+
108,
|
| 3707 |
+
1149,
|
| 3708 |
+
1912
|
| 3709 |
+
],
|
| 3710 |
+
"class_recalls": [
|
| 3711 |
+
0.13811297330850403,
|
| 3712 |
+
0.03119584055459272,
|
| 3713 |
+
0.32077051926298156,
|
| 3714 |
+
0.5063559322033898
|
| 3715 |
+
],
|
| 3716 |
+
"balanced_accuracy_fraction_exact": "86999251883/349241962464",
|
| 3717 |
+
"items_acc_sha256": "e5b60abf99607d4b08611eecbdfbe0e53a5b4eeb610578be49bd01fb141f6f81",
|
| 3718 |
+
"items_gold_sha256": "d87e03f33f63c1efd17611ee976a17b25f70df9c1ed59f02b31d581437bd6701",
|
| 3719 |
+
"predicted_class_percent": null,
|
| 3720 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3721 |
+
},
|
| 3722 |
+
"belebele_pol": {
|
| 3723 |
+
"n": 900,
|
| 3724 |
+
"balanced_accuracy_percent": 26.6,
|
| 3725 |
+
"classes": [
|
| 3726 |
+
"A",
|
| 3727 |
+
"B",
|
| 3728 |
+
"C",
|
| 3729 |
+
"D"
|
| 3730 |
+
],
|
| 3731 |
+
"gold_class_percent": [
|
| 3732 |
+
22.9,
|
| 3733 |
+
27.9,
|
| 3734 |
+
27.3,
|
| 3735 |
+
21.9
|
| 3736 |
+
],
|
| 3737 |
+
"class_totals": [
|
| 3738 |
+
206,
|
| 3739 |
+
251,
|
| 3740 |
+
246,
|
| 3741 |
+
197
|
| 3742 |
+
],
|
| 3743 |
+
"class_correct": [
|
| 3744 |
+
149,
|
| 3745 |
+
7,
|
| 3746 |
+
52,
|
| 3747 |
+
20
|
| 3748 |
+
],
|
| 3749 |
+
"class_recalls": [
|
| 3750 |
+
0.7233009708737864,
|
| 3751 |
+
0.027888446215139442,
|
| 3752 |
+
0.21138211382113822,
|
| 3753 |
+
0.10152284263959391
|
| 3754 |
+
],
|
| 3755 |
+
"balanced_accuracy_fraction_exact": "1333191163/5011552344",
|
| 3756 |
+
"items_acc_sha256": "6aa1fe5085f0190cca63ac2e873f2f42b139d3a6ba283657a58748bf86401ae7",
|
| 3757 |
+
"items_gold_sha256": "ae784c3f36b672f194365df64d33afeb4cef9377d4543b979580c743fe688dd4",
|
| 3758 |
+
"predicted_class_percent": null,
|
| 3759 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 3760 |
+
}
|
| 3761 |
+
}
|
| 3762 |
}
|
| 3763 |
+
},
|
| 3764 |
+
"source_extension_note": "Earlier diagnostic records preserved. New balanced values derived from primary correctness and the frozen ordered gold vectors; exact retained diagnostic reader verified against the primary reader. No new inference, prediction-margin diagnostic or confidence interval supplied."
|
| 3765 |
+
},
|
| 3766 |
+
"card_exporter_sha256": "50d7b0506c35b205d14d2fec138166c932000d1cfc366e7e1c4ee3841208bd28"
|
| 3767 |
}
|
README.md
CHANGED
|
@@ -31,15 +31,20 @@ tags:
|
|
| 31 |
| 30,000 | 31,457,280,000 | 50,205,910,554 | `wagi_30000.pt` |
|
| 32 |
| 35,000 | 36,700,160,000 | 55,448,790,554 | `wagi_35000.pt` |
|
| 33 |
| 40,000 | 41,943,040,000 | 60,691,670,554 | `wagi_40000.pt` |
|
|
|
|
| 34 |
|
| 35 |
Files are managed in the separate [checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints). **Historical archive snapshot, 12 November 2026, 18:51 UTC:** upload of the then-listed eight native checkpoint files completed at **18:16 UTC**. The archive is public; verify its exact revision, file list, `SHA256SUMS` and file sizes before use. This main repository contains documentation during training.
|
| 36 |
|
| 37 |
-
**Archive availability,
|
| 38 |
|
| 39 |
Token counts: **positions in this continuation = updates Γ 1,048,576**, counting each logical update once, not repeated compute after recovery. **Lineage lower bound = 18,863,257,682 inherited positions + positions in this continuation β 114,627,128 identified-copy positions**. The lineage figure is a source- and file-disjoint lower bound, **not globally unique tokens**. Exact evaluated safetensors identities in [EVALUATION.json](EVALUATION.json) are distinct from the native `.pt` file digests in the archive.
|
| 40 |
|
| 41 |
|
| 42 |
-
**Execution receipt reported by the operators,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
**Historical operator receipt, 12 November 2026, 18:51 UTC:** training was paused from approximately **16:48 UTC**, at reported progress around **update 40,000**, while allocation prices exceeded the approved ceilings. At that snapshot the public native archive ended at update **35,000**. The [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a) provides later telemetry; reported progress, saved recovery state and published files have separate receipts.
|
| 45 |
|
|
@@ -65,7 +70,7 @@ The [training configuration](TRAINING_CONFIGURATION.md) records the 120,000-upda
|
|
| 65 |
| Setting | Operator-selected recipe |
|
| 66 |
|---|---|
|
| 67 |
| Selected scheduling argument | `KROKI=120000`; confirmed runtime `--steps 120000` adoption at 11 November, 16:23:47 UTC, resuming from update 29,800 on H200; configured decay begins at update 96,000 |
|
| 68 |
-
| New-source allocation |
|
| 69 |
| Optimizer / schedule | Muon peak 0.02 plus AdamW peak 0.0006; 2,000-update warmup, then constant learning rates under the main-base plan; adoption receipt confirms LR 0.0006. The configured final-20% WSD decay to 0.00006 remains at update 96,000; an extension approaching it requires review |
|
| 70 |
| Checkpoints | Runtime `--ckpt-every 200` confirmed at the 16:23:47 UTC adoption; weights-only milestones every 5,000 updates. Native milestone files and full-state recovery files are distinguished in the separate checkpoint archive |
|
| 71 |
| Code | Launcher v5 SHA256 `796c71016c1e167bfd3ca98b78518f91c3502a96eac233dd7186ef6639fcc656`; exact command and converter pins in the configuration document |
|
|
@@ -122,39 +127,39 @@ The PL/EN balance will follow the approved fresh-data inventory. The user explic
|
|
| 122 |
|
| 123 |
New material requires licensing, PII filtering, contamination scans, grouped held-outs and deduplication against the base model's full data ancestry. A 1% document holdout is not necessarily 1% of tokens. Historical exposure in the starting weights cannot be undone by filtering new inputs; retrospective benchmark hits must be handled explicitly in evaluation denominators.
|
| 124 |
|
| 125 |
-
## Standard benchmark measurements β updates 5,000β
|
| 126 |
|
| 127 |
-
We thank **BenchTeam / Fabryka AI** for benchmark implementations, protocol work and evaluation proposals. Its independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is distinct from the **SlayerLab measurements below**, published in the [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a). The
|
| 128 |
|
| 129 |
-
The same nine full splits were scored zero-shot in float32 for 1.07 update 22,327, 1.08 update 1,709 and 1.09 updates 5,000, 10,000, 15,000, 20,000, 25,000, 30,000 and
|
| 130 |
|
| 131 |
-
| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 | 1.09, update 25,000 | 1.09, update 30,000 | 1.09, update 35,000 |
|
| 132 |
-
|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 133 |
-
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] | 33.79 [31.06, 36.52] | 33.02 [30.29, 35.67] | 35.15 [32.42, 37.88] |
|
| 134 |
-
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] | 51.17 [50.18, 52.14] | 52.00 [51.02, 52.98] | 52.48 [51.50, 53.46] |
|
| 135 |
-
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] | 71.55 [69.48, 73.61] | 70.67 [68.61, 72.74] | 71.27 [69.15, 73.29] |
|
| 136 |
-
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] | 55.88 [53.12, 58.64] | 57.14 [54.46, 59.83] | 56.75 [54.06, 59.43] |
|
| 137 |
-
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] | 52.20 [50.52, 53.91] | 60.67 [58.99, 62.35] | 59.88 [58.20, 61.56] |
|
| 138 |
-
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] | 37.20 [32.80, 41.40] | 36.20 [32.00, 40.40] | 34.40 [30.40, 38.60] |
|
| 139 |
-
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] | 25.33 [22.44, 28.22] | 24.44 [21.67, 27.33] | 24.33 [21.56, 27.11] |
|
| 140 |
-
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] | 43.94 [42.94, 44.91] | 45.75 [44.77, 46.73] | 46.63 [45.68, 47.63] |
|
| 141 |
-
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] | 24.26 [23.57, 24.96] | 24.71 [23.99, 25.42] | 23.28 [22.58, 23.97] |
|
| 142 |
|
| 143 |
Reference accuracies are context, not additional trained models or release thresholds:
|
| 144 |
|
| 145 |
| Reference strategy | Scope | Accuracy (%) |
|
| 146 |
-
|---|---|---:|
|
| 147 |
| Uniform choice among four answers | Four-option reference for ARC-Challenge, HellaSwag/PL, OpenBookQA, Belebele-PL and MMLU-PL | 25.00 |
|
| 148 |
| Uniform choice among two answers | BoolQ, PIQA and WinoGrande | 50.00 |
|
| 149 |
| Always answer yes | BoolQ validation majority class: 2,033 of 3,270 labels | 62.17 |
|
| 150 |
|
| 151 |
The majority-class reference is different from uniform guessing. A score near a reference does not prove that checkpoints are equal or that a model has no knowledge; individual confidence intervals are not paired tests. Class-balanced accuracy and answer-frequency diagnostics, when available, are auxiliary and do not replace the primary metrics above.
|
| 152 |
|
| 153 |
-
Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The
|
| 154 |
|
| 155 |
The [machine-readable aggregate record](EVALUATION.json) supplies exact values, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection, without prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 156 |
|
| 157 |
-
The
|
| 158 |
|
| 159 |
**Descriptive results; lineage contamination checks are incomplete.** New-input chunks c03 onward were cleaned against these nine Track tasks, c01βc02 were only partially checked, and the inherited 1.00β1.08 data was not scanned. This is not a guarantee that these tasks are held out from the model's whole training history. Belebele and MMLU-PL are near their four-choice 25% random baseline; this is not proof of absent knowledge. No overall average, ranking or release threshold is inferred. Different evaluators must not be silently substituted.
|
| 160 |
|
|
@@ -175,10 +180,11 @@ Balanced accuracy is the unweighted mean of recall across gold-answer classes. I
|
|
| 175 |
| 25,000 | 57.44 | 24.77 | 26.25 |
|
| 176 |
| 30,000 | 59.10 | 25.00 | 25.39 |
|
| 177 |
| 35,000 | 60.29 | 24.70 | 25.64 |
|
|
|
|
|
|
|
| 178 |
|
| 179 |
The corresponding uniform-choice reference is 50% for BoolQ and 25% for the two four-class tasks. These auxiliary values support inspection of class imbalance. They leave the primary metrics and their published intervals unchanged; a checkpoint-difference claim requires a separate paired analysis.
|
| 180 |
|
| 181 |
-
|
| 182 |
## Additional GSM8K and SST-2 measurements β update 10,000
|
| 183 |
|
| 184 |
SlayerLab measured this two-task subset on 11 November 2026 using the [pinned BenchTeam code](https://github.com/slayerlabs/benchmarks-fabryka-ai/tree/8b547007526971bd2fc35dfe56ff2c7d9b802568), lm-eval 0.4.9.1, the `slayer` adapter, float32, batch 1 and context limit 4,096. These are full evaluated splits, not a full 50-task run or a copy of the community's independent report. **This separate two-task protocol measures 1.09 update 10,000 only; it does not supply GSM8K or SST-2 results for updates 15,000, 20,000, 25,000, 30,000 or 35,000.** Releasing another checkpoint does not supply a missing measurement.
|
|
|
|
| 31 |
| 30,000 | 31,457,280,000 | 50,205,910,554 | `wagi_30000.pt` |
|
| 32 |
| 35,000 | 36,700,160,000 | 55,448,790,554 | `wagi_35000.pt` |
|
| 33 |
| 40,000 | 41,943,040,000 | 60,691,670,554 | `wagi_40000.pt` |
|
| 34 |
+
| 45,000 | 47,185,920,000 | 65,934,550,554 | `wagi_45000.pt` |
|
| 35 |
|
| 36 |
Files are managed in the separate [checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints). **Historical archive snapshot, 12 November 2026, 18:51 UTC:** upload of the then-listed eight native checkpoint files completed at **18:16 UTC**. The archive is public; verify its exact revision, file list, `SHA256SUMS` and file sizes before use. This main repository contains documentation during training.
|
| 37 |
|
| 38 |
+
**Archive availability, 16 November 2026:** the archive distinguishes nine weights-only `.pt` files, at updates 5,000 through 45,000, from the full update-35,000 recovery checkpoint `ckpt_35000.pt`, which also contains optimizer and queue state. No full recovery checkpoint after update 35,000 is published. These are native training checkpoints, not a Transformers `AutoModel` or safetensors release. Check the archive's exact revision, file list and checksums before use; its publication status is separate from the evaluation table below.
|
| 39 |
|
| 40 |
Token counts: **positions in this continuation = updates Γ 1,048,576**, counting each logical update once, not repeated compute after recovery. **Lineage lower bound = 18,863,257,682 inherited positions + positions in this continuation β 114,627,128 identified-copy positions**. The lineage figure is a source- and file-disjoint lower bound, **not globally unique tokens**. Exact evaluated safetensors identities in [EVALUATION.json](EVALUATION.json) are distinct from the native `.pt` file digests in the archive.
|
| 41 |
|
| 42 |
|
| 43 |
+
**Execution receipt reported by the operators, 16 November 2026, 08:45 UTC:** continued pretraining resumed from the full recovery state at **update 47,200** on **2 Γ NVIDIA RTX 4090** (launcher 08:36:38 UTC, trainer resume 08:37:07 UTC) with the same global batch of **256 sequences Γ 4,096 tokens per update** (micro-batch 2 Γ accumulation 64 Γ 2 GPUs); the first newly recorded training point, update 47,210, was logged at 08:45:33 UTC. Training alternates between **8 Γ B200/H200/B300** and **NVIDIA RTX 4090** GPUs depending on availability, always resuming from the latest full recovery state. Training is not finished; execution can advance beyond the latest published checkpoint.
|
| 44 |
+
|
| 45 |
+
**Operator receipt, 16 November 2026, 06:20 UTC:** on 15 November, continued pretraining resumed from **update 42,800** on **8 Γ NVIDIA B300** and saved a full recovery state at **update 47,200**; on 16 November between 06:18 and 06:20 UTC that state was copied outside the training machine with a verified SHA-256.
|
| 46 |
+
|
| 47 |
+
**Historical operator receipt, 14 November 2026, 01:50 UTC:** continued pretraining resumed from **update 40,000** on **2 Γ NVIDIA RTX 4090**.
|
| 48 |
|
| 49 |
**Historical operator receipt, 12 November 2026, 18:51 UTC:** training was paused from approximately **16:48 UTC**, at reported progress around **update 40,000**, while allocation prices exceeded the approved ceilings. At that snapshot the public native archive ended at update **35,000**. The [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a) provides later telemetry; reported progress, saved recovery state and published files have separate receipts.
|
| 50 |
|
|
|
|
| 70 |
| Setting | Operator-selected recipe |
|
| 71 |
|---|---|
|
| 72 |
| Selected scheduling argument | `KROKI=120000`; confirmed runtime `--steps 120000` adoption at 11 November, 16:23:47 UTC, resuming from update 29,800 on H200; configured decay begins at update 96,000 |
|
| 73 |
+
| New-source allocation | **20% PL / 80% EN** in segments c01βc15 (updates 0β75,000). On 12 November at **18:44 UTC**, the owner approved **30% PL / 70% EN** from **c16**; segments c16 and c17 were built that way (measured token shares 0.3000 PL in both), so updates **75,001β80,000** use **30% PL / 70% EN**. These are new-input bucket shares, not a measured cumulative model-language ratio |
|
| 74 |
| Optimizer / schedule | Muon peak 0.02 plus AdamW peak 0.0006; 2,000-update warmup, then constant learning rates under the main-base plan; adoption receipt confirms LR 0.0006. The configured final-20% WSD decay to 0.00006 remains at update 96,000; an extension approaching it requires review |
|
| 75 |
| Checkpoints | Runtime `--ckpt-every 200` confirmed at the 16:23:47 UTC adoption; weights-only milestones every 5,000 updates. Native milestone files and full-state recovery files are distinguished in the separate checkpoint archive |
|
| 76 |
| Code | Launcher v5 SHA256 `796c71016c1e167bfd3ca98b78518f91c3502a96eac233dd7186ef6639fcc656`; exact command and converter pins in the configuration document |
|
|
|
|
| 127 |
|
| 128 |
New material requires licensing, PII filtering, contamination scans, grouped held-outs and deduplication against the base model's full data ancestry. A 1% document holdout is not necessarily 1% of tokens. Historical exposure in the starting weights cannot be undone by filtering new inputs; retrospective benchmark hits must be handled explicitly in evaluation denominators.
|
| 129 |
|
| 130 |
+
## Standard benchmark measurements β updates 5,000β45,000
|
| 131 |
|
| 132 |
+
We thank **BenchTeam / Fabryka AI** for benchmark implementations, protocol work and evaluation proposals. Its independent [full benchmark report](https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md) is distinct from the **SlayerLab measurements below**, published in the [CPT Track history](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a). The nine listed 1.09 checkpoints are intermediate weights of the ongoing continuation, not its eventual budget-limited endpoint.
|
| 133 |
|
| 134 |
+
The same nine full splits were scored zero-shot in float32 for 1.07 update 22,327, 1.08 update 1,709 and 1.09 updates 5,000, 10,000, 15,000, 20,000, 25,000, 30,000, 35,000, 40,000 and 45,000. Evaluator SHA, dataset revisions, file digests, task protocols and sample counts match across these rows. Scores are percentages with **pointwise 95% confidence intervals**; the named metric is accuracy (`acc`) or length-normalized accuracy (`acc_norm`). WinoGrande uses partial-sentence likelihood; MMLU-PL is micro-averaged. Polish HellaSwag and MMLU use the OpenGPTX task definitions.
|
| 135 |
|
| 136 |
+
| Task | Metric | n | 1.07 | 1.08 | 1.09, update 5,000 | 1.09, update 10,000 | 1.09, update 15,000 | 1.09, update 20,000 | 1.09, update 25,000 | 1.09, update 30,000 | 1.09, update 35,000 | 1.09, update 40,000 | 1.09, update 45,000 |
|
| 137 |
+
|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
| 138 |
+
| ARC-Challenge | acc_norm | 1,172 | 27.30 [24.74, 29.86] | 30.03 [27.39, 32.68] | 29.35 [26.79, 32.00] | 30.46 [27.82, 33.11] | 32.51 [29.86, 35.24] | 33.11 [30.46, 35.84] | 33.79 [31.06, 36.52] | 33.02 [30.29, 35.67] | 35.15 [32.42, 37.88] | 34.98 [32.25, 37.80] | 34.81 [32.08, 37.54] |
|
| 139 |
+
| HellaSwag | acc_norm | 10,042 | 43.38 [42.43, 44.34] | 43.42 [42.43, 44.39] | 46.53 [45.55, 47.52] | 48.93 [47.97, 49.91] | 49.77 [48.80, 50.76] | 50.38 [49.39, 51.35] | 51.17 [50.18, 52.14] | 52.00 [51.02, 52.98] | 52.48 [51.50, 53.46] | 52.80 [51.81, 53.78] | 53.65 [52.69, 54.62] |
|
| 140 |
+
| PIQA | acc_norm | 1,838 | 66.97 [64.80, 69.15] | 66.21 [64.15, 68.34] | 68.28 [66.21, 70.40] | 68.99 [66.92, 71.11] | 68.99 [66.87, 71.06] | 70.78 [68.72, 72.85] | 71.55 [69.48, 73.61] | 70.67 [68.61, 72.74] | 71.27 [69.15, 73.29] | 71.22 [69.20, 73.29] | 70.29 [68.23, 72.31] |
|
| 141 |
+
| WinoGrande | acc | 1,267 | 53.59 [50.83, 56.35] | 53.67 [50.91, 56.43] | 54.93 [52.17, 57.70] | 54.93 [52.17, 57.62] | 56.75 [54.06, 59.43] | 55.01 [52.25, 57.77] | 55.88 [53.12, 58.64] | 57.14 [54.46, 59.83] | 56.75 [54.06, 59.43] | 57.70 [55.01, 60.38] | 59.04 [56.27, 61.72] |
|
| 142 |
+
| BoolQ | acc | 3,270 | 58.29 [56.61, 59.97] | 59.97 [58.26, 61.59] | 61.80 [60.18, 63.49] | 61.13 [59.45, 62.78] | 59.88 [58.20, 61.53] | 51.41 [49.69, 53.12] | 52.20 [50.52, 53.91] | 60.67 [58.99, 62.35] | 59.88 [58.20, 61.56] | 59.17 [57.46, 60.83] | 61.93 [60.24, 63.61] |
|
| 143 |
+
| OpenBookQA | acc_norm | 500 | 33.60 [29.40, 37.80] | 34.80 [30.60, 39.00] | 35.60 [31.40, 40.00] | 34.40 [30.20, 38.60] | 36.00 [31.80, 40.20] | 35.20 [31.00, 39.40] | 37.20 [32.80, 41.40] | 36.20 [32.00, 40.40] | 34.40 [30.40, 38.60] | 34.60 [30.40, 38.80] | 36.00 [31.80, 40.20] |
|
| 144 |
+
| Belebele, Polish | acc | 900 | 26.00 [23.11, 28.89] | 24.00 [21.22, 26.89] | 24.11 [21.33, 26.89] | 24.11 [21.33, 27.00] | 22.56 [19.89, 25.33] | 24.67 [21.89, 27.56] | 25.33 [22.44, 28.22] | 24.44 [21.67, 27.33] | 24.33 [21.56, 27.11] | 25.56 [22.78, 28.44] | 25.33 [22.56, 28.22] |
|
| 145 |
+
| HellaSwag-PL | acc_norm | 10,042 | 43.83 [42.84, 44.79] | 43.90 [42.90, 44.87] | 44.73 [43.76, 45.71] | 45.51 [44.51, 46.48] | 44.18 [43.21, 45.16] | 44.24 [43.26, 45.22] | 43.94 [42.94, 44.91] | 45.75 [44.77, 46.73] | 46.63 [45.68, 47.63] | 46.61 [45.65, 47.57] | 46.78 [45.79, 47.75] |
|
| 146 |
+
| MMLU-PL | acc | 14,042 | 24.01 [23.30, 24.71] | 23.04 [22.34, 23.74] | 24.10 [23.39, 24.82] | 23.32 [22.62, 24.03] | 23.50 [22.80, 24.21] | 23.86 [23.17, 24.58] | 24.26 [23.57, 24.96] | 24.71 [23.99, 25.42] | 23.28 [22.58, 23.97] | 24.24 [23.54, 24.97] | 25.74 [25.02, 26.46] |
|
| 147 |
|
| 148 |
Reference accuracies are context, not additional trained models or release thresholds:
|
| 149 |
|
| 150 |
| Reference strategy | Scope | Accuracy (%) |
|
| 151 |
+
|---|---|---:|---:|---:|
|
| 152 |
| Uniform choice among four answers | Four-option reference for ARC-Challenge, HellaSwag/PL, OpenBookQA, Belebele-PL and MMLU-PL | 25.00 |
|
| 153 |
| Uniform choice among two answers | BoolQ, PIQA and WinoGrande | 50.00 |
|
| 154 |
| Always answer yes | BoolQ validation majority class: 2,033 of 3,270 labels | 62.17 |
|
| 155 |
|
| 156 |
The majority-class reference is different from uniform guessing. A score near a reference does not prove that checkpoints are equal or that a model has no knowledge; individual confidence intervals are not paired tests. Class-balanced accuracy and answer-frequency diagnostics, when available, are auxiliary and do not replace the primary metrics above.
|
| 157 |
|
| 158 |
+
Intervals use the percentile bootstrap with 10,000 replicates and seed 20261110. The retained generator uses a model-specific stream initialized from the seed and the first eight hex digits of SHA256(model label), with acc then acc_norm in source task order. CI-generator SHA-256: `b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34`. **Items are treated as independent; subject, template and source clusters are not modeled.** These are intervals for individual results, not paired differences or training-seed variance; overlap is not a test of a difference.
|
| 159 |
|
| 160 |
The [machine-readable aggregate record](EVALUATION.json) supplies exact values, intervals, counts, dataset revisions and file digests, checkpoint and evaluator identities, and metric selection, without prompts, answers or per-item predictions. ARC-Challenge, OpenBookQA and MMLU-PL use test splits; HellaSwag, PIQA, WinoGrande, BoolQ and HellaSwag-PL use validation splits; Belebele uses the full Polish evaluation set.
|
| 161 |
|
| 162 |
+
The aggregate SHA-256 is `d21e1d32ab57a4edb261fdc83ab898d47810f6c2444d084ff384920af282172f`. All 9 previous model records retain their scores, intervals, counts and dataset pins; the update-40,000 and 45,000 records extend the nine-task comparison without replacing the independent two-task protocol below. The retained generator reproduces all previous aggregate records.
|
| 163 |
|
| 164 |
**Descriptive results; lineage contamination checks are incomplete.** New-input chunks c03 onward were cleaned against these nine Track tasks, c01βc02 were only partially checked, and the inherited 1.00β1.08 data was not scanned. This is not a guarantee that these tasks are held out from the model's whole training history. Belebele and MMLU-PL are near their four-choice 25% random baseline; this is not proof of absent knowledge. No overall average, ranking or release threshold is inferred. Different evaluators must not be silently substituted.
|
| 165 |
|
|
|
|
| 180 |
| 25,000 | 57.44 | 24.77 | 26.25 |
|
| 181 |
| 30,000 | 59.10 | 25.00 | 25.39 |
|
| 182 |
| 35,000 | 60.29 | 24.70 | 25.64 |
|
| 183 |
+
| 40,000 | 60.36 | 24.93 | 26.29 |
|
| 184 |
+
| 45,000 | 58.92 | 24.91 | 26.60 |
|
| 185 |
|
| 186 |
The corresponding uniform-choice reference is 50% for BoolQ and 25% for the two four-class tasks. These auxiliary values support inspection of class imbalance. They leave the primary metrics and their published intervals unchanged; a checkpoint-difference claim requires a separate paired analysis.
|
| 187 |
|
|
|
|
| 188 |
## Additional GSM8K and SST-2 measurements β update 10,000
|
| 189 |
|
| 190 |
SlayerLab measured this two-task subset on 11 November 2026 using the [pinned BenchTeam code](https://github.com/slayerlabs/benchmarks-fabryka-ai/tree/8b547007526971bd2fc35dfe56ff2c7d9b802568), lm-eval 0.4.9.1, the `slayer` adapter, float32, batch 1 and context limit 4,096. These are full evaluated splits, not a full 50-task run or a copy of the community's independent report. **This separate two-task protocol measures 1.09 update 10,000 only; it does not supply GSM8K or SST-2 results for updates 15,000, 20,000, 25,000, 30,000 or 35,000.** Releasing another checkpoint does not supply a missing measurement.
|