Evaluation at update 60000; training log to 85000 target
Browse files- EVALUATION.json +373 -4
- README.md +19 -19
- TRAINING_LOG.md +9 -4
- plots/arc-challenge.png +2 -2
- plots/belebele-polish.png +2 -2
- plots/benchmark-overview.pdf +0 -0
- plots/benchmark-overview.png +2 -2
- plots/benchmark-overview.svg +0 -0
- plots/benchmark-scores.csv +9 -0
- plots/boolq.png +2 -2
- plots/hellaswag-pl.png +2 -2
- plots/hellaswag-progress.png +2 -2
- plots/hellaswag-progress.svg +213 -192
- plots/hellaswag.png +2 -2
- plots/mmlu-pl.png +2 -2
- plots/openbookqa.png +2 -2
- plots/piqa.png +2 -2
- plots/winogrande.png +2 -2
EVALUATION.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
-
"status": "descriptive intermediate base checkpoint measurements through 1.09 update
|
| 5 |
-
"source_aggregate_sha256": "
|
| 6 |
"source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
|
@@ -3370,6 +3370,260 @@
|
|
| 3370 |
"correct_primary": 3491
|
| 3371 |
}
|
| 3372 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3373 |
}
|
| 3374 |
},
|
| 3375 |
"additional_measurements": {
|
|
@@ -3569,9 +3823,9 @@
|
|
| 3569 |
}
|
| 3570 |
}
|
| 3571 |
},
|
| 3572 |
-
"source_generator_note": "Retained CI generator reproduces all
|
| 3573 |
"auxiliary_diagnostics": {
|
| 3574 |
-
"source_aggregate_sha256": "
|
| 3575 |
"method": "unweighted mean of class recalls (balanced accuracy); collected after the primary measurements; not a replacement for primary accuracy",
|
| 3576 |
"units": "percent",
|
| 3577 |
"uncertainty": "confidence intervals not supplied for these auxiliary values; no formal checkpoint-difference conclusion",
|
|
@@ -4497,6 +4751,121 @@
|
|
| 4497 |
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 4498 |
}
|
| 4499 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4500 |
}
|
| 4501 |
},
|
| 4502 |
"source_extension_note": "Earlier diagnostic records preserved. New balanced values derived from primary correctness and the frozen ordered gold vectors; exact retained diagnostic reader verified against the primary reader. No new inference, prediction-margin diagnostic or confidence interval supplied."
|
|
|
|
| 1 |
{
|
| 2 |
"schema_version": 1,
|
| 3 |
"measurement_owner": "SlayerLab / Fabryka AI",
|
| 4 |
+
"status": "descriptive intermediate base checkpoint measurements through 1.09 update 60000; archive availability and execution state have separate receipts in the model card; this aggregate is not live telemetry; training-lineage contamination checks incomplete",
|
| 5 |
+
"source_aggregate_sha256": "426fec8c8e80a8c02d229b91509c53c5fcdd5b9e4386559d50a7d98239569032",
|
| 6 |
"source_generator_sha256": "b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34",
|
| 7 |
"benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md",
|
| 8 |
"community_report_is_separate_measurement": true,
|
|
|
|
| 3370 |
"correct_primary": 3491
|
| 3371 |
}
|
| 3372 |
}
|
| 3373 |
+
},
|
| 3374 |
+
"1.09-k60000": {
|
| 3375 |
+
"step": 60000,
|
| 3376 |
+
"weights_sha256": "6afec53cf7591679f902056d5773cb006fe45d41f0f0d3ebc71f035197d2baeb",
|
| 3377 |
+
"source_results_sha256": "6ee7e0469f36344f4e9c247eb1be8a7d130e5bc3139a389e53f06f3a58a1bedb",
|
| 3378 |
+
"evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016",
|
| 3379 |
+
"weights_published": false,
|
| 3380 |
+
"tasks": {
|
| 3381 |
+
"arc_challenge": {
|
| 3382 |
+
"n": 1172,
|
| 3383 |
+
"n_pelne": 1172,
|
| 3384 |
+
"dataset_repo": "allenai/ai2_arc",
|
| 3385 |
+
"dataset_sha": "210d026faf9955653af8916fad021475a3f00453",
|
| 3386 |
+
"pliki_sha256": {
|
| 3387 |
+
"ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
|
| 3388 |
+
},
|
| 3389 |
+
"acc": 0.323379,
|
| 3390 |
+
"acc_ci95": [
|
| 3391 |
+
0.296928,
|
| 3392 |
+
0.349829
|
| 3393 |
+
],
|
| 3394 |
+
"acc_norm": 0.360068,
|
| 3395 |
+
"acc_norm_ci95": [
|
| 3396 |
+
0.332765,
|
| 3397 |
+
0.388225
|
| 3398 |
+
],
|
| 3399 |
+
"split": "test",
|
| 3400 |
+
"correct_primary": 422
|
| 3401 |
+
},
|
| 3402 |
+
"hellaswag": {
|
| 3403 |
+
"n": 10042,
|
| 3404 |
+
"n_pelne": 10042,
|
| 3405 |
+
"dataset_repo": "Rowan/hellaswag",
|
| 3406 |
+
"dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
|
| 3407 |
+
"pliki_sha256": {
|
| 3408 |
+
"data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
|
| 3409 |
+
},
|
| 3410 |
+
"acc": 0.420733,
|
| 3411 |
+
"acc_ci95": [
|
| 3412 |
+
0.411073,
|
| 3413 |
+
0.430193
|
| 3414 |
+
],
|
| 3415 |
+
"acc_norm": 0.550687,
|
| 3416 |
+
"acc_norm_ci95": [
|
| 3417 |
+
0.541125,
|
| 3418 |
+
0.560446
|
| 3419 |
+
],
|
| 3420 |
+
"split": "validation",
|
| 3421 |
+
"correct_primary": 5530
|
| 3422 |
+
},
|
| 3423 |
+
"piqa": {
|
| 3424 |
+
"n": 1838,
|
| 3425 |
+
"n_pelne": 1838,
|
| 3426 |
+
"dataset_repo": "baber/piqa",
|
| 3427 |
+
"dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
|
| 3428 |
+
"pliki_sha256": {
|
| 3429 |
+
"piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
|
| 3430 |
+
},
|
| 3431 |
+
"acc": 0.713819,
|
| 3432 |
+
"acc_ci95": [
|
| 3433 |
+
0.693131,
|
| 3434 |
+
0.734494
|
| 3435 |
+
],
|
| 3436 |
+
"acc_norm": 0.720892,
|
| 3437 |
+
"acc_norm_ci95": [
|
| 3438 |
+
0.700218,
|
| 3439 |
+
0.741023
|
| 3440 |
+
],
|
| 3441 |
+
"split": "validation",
|
| 3442 |
+
"correct_primary": 1325
|
| 3443 |
+
},
|
| 3444 |
+
"winogrande": {
|
| 3445 |
+
"n": 1267,
|
| 3446 |
+
"n_pelne": 1267,
|
| 3447 |
+
"dataset_repo": "allenai/winogrande",
|
| 3448 |
+
"dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5",
|
| 3449 |
+
"pliki_sha256": {
|
| 3450 |
+
"winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1"
|
| 3451 |
+
},
|
| 3452 |
+
"acc": 0.574586,
|
| 3453 |
+
"acc_ci95": [
|
| 3454 |
+
0.547751,
|
| 3455 |
+
0.601421
|
| 3456 |
+
],
|
| 3457 |
+
"acc_norm": 0.574586,
|
| 3458 |
+
"acc_norm_ci95": [
|
| 3459 |
+
0.546961,
|
| 3460 |
+
0.601421
|
| 3461 |
+
],
|
| 3462 |
+
"split": "validation",
|
| 3463 |
+
"correct_primary": 728
|
| 3464 |
+
},
|
| 3465 |
+
"boolq": {
|
| 3466 |
+
"n": 3270,
|
| 3467 |
+
"n_pelne": 3270,
|
| 3468 |
+
"dataset_repo": "aps/super_glue",
|
| 3469 |
+
"dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646",
|
| 3470 |
+
"pliki_sha256": {
|
| 3471 |
+
"boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6"
|
| 3472 |
+
},
|
| 3473 |
+
"acc": 0.631804,
|
| 3474 |
+
"acc_ci95": [
|
| 3475 |
+
0.614985,
|
| 3476 |
+
0.648624
|
| 3477 |
+
],
|
| 3478 |
+
"acc_norm": 0.625994,
|
| 3479 |
+
"acc_norm_ci95": [
|
| 3480 |
+
0.608869,
|
| 3481 |
+
0.642508
|
| 3482 |
+
],
|
| 3483 |
+
"split": "validation",
|
| 3484 |
+
"correct_primary": 2066
|
| 3485 |
+
},
|
| 3486 |
+
"openbookqa": {
|
| 3487 |
+
"n": 500,
|
| 3488 |
+
"n_pelne": 500,
|
| 3489 |
+
"dataset_repo": "allenai/openbookqa",
|
| 3490 |
+
"dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454",
|
| 3491 |
+
"pliki_sha256": {
|
| 3492 |
+
"main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3"
|
| 3493 |
+
},
|
| 3494 |
+
"acc": 0.254,
|
| 3495 |
+
"acc_ci95": [
|
| 3496 |
+
0.216,
|
| 3497 |
+
0.294
|
| 3498 |
+
],
|
| 3499 |
+
"acc_norm": 0.35,
|
| 3500 |
+
"acc_norm_ci95": [
|
| 3501 |
+
0.31,
|
| 3502 |
+
0.392
|
| 3503 |
+
],
|
| 3504 |
+
"split": "test",
|
| 3505 |
+
"correct_primary": 175
|
| 3506 |
+
},
|
| 3507 |
+
"belebele_pol": {
|
| 3508 |
+
"n": 900,
|
| 3509 |
+
"n_pelne": 900,
|
| 3510 |
+
"dataset_repo": "facebook/belebele",
|
| 3511 |
+
"dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
| 3512 |
+
"pliki_sha256": {
|
| 3513 |
+
"data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec"
|
| 3514 |
+
},
|
| 3515 |
+
"acc": 0.247778,
|
| 3516 |
+
"acc_ci95": [
|
| 3517 |
+
0.22,
|
| 3518 |
+
0.276667
|
| 3519 |
+
],
|
| 3520 |
+
"acc_norm": 0.247778,
|
| 3521 |
+
"acc_norm_ci95": [
|
| 3522 |
+
0.22,
|
| 3523 |
+
0.275556
|
| 3524 |
+
],
|
| 3525 |
+
"split": "pol_Latn evaluation set",
|
| 3526 |
+
"correct_primary": 223
|
| 3527 |
+
},
|
| 3528 |
+
"hellaswag_pl": {
|
| 3529 |
+
"n": 10042,
|
| 3530 |
+
"n_pelne": 10042,
|
| 3531 |
+
"dataset_repo": "Eurolingua/hellaswagx",
|
| 3532 |
+
"dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b",
|
| 3533 |
+
"pliki_sha256": {
|
| 3534 |
+
"hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16"
|
| 3535 |
+
},
|
| 3536 |
+
"acc": 0.365166,
|
| 3537 |
+
"acc_ci95": [
|
| 3538 |
+
0.355706,
|
| 3539 |
+
0.374627
|
| 3540 |
+
],
|
| 3541 |
+
"acc_norm": 0.474806,
|
| 3542 |
+
"acc_norm_ci95": [
|
| 3543 |
+
0.465144,
|
| 3544 |
+
0.484664
|
| 3545 |
+
],
|
| 3546 |
+
"split": "validation",
|
| 3547 |
+
"correct_primary": 4768
|
| 3548 |
+
},
|
| 3549 |
+
"mmlu_pl": {
|
| 3550 |
+
"n": 14042,
|
| 3551 |
+
"n_pelne": 14042,
|
| 3552 |
+
"dataset_repo": "Eurolingua/mmlux",
|
| 3553 |
+
"dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882",
|
| 3554 |
+
"pliki_sha256": {
|
| 3555 |
+
"hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea",
|
| 3556 |
+
"hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38",
|
| 3557 |
+
"hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03",
|
| 3558 |
+
"hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96",
|
| 3559 |
+
"hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61",
|
| 3560 |
+
"hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48",
|
| 3561 |
+
"hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d",
|
| 3562 |
+
"hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70",
|
| 3563 |
+
"hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee",
|
| 3564 |
+
"hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606",
|
| 3565 |
+
"hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2",
|
| 3566 |
+
"hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a",
|
| 3567 |
+
"hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64",
|
| 3568 |
+
"hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b",
|
| 3569 |
+
"hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f",
|
| 3570 |
+
"hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667",
|
| 3571 |
+
"hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be",
|
| 3572 |
+
"hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f",
|
| 3573 |
+
"hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2",
|
| 3574 |
+
"hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec",
|
| 3575 |
+
"hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1",
|
| 3576 |
+
"hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4",
|
| 3577 |
+
"hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a",
|
| 3578 |
+
"hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d",
|
| 3579 |
+
"hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d",
|
| 3580 |
+
"hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735",
|
| 3581 |
+
"hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8",
|
| 3582 |
+
"hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f",
|
| 3583 |
+
"hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45",
|
| 3584 |
+
"hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf",
|
| 3585 |
+
"hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7",
|
| 3586 |
+
"hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0",
|
| 3587 |
+
"hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82",
|
| 3588 |
+
"hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4",
|
| 3589 |
+
"hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746",
|
| 3590 |
+
"hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96",
|
| 3591 |
+
"hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51",
|
| 3592 |
+
"hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608",
|
| 3593 |
+
"hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb",
|
| 3594 |
+
"hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb",
|
| 3595 |
+
"hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4",
|
| 3596 |
+
"hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809",
|
| 3597 |
+
"hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127",
|
| 3598 |
+
"hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b",
|
| 3599 |
+
"hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285",
|
| 3600 |
+
"hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639",
|
| 3601 |
+
"hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88",
|
| 3602 |
+
"hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e",
|
| 3603 |
+
"hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57",
|
| 3604 |
+
"hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080",
|
| 3605 |
+
"hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3",
|
| 3606 |
+
"hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1",
|
| 3607 |
+
"hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c",
|
| 3608 |
+
"hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a",
|
| 3609 |
+
"hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786",
|
| 3610 |
+
"hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7",
|
| 3611 |
+
"hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a"
|
| 3612 |
+
},
|
| 3613 |
+
"acc": 0.244552,
|
| 3614 |
+
"acc_ci95": [
|
| 3615 |
+
0.237359,
|
| 3616 |
+
0.251531
|
| 3617 |
+
],
|
| 3618 |
+
"acc_norm": 0.244552,
|
| 3619 |
+
"acc_norm_ci95": [
|
| 3620 |
+
0.237288,
|
| 3621 |
+
0.251816
|
| 3622 |
+
],
|
| 3623 |
+
"split": "test; micro-average over subjects",
|
| 3624 |
+
"correct_primary": 3434
|
| 3625 |
+
}
|
| 3626 |
+
}
|
| 3627 |
}
|
| 3628 |
},
|
| 3629 |
"additional_measurements": {
|
|
|
|
| 3823 |
}
|
| 3824 |
}
|
| 3825 |
},
|
| 3826 |
+
"source_generator_note": "Retained CI generator reproduces all 13 previous model records by value. Those published records were preserved unchanged; only the new checkpoint records were appended. The earlier missing stream/producer link is resolved by the retained generator and reproduction receipt.",
|
| 3827 |
"auxiliary_diagnostics": {
|
| 3828 |
+
"source_aggregate_sha256": "03833265b1f1d9f1085f65e6c9eeeffc17f4b0598cc5760760b0f6520bdee537",
|
| 3829 |
"method": "unweighted mean of class recalls (balanced accuracy); collected after the primary measurements; not a replacement for primary accuracy",
|
| 3830 |
"units": "percent",
|
| 3831 |
"uncertainty": "confidence intervals not supplied for these auxiliary values; no formal checkpoint-difference conclusion",
|
|
|
|
| 4751 |
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 4752 |
}
|
| 4753 |
}
|
| 4754 |
+
},
|
| 4755 |
+
"1.09-k60000": {
|
| 4756 |
+
"source_diagnostic_sha256": "ad596de32e65a15eb7e3d89c5f0c0edab72e81289addce657d3ba488343960c4",
|
| 4757 |
+
"source_diagnostic_kind": "derived balanced aggregate; no new model forward",
|
| 4758 |
+
"source_gold_diagnostic_sha256": "85cf33056bd30a85cf5411568ba6e2098eb31bbe9493d4156d800e98d6edcc1b",
|
| 4759 |
+
"source_track_results_sha256": "6ee7e0469f36344f4e9c247eb1be8a7d130e5bc3139a389e53f06f3a58a1bedb",
|
| 4760 |
+
"tasks": {
|
| 4761 |
+
"boolq": {
|
| 4762 |
+
"n": 3270,
|
| 4763 |
+
"balanced_accuracy_percent": 57.24,
|
| 4764 |
+
"classes": [
|
| 4765 |
+
"no",
|
| 4766 |
+
"yes"
|
| 4767 |
+
],
|
| 4768 |
+
"gold_class_percent": [
|
| 4769 |
+
37.8,
|
| 4770 |
+
62.2
|
| 4771 |
+
],
|
| 4772 |
+
"class_totals": [
|
| 4773 |
+
1237,
|
| 4774 |
+
2033
|
| 4775 |
+
],
|
| 4776 |
+
"class_correct": [
|
| 4777 |
+
406,
|
| 4778 |
+
1660
|
| 4779 |
+
],
|
| 4780 |
+
"class_recalls": [
|
| 4781 |
+
0.32821341956346,
|
| 4782 |
+
0.8165272995573045
|
| 4783 |
+
],
|
| 4784 |
+
"balanced_accuracy_fraction_exact": "1439409/2514821",
|
| 4785 |
+
"items_acc_sha256": "d51b638c2c55939679432f876c10191b0f76a7ba5d0d9fe7f2e983c1960ece55",
|
| 4786 |
+
"items_gold_sha256": "2404e078cb21f7e36be36272bdf4515a311d9104bf6c954eb091ffa20fd434a3",
|
| 4787 |
+
"predicted_class_percent": null,
|
| 4788 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 4789 |
+
},
|
| 4790 |
+
"mmlu_pl": {
|
| 4791 |
+
"n": 14042,
|
| 4792 |
+
"balanced_accuracy_percent": 24.82,
|
| 4793 |
+
"classes": [
|
| 4794 |
+
"A",
|
| 4795 |
+
"B",
|
| 4796 |
+
"C",
|
| 4797 |
+
"D"
|
| 4798 |
+
],
|
| 4799 |
+
"gold_class_percent": [
|
| 4800 |
+
22.9,
|
| 4801 |
+
24.7,
|
| 4802 |
+
25.5,
|
| 4803 |
+
26.9
|
| 4804 |
+
],
|
| 4805 |
+
"class_totals": [
|
| 4806 |
+
3222,
|
| 4807 |
+
3462,
|
| 4808 |
+
3582,
|
| 4809 |
+
3776
|
| 4810 |
+
],
|
| 4811 |
+
"class_correct": [
|
| 4812 |
+
495,
|
| 4813 |
+
2223,
|
| 4814 |
+
529,
|
| 4815 |
+
187
|
| 4816 |
+
],
|
| 4817 |
+
"class_recalls": [
|
| 4818 |
+
0.15363128491620112,
|
| 4819 |
+
0.6421143847487002,
|
| 4820 |
+
0.1476828587381351,
|
| 4821 |
+
0.04952330508474576
|
| 4822 |
+
],
|
| 4823 |
+
"balanced_accuracy_fraction_exact": "693560893919/2793935699712",
|
| 4824 |
+
"items_acc_sha256": "aae6d5cdb15bfe05151e28cf5d8ffbcf305a07d4ce92c99b78ea1c5eecbcf5c7",
|
| 4825 |
+
"items_gold_sha256": "d87e03f33f63c1efd17611ee976a17b25f70df9c1ed59f02b31d581437bd6701",
|
| 4826 |
+
"predicted_class_percent": null,
|
| 4827 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 4828 |
+
},
|
| 4829 |
+
"belebele_pol": {
|
| 4830 |
+
"n": 900,
|
| 4831 |
+
"balanced_accuracy_percent": 25.54,
|
| 4832 |
+
"classes": [
|
| 4833 |
+
"A",
|
| 4834 |
+
"B",
|
| 4835 |
+
"C",
|
| 4836 |
+
"D"
|
| 4837 |
+
],
|
| 4838 |
+
"gold_class_percent": [
|
| 4839 |
+
22.9,
|
| 4840 |
+
27.9,
|
| 4841 |
+
27.3,
|
| 4842 |
+
21.9
|
| 4843 |
+
],
|
| 4844 |
+
"class_totals": [
|
| 4845 |
+
206,
|
| 4846 |
+
251,
|
| 4847 |
+
246,
|
| 4848 |
+
197
|
| 4849 |
+
],
|
| 4850 |
+
"class_correct": [
|
| 4851 |
+
125,
|
| 4852 |
+
21,
|
| 4853 |
+
59,
|
| 4854 |
+
18
|
| 4855 |
+
],
|
| 4856 |
+
"class_recalls": [
|
| 4857 |
+
0.6067961165048543,
|
| 4858 |
+
0.08366533864541832,
|
| 4859 |
+
0.23983739837398374,
|
| 4860 |
+
0.09137055837563451
|
| 4861 |
+
],
|
| 4862 |
+
"balanced_accuracy_fraction_exact": "640018717/2505776172",
|
| 4863 |
+
"items_acc_sha256": "a49255585b343f8a1a2a257543ff251390011af7574ef32682d1bcfab0d6220e",
|
| 4864 |
+
"items_gold_sha256": "ae784c3f36b672f194365df64d33afeb4cef9377d4543b979580c743fe688dd4",
|
| 4865 |
+
"predicted_class_percent": null,
|
| 4866 |
+
"predicted_class_percent_note": "Prediction margins not measured for these new rows; balanced accuracy uses frozen gold and correctness."
|
| 4867 |
+
}
|
| 4868 |
+
}
|
| 4869 |
}
|
| 4870 |
},
|
| 4871 |
"source_extension_note": "Earlier diagnostic records preserved. New balanced values derived from primary correctness and the frozen ordered gold vectors; exact retained diagnostic reader verified against the primary reader. No new inference, prediction-margin diagnostic or confidence interval supplied."
|
README.md
CHANGED
|
@@ -17,7 +17,7 @@ datasets:
|
|
| 17 |
- SlayerLab/gollem-corpus-16b-pl
|
| 18 |
base_model: SlayerLab/Slayer1B-1.08
|
| 19 |
model-index:
|
| 20 |
-
- name: Slayer1B-1.09 update
|
| 21 |
results:
|
| 22 |
- task:
|
| 23 |
type: text-generation
|
|
@@ -30,7 +30,7 @@ model-index:
|
|
| 30 |
metrics:
|
| 31 |
- type: acc_norm
|
| 32 |
name: acc_norm (%)
|
| 33 |
-
value:
|
| 34 |
- task:
|
| 35 |
type: text-generation
|
| 36 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -42,7 +42,7 @@ model-index:
|
|
| 42 |
metrics:
|
| 43 |
- type: acc_norm
|
| 44 |
name: acc_norm (%)
|
| 45 |
-
value:
|
| 46 |
- task:
|
| 47 |
type: text-generation
|
| 48 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -54,7 +54,7 @@ model-index:
|
|
| 54 |
metrics:
|
| 55 |
- type: acc_norm
|
| 56 |
name: acc_norm (%)
|
| 57 |
-
value: 72.
|
| 58 |
- task:
|
| 59 |
type: text-generation
|
| 60 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -66,7 +66,7 @@ model-index:
|
|
| 66 |
metrics:
|
| 67 |
- type: acc
|
| 68 |
name: acc (%)
|
| 69 |
-
value: 57.
|
| 70 |
- task:
|
| 71 |
type: text-generation
|
| 72 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -78,7 +78,7 @@ model-index:
|
|
| 78 |
metrics:
|
| 79 |
- type: acc
|
| 80 |
name: acc (%)
|
| 81 |
-
value:
|
| 82 |
- task:
|
| 83 |
type: text-generation
|
| 84 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -90,7 +90,7 @@ model-index:
|
|
| 90 |
metrics:
|
| 91 |
- type: acc_norm
|
| 92 |
name: acc_norm (%)
|
| 93 |
-
value:
|
| 94 |
- task:
|
| 95 |
type: text-generation
|
| 96 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -102,7 +102,7 @@ model-index:
|
|
| 102 |
metrics:
|
| 103 |
- type: acc
|
| 104 |
name: acc (%)
|
| 105 |
-
value:
|
| 106 |
- task:
|
| 107 |
type: text-generation
|
| 108 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -114,7 +114,7 @@ model-index:
|
|
| 114 |
metrics:
|
| 115 |
- type: acc_norm
|
| 116 |
name: acc_norm (%)
|
| 117 |
-
value: 47.
|
| 118 |
- task:
|
| 119 |
type: text-generation
|
| 120 |
name: Zero-shot multiple-choice evaluation
|
|
@@ -126,7 +126,7 @@ model-index:
|
|
| 126 |
metrics:
|
| 127 |
- type: acc
|
| 128 |
name: acc (%)
|
| 129 |
-
value: 24.
|
| 130 |
---
|
| 131 |
|
| 132 |
# Slayer1B 1.09
|
|
@@ -166,7 +166,7 @@ Start directly from this **Slayer1B-1.09 base model**, pinned to the released up
|
|
| 166 |
|
| 167 |
## Is this the right model for your task?
|
| 168 |
|
| 169 |
-
Choose Slayer1B 1.09 when you want a small Polish–English **starting checkpoint** for text completion, model research, or your own post-training. The public archive contains weights-only checkpoints every 5,000 updates from 5,000 through
|
| 170 |
|
| 171 |
For direct question answering, following instructions, tool calling, or agent workflows, use an instruction-tuned model instead. Slayer1B 1.09 predicts continuations from a text prompt; it may repeat itself and does not reliably follow requests. Its 4K context also makes it a poor fit for long-document tasks without additional adaptation.
|
| 172 |
|
|
@@ -201,17 +201,17 @@ print(prompt + continuation)
|
|
| 201 |
|
| 202 |
These are zero-shot, float32 multiple-choice evaluations. The 1.08 and 1.09 columns use the same documented Slayer evaluation setup; scores are percentages. WinoGrande uses accuracy; the other listed tasks use length-normalized accuracy.
|
| 203 |
|
| 204 |
-
| Benchmark | 1.08 | 1.09, update
|
| 205 |
|---|---:|---:|
|
| 206 |
-
| ARC-Challenge | 30.0 |
|
| 207 |
-
| HellaSwag | 43.4 |
|
| 208 |
-
| PIQA | 66.2 | 72.
|
| 209 |
-
| WinoGrande | 53.7 | 57.
|
| 210 |
| HellaSwag-PL | 43.9 | 47.5 |
|
| 211 |
|
| 212 |
-
On HellaSwag, the measured score rose from 43.4% to
|
| 213 |
|
| 214 |
-
Polish knowledge measurements remain limited: at update
|
| 215 |
|
| 216 |

|
| 217 |
|
|
@@ -219,7 +219,7 @@ Polish knowledge measurements remain limited: at update 55,000, Belebele-PL is *
|
|
| 219 |
|
| 220 |
Version 1.09 starts from the released Slayer1B 1.08 checkpoint and continues pretraining. Through the evaluated update-45,000 checkpoint, new source allocation is documented as 80% English and 20% Polish. Selected inputs include GoLLeM web, FinePDFs-Edu, and FineWeb-Edu. The 4,096-token context and training recipe are described in [TRAINING_CONFIGURATION.md](TRAINING_CONFIGURATION.md); source selection, filtering, and licensing notes are in [DATA_CARD.md](DATA_CARD.md).
|
| 221 |
|
| 222 |
-
At update
|
| 223 |
|
| 224 |
## Limitations
|
| 225 |
|
|
|
|
| 17 |
- SlayerLab/gollem-corpus-16b-pl
|
| 18 |
base_model: SlayerLab/Slayer1B-1.08
|
| 19 |
model-index:
|
| 20 |
+
- name: Slayer1B-1.09 update 60000
|
| 21 |
results:
|
| 22 |
- task:
|
| 23 |
type: text-generation
|
|
|
|
| 30 |
metrics:
|
| 31 |
- type: acc_norm
|
| 32 |
name: acc_norm (%)
|
| 33 |
+
value: 36.01
|
| 34 |
- task:
|
| 35 |
type: text-generation
|
| 36 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 42 |
metrics:
|
| 43 |
- type: acc_norm
|
| 44 |
name: acc_norm (%)
|
| 45 |
+
value: 55.07
|
| 46 |
- task:
|
| 47 |
type: text-generation
|
| 48 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 54 |
metrics:
|
| 55 |
- type: acc_norm
|
| 56 |
name: acc_norm (%)
|
| 57 |
+
value: 72.09
|
| 58 |
- task:
|
| 59 |
type: text-generation
|
| 60 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 66 |
metrics:
|
| 67 |
- type: acc
|
| 68 |
name: acc (%)
|
| 69 |
+
value: 57.46
|
| 70 |
- task:
|
| 71 |
type: text-generation
|
| 72 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 78 |
metrics:
|
| 79 |
- type: acc
|
| 80 |
name: acc (%)
|
| 81 |
+
value: 63.18
|
| 82 |
- task:
|
| 83 |
type: text-generation
|
| 84 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 90 |
metrics:
|
| 91 |
- type: acc_norm
|
| 92 |
name: acc_norm (%)
|
| 93 |
+
value: 35.0
|
| 94 |
- task:
|
| 95 |
type: text-generation
|
| 96 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 102 |
metrics:
|
| 103 |
- type: acc
|
| 104 |
name: acc (%)
|
| 105 |
+
value: 24.78
|
| 106 |
- task:
|
| 107 |
type: text-generation
|
| 108 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 114 |
metrics:
|
| 115 |
- type: acc_norm
|
| 116 |
name: acc_norm (%)
|
| 117 |
+
value: 47.48
|
| 118 |
- task:
|
| 119 |
type: text-generation
|
| 120 |
name: Zero-shot multiple-choice evaluation
|
|
|
|
| 126 |
metrics:
|
| 127 |
- type: acc
|
| 128 |
name: acc (%)
|
| 129 |
+
value: 24.46
|
| 130 |
---
|
| 131 |
|
| 132 |
# Slayer1B 1.09
|
|
|
|
| 166 |
|
| 167 |
## Is this the right model for your task?
|
| 168 |
|
| 169 |
+
Choose Slayer1B 1.09 when you want a small Polish–English **starting checkpoint** for text completion, model research, or your own post-training. The public archive contains weights-only checkpoints every 5,000 updates from 5,000 through 60,000, so you can inspect how measured benchmark scores change during this continuation.
|
| 170 |
|
| 171 |
For direct question answering, following instructions, tool calling, or agent workflows, use an instruction-tuned model instead. Slayer1B 1.09 predicts continuations from a text prompt; it may repeat itself and does not reliably follow requests. Its 4K context also makes it a poor fit for long-document tasks without additional adaptation.
|
| 172 |
|
|
|
|
| 201 |
|
| 202 |
These are zero-shot, float32 multiple-choice evaluations. The 1.08 and 1.09 columns use the same documented Slayer evaluation setup; scores are percentages. WinoGrande uses accuracy; the other listed tasks use length-normalized accuracy.
|
| 203 |
|
| 204 |
+
| Benchmark | 1.08 | 1.09, update 60,000 |
|
| 205 |
|---|---:|---:|
|
| 206 |
+
| ARC-Challenge | 30.0 | 36.0 |
|
| 207 |
+
| HellaSwag | 43.4 | 55.1 |
|
| 208 |
+
| PIQA | 66.2 | 72.1 |
|
| 209 |
+
| WinoGrande | 53.7 | 57.5 |
|
| 210 |
| HellaSwag-PL | 43.9 | 47.5 |
|
| 211 |
|
| 212 |
+
On HellaSwag, the measured score rose from 43.4% to 55.1% across these two checkpoints. This is a descriptive checkpoint comparison, not a controlled experiment isolating the effect of the new training data. The [nine-task overview](plots/benchmark-overview.png) includes sample counts and pointwise 95% confidence intervals; [EVALUATION.json](EVALUATION.json) records exact dataset revisions, metrics, and checkpoint identities.
|
| 213 |
|
| 214 |
+
Polish knowledge measurements remain limited: at update 60,000, Belebele-PL is **24.78%** and MMLU-PL is **24.46%**, near the 25% uniform-choice reference. These results do not establish strong Polish question answering. Training-lineage contamination checks are incomplete, so benchmark scores are descriptive rather than proof that every evaluation example was unseen during training. GSM8K and SST-2 were measured under a separate protocol at update 10,000; details are in [EVALUATION.json](EVALUATION.json).
|
| 215 |
|
| 216 |

|
| 217 |
|
|
|
|
| 219 |
|
| 220 |
Version 1.09 starts from the released Slayer1B 1.08 checkpoint and continues pretraining. Through the evaluated update-45,000 checkpoint, new source allocation is documented as 80% English and 20% Polish. Selected inputs include GoLLeM web, FinePDFs-Edu, and FineWeb-Edu. The 4,096-token context and training recipe are described in [TRAINING_CONFIGURATION.md](TRAINING_CONFIGURATION.md); source selection, filtering, and licensing notes are in [DATA_CARD.md](DATA_CARD.md).
|
| 221 |
|
| 222 |
+
At update 60,000, the documented cumulative exposure estimate is 81.7B token occurrences, including inherited training and a known duplicate correction. This is not a count of globally unique tokens. See the [training log](TRAINING_LOG.md), [mix by stage](TRAINING_MIX.md), and the [training telemetry](https://track.fabryka.ai/run/71cf98ec-7c60-54a8-ac47-0ea7a23f573a). The public telemetry stopped updating after update 50,750; this card and the training log are the current record. The benchmark table above is specifically for update 60,000.
|
| 223 |
|
| 224 |
## Limitations
|
| 225 |
|
TRAINING_LOG.md
CHANGED
|
@@ -21,11 +21,16 @@ Generated from the model card before its reader-focused rewrite on 16 November 2
|
|
| 21 |
| 35,000 | 36,700,160,000 | 55,448,790,554 | `wagi_35000.pt` |
|
| 22 |
| 40,000 | 41,943,040,000 | 60,691,670,554 | `wagi_40000.pt` |
|
| 23 |
| 45,000 | 47,185,920,000 | 65,934,550,554 | `wagi_45000.pt` |
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
Files are managed in the separate [checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints). **Historical archive snapshot, 12 November 2026, 18:51 UTC:** upload of the then-listed eight native checkpoint files completed at **18:16 UTC**. The archive is public; verify its exact revision, file list, `SHA256SUMS` and file sizes before use. This main repository contains documentation during training.
|
| 26 |
|
| 27 |
**Archive availability, 16 November 2026:** the archive distinguishes nine weights-only `.pt` files, at updates 5,000 through 45,000, from the full update-35,000 recovery checkpoint `ckpt_35000.pt`, which also contains optimizer and queue state. No full recovery checkpoint after update 35,000 is published. These are native training checkpoints, not a Transformers `AutoModel` or safetensors release. Check the archive's exact revision, file list and checksums before use; its publication status is separate from the evaluation table below.
|
| 28 |
|
|
|
|
|
|
|
| 29 |
Token counts: **positions in this continuation = updates × 1,048,576**, counting each logical update once, not repeated compute after recovery. **Lineage lower bound = 18,863,257,682 inherited positions + positions in this continuation − 114,627,128 identified-copy positions**. The lineage figure is a source- and file-disjoint lower bound, **not globally unique tokens**. Exact evaluated safetensors identities in [EVALUATION.json](EVALUATION.json) are distinct from the native `.pt` file digests in the archive.
|
| 30 |
|
| 31 |
|
|
@@ -42,7 +47,7 @@ Token counts: **positions in this continuation = updates × 1,048,576**, countin
|
|
| 42 |
|
| 43 |
**Training objective — 11 November 2026, 15:29 UTC:** the objective remains maximum useful affordable exposure with constant learning rates after warmup and no planned in-run terminal decay, retaining cost guards and the operational reserve for a recoverable checkpoint, transfer and safe shutdown. An extension approaching the configured decay boundary requires review. Stop at a verified full checkpoint when the real budget or allocation requires it; a 5,000-update boundary is not required. Positions per dollar measure operating efficiency, not model quality.
|
| 44 |
|
| 45 |
-
**Recipe adoption — 11 November 2026:** operators confirmed the new recipe at **16:23:47 UTC**, resuming from **update 29,800 on H200**, with **`--steps 120000 --ckpt-every 200`**, LR **0.0006** and configured WSD decay beginning at update **96,000**.
|
| 46 |
|
| 47 |
A separate, continuously curated option targets approximately **10B tokens of quality-selected “gold” data at context 4,096**. It remains isolated from main-base inputs and evaluation material, with its own manifest and licensing/admission gate; books selected with Arkadiusz Słota require title-level license approval. A later finishing run with decay needs its own run and budget decision. Preliminary exclusion lists are not a complete packed 10B corpus or a release, and quality-based selection is not evidence of a measured quality gain. See the [data card](DATA_CARD.md) for the dated inventory and isolation boundary.
|
| 48 |
|
|
@@ -52,21 +57,21 @@ The unchanged architecture is specified as **1,073,912,851 parameters**, 20 laye
|
|
| 52 |
|
| 53 |
Architecture and tokenizer lineage includes [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md). The selected 1.08 base and the separate historical 1.08b ablation are different checkpoints; the continuation described here starts from 1.08.
|
| 54 |
|
| 55 |
-
The [training configuration](TRAINING_CONFIGURATION.md) records the 120,000-update scheduling recipe and its adoption. The operational target of
|
| 56 |
|
| 57 |
## Current CPT configuration and budget-limited endpoint
|
| 58 |
|
| 59 |
| Setting | Operator-selected recipe |
|
| 60 |
|---|---|
|
| 61 |
| Selected scheduling argument | `KROKI=120000`; confirmed runtime `--steps 120000` adoption at 11 November, 16:23:47 UTC, resuming from update 29,800 on H200; configured decay begins at update 96,000 |
|
| 62 |
-
| New-source allocation | **20% PL / 80% EN** in segments c01–c15 (updates 0–75,000). On 12 November at **18:44 UTC**, the owner approved **30% PL / 70% EN** from **c16**; segments c16 and c17 were built that way (measured token shares 0.3000 PL in both), so updates **75,001–
|
| 63 |
| Optimizer / schedule | Muon peak 0.02 plus AdamW peak 0.0006; 2,000-update warmup, then constant learning rates under the main-base plan; adoption receipt confirms LR 0.0006. The configured final-20% WSD decay to 0.00006 remains at update 96,000; an extension approaching it requires review |
|
| 64 |
| Checkpoints | Runtime `--ckpt-every 200` confirmed at the 16:23:47 UTC adoption; weights-only milestones every 5,000 updates. Native milestone files and full-state recovery files are distinguished in the separate checkpoint archive |
|
| 65 |
| Code | Launcher v5 SHA256 `796c71016c1e167bfd3ca98b78518f91c3502a96eac233dd7186ef6639fcc656`; exact command and converter pins in the configuration document |
|
| 66 |
|
| 67 |
Operators must review actual available funds, prices and allocation changes without assuming that the decay boundary is permanently unaffordable. An extended run must be reviewed before entering decay; do not shorten `KROKI` to a budget estimate and bring decay forward. Runtime changes need the operator's approved restart procedure and receipt. The trainer and v5 launcher are unchanged; the operator wrapper's argument ceiling was separately approved for the new recipe.
|
| 68 |
|
| 69 |
-
The reviewed inherited count is **18,863,257,682 source/file-disjoint positions**. The conservative c01 correction is **114,627,128 positions**, computed from two exclusion-list counts. The training target is **over 100B source- and file-disjoint positions in the 1.00–1.09 lineage**; the formula reaches that threshold at update **77,488**, and the
|
| 70 |
|
| 71 |
## Initial CPT execution evidence
|
| 72 |
|
|
|
|
| 21 |
| 35,000 | 36,700,160,000 | 55,448,790,554 | `wagi_35000.pt` |
|
| 22 |
| 40,000 | 41,943,040,000 | 60,691,670,554 | `wagi_40000.pt` |
|
| 23 |
| 45,000 | 47,185,920,000 | 65,934,550,554 | `wagi_45000.pt` |
|
| 24 |
+
| 50,000 | 52,428,800,000 | 71,177,430,554 | `wagi_50000.pt` |
|
| 25 |
+
| 55,000 | 57,671,680,000 | 76,420,310,554 | `wagi_55000.pt` |
|
| 26 |
+
| 60,000 | 62,914,560,000 | 81,663,190,554 | `wagi_60000.pt` |
|
| 27 |
|
| 28 |
Files are managed in the separate [checkpoint archive](https://huggingface.co/SlayerLab/Slayer1B-1.09-checkpoints). **Historical archive snapshot, 12 November 2026, 18:51 UTC:** upload of the then-listed eight native checkpoint files completed at **18:16 UTC**. The archive is public; verify its exact revision, file list, `SHA256SUMS` and file sizes before use. This main repository contains documentation during training.
|
| 29 |
|
| 30 |
**Archive availability, 16 November 2026:** the archive distinguishes nine weights-only `.pt` files, at updates 5,000 through 45,000, from the full update-35,000 recovery checkpoint `ckpt_35000.pt`, which also contains optimizer and queue state. No full recovery checkpoint after update 35,000 is published. These are native training checkpoints, not a Transformers `AutoModel` or safetensors release. Check the archive's exact revision, file list and checksums before use; its publication status is separate from the evaluation table below.
|
| 31 |
|
| 32 |
+
**Archive availability, 18 November 2026:** archive revision `dc3b6a9b0f87305da7ceca39ca27a79fa60b34cd` holds twelve weights-only `.pt` files, at updates 5,000 through 60,000, and the full update-35,000 recovery checkpoint. Full calibrated validation at update 60,000 was not measured. Update 60,000 is the last checkpoint of the 8 × NVIDIA B200 segment; training continued on 8 × NVIDIA H200 from that full recovery state.
|
| 33 |
+
|
| 34 |
Token counts: **positions in this continuation = updates × 1,048,576**, counting each logical update once, not repeated compute after recovery. **Lineage lower bound = 18,863,257,682 inherited positions + positions in this continuation − 114,627,128 identified-copy positions**. The lineage figure is a source- and file-disjoint lower bound, **not globally unique tokens**. Exact evaluated safetensors identities in [EVALUATION.json](EVALUATION.json) are distinct from the native `.pt` file digests in the archive.
|
| 35 |
|
| 36 |
|
|
|
|
| 47 |
|
| 48 |
**Training objective — 11 November 2026, 15:29 UTC:** the objective remains maximum useful affordable exposure with constant learning rates after warmup and no planned in-run terminal decay, retaining cost guards and the operational reserve for a recoverable checkpoint, transfer and safe shutdown. An extension approaching the configured decay boundary requires review. Stop at a verified full checkpoint when the real budget or allocation requires it; a 5,000-update boundary is not required. Positions per dollar measure operating efficiency, not model quality.
|
| 49 |
|
| 50 |
+
**Recipe adoption — 11 November 2026:** operators confirmed the new recipe at **16:23:47 UTC**, resuming from **update 29,800 on H200**, with **`--steps 120000 --ckpt-every 200`**, LR **0.0006** and configured WSD decay beginning at update **96,000**. On **17 November 2026 at 23:42 UTC** the operational target was raised from 80,000 to **85,000 updates**, the end of the built input queue (segment c17), corresponding to **107,877,590,554 source- and file-disjoint lineage positions** under the stated accounting. The main run still ends at a constant learning rate; any decay is a separate later run. The 120,000-update argument controls the scheduler. Allocation and funding can interrupt progress before the target.
|
| 51 |
|
| 52 |
A separate, continuously curated option targets approximately **10B tokens of quality-selected “gold” data at context 4,096**. It remains isolated from main-base inputs and evaluation material, with its own manifest and licensing/admission gate; books selected with Arkadiusz Słota require title-level license approval. A later finishing run with decay needs its own run and budget decision. Preliminary exclusion lists are not a complete packed 10B corpus or a release, and quality-based selection is not evidence of a measured quality gain. See the [data card](DATA_CARD.md) for the dated inventory and isolation boundary.
|
| 53 |
|
|
|
|
| 57 |
|
| 58 |
Architecture and tokenizer lineage includes [SlayerLab/Slayer1B-1.07](https://huggingface.co/SlayerLab/Slayer1B-1.07/blob/a090b54e658fb57dc5166c1fcc90cc6bf5c80106/README.md). The selected 1.08 base and the separate historical 1.08b ablation are different checkpoints; the continuation described here starts from 1.08.
|
| 59 |
|
| 60 |
+
The [training configuration](TRAINING_CONFIGURATION.md) records the 120,000-update scheduling recipe and its adoption. The operational target of 85,000 updates and the actual completed count are separate fields. Completed 1.08 and the abandoned first attempt retain their own historical records.
|
| 61 |
|
| 62 |
## Current CPT configuration and budget-limited endpoint
|
| 63 |
|
| 64 |
| Setting | Operator-selected recipe |
|
| 65 |
|---|---|
|
| 66 |
| Selected scheduling argument | `KROKI=120000`; confirmed runtime `--steps 120000` adoption at 11 November, 16:23:47 UTC, resuming from update 29,800 on H200; configured decay begins at update 96,000 |
|
| 67 |
+
| New-source allocation | **20% PL / 80% EN** in segments c01–c15 (updates 0–75,000). On 12 November at **18:44 UTC**, the owner approved **30% PL / 70% EN** from **c16**; segments c16 and c17 were built that way (measured token shares 0.3000 PL in both), so updates **75,001–85,000** use **30% PL / 70% EN**. These are new-input bucket shares, not a measured cumulative model-language ratio |
|
| 68 |
| Optimizer / schedule | Muon peak 0.02 plus AdamW peak 0.0006; 2,000-update warmup, then constant learning rates under the main-base plan; adoption receipt confirms LR 0.0006. The configured final-20% WSD decay to 0.00006 remains at update 96,000; an extension approaching it requires review |
|
| 69 |
| Checkpoints | Runtime `--ckpt-every 200` confirmed at the 16:23:47 UTC adoption; weights-only milestones every 5,000 updates. Native milestone files and full-state recovery files are distinguished in the separate checkpoint archive |
|
| 70 |
| Code | Launcher v5 SHA256 `796c71016c1e167bfd3ca98b78518f91c3502a96eac233dd7186ef6639fcc656`; exact command and converter pins in the configuration document |
|
| 71 |
|
| 72 |
Operators must review actual available funds, prices and allocation changes without assuming that the decay boundary is permanently unaffordable. An extended run must be reviewed before entering decay; do not shorten `KROKI` to a budget estimate and bring decay forward. Runtime changes need the operator's approved restart procedure and receipt. The trainer and v5 launcher are unchanged; the operator wrapper's argument ceiling was separately approved for the new recipe.
|
| 73 |
|
| 74 |
+
The reviewed inherited count is **18,863,257,682 source/file-disjoint positions**. The conservative c01 correction is **114,627,128 positions**, computed from two exclusion-list counts. The training target is **over 100B source- and file-disjoint positions in the 1.00–1.09 lineage**; the formula reaches that threshold at update **77,488**, and the 85,000-update plan corresponds to approximately **107.9B**. Historical content-level near-duplicates remain unresolved. The [data card](DATA_CARD.md) separately records prepared input capacity.
|
| 75 |
|
| 76 |
## Initial CPT execution evidence
|
| 77 |
|
plots/arc-challenge.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/belebele-polish.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/benchmark-overview.pdf
CHANGED
|
Binary files a/plots/benchmark-overview.pdf and b/plots/benchmark-overview.pdf differ
|
|
|
plots/benchmark-overview.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/benchmark-overview.svg
CHANGED
|
|
|
|
plots/benchmark-scores.csv
CHANGED
|
@@ -12,6 +12,7 @@ ARC-Challenge,acc_norm,1172,"1.09, update 40000",34.98293515358362,32.2526,37.79
|
|
| 12 |
ARC-Challenge,acc_norm,1172,"1.09, update 45000",34.81228668941979,32.081900000000005,37.5427
|
| 13 |
ARC-Challenge,acc_norm,1172,"1.09, update 50000",36.43344709897611,33.6177,39.2491
|
| 14 |
ARC-Challenge,acc_norm,1172,"1.09, update 55000",34.81228668941979,32.081900000000005,37.5427
|
|
|
|
| 15 |
HellaSwag,acc_norm,10042,1.07,43.37781318462458,42.431799999999996,44.3438
|
| 16 |
HellaSwag,acc_norm,10042,1.08,43.41764588727345,42.431799999999996,44.3935
|
| 17 |
HellaSwag,acc_norm,10042,"1.09, update 5000",46.5345548695479,45.5487,47.5204
|
|
@@ -25,6 +26,7 @@ HellaSwag,acc_norm,10042,"1.09, update 40000",52.79824736108345,51.8124000000000
|
|
| 25 |
HellaSwag,acc_norm,10042,"1.09, update 45000",53.654650468034255,52.6887,54.620599999999996
|
| 26 |
HellaSwag,acc_norm,10042,"1.09, update 50000",54.09281019717188,53.1269,55.0588
|
| 27 |
HellaSwag,acc_norm,10042,"1.09, update 55000",54.690300736905,53.7244,55.6562
|
|
|
|
| 28 |
PIQA,acc_norm,1838,1.07,66.97497279651796,64.7987,69.1513
|
| 29 |
PIQA,acc_norm,1838,1.08,66.2132752992383,64.1458,68.33510000000001
|
| 30 |
PIQA,acc_norm,1838,"1.09, update 5000",68.28073993471165,66.2119,70.4026
|
|
@@ -38,6 +40,7 @@ PIQA,acc_norm,1838,"1.09, update 40000",71.21871599564744,69.20429999999999,73.2
|
|
| 38 |
PIQA,acc_norm,1838,"1.09, update 45000",70.29379760609358,68.2263,72.3069
|
| 39 |
PIQA,acc_norm,1838,"1.09, update 50000",71.05549510337323,68.9336,73.1774
|
| 40 |
PIQA,acc_norm,1838,"1.09, update 55000",72.30685527747552,70.29379999999999,74.3199
|
|
|
|
| 41 |
WinoGrande,acc,1267,1.07,53.591160220994475,50.828700000000005,56.3536
|
| 42 |
WinoGrande,acc,1267,1.08,53.67008681925809,50.9077,56.4325
|
| 43 |
WinoGrande,acc,1267,"1.09, update 5000",54.93291239147593,52.1705,57.6953
|
|
@@ -51,6 +54,7 @@ WinoGrande,acc,1267,"1.09, update 40000",57.69534333070245,55.0118,60.3788
|
|
| 51 |
WinoGrande,acc,1267,"1.09, update 45000",59.0370955011839,56.2747,61.720600000000005
|
| 52 |
WinoGrande,acc,1267,"1.09, update 50000",56.98500394632991,54.143600000000006,59.670500000000004
|
| 53 |
WinoGrande,acc,1267,"1.09, update 55000",57.61641673243883,54.9329,60.3788
|
|
|
|
| 54 |
BoolQ,acc,3270,1.07,58.28746177370031,56.6055,59.96939999999999
|
| 55 |
BoolQ,acc,3270,1.08,59.969418960244646,58.2569,61.590199999999996
|
| 56 |
BoolQ,acc,3270,"1.09, update 5000",61.80428134556575,60.1827,63.486200000000004
|
|
@@ -64,6 +68,7 @@ BoolQ,acc,3270,"1.09, update 40000",59.174311926605505,57.4618,60.82570000000000
|
|
| 64 |
BoolQ,acc,3270,"1.09, update 45000",61.92660550458716,60.244600000000005,63.6086
|
| 65 |
BoolQ,acc,3270,"1.09, update 50000",61.13149847094801,59.4495,62.8135
|
| 66 |
BoolQ,acc,3270,"1.09, update 55000",62.782874617737,61.1315,64.4037
|
|
|
|
| 67 |
OpenBookQA,acc_norm,500,1.07,33.6,29.4,37.8
|
| 68 |
OpenBookQA,acc_norm,500,1.08,34.8,30.599999999999998,39.0
|
| 69 |
OpenBookQA,acc_norm,500,"1.09, update 5000",35.6,31.4,40.0
|
|
@@ -77,6 +82,7 @@ OpenBookQA,acc_norm,500,"1.09, update 40000",34.6,30.4,38.800000000000004
|
|
| 77 |
OpenBookQA,acc_norm,500,"1.09, update 45000",36.0,31.8,40.2
|
| 78 |
OpenBookQA,acc_norm,500,"1.09, update 50000",35.6,31.4,39.800000000000004
|
| 79 |
OpenBookQA,acc_norm,500,"1.09, update 55000",34.4,30.2,38.4
|
|
|
|
| 80 |
"Belebele, Polish",acc,900,1.07,26.0,23.1111,28.8889
|
| 81 |
"Belebele, Polish",acc,900,1.08,24.0,21.2222,26.8889
|
| 82 |
"Belebele, Polish",acc,900,"1.09, update 5000",24.11111111111111,21.3333,26.8889
|
|
@@ -90,6 +96,7 @@ OpenBookQA,acc_norm,500,"1.09, update 55000",34.4,30.2,38.4
|
|
| 90 |
"Belebele, Polish",acc,900,"1.09, update 45000",25.333333333333332,22.555600000000002,28.222199999999997
|
| 91 |
"Belebele, Polish",acc,900,"1.09, update 50000",25.77777777777778,22.8889,28.6667
|
| 92 |
"Belebele, Polish",acc,900,"1.09, update 55000",25.555555555555557,22.775000000000002,28.444399999999998
|
|
|
|
| 93 |
HellaSwag-PL,acc_norm,10042,1.07,43.825931089424415,42.8401,44.792100000000005
|
| 94 |
HellaSwag-PL,acc_norm,10042,1.08,43.89563831905995,42.8998,44.8715
|
| 95 |
HellaSwag-PL,acc_norm,10042,"1.09, update 5000",44.732125074686316,43.7562,45.708
|
|
@@ -103,6 +110,7 @@ HellaSwag-PL,acc_norm,10042,"1.09, update 40000",46.61422027484565,45.6483000000
|
|
| 103 |
HellaSwag-PL,acc_norm,10042,"1.09, update 45000",46.78350926110337,45.7877,47.7495
|
| 104 |
HellaSwag-PL,acc_norm,10042,"1.09, update 50000",47.51045608444533,46.5246,48.4963
|
| 105 |
HellaSwag-PL,acc_norm,10042,"1.09, update 55000",47.54033061143198,46.5744,48.5262
|
|
|
|
| 106 |
MMLU-PL,acc,14042,1.07,24.013673265916538,23.3015,24.7116
|
| 107 |
MMLU-PL,acc,14042,1.08,23.038028770830365,22.3401,23.743100000000002
|
| 108 |
MMLU-PL,acc,14042,"1.09, update 5000",24.099131177894886,23.387,24.8184
|
|
@@ -116,6 +124,7 @@ MMLU-PL,acc,14042,"1.09, update 40000",24.241561031192138,23.5437,24.968
|
|
| 116 |
MMLU-PL,acc,14042,"1.09, update 45000",25.737074490813274,25.0247,26.4635
|
| 117 |
MMLU-PL,acc,14042,"1.09, update 50000",24.683093576413615,23.9709,25.4095
|
| 118 |
MMLU-PL,acc,14042,"1.09, update 55000",24.86113089303518,24.149,25.5733
|
|
|
|
| 119 |
GSM8K,"exact_match,strict-match",1319,1.07,1.5163002274450341,0.9097801364670205,2.1986353297952994
|
| 120 |
GSM8K,"exact_match,strict-match",1319,1.08,0.9097801364670205,0.45489006823351025,1.4404852160727823
|
| 121 |
GSM8K,"exact_match,strict-match",1319,"1.09, update 10000",0.9855951478392722,0.45489006823351025,1.5163002274450341
|
|
|
|
| 12 |
ARC-Challenge,acc_norm,1172,"1.09, update 45000",34.81228668941979,32.081900000000005,37.5427
|
| 13 |
ARC-Challenge,acc_norm,1172,"1.09, update 50000",36.43344709897611,33.6177,39.2491
|
| 14 |
ARC-Challenge,acc_norm,1172,"1.09, update 55000",34.81228668941979,32.081900000000005,37.5427
|
| 15 |
+
ARC-Challenge,acc_norm,1172,"1.09, update 60000",36.00682593856655,33.2765,38.8225
|
| 16 |
HellaSwag,acc_norm,10042,1.07,43.37781318462458,42.431799999999996,44.3438
|
| 17 |
HellaSwag,acc_norm,10042,1.08,43.41764588727345,42.431799999999996,44.3935
|
| 18 |
HellaSwag,acc_norm,10042,"1.09, update 5000",46.5345548695479,45.5487,47.5204
|
|
|
|
| 26 |
HellaSwag,acc_norm,10042,"1.09, update 45000",53.654650468034255,52.6887,54.620599999999996
|
| 27 |
HellaSwag,acc_norm,10042,"1.09, update 50000",54.09281019717188,53.1269,55.0588
|
| 28 |
HellaSwag,acc_norm,10042,"1.09, update 55000",54.690300736905,53.7244,55.6562
|
| 29 |
+
HellaSwag,acc_norm,10042,"1.09, update 60000",55.06871141206931,54.1125,56.0446
|
| 30 |
PIQA,acc_norm,1838,1.07,66.97497279651796,64.7987,69.1513
|
| 31 |
PIQA,acc_norm,1838,1.08,66.2132752992383,64.1458,68.33510000000001
|
| 32 |
PIQA,acc_norm,1838,"1.09, update 5000",68.28073993471165,66.2119,70.4026
|
|
|
|
| 40 |
PIQA,acc_norm,1838,"1.09, update 45000",70.29379760609358,68.2263,72.3069
|
| 41 |
PIQA,acc_norm,1838,"1.09, update 50000",71.05549510337323,68.9336,73.1774
|
| 42 |
PIQA,acc_norm,1838,"1.09, update 55000",72.30685527747552,70.29379999999999,74.3199
|
| 43 |
+
PIQA,acc_norm,1838,"1.09, update 60000",72.0892274211099,70.0218,74.1023
|
| 44 |
WinoGrande,acc,1267,1.07,53.591160220994475,50.828700000000005,56.3536
|
| 45 |
WinoGrande,acc,1267,1.08,53.67008681925809,50.9077,56.4325
|
| 46 |
WinoGrande,acc,1267,"1.09, update 5000",54.93291239147593,52.1705,57.6953
|
|
|
|
| 54 |
WinoGrande,acc,1267,"1.09, update 45000",59.0370955011839,56.2747,61.720600000000005
|
| 55 |
WinoGrande,acc,1267,"1.09, update 50000",56.98500394632991,54.143600000000006,59.670500000000004
|
| 56 |
WinoGrande,acc,1267,"1.09, update 55000",57.61641673243883,54.9329,60.3788
|
| 57 |
+
WinoGrande,acc,1267,"1.09, update 60000",57.4585635359116,54.7751,60.1421
|
| 58 |
BoolQ,acc,3270,1.07,58.28746177370031,56.6055,59.96939999999999
|
| 59 |
BoolQ,acc,3270,1.08,59.969418960244646,58.2569,61.590199999999996
|
| 60 |
BoolQ,acc,3270,"1.09, update 5000",61.80428134556575,60.1827,63.486200000000004
|
|
|
|
| 68 |
BoolQ,acc,3270,"1.09, update 45000",61.92660550458716,60.244600000000005,63.6086
|
| 69 |
BoolQ,acc,3270,"1.09, update 50000",61.13149847094801,59.4495,62.8135
|
| 70 |
BoolQ,acc,3270,"1.09, update 55000",62.782874617737,61.1315,64.4037
|
| 71 |
+
BoolQ,acc,3270,"1.09, update 60000",63.18042813455658,61.4985,64.8624
|
| 72 |
OpenBookQA,acc_norm,500,1.07,33.6,29.4,37.8
|
| 73 |
OpenBookQA,acc_norm,500,1.08,34.8,30.599999999999998,39.0
|
| 74 |
OpenBookQA,acc_norm,500,"1.09, update 5000",35.6,31.4,40.0
|
|
|
|
| 82 |
OpenBookQA,acc_norm,500,"1.09, update 45000",36.0,31.8,40.2
|
| 83 |
OpenBookQA,acc_norm,500,"1.09, update 50000",35.6,31.4,39.800000000000004
|
| 84 |
OpenBookQA,acc_norm,500,"1.09, update 55000",34.4,30.2,38.4
|
| 85 |
+
OpenBookQA,acc_norm,500,"1.09, update 60000",35.0,31.0,39.2
|
| 86 |
"Belebele, Polish",acc,900,1.07,26.0,23.1111,28.8889
|
| 87 |
"Belebele, Polish",acc,900,1.08,24.0,21.2222,26.8889
|
| 88 |
"Belebele, Polish",acc,900,"1.09, update 5000",24.11111111111111,21.3333,26.8889
|
|
|
|
| 96 |
"Belebele, Polish",acc,900,"1.09, update 45000",25.333333333333332,22.555600000000002,28.222199999999997
|
| 97 |
"Belebele, Polish",acc,900,"1.09, update 50000",25.77777777777778,22.8889,28.6667
|
| 98 |
"Belebele, Polish",acc,900,"1.09, update 55000",25.555555555555557,22.775000000000002,28.444399999999998
|
| 99 |
+
"Belebele, Polish",acc,900,"1.09, update 60000",24.77777777777778,22.0,27.6667
|
| 100 |
HellaSwag-PL,acc_norm,10042,1.07,43.825931089424415,42.8401,44.792100000000005
|
| 101 |
HellaSwag-PL,acc_norm,10042,1.08,43.89563831905995,42.8998,44.8715
|
| 102 |
HellaSwag-PL,acc_norm,10042,"1.09, update 5000",44.732125074686316,43.7562,45.708
|
|
|
|
| 110 |
HellaSwag-PL,acc_norm,10042,"1.09, update 45000",46.78350926110337,45.7877,47.7495
|
| 111 |
HellaSwag-PL,acc_norm,10042,"1.09, update 50000",47.51045608444533,46.5246,48.4963
|
| 112 |
HellaSwag-PL,acc_norm,10042,"1.09, update 55000",47.54033061143198,46.5744,48.5262
|
| 113 |
+
HellaSwag-PL,acc_norm,10042,"1.09, update 60000",47.48058155745867,46.5144,48.4664
|
| 114 |
MMLU-PL,acc,14042,1.07,24.013673265916538,23.3015,24.7116
|
| 115 |
MMLU-PL,acc,14042,1.08,23.038028770830365,22.3401,23.743100000000002
|
| 116 |
MMLU-PL,acc,14042,"1.09, update 5000",24.099131177894886,23.387,24.8184
|
|
|
|
| 124 |
MMLU-PL,acc,14042,"1.09, update 45000",25.737074490813274,25.0247,26.4635
|
| 125 |
MMLU-PL,acc,14042,"1.09, update 50000",24.683093576413615,23.9709,25.4095
|
| 126 |
MMLU-PL,acc,14042,"1.09, update 55000",24.86113089303518,24.149,25.5733
|
| 127 |
+
MMLU-PL,acc,14042,"1.09, update 60000",24.455205811138015,23.735899999999997,25.153100000000002
|
| 128 |
GSM8K,"exact_match,strict-match",1319,1.07,1.5163002274450341,0.9097801364670205,2.1986353297952994
|
| 129 |
GSM8K,"exact_match,strict-match",1319,1.08,0.9097801364670205,0.45489006823351025,1.4404852160727823
|
| 130 |
GSM8K,"exact_match,strict-match",1319,"1.09, update 10000",0.9855951478392722,0.45489006823351025,1.5163002274450341
|
plots/boolq.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/hellaswag-pl.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/hellaswag-progress.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/hellaswag-progress.svg
CHANGED
|
|
|
|
plots/hellaswag.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/mmlu-pl.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/openbookqa.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/piqa.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
plots/winogrande.png
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|