| { |
| "schema_version": 1, |
| "measurement_owner": "SlayerLab / Fabryka AI", |
| "status": "descriptive; training-lineage contamination scan pending", |
| "source_aggregate_sha256": "fc6f67a20bdc6ada687349d35de26f6811e27766dfe1aa197082000c03535796", |
| "source_generator_sha256": null, |
| "benchmark_community_report": "https://github.com/slayerlabs/benchmarks-fabryka-ai/blob/8b547007526971bd2fc35dfe56ff2c7d9b802568/research/results/full-run-2026-11/README.md", |
| "community_report_is_separate_measurement": true, |
| "protocol": { |
| "few_shot": 0, |
| "dtype": "float32", |
| "context_limit": 4096, |
| "initial_token_id": 0, |
| "special_tokens": false, |
| "truncation": "left", |
| "scoring": "summed continuation-token log likelihood; task-specific partial scoring for WinoGrande", |
| "metric_units": "fraction; multiply by 100 for card percentages", |
| "primary_metrics": { |
| "arc_challenge": "acc_norm", |
| "hellaswag": "acc_norm", |
| "piqa": "acc_norm", |
| "winogrande": "acc", |
| "boolq": "acc", |
| "openbookqa": "acc_norm", |
| "belebele_pol": "acc", |
| "hellaswag_pl": "acc_norm", |
| "mmlu_pl": "acc" |
| }, |
| "mmlu_aggregation": "micro-average over questions", |
| "polish_task_definitions": "OpenGPTX ogx_hellaswagx_pl and ogx_mmlux_pl; source revision 0018eb9b", |
| "task_protocols": { |
| "arc_challenge": "lm-eval arc_challenge: ctx='Question: {question}\\nAnswer:', kont=' '+choices.text[i], gold=choices.label.index(answerKey); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "hellaswag": "lm-eval hellaswag: process_docs (preprocess(activity_label+': '+ctx_a+' '+ctx_b.capitalize()), endings preprocess), ctx=query, kont=' '+ending, gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "piqa": "lm-eval piqa (baber/piqa): ctx='Question: {goal}\\nAnswer:', kont=' '+[sol1,sol2][i], gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "winogrande": "lm-eval winogrande (partial scoring): ctx=sentence[:idx('_')]+option_i, kont=' '+sentence[idx+1:].strip() (wspólna), gold={'1':0,'2':1}[answer]; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "boolq": "lm-eval boolq v2: ctx='{passage}\\nQuestion: {question}?\\nAnswer:', kont=' no'|' yes', gold=int(label); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "openbookqa": "lm-eval openbookqa: ctx=question_stem, kont=' '+choices.text[i], gold=choices.label.index(answerKey.lstrip()); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "belebele_pol": "lm-eval belebele_pol_Latn 0-shot: ctx='P: {flores_passage}\\nQ: {question.strip()}\\nA: {mc_answer1}\\nB: ...\\nC: ...\\nD: {mc_answer4}\\nAnswer:', kont=' A'..' D', gold=int(correct_answer_num)-1; 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "hellaswag_pl": "OpenGPTX ogx_hellaswagx_pl @0018eb9b (= lm-eval hellaswag process_docs na polskich polach); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32", |
| "mmlu_pl": "OpenGPTX ogx_mmlux_pl-* @0018eb9b 0-shot: ctx=description(PL, doklejone BEZ separatora, jak w forku)+'{question.strip()}\\nA. {c0}\\nB. {c1}\\nC. {c2}\\nD. {c3}\\nOdpowiedź:', kont=' A'..' D', gold=answer; acc = mikro po 14042 pozycjach (jak grupa lm-eval mmlu, wagi = rozmiary); 0-shot; LL = suma log p tokenów kontynuacji; start id 0; add_special_tokens=False; _encode_pair jak HFLM; obcięcie z lewej do block; fp32" |
| } |
| }, |
| "ci": { |
| "level": 0.95, |
| "method": "percentile bootstrap via Binomial(n,p)/n for binary item scores", |
| "replicates": 10000, |
| "base_seed": 20261110, |
| "seed_scope": "Updated aggregate records seed 20261110; stream derivation is not supplied. Previous aggregate documented a model-specific stream.", |
| "assumption": "items treated as independent; no subject, template or source clusters", |
| "interpretation": "pointwise interval for one result; not a paired interval for model differences; not training-seed variance" |
| }, |
| "models": { |
| "1.07": { |
| "step": 22327, |
| "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5", |
| "weights_published": true, |
| "source_results_sha256": "9d89a9cdc1ea66723c9c709930d987900609bee84de5598dba400231407babf2", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.261945, |
| "acc_ci95": [ |
| 0.237201, |
| 0.286689 |
| ], |
| "acc_norm": 0.273038, |
| "acc_norm_ci95": [ |
| 0.24744, |
| 0.298635 |
| ], |
| "split": "test", |
| "correct_primary": 320 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.351324, |
| "acc_ci95": [ |
| 0.341762, |
| 0.360486 |
| ], |
| "acc_norm": 0.433778, |
| "acc_norm_ci95": [ |
| 0.424318, |
| 0.443438 |
| ], |
| "split": "validation", |
| "correct_primary": 4356 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.670294, |
| "acc_ci95": [ |
| 0.648531, |
| 0.691526 |
| ], |
| "acc_norm": 0.66975, |
| "acc_norm_ci95": [ |
| 0.647987, |
| 0.691513 |
| ], |
| "split": "validation", |
| "correct_primary": 1231 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.535912, |
| "acc_ci95": [ |
| 0.508287, |
| 0.563536 |
| ], |
| "acc_norm": 0.535912, |
| "acc_norm_ci95": [ |
| 0.509077, |
| 0.563536 |
| ], |
| "split": "validation", |
| "correct_primary": 679 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.582875, |
| "acc_ci95": [ |
| 0.566055, |
| 0.599694 |
| ], |
| "acc_norm": 0.621101, |
| "acc_norm_ci95": [ |
| 0.604281, |
| 0.637615 |
| ], |
| "split": "validation", |
| "correct_primary": 1906 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.2, |
| "acc_ci95": [ |
| 0.166, |
| 0.236 |
| ], |
| "acc_norm": 0.336, |
| "acc_norm_ci95": [ |
| 0.294, |
| 0.378 |
| ], |
| "split": "test", |
| "correct_primary": 168 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.26, |
| "acc_ci95": [ |
| 0.231111, |
| 0.288889 |
| ], |
| "acc_norm": 0.26, |
| "acc_norm_ci95": [ |
| 0.232222, |
| 0.288889 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 234 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.345051, |
| "acc_ci95": [ |
| 0.335986, |
| 0.354611 |
| ], |
| "acc_norm": 0.438259, |
| "acc_norm_ci95": [ |
| 0.428401, |
| 0.447921 |
| ], |
| "split": "validation", |
| "correct_primary": 4401 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.240137, |
| "acc_ci95": [ |
| 0.233015, |
| 0.247116 |
| ], |
| "acc_norm": 0.240137, |
| "acc_norm_ci95": [ |
| 0.233158, |
| 0.247187 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3372 |
| } |
| } |
| }, |
| "1.08": { |
| "step": 1709, |
| "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4", |
| "weights_published": true, |
| "source_results_sha256": "108e43d0ac726f946c0480ff00319eb1811a3e254f05b1540156b0db11d1566e", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.271331, |
| "acc_ci95": [ |
| 0.245734, |
| 0.296928 |
| ], |
| "acc_norm": 0.300341, |
| "acc_norm_ci95": [ |
| 0.273891, |
| 0.326792 |
| ], |
| "split": "test", |
| "correct_primary": 352 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.350926, |
| "acc_ci95": [ |
| 0.341765, |
| 0.360486 |
| ], |
| "acc_norm": 0.434176, |
| "acc_norm_ci95": [ |
| 0.424318, |
| 0.443935 |
| ], |
| "split": "validation", |
| "correct_primary": 4360 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.681719, |
| "acc_ci95": [ |
| 0.661045, |
| 0.702938 |
| ], |
| "acc_norm": 0.662133, |
| "acc_norm_ci95": [ |
| 0.641458, |
| 0.683351 |
| ], |
| "split": "validation", |
| "correct_primary": 1217 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.536701, |
| "acc_ci95": [ |
| 0.509077, |
| 0.564325 |
| ], |
| "acc_norm": 0.536701, |
| "acc_norm_ci95": [ |
| 0.508287, |
| 0.563536 |
| ], |
| "split": "validation", |
| "correct_primary": 680 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.599694, |
| "acc_ci95": [ |
| 0.582569, |
| 0.615902 |
| ], |
| "acc_norm": 0.620489, |
| "acc_norm_ci95": [ |
| 0.603364, |
| 0.637309 |
| ], |
| "split": "validation", |
| "correct_primary": 1961 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.23, |
| "acc_ci95": [ |
| 0.194, |
| 0.266 |
| ], |
| "acc_norm": 0.348, |
| "acc_norm_ci95": [ |
| 0.306, |
| 0.39 |
| ], |
| "split": "test", |
| "correct_primary": 174 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.24, |
| "acc_ci95": [ |
| 0.212222, |
| 0.268889 |
| ], |
| "acc_norm": 0.24, |
| "acc_norm_ci95": [ |
| 0.212222, |
| 0.267778 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 216 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.352221, |
| "acc_ci95": [ |
| 0.342661, |
| 0.361581 |
| ], |
| "acc_norm": 0.438956, |
| "acc_norm_ci95": [ |
| 0.428998, |
| 0.448715 |
| ], |
| "split": "validation", |
| "correct_primary": 4408 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.23038, |
| "acc_ci95": [ |
| 0.223401, |
| 0.237431 |
| ], |
| "acc_norm": 0.23038, |
| "acc_norm_ci95": [ |
| 0.223259, |
| 0.237288 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3235 |
| } |
| } |
| }, |
| "1.09-k5000": { |
| "step": 5000, |
| "weights_sha256": "ccd7fc17b834d1de82ac53d0252085cbab567ac96676d0e263cd463a05745631", |
| "weights_published": false, |
| "source_results_sha256": "ec8d493463d99687f85a06654cef7706b3d37f0c9b9c3e760512f2baac34d632", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.288396, |
| "acc_ci95": [ |
| 0.262799, |
| 0.314846 |
| ], |
| "acc_norm": 0.293515, |
| "acc_norm_ci95": [ |
| 0.267918, |
| 0.319966 |
| ], |
| "split": "test", |
| "correct_primary": 344 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.370942, |
| "acc_ci95": [ |
| 0.361781, |
| 0.380303 |
| ], |
| "acc_norm": 0.465346, |
| "acc_norm_ci95": [ |
| 0.455487, |
| 0.475204 |
| ], |
| "split": "validation", |
| "correct_primary": 4673 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.688248, |
| "acc_ci95": [ |
| 0.667029, |
| 0.708923 |
| ], |
| "acc_norm": 0.682807, |
| "acc_norm_ci95": [ |
| 0.662119, |
| 0.704026 |
| ], |
| "split": "validation", |
| "correct_primary": 1255 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.549329, |
| "acc_ci95": [ |
| 0.521705, |
| 0.576953 |
| ], |
| "acc_norm": 0.549329, |
| "acc_norm_ci95": [ |
| 0.521705, |
| 0.576953 |
| ], |
| "split": "validation", |
| "correct_primary": 696 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.618043, |
| "acc_ci95": [ |
| 0.601827, |
| 0.634862 |
| ], |
| "acc_norm": 0.623242, |
| "acc_norm_ci95": [ |
| 0.606728, |
| 0.640061 |
| ], |
| "split": "validation", |
| "correct_primary": 2021 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.252, |
| "acc_ci95": [ |
| 0.216, |
| 0.29 |
| ], |
| "acc_norm": 0.356, |
| "acc_norm_ci95": [ |
| 0.314, |
| 0.4 |
| ], |
| "split": "test", |
| "correct_primary": 178 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.241111, |
| "acc_ci95": [ |
| 0.213333, |
| 0.268889 |
| ], |
| "acc_norm": 0.241111, |
| "acc_norm_ci95": [ |
| 0.213333, |
| 0.27 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 217 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.349632, |
| "acc_ci95": [ |
| 0.340271, |
| 0.358992 |
| ], |
| "acc_norm": 0.447321, |
| "acc_norm_ci95": [ |
| 0.437562, |
| 0.45708 |
| ], |
| "split": "validation", |
| "correct_primary": 4492 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.240991, |
| "acc_ci95": [ |
| 0.23387, |
| 0.248184 |
| ], |
| "acc_norm": 0.240991, |
| "acc_norm_ci95": [ |
| 0.23387, |
| 0.248042 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3384 |
| } |
| } |
| }, |
| "1.09-k10000": { |
| "step": 10000, |
| "weights_sha256": "12939d84ef0d06b350773fb30837390306d899ded315d1721b52c7615a87317a", |
| "weights_published": false, |
| "source_results_sha256": "3fff987f3b9d74eaba1e121a6a769247fbc12c79043bd5cb5d442c8dafeb9a69", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.292662, |
| "acc_ci95": [ |
| 0.267044, |
| 0.319113 |
| ], |
| "acc_norm": 0.304608, |
| "acc_norm_ci95": [ |
| 0.278157, |
| 0.331058 |
| ], |
| "split": "test", |
| "correct_primary": 357 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.386178, |
| "acc_ci95": [ |
| 0.376817, |
| 0.39574 |
| ], |
| "acc_norm": 0.489345, |
| "acc_norm_ci95": [ |
| 0.479685, |
| 0.499104 |
| ], |
| "split": "validation", |
| "correct_primary": 4914 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.696953, |
| "acc_ci95": [ |
| 0.675734, |
| 0.717628 |
| ], |
| "acc_norm": 0.68988, |
| "acc_norm_ci95": [ |
| 0.669206, |
| 0.711099 |
| ], |
| "split": "validation", |
| "correct_primary": 1268 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.549329, |
| "acc_ci95": [ |
| 0.521705, |
| 0.576164 |
| ], |
| "acc_norm": 0.549329, |
| "acc_norm_ci95": [ |
| 0.521705, |
| 0.576953 |
| ], |
| "split": "validation", |
| "correct_primary": 696 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.611315, |
| "acc_ci95": [ |
| 0.594495, |
| 0.627829 |
| ], |
| "acc_norm": 0.62263, |
| "acc_norm_ci95": [ |
| 0.60581, |
| 0.639144 |
| ], |
| "split": "validation", |
| "correct_primary": 1999 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.242, |
| "acc_ci95": [ |
| 0.206, |
| 0.28 |
| ], |
| "acc_norm": 0.344, |
| "acc_norm_ci95": [ |
| 0.302, |
| 0.386 |
| ], |
| "split": "test", |
| "correct_primary": 172 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.241111, |
| "acc_ci95": [ |
| 0.213333, |
| 0.27 |
| ], |
| "acc_norm": 0.241111, |
| "acc_norm_ci95": [ |
| 0.213333, |
| 0.27 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 217 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.354013, |
| "acc_ci95": [ |
| 0.344652, |
| 0.363274 |
| ], |
| "acc_norm": 0.455089, |
| "acc_norm_ci95": [ |
| 0.44513, |
| 0.464848 |
| ], |
| "split": "validation", |
| "correct_primary": 4570 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.233229, |
| "acc_ci95": [ |
| 0.22625, |
| 0.240279 |
| ], |
| "acc_norm": 0.233229, |
| "acc_norm_ci95": [ |
| 0.22625, |
| 0.240422 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3275 |
| } |
| } |
| }, |
| "1.09-k15000": { |
| "step": 15000, |
| "weights_sha256": "d5e3927cc5bfe380f6acb9a2de089ef776ae61a57a70d5a719da9f4da3932f9d", |
| "weights_published": false, |
| "source_results_sha256": "1bbb4ce856f45ed4ebacc1911431b749f6df4ed8f5caf292437c9424f03959bd", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.309727, |
| "acc_ci95": [ |
| 0.283276, |
| 0.337031 |
| ], |
| "acc_norm": 0.325085, |
| "acc_norm_ci95": [ |
| 0.298635, |
| 0.352389 |
| ], |
| "split": "test", |
| "correct_primary": 381 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.387771, |
| "acc_ci95": [ |
| 0.378411, |
| 0.39753 |
| ], |
| "acc_norm": 0.49771, |
| "acc_norm_ci95": [ |
| 0.487951, |
| 0.507568 |
| ], |
| "split": "validation", |
| "correct_primary": 4998 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.695321, |
| "acc_ci95": [ |
| 0.674646, |
| 0.71654 |
| ], |
| "acc_norm": 0.68988, |
| "acc_norm_ci95": [ |
| 0.668662, |
| 0.710555 |
| ], |
| "split": "validation", |
| "correct_primary": 1268 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.567482, |
| "acc_ci95": [ |
| 0.540647, |
| 0.594317 |
| ], |
| "acc_norm": 0.567482, |
| "acc_norm_ci95": [ |
| 0.539858, |
| 0.594317 |
| ], |
| "split": "validation", |
| "correct_primary": 719 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.598777, |
| "acc_ci95": [ |
| 0.581957, |
| 0.615291 |
| ], |
| "acc_norm": 0.619572, |
| "acc_norm_ci95": [ |
| 0.603058, |
| 0.636697 |
| ], |
| "split": "validation", |
| "correct_primary": 1958 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.252, |
| "acc_ci95": [ |
| 0.214, |
| 0.29 |
| ], |
| "acc_norm": 0.36, |
| "acc_norm_ci95": [ |
| 0.318, |
| 0.402 |
| ], |
| "split": "test", |
| "correct_primary": 180 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.225556, |
| "acc_ci95": [ |
| 0.198889, |
| 0.253333 |
| ], |
| "acc_norm": 0.225556, |
| "acc_norm_ci95": [ |
| 0.197778, |
| 0.253333 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 203 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.34286, |
| "acc_ci95": [ |
| 0.333599, |
| 0.352121 |
| ], |
| "acc_norm": 0.441844, |
| "acc_norm_ci95": [ |
| 0.432085, |
| 0.451603 |
| ], |
| "split": "validation", |
| "correct_primary": 4437 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.235009, |
| "acc_ci95": [ |
| 0.227959, |
| 0.24206 |
| ], |
| "acc_norm": 0.235009, |
| "acc_norm_ci95": [ |
| 0.22803, |
| 0.242273 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3300 |
| } |
| } |
| }, |
| "1.09-k20000": { |
| "step": 20000, |
| "weights_sha256": "8bb1927755ae856a5f9f9e63ac2d3f94cd750a8059eeca81be86cf820abc2461", |
| "weights_published": false, |
| "source_results_sha256": "a6225b2af295c846852314f88c0e08f3a71d67d3cd67a53c65bdd87b989a018f", |
| "evaluator_sha256": "ae6658f9c83a484e373dc6d14476a7aeb98a905c27cd6d97ea234fb56140e016", |
| "tasks": { |
| "arc_challenge": { |
| "n": 1172, |
| "n_pelne": 1172, |
| "dataset_repo": "allenai/ai2_arc", |
| "dataset_sha": "210d026faf9955653af8916fad021475a3f00453", |
| "pliki_sha256": { |
| "ARC-Challenge/test-00000-of-00001.parquet": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "acc": 0.306314, |
| "acc_ci95": [ |
| 0.280717, |
| 0.332765 |
| ], |
| "acc_norm": 0.331058, |
| "acc_norm_ci95": [ |
| 0.304608, |
| 0.358362 |
| ], |
| "split": "test", |
| "correct_primary": 388 |
| }, |
| "hellaswag": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Rowan/hellaswag", |
| "dataset_sha": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "pliki_sha256": { |
| "data/validation-00000-of-00001.parquet": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "acc": 0.397232, |
| "acc_ci95": [ |
| 0.387473, |
| 0.406592 |
| ], |
| "acc_norm": 0.503784, |
| "acc_norm_ci95": [ |
| 0.493926, |
| 0.513543 |
| ], |
| "split": "validation", |
| "correct_primary": 5059 |
| }, |
| "piqa": { |
| "n": 1838, |
| "n_pelne": 1838, |
| "dataset_repo": "baber/piqa", |
| "dataset_sha": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "pliki_sha256": { |
| "piqa_validation.parquet": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "acc": 0.702938, |
| "acc_ci95": [ |
| 0.682263, |
| 0.723069 |
| ], |
| "acc_norm": 0.707835, |
| "acc_norm_ci95": [ |
| 0.68716, |
| 0.728509 |
| ], |
| "split": "validation", |
| "correct_primary": 1301 |
| }, |
| "winogrande": { |
| "n": 1267, |
| "n_pelne": 1267, |
| "dataset_repo": "allenai/winogrande", |
| "dataset_sha": "01e74176c63542e6b0bcb004dcdea22d94fb67b5", |
| "pliki_sha256": { |
| "winogrande_xl/validation-00000-of-00001.parquet": "f9d914d1818c0aba98cabdf8af2bc4bd943c462b9c5e7062647e05c2ce3315d1" |
| }, |
| "acc": 0.550118, |
| "acc_ci95": [ |
| 0.522494, |
| 0.577743 |
| ], |
| "acc_norm": 0.550118, |
| "acc_norm_ci95": [ |
| 0.523283, |
| 0.577743 |
| ], |
| "split": "validation", |
| "correct_primary": 697 |
| }, |
| "boolq": { |
| "n": 3270, |
| "n_pelne": 3270, |
| "dataset_repo": "aps/super_glue", |
| "dataset_sha": "3de24cf8022e94f4ee4b9d55a6f539891524d646", |
| "pliki_sha256": { |
| "boolq/validation-00000-of-00001.parquet": "f46438b8cce048184552a2bfcdb8252eadb3a2602f0c22334caccd9b6ea547d6" |
| }, |
| "acc": 0.514067, |
| "acc_ci95": [ |
| 0.496942, |
| 0.531193 |
| ], |
| "acc_norm": 0.617737, |
| "acc_norm_ci95": [ |
| 0.601223, |
| 0.634564 |
| ], |
| "split": "validation", |
| "correct_primary": 1681 |
| }, |
| "openbookqa": { |
| "n": 500, |
| "n_pelne": 500, |
| "dataset_repo": "allenai/openbookqa", |
| "dataset_sha": "388097ea7776314e93a529163e0fea805b8a6454", |
| "pliki_sha256": { |
| "main/test-00000-of-00001.parquet": "cd5483e366daa230c1c87bbdc512d8b7229f14f6dd04d19fc8b1a3855aaaa8a3" |
| }, |
| "acc": 0.244, |
| "acc_ci95": [ |
| 0.208, |
| 0.282 |
| ], |
| "acc_norm": 0.352, |
| "acc_norm_ci95": [ |
| 0.31, |
| 0.394 |
| ], |
| "split": "test", |
| "correct_primary": 176 |
| }, |
| "belebele_pol": { |
| "n": 900, |
| "n_pelne": 900, |
| "dataset_repo": "facebook/belebele", |
| "dataset_sha": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", |
| "pliki_sha256": { |
| "data/pol_Latn.jsonl": "1699bb60b65e258d71989c541027c5439eb36a4f5032a5be392137bf5a1b6bec" |
| }, |
| "acc": 0.246667, |
| "acc_ci95": [ |
| 0.218889, |
| 0.275556 |
| ], |
| "acc_norm": 0.246667, |
| "acc_norm_ci95": [ |
| 0.218889, |
| 0.275556 |
| ], |
| "split": "pol_Latn evaluation set", |
| "correct_primary": 222 |
| }, |
| "hellaswag_pl": { |
| "n": 10042, |
| "n_pelne": 10042, |
| "dataset_repo": "Eurolingua/hellaswagx", |
| "dataset_sha": "c38911a204759866dbcde35eeb584c36caf11a8b", |
| "pliki_sha256": { |
| "hellaswag_PL_validation.jsonl": "acdfc6d397885a4c86de178a6e5f9b7e9b90a7ff81ffcd6e5e0d3d0788dd3a16" |
| }, |
| "acc": 0.345549, |
| "acc_ci95": [ |
| 0.336288, |
| 0.355009 |
| ], |
| "acc_norm": 0.442442, |
| "acc_norm_ci95": [ |
| 0.432583, |
| 0.452201 |
| ], |
| "split": "validation", |
| "correct_primary": 4443 |
| }, |
| "mmlu_pl": { |
| "n": 14042, |
| "n_pelne": 14042, |
| "dataset_repo": "Eurolingua/mmlux", |
| "dataset_sha": "7fb62cf05c1ce9ea550dbfd2ea6c145204344882", |
| "pliki_sha256": { |
| "hendrycks_abstract_algebra_PL_test.jsonl": "4905c6ef8d306c2ff1e1771ef8c15b63152bb88effec11844217893b81fcc3ea", |
| "hendrycks_anatomy_PL_test.jsonl": "5cba9bc54a6f0bdaddcd2e0e23155fccfeab469b561f03cc571e1387abeb9a38", |
| "hendrycks_astronomy_PL_test.jsonl": "00dc5fca7a8be3d8bef9e24ad9018e5417fb07703ebd2d0039c2c82418272b03", |
| "hendrycks_business_ethics_PL_test.jsonl": "9c9542880c720e2d0f5a768ab2d6602bdf5c394d66e51ff352af79aaca6bbd96", |
| "hendrycks_clinical_knowledge_PL_test.jsonl": "79fd5fd7a17dc7c7d84a9e9b138cf13c2dcaf6ab332eb1a0f299a7b52dc89c61", |
| "hendrycks_college_biology_PL_test.jsonl": "b49d766fb2c4160809d3907b846cd6c5c609299f47d9096bcf3208b843392c48", |
| "hendrycks_college_chemistry_PL_test.jsonl": "36c6fd391411789310dca0bbd36aec9aae8784df2bac2e6838e0895eebd97f1d", |
| "hendrycks_college_computer_science_PL_test.jsonl": "90ab29151dc4b8c7e13893516474201b73e8f9765cdf87225b6b9cacdd540e70", |
| "hendrycks_college_mathematics_PL_test.jsonl": "84411ee6464d01f24f2482bccb451635fe75d8169e33932ce6e067c1be3d37ee", |
| "hendrycks_college_medicine_PL_test.jsonl": "faa451d891c8a67b15bae4921c49a8afea448ac7b1b6095646c3570ac3040606", |
| "hendrycks_college_physics_PL_test.jsonl": "bfcaac4cb6d13e7cbcc6ebfb7ea6a9ab0d0ec07613b1b10ca679e632106b4bc2", |
| "hendrycks_computer_security_PL_test.jsonl": "c8324fb94b5b4b7e17440520ea20a22d8c03c3b86da81824059602982463cf8a", |
| "hendrycks_conceptual_physics_PL_test.jsonl": "5227e303a5437f437a4ad9d7eb75917d36051ce635af9f43a92d9525918eac64", |
| "hendrycks_econometrics_PL_test.jsonl": "9bf1d2a2c9eb4f5692b3876919dbc4e679d43d3ea2b64dc8fa2c17703a3e696b", |
| "hendrycks_electrical_engineering_PL_test.jsonl": "a431be1d342cc1eca6aa408688b78751c74621056a4fa4c31e7dfda443e4be6f", |
| "hendrycks_elementary_mathematics_PL_test.jsonl": "9cdb8692fc08d6d8ac5dec24d89f386863b6a02480eb3b8c5be641f15700e667", |
| "hendrycks_formal_logic_PL_test.jsonl": "0f30d48346a0cd2f136463b283d8bd3fe085de67da2a34d90cac689740f9b5be", |
| "hendrycks_global_facts_PL_test.jsonl": "21c753017550f0d31d7c2cdbe82637c877682a9edf8ab913de720e79c82b374f", |
| "hendrycks_high_school_biology_PL_test.jsonl": "7b086933dd97cfc392fba5f78c3523ddc0302cf2b7caed7428a75bfdf9fbb7f2", |
| "hendrycks_high_school_chemistry_PL_test.jsonl": "85f836040875313d60cf9241bd9e827a3cdb770cf159ddf3f7032edb293871ec", |
| "hendrycks_high_school_computer_science_PL_test.jsonl": "034b7b497ada5cc1146c6c5fa02291094d5da512bc0c618ae2b967bd941a70c1", |
| "hendrycks_high_school_european_history_PL_test.jsonl": "023ce8f773cb5dccd86658165a86a5bc1b0d83d53cede5a22c8f7a027582e2a4", |
| "hendrycks_high_school_geography_PL_test.jsonl": "3cce3d6defc5cad9742f9db4407ec5176e6b5d1634b0f8ecb4cfe9f4cabfbe5a", |
| "hendrycks_high_school_government_and_politics_PL_test.jsonl": "9c73c16602cd663acf3b08cdecef6aa8a2c019ed647f02cb970f078d88e3ec7d", |
| "hendrycks_high_school_macroeconomics_PL_test.jsonl": "9425c91f59bd35355262f0c60b04b479a2e700064751efddadb9a28cbc83974d", |
| "hendrycks_high_school_mathematics_PL_test.jsonl": "80c73fa2b882104047260829a57cdca588d1ec9ba690b8a8a5cb87f65283e735", |
| "hendrycks_high_school_microeconomics_PL_test.jsonl": "aebc508914a9b32f5dd1bdc3ebd1375a619791214cb67968ccc6d5712675cbe8", |
| "hendrycks_high_school_physics_PL_test.jsonl": "bba7751511cada0b7b0b07133f2a961e5f046e174e201628a6722a279ab4e47f", |
| "hendrycks_high_school_psychology_PL_test.jsonl": "adf5943b42493ae59f9310028a45adffaace2359d52b8cbb1fa9cc41d6371e45", |
| "hendrycks_high_school_statistics_PL_test.jsonl": "14ffdec0e35d7437d7b2243c3bcbf606843df9ac4eaa044cae95faa3f6e41fdf", |
| "hendrycks_high_school_us_history_PL_test.jsonl": "d8a7946dbf1860b1e99ff0d1948ede18500cc06984ef68786ffed6a8160731f7", |
| "hendrycks_high_school_world_history_PL_test.jsonl": "ae943acd33cf4b23fd3d70d4cc5b0d16f073574c547de91bfc157d552bf63ec0", |
| "hendrycks_human_aging_PL_test.jsonl": "21df1fd8d194a6e453e71e1583626fc723f1c9b44cc571ef7d6074450754ed82", |
| "hendrycks_human_sexuality_PL_test.jsonl": "d7c467fcee3671069986948b69e6ec69c2630173207025ce4925719bb1a880d4", |
| "hendrycks_international_law_PL_test.jsonl": "f0b6f86e54390b63fda0ecf07c7c55f87d30864f6a698c9819f8d8e6eca4d746", |
| "hendrycks_jurisprudence_PL_test.jsonl": "9d7f42f41510da5d835d8df834c4bf6bcd2e03e658cc906ec334810b30ebbe96", |
| "hendrycks_logical_fallacies_PL_test.jsonl": "8b7cd34e9126adcfbc4b9b7bee8345d97352ec892221a0e98fa268210b04ee51", |
| "hendrycks_machine_learning_PL_test.jsonl": "f654c60a51877a9d269471ab01b54b0d4d20f9cc07f7fb57512fb3e90d5a8608", |
| "hendrycks_management_PL_test.jsonl": "628374b8f9beabefe43c2b1dda2c07760eb455d2e15f9e58f58506d61f0fc3bb", |
| "hendrycks_marketing_PL_test.jsonl": "31dc1eef1527c98642080c0b3866ea113bda77c2a03c5155aafe382206ffacdb", |
| "hendrycks_medical_genetics_PL_test.jsonl": "f05fcc33d029460a7f7916ec83debfc746ced5ea931f7a56b63fe1245819e4a4", |
| "hendrycks_miscellaneous_PL_test.jsonl": "94be7854c05aef25ebca4ae1dc562f37172b5346a14fb760b8e89eefea588809", |
| "hendrycks_moral_disputes_PL_test.jsonl": "245ad1cf635274e5ac81559542cb21638b7c729e8a811f76271f4301937d6127", |
| "hendrycks_moral_scenarios_PL_test.jsonl": "d60b08bb71451594385db5ff6ffdef4ca11fec801dc86d681e68f1727a690f9b", |
| "hendrycks_nutrition_PL_test.jsonl": "3cfe618d8e23e3c8e5c752544f07a71b90b4c84d4bc5d673026e8be4de4bf285", |
| "hendrycks_philosophy_PL_test.jsonl": "a1939f9773085bdae7e127fce099afc4153ec4cbe5f66e126f12599feaf76639", |
| "hendrycks_prehistory_PL_test.jsonl": "42eaf871b859f22321dfdec21b6b6b4a3c8f1435c2d4cd2da46dc9fb5ceced88", |
| "hendrycks_professional_accounting_PL_test.jsonl": "1fb6440616a195b94d3de760af5cacd05656667dfac595e2601504cda407c25e", |
| "hendrycks_professional_law_PL_test.jsonl": "fb86b175ecdfe8b56d4991bf6d6705a58f1d89a3efda2a8eb5f01f8082b21e57", |
| "hendrycks_professional_medicine_PL_test.jsonl": "ec818eb13a66e631ab82a10bae1536377dcd4bcadf2321c964b7718c60992080", |
| "hendrycks_professional_psychology_PL_test.jsonl": "72ddab1aaddc26266f4964941334a84d9243f63c329d2d9349bdcaa20efa6bc3", |
| "hendrycks_public_relations_PL_test.jsonl": "2e16835d88ba1ebb6f70a526bd85b9830cb1d3e14e2358a43ce15a3d548232e1", |
| "hendrycks_security_studies_PL_test.jsonl": "c476a634f4f48ff025869158057de726a8080bee253d88921d90252204aa3e4c", |
| "hendrycks_sociology_PL_test.jsonl": "dd56860da95ed793943c9c201f9aa8f5aa8d83753b1bbcbc440fff7ac30f881a", |
| "hendrycks_us_foreign_policy_PL_test.jsonl": "95183b3fcd02d17f5bb7d359b709b3fde13bf6f0f0f1375ab676b0e7dda36786", |
| "hendrycks_virology_PL_test.jsonl": "99e2c6f80907315317619f8c92f8d501ae484619c1169e36d9ad8fa59abfbda7", |
| "hendrycks_world_religions_PL_test.jsonl": "4631dbc58d379cbc46751515f36af682f8bf53db36f3345dbe090d5a00b7b24a" |
| }, |
| "acc": 0.238641, |
| "acc_ci95": [ |
| 0.231662, |
| 0.245763 |
| ], |
| "acc_norm": 0.238641, |
| "acc_norm_ci95": [ |
| 0.231591, |
| 0.245834 |
| ], |
| "split": "test; micro-average over subjects", |
| "correct_primary": 3351 |
| } |
| } |
| } |
| }, |
| "additional_measurements": { |
| "benchteam_gsm8k_sst2_20261111": { |
| "measurement_owner": "SlayerLab / Fabryka AI", |
| "status": "descriptive; training-lineage contamination scan pending", |
| "measured_utc_date": "2026-11-11", |
| "source_manifest_sha256": "229c86053bda604716282e314e584f2eac24d870dc5f99e102328536ecd9d95e", |
| "protocol": { |
| "source_repository": "https://github.com/slayerlabs/benchmarks-fabryka-ai", |
| "inventory_commit": "8b547007526971bd2fc35dfe56ff2c7d9b802568", |
| "source_protocol_id": "bench50-v0.3.1-fullB", |
| "scope": "two-task subset, full evaluated splits; not a full 50-task result", |
| "model_adapter": "slayer", |
| "lm_eval_version": "0.4.9.1", |
| "dtype": "float32", |
| "batch_size": 1, |
| "context_limit": 4096, |
| "task_settings": { |
| "gsm8k": { |
| "task_id": "T33", |
| "split": "test", |
| "few_shot": 5, |
| "metric": "exact_match,strict-match" |
| }, |
| "sst2": { |
| "task_id": "T21", |
| "split": "validation", |
| "few_shot": 0, |
| "metric": "acc,none" |
| } |
| }, |
| "metric_units": "fraction; multiply by 100 for card percentages", |
| "ci_interpretation": "pointwise intervals for individual scores as reported; not confidence intervals for model differences or training-seed variance", |
| "dataset_identity": "source_id_digest pins the evaluated item set; dataset revisions are not supplied in these aggregate reports" |
| }, |
| "models": { |
| "1.07": { |
| "step": 22327, |
| "weights_published": true, |
| "repo": "SlayerLab/Slayer1B-1.07", |
| "revision": "a090b54e658fb57dc5166c1fcc90cc6bf5c80106", |
| "weights_sha256": "d826b4a11bf4eaccee10fe43ec19a4f6d515dfc5f3e5c859adb631f1a0b76ad5", |
| "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.07-a090b54e-20261111T012858Z", |
| "source_aggregate_sha256": "4639f14fc11b2c254f91fde7b7f606cd90a8480a443df37c6671efb3c09c8c7b", |
| "tasks": { |
| "gsm8k": { |
| "task_id": "T33", |
| "metric": "exact_match,strict-match", |
| "value": 0.015163002274450341, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.009097801364670205, |
| "upper": 0.021986353297952996, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 1319, |
| "n_population": 1319, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "d20e0d2b1f0e21915f857c6c4218f43f4c1db4c2799e28a78c3618d4fcfcf1b6", |
| "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0" |
| }, |
| "sst2": { |
| "task_id": "T21", |
| "metric": "acc,none", |
| "value": 0.5103211009174312, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.47706422018348627, |
| "upper": 0.5435779816513762, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 872, |
| "n_population": 872, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "f1965fb1a3651c6ab526a82d9bc35504b8c4f92c4789b471d61fa6a37e54c83f", |
| "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7" |
| } |
| } |
| }, |
| "1.08": { |
| "step": 1709, |
| "weights_published": true, |
| "repo": "SlayerLab/Slayer1B-1.08", |
| "revision": "5f854a0cc7160594d54d1788d37a44e8ebabe16c", |
| "weights_sha256": "1c510d9a59d3b9e62360d1db57e08fa450fb9b09df849df7f2eef132ea6715a4", |
| "source_run_id": "niezalezny-SlayerLab_Slayer1B-1.08-5f854a0c-20261111T023631Z", |
| "source_aggregate_sha256": "3ee230fb5029cf9c6424c27bae2b60dfd345998b2bee42021cb53e625021615b", |
| "tasks": { |
| "gsm8k": { |
| "task_id": "T33", |
| "metric": "exact_match,strict-match", |
| "value": 0.009097801364670205, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.004548900682335102, |
| "upper": 0.014404852160727824, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 1319, |
| "n_population": 1319, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "7e2f64e0958f9e5897debe2804cfb25c804b4abe1b8bdd440f7885d16b98f3e6", |
| "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0" |
| }, |
| "sst2": { |
| "task_id": "T21", |
| "metric": "acc,none", |
| "value": 0.5045871559633027, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.4713302752293578, |
| "upper": 0.536697247706422, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 872, |
| "n_population": 872, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "1e1a7b5d6c41dff7140208736213f308a5be62986bef1e9bf5b7aeb3e176a48a", |
| "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7" |
| } |
| } |
| }, |
| "1.09-k10000": { |
| "step": 10000, |
| "weights_published": false, |
| "repo": null, |
| "revision": null, |
| "weights_sha256": "12939d84ef0d06b350773fb30837390306d899ded315d1721b52c7615a87317a", |
| "source_run_id": "niezalezny-Slayer1B-1.09-k10000-lokalny-lokalny-20261111T025334Z", |
| "source_aggregate_sha256": "042b2639bc325aeb7fb60787bc8205d1384557e37dbb7b01ba5489db41ebdfe8", |
| "tasks": { |
| "gsm8k": { |
| "task_id": "T33", |
| "metric": "exact_match,strict-match", |
| "value": 0.009855951478392721, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.004548900682335102, |
| "upper": 0.015163002274450341, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 1319, |
| "n_population": 1319, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "5cf7805fa9c7d3d65d6af44920b6659287c621294efe4d7cde62f2055e65562e", |
| "source_id_digest": "a32d2c4b15b4e22c87fc4e4717cf476baa2aa90a17f61230cef747c0a3bb61d0" |
| }, |
| "sst2": { |
| "task_id": "T21", |
| "metric": "acc,none", |
| "value": 0.5825688073394495, |
| "unit": "score", |
| "ci95": { |
| "lower": 0.5493119266055045, |
| "upper": 0.6135321100917431, |
| "level": 0.95, |
| "method": "paired_bootstrap", |
| "replicates": 10000, |
| "seed": 20261028 |
| }, |
| "n_scored": 872, |
| "n_population": 872, |
| "contamination": "unscanned", |
| "state": "ok", |
| "runner_sha256": "c906036b4e597e2c17ad7f1f6441e72fbf8e066946c2a7731966612674e3995e", |
| "results_sha256": "d47706c62088fe772f1c26d2ed13f52dfecc8d832bbdecd14dd24c89086e9659", |
| "source_id_digest": "7a716d60452e40d779aaad8da40aa11cdfb66cf0a069b6f0a5c5b38c78819ce7" |
| } |
| } |
| } |
| } |
| } |
| }, |
| "source_generator_note": "The updated approved aggregate does not supply its CI-generator SHA; evaluator SHA is retained per model. Previous aggregate: ce0822a324f805bb61f28cc8ea3197e6a9abd97a4591239349dc60141f5289db; previous CI generator: b792b5a7b4f5218acff4b3d5e9fb63c6f11d39fa26cca6fbd467758c90555a34." |
| } |
|
|
