| { | |
| "tasks": { | |
| "hellaswag": { | |
| "n": 10042, | |
| "seconds": 146.01327709999168, | |
| "routes": { | |
| "cached/language+evidence": 2973, | |
| "cached/language": 7068, | |
| "cached/exact-arithmetic": 1 | |
| }, | |
| "changed_system_predictions": 0, | |
| "fresh_sql_parity_checks": 3, | |
| "maximum_likelihood_error": 0.0, | |
| "prediction_sha256": "ae5e8d2fb43e21a1810c4e8518e5a54256cf7ea66b77a273f1229b753360c7c9", | |
| "primary": "char", | |
| "native_correct": 2809, | |
| "native_percent": 27.972515435172276, | |
| "previous_native_correct": 2809, | |
| "previous_native_percent": 27.972515435172276, | |
| "native_change_points": 0.0, | |
| "assisted_correct": 2805, | |
| "assisted_percent": 27.9326827325234, | |
| "previous_assisted_percent": 27.93, | |
| "assisted_change_points": 0.0 | |
| }, | |
| "arc_easy": { | |
| "n": 2376, | |
| "seconds": 22.353922400041483, | |
| "routes": { | |
| "cached/language+evidence": 2374, | |
| "cached/exact-arithmetic": 1, | |
| "cached/language": 1 | |
| }, | |
| "changed_system_predictions": 0, | |
| "fresh_sql_parity_checks": 3, | |
| "maximum_likelihood_error": 0.0, | |
| "prediction_sha256": "f9b4a6ca075ad0110315bb1dd19e8eac4b6df47252730596b25c6933dae521d9", | |
| "primary": "char", | |
| "native_correct": 632, | |
| "native_percent": 26.5993265993266, | |
| "previous_native_correct": 632, | |
| "previous_native_percent": 26.5993265993266, | |
| "native_change_points": 0.0, | |
| "assisted_correct": 1476, | |
| "assisted_percent": 62.121212121212125, | |
| "previous_assisted_percent": 62.12, | |
| "assisted_change_points": 0.0 | |
| }, | |
| "arc_challenge": { | |
| "n": 1172, | |
| "seconds": 11.865461300010793, | |
| "routes": { | |
| "cached/language+evidence": 1169, | |
| "cached/exact-arithmetic": 2, | |
| "cached/language": 1 | |
| }, | |
| "changed_system_predictions": 0, | |
| "fresh_sql_parity_checks": 3, | |
| "maximum_likelihood_error": 0.0, | |
| "prediction_sha256": "301b66946ce1cda420019aca3e1240bc9c262f2cb4f73e909b3ad19e61f76675", | |
| "primary": "char", | |
| "native_correct": 260, | |
| "native_percent": 22.18430034129693, | |
| "previous_native_correct": 260, | |
| "previous_native_percent": 22.18430034129693, | |
| "native_change_points": 0.0, | |
| "assisted_correct": 371, | |
| "assisted_percent": 31.655290102389078, | |
| "previous_assisted_percent": 31.66, | |
| "assisted_change_points": -0.0 | |
| }, | |
| "piqa": { | |
| "n": 1838, | |
| "seconds": 10.459679999970831, | |
| "routes": { | |
| "cached/language": 433, | |
| "cached/language+evidence": 1405 | |
| }, | |
| "changed_system_predictions": 0, | |
| "fresh_sql_parity_checks": 3, | |
| "maximum_likelihood_error": 0.0, | |
| "prediction_sha256": "cf2a6f1adc18959b67dce5af1fa668e7f4301d7eb9b25d3918fdc221c8369597", | |
| "primary": "char", | |
| "native_correct": 899, | |
| "native_percent": 48.91186071817192, | |
| "previous_native_correct": 899, | |
| "previous_native_percent": 48.91186071817192, | |
| "native_change_points": 0.0, | |
| "assisted_correct": 1078, | |
| "assisted_percent": 58.65070729053319, | |
| "previous_assisted_percent": 58.65, | |
| "assisted_change_points": 0.0 | |
| }, | |
| "arithmark3": { | |
| "n": 1000, | |
| "seconds": 9.040633900032844, | |
| "routes": { | |
| "cached/exact-arithmetic": 680, | |
| "cached/language+evidence": 320 | |
| }, | |
| "changed_system_predictions": 0, | |
| "fresh_sql_parity_checks": 3, | |
| "maximum_likelihood_error": 0.0, | |
| "prediction_sha256": "9cb2c70adfac81b8fde8c1fa2079553530d667b15d3190db54a7b76dce26f83d", | |
| "primary": "token", | |
| "native_correct": 378, | |
| "native_percent": 37.8, | |
| "previous_native_correct": 378, | |
| "previous_native_percent": 37.8, | |
| "native_change_points": 0.0, | |
| "assisted_correct": 695, | |
| "assisted_percent": 69.5, | |
| "previous_assisted_percent": 69.5, | |
| "assisted_change_points": 0.0 | |
| } | |
| }, | |
| "total_questions": 16428, | |
| "seconds": 200.08493640000233, | |
| "native_mean_percent": 32.693600618793546, | |
| "assisted_mean_percent": 49.97197844933156, | |
| "current_index_formula": "Chance-normalized HellaSwag + combined ARC + PIQA + 0.65*ArithMark, divided by 3.65", | |
| "local_index_native": 3.3067104959787845, | |
| "local_index_assisted": 24.373331426529237, | |
| "whole_system_evaluation_cached": true, | |
| "fresh_likelihoods": true, | |
| "training": false, | |
| "official_submission": false, | |
| "freeze_sha256": "d1f3b91f49fe8036b10a814daca5eb20e9e7c64f7551e86cc475a8c14373b1e6", | |
| "sql_parity_sha256": "27cc34d0f40d3f369c42d6f17a9ef3b42bb94c0f514f21037af7be08235128c1" | |
| } | |