| { |
| "v2|typed_en|single-question": { |
| "n": 2000, |
| "correct": 1510, |
| "accuracy": 0.755, |
| "nll": 0.6463324192750852, |
| "kl": 0.13270933329166798, |
| "ce_soft": 0.9000678889159969, |
| "brier": 0.07002914815029006, |
| "brier_mean_k": 0.02004525517631372, |
| "brier_onehot": 0.3676075399939686, |
| "ece10": 0.12290431407094005, |
| "mean_conf": 0.6332694019079208, |
| "by_type": { |
| "choice": { |
| "correct": 411, |
| "n": 600, |
| "accuracy": 0.685 |
| }, |
| "noul": { |
| "correct": 503, |
| "n": 600, |
| "accuracy": 0.8383333333333334 |
| }, |
| "score": { |
| "correct": 596, |
| "n": 800, |
| "accuracy": 0.745 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7335, |
| 0.7765 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.24199143103868068, |
| "mae_vs_gold_label": 0.4274223285297194, |
| "n": 800 |
| } |
| }, |
| "v2|typed_en|full-case context": { |
| "n": 2000, |
| "correct": 857, |
| "accuracy": 0.4285, |
| "nll": 1.2097812919002324, |
| "kl": 0.4959933924254989, |
| "ce_soft": 1.2633519480498279, |
| "brier": 0.26928889266663847, |
| "brier_mean_k": 0.08697066222849263, |
| "brier_onehot": 0.6970586865702736, |
| "ece10": 0.11118127906322481, |
| "mean_conf": 0.5374737640917301, |
| "by_type": { |
| "choice": { |
| "correct": 198, |
| "n": 600, |
| "accuracy": 0.33 |
| }, |
| "noul": { |
| "correct": 298, |
| "n": 600, |
| "accuracy": 0.49666666666666665 |
| }, |
| "score": { |
| "correct": 361, |
| "n": 800, |
| "accuracy": 0.45125 |
| } |
| }, |
| "ci95_cases": [ |
| 0.407, |
| 0.45 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5068433761781231, |
| "mae_vs_gold_label": 0.6975457973429002, |
| "n": 800 |
| } |
| }, |
| "v2|typed_en|full-case joint": { |
| "n": 2000, |
| "correct": 706, |
| "accuracy": 0.353, |
| "nll": 1.217202288315301, |
| "kl": 0.4577465691470793, |
| "ce_soft": 1.2251051247714082, |
| "brier": 0.2549271664455238, |
| "brier_mean_k": 0.07846016753062504, |
| "brier_onehot": 0.7019905508599895, |
| "ece10": 0.12446078114211558, |
| "mean_conf": 0.4774607811421156, |
| "by_type": { |
| "choice": { |
| "correct": 193, |
| "n": 600, |
| "accuracy": 0.32166666666666666 |
| }, |
| "noul": { |
| "correct": 286, |
| "n": 600, |
| "accuracy": 0.4766666666666667 |
| }, |
| "score": { |
| "correct": 227, |
| "n": 800, |
| "accuracy": 0.28375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.3315, |
| 0.374 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.7265793839785138, |
| "mae_vs_gold_label": 0.9440566271264106, |
| "n": 800 |
| } |
| }, |
| "v2|typed_en|single-question vs full-case context": { |
| "n_common": 2000, |
| "only_b_right": 144, |
| "only_a_right": 797, |
| "p_mcnemar_exact": 3.605359197992819e-110 |
| }, |
| "v2|typed_en|full-case context vs full-case joint": { |
| "n_common": 2000, |
| "only_b_right": 302, |
| "only_a_right": 453, |
| "p_mcnemar_exact": 4.334154965648315e-08 |
| }, |
| "v2|typed_es|single-question": { |
| "n": 1985, |
| "correct": 1482, |
| "accuracy": 0.7465994962216624, |
| "nll": 0.6550145421374155, |
| "kl": 0.13250811726703088, |
| "ce_soft": 0.9007772998807515, |
| "brier": 0.07028015084314279, |
| "brier_mean_k": 0.01999555773940268, |
| "brier_onehot": 0.3732510221786229, |
| "ece10": 0.1152207864471887, |
| "mean_conf": 0.6313787097744737, |
| "by_type": { |
| "choice": { |
| "correct": 401, |
| "n": 594, |
| "accuracy": 0.6750841750841751 |
| }, |
| "noul": { |
| "correct": 497, |
| "n": 597, |
| "accuracy": 0.8324958123953099 |
| }, |
| "score": { |
| "correct": 584, |
| "n": 794, |
| "accuracy": 0.7355163727959698 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7249370277078085, |
| 0.7677581863979849 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.23905470773449672, |
| "mae_vs_gold_label": 0.4315963490568928, |
| "n": 794 |
| } |
| }, |
| "v2|typed_es|full-case context": { |
| "n": 1985, |
| "correct": 832, |
| "accuracy": 0.41914357682619646, |
| "nll": 1.2000239557178545, |
| "kl": 0.47456987091550584, |
| "ce_soft": 1.2428390535292264, |
| "brier": 0.26059998598083783, |
| "brier_mean_k": 0.08474363410961229, |
| "brier_onehot": 0.6933349301776697, |
| "ece10": 0.10654264947479258, |
| "mean_conf": 0.523837870787313, |
| "by_type": { |
| "choice": { |
| "correct": 236, |
| "n": 594, |
| "accuracy": 0.39730639730639733 |
| }, |
| "noul": { |
| "correct": 295, |
| "n": 597, |
| "accuracy": 0.49413735343383586 |
| }, |
| "score": { |
| "correct": 301, |
| "n": 794, |
| "accuracy": 0.3790931989924433 |
| } |
| }, |
| "ci95_cases": [ |
| 0.39848866498740554, |
| 0.4397984886649874 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5519418643797106, |
| "mae_vs_gold_label": 0.7501145362577187, |
| "n": 794 |
| } |
| }, |
| "v2|typed_es|full-case joint": { |
| "n": 1985, |
| "correct": 673, |
| "accuracy": 0.33904282115869017, |
| "nll": 1.2040304828794082, |
| "kl": 0.44507417616530687, |
| "ce_soft": 1.2133433587790274, |
| "brier": 0.24717614322069642, |
| "brier_mean_k": 0.07750307326535547, |
| "brier_onehot": 0.6933235732686921, |
| "ece10": 0.11217208232176995, |
| "mean_conf": 0.4512149034804601, |
| "by_type": { |
| "choice": { |
| "correct": 167, |
| "n": 594, |
| "accuracy": 0.28114478114478114 |
| }, |
| "noul": { |
| "correct": 284, |
| "n": 597, |
| "accuracy": 0.47571189279731996 |
| }, |
| "score": { |
| "correct": 222, |
| "n": 794, |
| "accuracy": 0.2795969773299748 |
| } |
| }, |
| "ci95_cases": [ |
| 0.31738035264483627, |
| 0.3602015113350126 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.6897287760172528, |
| "mae_vs_gold_label": 0.921904983332085, |
| "n": 794 |
| } |
| }, |
| "v2|typed_es|single-question vs full-case context": { |
| "n_common": 1985, |
| "only_b_right": 119, |
| "only_a_right": 769, |
| "p_mcnemar_exact": 3.7714035666656705e-117 |
| }, |
| "v2|typed_es|full-case context vs full-case joint": { |
| "n_common": 1985, |
| "only_b_right": 254, |
| "only_a_right": 413, |
| "p_mcnemar_exact": 7.935736100989516e-10 |
| }, |
| "v2|telepatia_es|single-question": { |
| "n": 2000, |
| "correct": 1492, |
| "accuracy": 0.746, |
| "nll": 0.6768666282900727, |
| "kl": 0.137470178826373, |
| "ce_soft": 0.9048287344507019, |
| "brier": 0.07226937229140096, |
| "brier_mean_k": 0.020558110269673965, |
| "brier_onehot": 0.3838231707888498, |
| "ece10": 0.1303335434347391, |
| "mean_conf": 0.6159254869967699, |
| "by_type": { |
| "choice": { |
| "correct": 415, |
| "n": 600, |
| "accuracy": 0.6916666666666667 |
| }, |
| "noul": { |
| "correct": 502, |
| "n": 600, |
| "accuracy": 0.8366666666666667 |
| }, |
| "score": { |
| "correct": 575, |
| "n": 800, |
| "accuracy": 0.71875 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7245, |
| 0.768 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.26117300274514005, |
| "mae_vs_gold_label": 0.44825553921662503, |
| "n": 800 |
| } |
| }, |
| "v2|telepatia_es|full-case context": { |
| "n": 2000, |
| "correct": 823, |
| "accuracy": 0.4115, |
| "nll": 1.189682075850548, |
| "kl": 0.4673731803374047, |
| "ce_soft": 1.2347317359617336, |
| "brier": 0.26037437617486775, |
| "brier_mean_k": 0.08499574423415869, |
| "brier_onehot": 0.6928821846712389, |
| "ece10": 0.1040707949474454, |
| "mean_conf": 0.5111226370260119, |
| "by_type": { |
| "choice": { |
| "correct": 199, |
| "n": 600, |
| "accuracy": 0.33166666666666667 |
| }, |
| "noul": { |
| "correct": 291, |
| "n": 600, |
| "accuracy": 0.485 |
| }, |
| "score": { |
| "correct": 333, |
| "n": 800, |
| "accuracy": 0.41625 |
| } |
| }, |
| "ci95_cases": [ |
| 0.39, |
| 0.433 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5304797743702488, |
| "mae_vs_gold_label": 0.7196103962347843, |
| "n": 800 |
| } |
| }, |
| "v2|telepatia_es|full-case joint": { |
| "n": 2000, |
| "correct": 636, |
| "accuracy": 0.318, |
| "nll": 1.2328385029619342, |
| "kl": 0.4669040681698344, |
| "ce_soft": 1.2342626237941632, |
| "brier": 0.2603068636325369, |
| "brier_mean_k": 0.08182894089557148, |
| "brier_onehot": 0.7105703100725699, |
| "ece10": 0.15144768108427525, |
| "mean_conf": 0.46944768108427526, |
| "by_type": { |
| "choice": { |
| "correct": 125, |
| "n": 600, |
| "accuracy": 0.20833333333333334 |
| }, |
| "noul": { |
| "correct": 298, |
| "n": 600, |
| "accuracy": 0.49666666666666665 |
| }, |
| "score": { |
| "correct": 213, |
| "n": 800, |
| "accuracy": 0.26625 |
| } |
| }, |
| "ci95_cases": [ |
| 0.296, |
| 0.34 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.6871891529425792, |
| "mae_vs_gold_label": 0.913369414373301, |
| "n": 800 |
| } |
| }, |
| "v2|telepatia_es|single-question vs full-case context": { |
| "n_common": 2000, |
| "only_b_right": 126, |
| "only_a_right": 795, |
| "p_mcnemar_exact": 2.2596591083909646e-119 |
| }, |
| "v2|telepatia_es|full-case context vs full-case joint": { |
| "n_common": 2000, |
| "only_b_right": 261, |
| "only_a_right": 448, |
| "p_mcnemar_exact": 2.1310615747335922e-12 |
| }, |
| "v3|typed_en|single-question": { |
| "n": 2000, |
| "correct": 1528, |
| "accuracy": 0.764, |
| "nll": 0.6348000373513565, |
| "kl": 0.12133635641347341, |
| "ce_soft": 0.8886949120378023, |
| "brier": 0.0651098222513845, |
| "brier_mean_k": 0.019005062125487936, |
| "brier_onehot": 0.3611829084554942, |
| "ece10": 0.13159668615460393, |
| "mean_conf": 0.632403313845396, |
| "by_type": { |
| "choice": { |
| "correct": 429, |
| "n": 600, |
| "accuracy": 0.715 |
| }, |
| "noul": { |
| "correct": 503, |
| "n": 600, |
| "accuracy": 0.8383333333333334 |
| }, |
| "score": { |
| "correct": 596, |
| "n": 800, |
| "accuracy": 0.745 |
| } |
| }, |
| "ci95_cases": [ |
| 0.742, |
| 0.7855 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.23437388759128985, |
| "mae_vs_gold_label": 0.42270025751247886, |
| "n": 800 |
| } |
| }, |
| "v3|typed_en|full-case context": { |
| "n": 2000, |
| "correct": 831, |
| "accuracy": 0.4155, |
| "nll": 1.1946701619356137, |
| "kl": 0.48748056433424275, |
| "ce_soft": 1.2548391199585716, |
| "brier": 0.26584338076551384, |
| "brier_mean_k": 0.08537406918322844, |
| "brier_onehot": 0.6924497841014353, |
| "ece10": 0.11764439262449741, |
| "mean_conf": 0.5266776051074267, |
| "by_type": { |
| "choice": { |
| "correct": 153, |
| "n": 600, |
| "accuracy": 0.255 |
| }, |
| "noul": { |
| "correct": 295, |
| "n": 600, |
| "accuracy": 0.49166666666666664 |
| }, |
| "score": { |
| "correct": 383, |
| "n": 800, |
| "accuracy": 0.47875 |
| } |
| }, |
| "ci95_cases": [ |
| 0.3945, |
| 0.4365 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5401024079043407, |
| "mae_vs_gold_label": 0.7221055129170417, |
| "n": 800 |
| } |
| }, |
| "v3|typed_en|full-case joint": { |
| "n": 2000, |
| "correct": 742, |
| "accuracy": 0.371, |
| "nll": 1.1821842679997898, |
| "kl": 0.43716691089620746, |
| "ce_soft": 1.2045254665205365, |
| "brier": 0.2458284892200497, |
| "brier_mean_k": 0.0777216018771812, |
| "brier_onehot": 0.6827152191861325, |
| "ece10": 0.08931016900390386, |
| "mean_conf": 0.44944229010492565, |
| "by_type": { |
| "choice": { |
| "correct": 172, |
| "n": 600, |
| "accuracy": 0.2866666666666667 |
| }, |
| "noul": { |
| "correct": 276, |
| "n": 600, |
| "accuracy": 0.46 |
| }, |
| "score": { |
| "correct": 294, |
| "n": 800, |
| "accuracy": 0.3675 |
| } |
| }, |
| "ci95_cases": [ |
| 0.349, |
| 0.393 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.6182189710583621, |
| "mae_vs_gold_label": 0.8307282497920095, |
| "n": 800 |
| } |
| }, |
| "v3|typed_en|single-question vs full-case context": { |
| "n_common": 2000, |
| "only_b_right": 150, |
| "only_a_right": 847, |
| "p_mcnemar_exact": 1.495086609422151e-118 |
| }, |
| "v3|typed_en|full-case context vs full-case joint": { |
| "n_common": 2000, |
| "only_b_right": 385, |
| "only_a_right": 474, |
| "p_mcnemar_exact": 0.0026569907313413405 |
| }, |
| "v3|typed_es|single-question": { |
| "n": 1985, |
| "correct": 1496, |
| "accuracy": 0.7536523929471033, |
| "nll": 0.6394842117466736, |
| "kl": 0.11898906052638147, |
| "ce_soft": 0.887258243140102, |
| "brier": 0.06475655764622247, |
| "brier_mean_k": 0.018860197074895686, |
| "brier_onehot": 0.36670251099112927, |
| "ece10": 0.12293764144107136, |
| "mean_conf": 0.6313106105219507, |
| "by_type": { |
| "choice": { |
| "correct": 415, |
| "n": 594, |
| "accuracy": 0.6986531986531986 |
| }, |
| "noul": { |
| "correct": 494, |
| "n": 597, |
| "accuracy": 0.8274706867671692 |
| }, |
| "score": { |
| "correct": 587, |
| "n": 794, |
| "accuracy": 0.739294710327456 |
| } |
| }, |
| "ci95_cases": [ |
| 0.7324937027707809, |
| 0.7743073047858942 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.23596503181925557, |
| "mae_vs_gold_label": 0.4296506547049947, |
| "n": 794 |
| } |
| }, |
| "v3|typed_es|full-case context": { |
| "n": 1985, |
| "correct": 859, |
| "accuracy": 0.4327455919395466, |
| "nll": 1.162280408384133, |
| "kl": 0.4496600929573715, |
| "ce_soft": 1.217929275571092, |
| "brier": 0.2505504103004089, |
| "brier_mean_k": 0.08182399037164234, |
| "brier_onehot": 0.6788902020159528, |
| "ece10": 0.09863890937203425, |
| "mean_conf": 0.5196653394690088, |
| "by_type": { |
| "choice": { |
| "correct": 211, |
| "n": 594, |
| "accuracy": 0.3552188552188552 |
| }, |
| "noul": { |
| "correct": 296, |
| "n": 597, |
| "accuracy": 0.49581239530988275 |
| }, |
| "score": { |
| "correct": 352, |
| "n": 794, |
| "accuracy": 0.4433249370277078 |
| } |
| }, |
| "ci95_cases": [ |
| 0.4105793450881612, |
| 0.4544080604534005 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5462041908321706, |
| "mae_vs_gold_label": 0.7497829019000534, |
| "n": 794 |
| } |
| }, |
| "v3|typed_es|full-case joint": { |
| "n": 1985, |
| "correct": 763, |
| "accuracy": 0.38438287153652395, |
| "nll": 1.172466296474459, |
| "kl": 0.42073980961851604, |
| "ce_soft": 1.1890089922322367, |
| "brier": 0.23343222723881618, |
| "brier_mean_k": 0.07347218898601528, |
| "brier_onehot": 0.6736496081085933, |
| "ece10": 0.07710073747923152, |
| "mean_conf": 0.4243709911927768, |
| "by_type": { |
| "choice": { |
| "correct": 225, |
| "n": 594, |
| "accuracy": 0.3787878787878788 |
| }, |
| "noul": { |
| "correct": 286, |
| "n": 597, |
| "accuracy": 0.47906197654941374 |
| }, |
| "score": { |
| "correct": 252, |
| "n": 794, |
| "accuracy": 0.31738035264483627 |
| } |
| }, |
| "ci95_cases": [ |
| 0.36171284634760703, |
| 0.4070528967254408 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.6776316444522275, |
| "mae_vs_gold_label": 0.9095255444835055, |
| "n": 794 |
| } |
| }, |
| "v3|typed_es|single-question vs full-case context": { |
| "n_common": 1985, |
| "only_b_right": 137, |
| "only_a_right": 774, |
| "p_mcnemar_exact": 1.6584466757134775e-108 |
| }, |
| "v3|typed_es|full-case context vs full-case joint": { |
| "n_common": 1985, |
| "only_b_right": 320, |
| "only_a_right": 416, |
| "p_mcnemar_exact": 0.0004544874488207351 |
| }, |
| "v3|telepatia_es|single-question": { |
| "n": 2000, |
| "correct": 1526, |
| "accuracy": 0.763, |
| "nll": 0.6385563440375813, |
| "kl": 0.11925372995655213, |
| "ce_soft": 0.886612285580881, |
| "brier": 0.06393994905918735, |
| "brier_mean_k": 0.018569114373183912, |
| "brier_onehot": 0.36353173992639604, |
| "ece10": 0.13380355775356292, |
| "mean_conf": 0.6294715915620327, |
| "by_type": { |
| "choice": { |
| "correct": 438, |
| "n": 600, |
| "accuracy": 0.73 |
| }, |
| "noul": { |
| "correct": 498, |
| "n": 600, |
| "accuracy": 0.83 |
| }, |
| "score": { |
| "correct": 590, |
| "n": 800, |
| "accuracy": 0.7375 |
| } |
| }, |
| "ci95_cases": [ |
| 0.742, |
| 0.785 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.23628337060784574, |
| "mae_vs_gold_label": 0.42863585742670696, |
| "n": 800 |
| } |
| }, |
| "v3|telepatia_es|full-case context": { |
| "n": 2000, |
| "correct": 878, |
| "accuracy": 0.439, |
| "nll": 1.1470116522285891, |
| "kl": 0.4409125760498499, |
| "ce_soft": 1.2082711316741788, |
| "brier": 0.24666331201318742, |
| "brier_mean_k": 0.08078183830888044, |
| "brier_onehot": 0.671401023710438, |
| "ece10": 0.09246306705474854, |
| "mean_conf": 0.5134267296791076, |
| "by_type": { |
| "choice": { |
| "correct": 223, |
| "n": 600, |
| "accuracy": 0.37166666666666665 |
| }, |
| "noul": { |
| "correct": 296, |
| "n": 600, |
| "accuracy": 0.49333333333333335 |
| }, |
| "score": { |
| "correct": 359, |
| "n": 800, |
| "accuracy": 0.44875 |
| } |
| }, |
| "ci95_cases": [ |
| 0.4185, |
| 0.4595 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.5391987730165998, |
| "mae_vs_gold_label": 0.7372078359872103, |
| "n": 800 |
| } |
| }, |
| "v3|telepatia_es|full-case joint": { |
| "n": 2000, |
| "correct": 745, |
| "accuracy": 0.3725, |
| "nll": 1.1746715414243962, |
| "kl": 0.42382525863971043, |
| "ce_soft": 1.1911838142640392, |
| "brier": 0.235579264029217, |
| "brier_mean_k": 0.07444738062797601, |
| "brier_onehot": 0.6762935895205119, |
| "ece10": 0.08313308775424957, |
| "mean_conf": 0.4297241877913475, |
| "by_type": { |
| "choice": { |
| "correct": 230, |
| "n": 600, |
| "accuracy": 0.38333333333333336 |
| }, |
| "noul": { |
| "correct": 279, |
| "n": 600, |
| "accuracy": 0.465 |
| }, |
| "score": { |
| "correct": 236, |
| "n": 800, |
| "accuracy": 0.295 |
| } |
| }, |
| "ci95_cases": [ |
| 0.35, |
| 0.3945 |
| ], |
| "score_mae": { |
| "mae_vs_gold_expectation": 0.6721788485455155, |
| "mae_vs_gold_label": 0.8984525854373351, |
| "n": 800 |
| } |
| }, |
| "v3|telepatia_es|single-question vs full-case context": { |
| "n_common": 2000, |
| "only_b_right": 136, |
| "only_a_right": 784, |
| "p_mcnemar_exact": 2.4222306432783787e-111 |
| }, |
| "v3|telepatia_es|full-case context vs full-case joint": { |
| "n_common": 2000, |
| "only_b_right": 330, |
| "only_a_right": 463, |
| "p_mcnemar_exact": 2.6285909419854747e-06 |
| } |
| } |
