{
 "v2|typed_en|single-question": {
  "n": 2000,
  "correct": 1510,
  "accuracy": 0.755,
  "nll": 0.6463324192750852,
  "kl": 0.13270933329166798,
  "ce_soft": 0.9000678889159969,
  "brier": 0.07002914815029006,
  "brier_mean_k": 0.02004525517631372,
  "brier_onehot": 0.3676075399939686,
  "ece10": 0.12290431407094005,
  "mean_conf": 0.6332694019079208,
  "by_type": {
   "choice": {
    "correct": 411,
    "n": 600,
    "accuracy": 0.685
   },
   "noul": {
    "correct": 503,
    "n": 600,
    "accuracy": 0.8383333333333334
   },
   "score": {
    "correct": 596,
    "n": 800,
    "accuracy": 0.745
   }
  },
  "ci95_cases": [
   0.7335,
   0.7765
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.24199143103868068,
   "mae_vs_gold_label": 0.4274223285297194,
   "n": 800
  }
 },
 "v2|typed_en|full-case context": {
  "n": 2000,
  "correct": 857,
  "accuracy": 0.4285,
  "nll": 1.2097812919002324,
  "kl": 0.4959933924254989,
  "ce_soft": 1.2633519480498279,
  "brier": 0.26928889266663847,
  "brier_mean_k": 0.08697066222849263,
  "brier_onehot": 0.6970586865702736,
  "ece10": 0.11118127906322481,
  "mean_conf": 0.5374737640917301,
  "by_type": {
   "choice": {
    "correct": 198,
    "n": 600,
    "accuracy": 0.33
   },
   "noul": {
    "correct": 298,
    "n": 600,
    "accuracy": 0.49666666666666665
   },
   "score": {
    "correct": 361,
    "n": 800,
    "accuracy": 0.45125
   }
  },
  "ci95_cases": [
   0.407,
   0.45
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5068433761781231,
   "mae_vs_gold_label": 0.6975457973429002,
   "n": 800
  }
 },
 "v2|typed_en|full-case joint": {
  "n": 2000,
  "correct": 706,
  "accuracy": 0.353,
  "nll": 1.217202288315301,
  "kl": 0.4577465691470793,
  "ce_soft": 1.2251051247714082,
  "brier": 0.2549271664455238,
  "brier_mean_k": 0.07846016753062504,
  "brier_onehot": 0.7019905508599895,
  "ece10": 0.12446078114211558,
  "mean_conf": 0.4774607811421156,
  "by_type": {
   "choice": {
    "correct": 193,
    "n": 600,
    "accuracy": 0.32166666666666666
   },
   "noul": {
    "correct": 286,
    "n": 600,
    "accuracy": 0.4766666666666667
   },
   "score": {
    "correct": 227,
    "n": 800,
    "accuracy": 0.28375
   }
  },
  "ci95_cases": [
   0.3315,
   0.374
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.7265793839785138,
   "mae_vs_gold_label": 0.9440566271264106,
   "n": 800
  }
 },
 "v2|typed_en|single-question vs full-case context": {
  "n_common": 2000,
  "only_b_right": 144,
  "only_a_right": 797,
  "p_mcnemar_exact": 3.605359197992819e-110
 },
 "v2|typed_en|full-case context vs full-case joint": {
  "n_common": 2000,
  "only_b_right": 302,
  "only_a_right": 453,
  "p_mcnemar_exact": 4.334154965648315e-08
 },
 "v2|typed_es|single-question": {
  "n": 1985,
  "correct": 1482,
  "accuracy": 0.7465994962216624,
  "nll": 0.6550145421374155,
  "kl": 0.13250811726703088,
  "ce_soft": 0.9007772998807515,
  "brier": 0.07028015084314279,
  "brier_mean_k": 0.01999555773940268,
  "brier_onehot": 0.3732510221786229,
  "ece10": 0.1152207864471887,
  "mean_conf": 0.6313787097744737,
  "by_type": {
   "choice": {
    "correct": 401,
    "n": 594,
    "accuracy": 0.6750841750841751
   },
   "noul": {
    "correct": 497,
    "n": 597,
    "accuracy": 0.8324958123953099
   },
   "score": {
    "correct": 584,
    "n": 794,
    "accuracy": 0.7355163727959698
   }
  },
  "ci95_cases": [
   0.7249370277078085,
   0.7677581863979849
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.23905470773449672,
   "mae_vs_gold_label": 0.4315963490568928,
   "n": 794
  }
 },
 "v2|typed_es|full-case context": {
  "n": 1985,
  "correct": 832,
  "accuracy": 0.41914357682619646,
  "nll": 1.2000239557178545,
  "kl": 0.47456987091550584,
  "ce_soft": 1.2428390535292264,
  "brier": 0.26059998598083783,
  "brier_mean_k": 0.08474363410961229,
  "brier_onehot": 0.6933349301776697,
  "ece10": 0.10654264947479258,
  "mean_conf": 0.523837870787313,
  "by_type": {
   "choice": {
    "correct": 236,
    "n": 594,
    "accuracy": 0.39730639730639733
   },
   "noul": {
    "correct": 295,
    "n": 597,
    "accuracy": 0.49413735343383586
   },
   "score": {
    "correct": 301,
    "n": 794,
    "accuracy": 0.3790931989924433
   }
  },
  "ci95_cases": [
   0.39848866498740554,
   0.4397984886649874
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5519418643797106,
   "mae_vs_gold_label": 0.7501145362577187,
   "n": 794
  }
 },
 "v2|typed_es|full-case joint": {
  "n": 1985,
  "correct": 673,
  "accuracy": 0.33904282115869017,
  "nll": 1.2040304828794082,
  "kl": 0.44507417616530687,
  "ce_soft": 1.2133433587790274,
  "brier": 0.24717614322069642,
  "brier_mean_k": 0.07750307326535547,
  "brier_onehot": 0.6933235732686921,
  "ece10": 0.11217208232176995,
  "mean_conf": 0.4512149034804601,
  "by_type": {
   "choice": {
    "correct": 167,
    "n": 594,
    "accuracy": 0.28114478114478114
   },
   "noul": {
    "correct": 284,
    "n": 597,
    "accuracy": 0.47571189279731996
   },
   "score": {
    "correct": 222,
    "n": 794,
    "accuracy": 0.2795969773299748
   }
  },
  "ci95_cases": [
   0.31738035264483627,
   0.3602015113350126
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.6897287760172528,
   "mae_vs_gold_label": 0.921904983332085,
   "n": 794
  }
 },
 "v2|typed_es|single-question vs full-case context": {
  "n_common": 1985,
  "only_b_right": 119,
  "only_a_right": 769,
  "p_mcnemar_exact": 3.7714035666656705e-117
 },
 "v2|typed_es|full-case context vs full-case joint": {
  "n_common": 1985,
  "only_b_right": 254,
  "only_a_right": 413,
  "p_mcnemar_exact": 7.935736100989516e-10
 },
 "v2|telepatia_es|single-question": {
  "n": 2000,
  "correct": 1492,
  "accuracy": 0.746,
  "nll": 0.6768666282900727,
  "kl": 0.137470178826373,
  "ce_soft": 0.9048287344507019,
  "brier": 0.07226937229140096,
  "brier_mean_k": 0.020558110269673965,
  "brier_onehot": 0.3838231707888498,
  "ece10": 0.1303335434347391,
  "mean_conf": 0.6159254869967699,
  "by_type": {
   "choice": {
    "correct": 415,
    "n": 600,
    "accuracy": 0.6916666666666667
   },
   "noul": {
    "correct": 502,
    "n": 600,
    "accuracy": 0.8366666666666667
   },
   "score": {
    "correct": 575,
    "n": 800,
    "accuracy": 0.71875
   }
  },
  "ci95_cases": [
   0.7245,
   0.768
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.26117300274514005,
   "mae_vs_gold_label": 0.44825553921662503,
   "n": 800
  }
 },
 "v2|telepatia_es|full-case context": {
  "n": 2000,
  "correct": 823,
  "accuracy": 0.4115,
  "nll": 1.189682075850548,
  "kl": 0.4673731803374047,
  "ce_soft": 1.2347317359617336,
  "brier": 0.26037437617486775,
  "brier_mean_k": 0.08499574423415869,
  "brier_onehot": 0.6928821846712389,
  "ece10": 0.1040707949474454,
  "mean_conf": 0.5111226370260119,
  "by_type": {
   "choice": {
    "correct": 199,
    "n": 600,
    "accuracy": 0.33166666666666667
   },
   "noul": {
    "correct": 291,
    "n": 600,
    "accuracy": 0.485
   },
   "score": {
    "correct": 333,
    "n": 800,
    "accuracy": 0.41625
   }
  },
  "ci95_cases": [
   0.39,
   0.433
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5304797743702488,
   "mae_vs_gold_label": 0.7196103962347843,
   "n": 800
  }
 },
 "v2|telepatia_es|full-case joint": {
  "n": 2000,
  "correct": 636,
  "accuracy": 0.318,
  "nll": 1.2328385029619342,
  "kl": 0.4669040681698344,
  "ce_soft": 1.2342626237941632,
  "brier": 0.2603068636325369,
  "brier_mean_k": 0.08182894089557148,
  "brier_onehot": 0.7105703100725699,
  "ece10": 0.15144768108427525,
  "mean_conf": 0.46944768108427526,
  "by_type": {
   "choice": {
    "correct": 125,
    "n": 600,
    "accuracy": 0.20833333333333334
   },
   "noul": {
    "correct": 298,
    "n": 600,
    "accuracy": 0.49666666666666665
   },
   "score": {
    "correct": 213,
    "n": 800,
    "accuracy": 0.26625
   }
  },
  "ci95_cases": [
   0.296,
   0.34
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.6871891529425792,
   "mae_vs_gold_label": 0.913369414373301,
   "n": 800
  }
 },
 "v2|telepatia_es|single-question vs full-case context": {
  "n_common": 2000,
  "only_b_right": 126,
  "only_a_right": 795,
  "p_mcnemar_exact": 2.2596591083909646e-119
 },
 "v2|telepatia_es|full-case context vs full-case joint": {
  "n_common": 2000,
  "only_b_right": 261,
  "only_a_right": 448,
  "p_mcnemar_exact": 2.1310615747335922e-12
 },
 "v3|typed_en|single-question": {
  "n": 2000,
  "correct": 1528,
  "accuracy": 0.764,
  "nll": 0.6348000373513565,
  "kl": 0.12133635641347341,
  "ce_soft": 0.8886949120378023,
  "brier": 0.0651098222513845,
  "brier_mean_k": 0.019005062125487936,
  "brier_onehot": 0.3611829084554942,
  "ece10": 0.13159668615460393,
  "mean_conf": 0.632403313845396,
  "by_type": {
   "choice": {
    "correct": 429,
    "n": 600,
    "accuracy": 0.715
   },
   "noul": {
    "correct": 503,
    "n": 600,
    "accuracy": 0.8383333333333334
   },
   "score": {
    "correct": 596,
    "n": 800,
    "accuracy": 0.745
   }
  },
  "ci95_cases": [
   0.742,
   0.7855
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.23437388759128985,
   "mae_vs_gold_label": 0.42270025751247886,
   "n": 800
  }
 },
 "v3|typed_en|full-case context": {
  "n": 2000,
  "correct": 831,
  "accuracy": 0.4155,
  "nll": 1.1946701619356137,
  "kl": 0.48748056433424275,
  "ce_soft": 1.2548391199585716,
  "brier": 0.26584338076551384,
  "brier_mean_k": 0.08537406918322844,
  "brier_onehot": 0.6924497841014353,
  "ece10": 0.11764439262449741,
  "mean_conf": 0.5266776051074267,
  "by_type": {
   "choice": {
    "correct": 153,
    "n": 600,
    "accuracy": 0.255
   },
   "noul": {
    "correct": 295,
    "n": 600,
    "accuracy": 0.49166666666666664
   },
   "score": {
    "correct": 383,
    "n": 800,
    "accuracy": 0.47875
   }
  },
  "ci95_cases": [
   0.3945,
   0.4365
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5401024079043407,
   "mae_vs_gold_label": 0.7221055129170417,
   "n": 800
  }
 },
 "v3|typed_en|full-case joint": {
  "n": 2000,
  "correct": 742,
  "accuracy": 0.371,
  "nll": 1.1821842679997898,
  "kl": 0.43716691089620746,
  "ce_soft": 1.2045254665205365,
  "brier": 0.2458284892200497,
  "brier_mean_k": 0.0777216018771812,
  "brier_onehot": 0.6827152191861325,
  "ece10": 0.08931016900390386,
  "mean_conf": 0.44944229010492565,
  "by_type": {
   "choice": {
    "correct": 172,
    "n": 600,
    "accuracy": 0.2866666666666667
   },
   "noul": {
    "correct": 276,
    "n": 600,
    "accuracy": 0.46
   },
   "score": {
    "correct": 294,
    "n": 800,
    "accuracy": 0.3675
   }
  },
  "ci95_cases": [
   0.349,
   0.393
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.6182189710583621,
   "mae_vs_gold_label": 0.8307282497920095,
   "n": 800
  }
 },
 "v3|typed_en|single-question vs full-case context": {
  "n_common": 2000,
  "only_b_right": 150,
  "only_a_right": 847,
  "p_mcnemar_exact": 1.495086609422151e-118
 },
 "v3|typed_en|full-case context vs full-case joint": {
  "n_common": 2000,
  "only_b_right": 385,
  "only_a_right": 474,
  "p_mcnemar_exact": 0.0026569907313413405
 },
 "v3|typed_es|single-question": {
  "n": 1985,
  "correct": 1496,
  "accuracy": 0.7536523929471033,
  "nll": 0.6394842117466736,
  "kl": 0.11898906052638147,
  "ce_soft": 0.887258243140102,
  "brier": 0.06475655764622247,
  "brier_mean_k": 0.018860197074895686,
  "brier_onehot": 0.36670251099112927,
  "ece10": 0.12293764144107136,
  "mean_conf": 0.6313106105219507,
  "by_type": {
   "choice": {
    "correct": 415,
    "n": 594,
    "accuracy": 0.6986531986531986
   },
   "noul": {
    "correct": 494,
    "n": 597,
    "accuracy": 0.8274706867671692
   },
   "score": {
    "correct": 587,
    "n": 794,
    "accuracy": 0.739294710327456
   }
  },
  "ci95_cases": [
   0.7324937027707809,
   0.7743073047858942
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.23596503181925557,
   "mae_vs_gold_label": 0.4296506547049947,
   "n": 794
  }
 },
 "v3|typed_es|full-case context": {
  "n": 1985,
  "correct": 859,
  "accuracy": 0.4327455919395466,
  "nll": 1.162280408384133,
  "kl": 0.4496600929573715,
  "ce_soft": 1.217929275571092,
  "brier": 0.2505504103004089,
  "brier_mean_k": 0.08182399037164234,
  "brier_onehot": 0.6788902020159528,
  "ece10": 0.09863890937203425,
  "mean_conf": 0.5196653394690088,
  "by_type": {
   "choice": {
    "correct": 211,
    "n": 594,
    "accuracy": 0.3552188552188552
   },
   "noul": {
    "correct": 296,
    "n": 597,
    "accuracy": 0.49581239530988275
   },
   "score": {
    "correct": 352,
    "n": 794,
    "accuracy": 0.4433249370277078
   }
  },
  "ci95_cases": [
   0.4105793450881612,
   0.4544080604534005
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5462041908321706,
   "mae_vs_gold_label": 0.7497829019000534,
   "n": 794
  }
 },
 "v3|typed_es|full-case joint": {
  "n": 1985,
  "correct": 763,
  "accuracy": 0.38438287153652395,
  "nll": 1.172466296474459,
  "kl": 0.42073980961851604,
  "ce_soft": 1.1890089922322367,
  "brier": 0.23343222723881618,
  "brier_mean_k": 0.07347218898601528,
  "brier_onehot": 0.6736496081085933,
  "ece10": 0.07710073747923152,
  "mean_conf": 0.4243709911927768,
  "by_type": {
   "choice": {
    "correct": 225,
    "n": 594,
    "accuracy": 0.3787878787878788
   },
   "noul": {
    "correct": 286,
    "n": 597,
    "accuracy": 0.47906197654941374
   },
   "score": {
    "correct": 252,
    "n": 794,
    "accuracy": 0.31738035264483627
   }
  },
  "ci95_cases": [
   0.36171284634760703,
   0.4070528967254408
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.6776316444522275,
   "mae_vs_gold_label": 0.9095255444835055,
   "n": 794
  }
 },
 "v3|typed_es|single-question vs full-case context": {
  "n_common": 1985,
  "only_b_right": 137,
  "only_a_right": 774,
  "p_mcnemar_exact": 1.6584466757134775e-108
 },
 "v3|typed_es|full-case context vs full-case joint": {
  "n_common": 1985,
  "only_b_right": 320,
  "only_a_right": 416,
  "p_mcnemar_exact": 0.0004544874488207351
 },
 "v3|telepatia_es|single-question": {
  "n": 2000,
  "correct": 1526,
  "accuracy": 0.763,
  "nll": 0.6385563440375813,
  "kl": 0.11925372995655213,
  "ce_soft": 0.886612285580881,
  "brier": 0.06393994905918735,
  "brier_mean_k": 0.018569114373183912,
  "brier_onehot": 0.36353173992639604,
  "ece10": 0.13380355775356292,
  "mean_conf": 0.6294715915620327,
  "by_type": {
   "choice": {
    "correct": 438,
    "n": 600,
    "accuracy": 0.73
   },
   "noul": {
    "correct": 498,
    "n": 600,
    "accuracy": 0.83
   },
   "score": {
    "correct": 590,
    "n": 800,
    "accuracy": 0.7375
   }
  },
  "ci95_cases": [
   0.742,
   0.785
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.23628337060784574,
   "mae_vs_gold_label": 0.42863585742670696,
   "n": 800
  }
 },
 "v3|telepatia_es|full-case context": {
  "n": 2000,
  "correct": 878,
  "accuracy": 0.439,
  "nll": 1.1470116522285891,
  "kl": 0.4409125760498499,
  "ce_soft": 1.2082711316741788,
  "brier": 0.24666331201318742,
  "brier_mean_k": 0.08078183830888044,
  "brier_onehot": 0.671401023710438,
  "ece10": 0.09246306705474854,
  "mean_conf": 0.5134267296791076,
  "by_type": {
   "choice": {
    "correct": 223,
    "n": 600,
    "accuracy": 0.37166666666666665
   },
   "noul": {
    "correct": 296,
    "n": 600,
    "accuracy": 0.49333333333333335
   },
   "score": {
    "correct": 359,
    "n": 800,
    "accuracy": 0.44875
   }
  },
  "ci95_cases": [
   0.4185,
   0.4595
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.5391987730165998,
   "mae_vs_gold_label": 0.7372078359872103,
   "n": 800
  }
 },
 "v3|telepatia_es|full-case joint": {
  "n": 2000,
  "correct": 745,
  "accuracy": 0.3725,
  "nll": 1.1746715414243962,
  "kl": 0.42382525863971043,
  "ce_soft": 1.1911838142640392,
  "brier": 0.235579264029217,
  "brier_mean_k": 0.07444738062797601,
  "brier_onehot": 0.6762935895205119,
  "ece10": 0.08313308775424957,
  "mean_conf": 0.4297241877913475,
  "by_type": {
   "choice": {
    "correct": 230,
    "n": 600,
    "accuracy": 0.38333333333333336
   },
   "noul": {
    "correct": 279,
    "n": 600,
    "accuracy": 0.465
   },
   "score": {
    "correct": 236,
    "n": 800,
    "accuracy": 0.295
   }
  },
  "ci95_cases": [
   0.35,
   0.3945
  ],
  "score_mae": {
   "mae_vs_gold_expectation": 0.6721788485455155,
   "mae_vs_gold_label": 0.8984525854373351,
   "n": 800
  }
 },
 "v3|telepatia_es|single-question vs full-case context": {
  "n_common": 2000,
  "only_b_right": 136,
  "only_a_right": 784,
  "p_mcnemar_exact": 2.4222306432783787e-111
 },
 "v3|telepatia_es|full-case context vs full-case joint": {
  "n_common": 2000,
  "only_b_right": 330,
  "only_a_right": 463,
  "p_mcnemar_exact": 2.6285909419854747e-06
 }
}