{
  "primary": "mean-pointer-53",
  "candidate_validation": {
    "linear-53": {
      "epoch": 14,
      "selection_score": 0.5044827221506452,
      "held_choice_macro": 0.528421052631579,
      "macro_nll": 1.1099640677586602,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.5608695652173913,
          "nll": 1.0465427660423776
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5754716981132075,
          "nll": 1.1738272307194628
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.7,
          "nll": 1.0270407646894455
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8529411764705882,
          "nll": 0.4446065302061684
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.15,
          "nll": 1.629424986243248
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.35,
          "nll": 1.2751261278986932
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5,
          "nll": 0.6992056747277577
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.25,
          "nll": 1.5839384615421295
        }
      }
    },
    "linear-54": {
      "epoch": 34,
      "selection_score": 0.5183229493170265,
      "held_choice_macro": 0.5314912280701753,
      "macro_nll": 1.0896898020199184,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.6130434782608696,
          "nll": 0.9422148072930134
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.589622641509434,
          "nll": 1.2442824535845023
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.7,
          "nll": 0.8913424015045166
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.7647058823529411,
          "nll": 0.4467054825166569
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.2,
          "nll": 1.6136522948741914
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 1.3385897763073444
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.48333333333333334,
          "nll": 0.7032629604140918
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.375,
          "nll": 1.5374682396650314
        }
      }
    },
    "mlp-53": {
      "epoch": 6,
      "selection_score": 0.5138907370728611,
      "held_choice_macro": 0.48956140350877186,
      "macro_nll": 1.077804449118502,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7739130434782608,
          "nll": 0.6720642662447694
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5566037735849056,
          "nll": 1.4199528159718644
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.75,
          "nll": 0.6397793352603912
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.7352941176470589,
          "nll": 0.4767963891191518
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.35,
          "nll": 1.4700219124555587
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.45,
          "nll": 1.5610679149627686
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5083333333333333,
          "nll": 0.707912593583266
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.2,
          "nll": 1.6748403653502464
        }
      }
    },
    "mlp-54": {
      "epoch": 8,
      "selection_score": 0.5333128056714678,
      "held_choice_macro": 0.5076315789473684,
      "macro_nll": 1.1165033057127243,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7521739130434782,
          "nll": 0.6441215125966625
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5566037735849056,
          "nll": 1.4465161976871248
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.7,
          "nll": 0.6828836982138455
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.7941176470588235,
          "nll": 0.4847725949195378
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 1.4290357738733293
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.45,
          "nll": 1.907752612605691
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5666666666666667,
          "nll": 0.6949161979059378
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.25,
          "nll": 1.6420278578996659
        }
      }
    },
    "pointer-53": {
      "epoch": 13,
      "selection_score": 0.5520907768141582,
      "held_choice_macro": 0.4715789473684211,
      "macro_nll": 1.3834691288226701,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7478260869565218,
          "nll": 0.9239768173341221
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5235849056603774,
          "nll": 2.117477944044271
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.7,
          "nll": 0.804490776755847
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.6470588235294118,
          "nll": 0.8704288248597069
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.6,
          "nll": 1.6172646887600421
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.55,
          "nll": 2.854619988286868
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.6583333333333333,
          "nll": 0.6809215102344751
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.525,
          "nll": 1.1985724803060294
        }
      }
    },
    "pointer-54": {
      "epoch": 10,
      "selection_score": 0.5517817401560166,
      "held_choice_macro": 0.4882456140350877,
      "macro_nll": 1.3144647585848324,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7086956521739131,
          "nll": 0.9556937654842017
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5283018867924528,
          "nll": 1.951292751427265
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.8,
          "nll": 0.6337956939918513
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8235294117647058,
          "nll": 0.539082182550213
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.5,
          "nll": 1.3860241921618581
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.45,
          "nll": 2.504596093390137
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.525,
          "nll": 0.9305559858058889
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.5,
          "nll": 1.6146774038672447
        }
      }
    },
    "mean-linear-53": {
      "epoch": 16,
      "selection_score": 0.5309960162554723,
      "held_choice_macro": 0.5455263157894736,
      "macro_nll": 1.0970576704259016,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.6956521739130435,
          "nll": 0.8432047027689608
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.589622641509434,
          "nll": 1.1514650333918772
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.65,
          "nll": 1.0076918490231037
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8529411764705882,
          "nll": 0.44652661242905783
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.35,
          "nll": 1.5400688737630843
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.3,
          "nll": 1.4780883759260177
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5166666666666667,
          "nll": 0.6931067710121472
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.25,
          "nll": 1.616309145092964
        }
      }
    },
    "mean-linear-54": {
      "epoch": 55,
      "selection_score": 0.5799733508964227,
      "held_choice_macro": 0.6407894736842105,
      "macro_nll": 1.1015072169769422,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7478260869565218,
          "nll": 0.6880949961775711
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5990566037735849,
          "nll": 1.3095410831913985
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.65,
          "nll": 0.838571004755795
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8529411764705882,
          "nll": 0.4153114815845209
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 1.4946045517921447
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.2,
          "nll": 1.7757694333791734
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.48333333333333334,
          "nll": 0.6950855811436971
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.3,
          "nll": 1.595079603791237
        }
      }
    },
    "mean-mlp-53": {
      "epoch": 6,
      "selection_score": 0.5509324782544598,
      "held_choice_macro": 0.5543859649122808,
      "macro_nll": 1.0166994906591755,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.8,
          "nll": 0.5715492898030788
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.6179245283018868,
          "nll": 1.183610129337055
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.8,
          "nll": 0.7166014191228897
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8823529411764706,
          "nll": 0.4464240033617791
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 1.4142736732959746
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.25,
          "nll": 1.4414601519703865
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5,
          "nll": 0.7240864309171836
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.2,
          "nll": 1.6355908274650575
        }
      }
    },
    "mean-mlp-54": {
      "epoch": 18,
      "selection_score": 0.5471455332705164,
      "held_choice_macro": 0.5796491228070175,
      "macro_nll": 1.224430591007334,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.8304347826086956,
          "nll": 0.4756355702644535
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5990566037735849,
          "nll": 1.701947223364804
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.75,
          "nll": 0.7800838805977037
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.6470588235294118,
          "nll": 0.542199343593572
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.35,
          "nll": 1.4540860489010812
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.25,
          "nll": 2.4939859665930273
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.525,
          "nll": 0.6896170216302077
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.25,
          "nll": 1.6578896731138228
        }
      }
    },
    "mean-pointer-53": {
      "epoch": 17,
      "selection_score": 0.5826073175265529,
      "held_choice_macro": 0.5207894736842105,
      "macro_nll": 1.5221892253719584,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7869565217391304,
          "nll": 0.8957058449639802
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5707547169811321,
          "nll": 2.281843843924606
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.9,
          "nll": 0.33590104000057863
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8823529411764706,
          "nll": 0.5906203307248862
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.45,
          "nll": 1.8350471680052578
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.25,
          "nll": 4.459872342599556
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.6666666666666666,
          "nll": 0.6472956269669036
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.575,
          "nll": 1.1312276057898998
        }
      }
    },
    "mean-pointer-54": {
      "epoch": 10,
      "selection_score": 0.5596496996968123,
      "held_choice_macro": 0.5357894736842105,
      "macro_nll": 1.265137420782477,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.782608695652174,
          "nll": 0.8111066133210167
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5849056603773585,
          "nll": 1.7987122467374714
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.75,
          "nll": 0.6093474811718806
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.7352941176470589,
          "nll": 0.5761815706928781
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.45,
          "nll": 1.3209631159901618
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 2.8080272178631276
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5166666666666667,
          "nll": 0.7050159197300673
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.45,
          "nll": 1.491745200753212
        }
      }
    },
    "qv4": {
      "epoch": 2,
      "selection_score": 0.5625868058894038,
      "held_choice_macro": 0.47684210526315784,
      "macro_nll": 1.9873767214783178,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7608695652173914,
          "nll": 1.4296370394480777
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5188679245283019,
          "nll": 3.724907393212323
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.85,
          "nll": 1.1244477602188225
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.7941176470588235,
          "nll": 0.7729704237772256
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 2.262365103792399
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.55,
          "nll": 4.569536149702321
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.5083333333333333,
          "nll": 1.0399302201888834
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.675,
          "nll": 0.9752196814864874
        }
      }
    },
    "all8": {
      "epoch": 0,
      "selection_score": 0.5799733508964227,
      "held_choice_macro": 0.6407894736842105,
      "macro_nll": 1.101507178779253,
      "sources": {
        "massive_seen": {
          "questions": 230,
          "accuracy": 0.7478260869565218,
          "nll": 0.6880949599410245
        },
        "massive_unseen": {
          "questions": 212,
          "accuracy": 0.5990566037735849,
          "nll": 1.309541177606421
        },
        "agnews": {
          "questions": 20,
          "accuracy": 0.65,
          "nll": 0.8385712005198002
        },
        "agnews_yn": {
          "questions": 34,
          "accuracy": 0.8529411764705882,
          "nll": 0.4153115503928241
        },
        "sst5": {
          "questions": 20,
          "accuracy": 0.4,
          "nll": 1.4946038603782654
        },
        "mnli": {
          "questions": 20,
          "accuracy": 0.2,
          "nll": 1.775769367814064
        },
        "regression_native_threshold": {
          "questions": 120,
          "accuracy": 0.48333333333333334,
          "nll": 0.695085575679938
        },
        "regression_native_score": {
          "questions": 40,
          "accuracy": 0.3,
          "nll": 1.5950797379016877
        }
      }
    }
  },
  "dataset_sha256": "5c8b20981248dad24cfa79d7493264db49af04923207b41855697cb1cbf23a5e",
  "checkpoint_sha256": "5ad9d0fbc6196733389bc9beb64af27f5ee5d0f55e11ff9c773c29023f71e155",
  "test_opened": false,
  "tie_break": "NLL rounded to four decimals, then prefer frozen over an equivalent adapter"
}