| { |
| "primary": "mean-pointer-53", |
| "candidate_validation": { |
| "linear-53": { |
| "epoch": 14, |
| "selection_score": 0.5044827221506452, |
| "held_choice_macro": 0.528421052631579, |
| "macro_nll": 1.1099640677586602, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.5608695652173913, |
| "nll": 1.0465427660423776 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5754716981132075, |
| "nll": 1.1738272307194628 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.7, |
| "nll": 1.0270407646894455 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8529411764705882, |
| "nll": 0.4446065302061684 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.15, |
| "nll": 1.629424986243248 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.35, |
| "nll": 1.2751261278986932 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5, |
| "nll": 0.6992056747277577 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.25, |
| "nll": 1.5839384615421295 |
| } |
| } |
| }, |
| "linear-54": { |
| "epoch": 34, |
| "selection_score": 0.5183229493170265, |
| "held_choice_macro": 0.5314912280701753, |
| "macro_nll": 1.0896898020199184, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.6130434782608696, |
| "nll": 0.9422148072930134 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.589622641509434, |
| "nll": 1.2442824535845023 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.7, |
| "nll": 0.8913424015045166 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.7647058823529411, |
| "nll": 0.4467054825166569 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.2, |
| "nll": 1.6136522948741914 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 1.3385897763073444 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.48333333333333334, |
| "nll": 0.7032629604140918 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.375, |
| "nll": 1.5374682396650314 |
| } |
| } |
| }, |
| "mlp-53": { |
| "epoch": 6, |
| "selection_score": 0.5138907370728611, |
| "held_choice_macro": 0.48956140350877186, |
| "macro_nll": 1.077804449118502, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7739130434782608, |
| "nll": 0.6720642662447694 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5566037735849056, |
| "nll": 1.4199528159718644 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.75, |
| "nll": 0.6397793352603912 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.7352941176470589, |
| "nll": 0.4767963891191518 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.35, |
| "nll": 1.4700219124555587 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.45, |
| "nll": 1.5610679149627686 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5083333333333333, |
| "nll": 0.707912593583266 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.2, |
| "nll": 1.6748403653502464 |
| } |
| } |
| }, |
| "mlp-54": { |
| "epoch": 8, |
| "selection_score": 0.5333128056714678, |
| "held_choice_macro": 0.5076315789473684, |
| "macro_nll": 1.1165033057127243, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7521739130434782, |
| "nll": 0.6441215125966625 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5566037735849056, |
| "nll": 1.4465161976871248 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.7, |
| "nll": 0.6828836982138455 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.7941176470588235, |
| "nll": 0.4847725949195378 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 1.4290357738733293 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.45, |
| "nll": 1.907752612605691 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5666666666666667, |
| "nll": 0.6949161979059378 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.25, |
| "nll": 1.6420278578996659 |
| } |
| } |
| }, |
| "pointer-53": { |
| "epoch": 13, |
| "selection_score": 0.5520907768141582, |
| "held_choice_macro": 0.4715789473684211, |
| "macro_nll": 1.3834691288226701, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7478260869565218, |
| "nll": 0.9239768173341221 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5235849056603774, |
| "nll": 2.117477944044271 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.7, |
| "nll": 0.804490776755847 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.6470588235294118, |
| "nll": 0.8704288248597069 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.6, |
| "nll": 1.6172646887600421 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.55, |
| "nll": 2.854619988286868 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.6583333333333333, |
| "nll": 0.6809215102344751 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.525, |
| "nll": 1.1985724803060294 |
| } |
| } |
| }, |
| "pointer-54": { |
| "epoch": 10, |
| "selection_score": 0.5517817401560166, |
| "held_choice_macro": 0.4882456140350877, |
| "macro_nll": 1.3144647585848324, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7086956521739131, |
| "nll": 0.9556937654842017 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5283018867924528, |
| "nll": 1.951292751427265 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.8, |
| "nll": 0.6337956939918513 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8235294117647058, |
| "nll": 0.539082182550213 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.5, |
| "nll": 1.3860241921618581 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.45, |
| "nll": 2.504596093390137 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.525, |
| "nll": 0.9305559858058889 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.5, |
| "nll": 1.6146774038672447 |
| } |
| } |
| }, |
| "mean-linear-53": { |
| "epoch": 16, |
| "selection_score": 0.5309960162554723, |
| "held_choice_macro": 0.5455263157894736, |
| "macro_nll": 1.0970576704259016, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.6956521739130435, |
| "nll": 0.8432047027689608 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.589622641509434, |
| "nll": 1.1514650333918772 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.65, |
| "nll": 1.0076918490231037 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8529411764705882, |
| "nll": 0.44652661242905783 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.35, |
| "nll": 1.5400688737630843 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.3, |
| "nll": 1.4780883759260177 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5166666666666667, |
| "nll": 0.6931067710121472 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.25, |
| "nll": 1.616309145092964 |
| } |
| } |
| }, |
| "mean-linear-54": { |
| "epoch": 55, |
| "selection_score": 0.5799733508964227, |
| "held_choice_macro": 0.6407894736842105, |
| "macro_nll": 1.1015072169769422, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7478260869565218, |
| "nll": 0.6880949961775711 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5990566037735849, |
| "nll": 1.3095410831913985 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.65, |
| "nll": 0.838571004755795 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8529411764705882, |
| "nll": 0.4153114815845209 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 1.4946045517921447 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.2, |
| "nll": 1.7757694333791734 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.48333333333333334, |
| "nll": 0.6950855811436971 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.3, |
| "nll": 1.595079603791237 |
| } |
| } |
| }, |
| "mean-mlp-53": { |
| "epoch": 6, |
| "selection_score": 0.5509324782544598, |
| "held_choice_macro": 0.5543859649122808, |
| "macro_nll": 1.0166994906591755, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.8, |
| "nll": 0.5715492898030788 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.6179245283018868, |
| "nll": 1.183610129337055 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.8, |
| "nll": 0.7166014191228897 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8823529411764706, |
| "nll": 0.4464240033617791 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 1.4142736732959746 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.25, |
| "nll": 1.4414601519703865 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5, |
| "nll": 0.7240864309171836 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.2, |
| "nll": 1.6355908274650575 |
| } |
| } |
| }, |
| "mean-mlp-54": { |
| "epoch": 18, |
| "selection_score": 0.5471455332705164, |
| "held_choice_macro": 0.5796491228070175, |
| "macro_nll": 1.224430591007334, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.8304347826086956, |
| "nll": 0.4756355702644535 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5990566037735849, |
| "nll": 1.701947223364804 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.75, |
| "nll": 0.7800838805977037 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.6470588235294118, |
| "nll": 0.542199343593572 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.35, |
| "nll": 1.4540860489010812 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.25, |
| "nll": 2.4939859665930273 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.525, |
| "nll": 0.6896170216302077 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.25, |
| "nll": 1.6578896731138228 |
| } |
| } |
| }, |
| "mean-pointer-53": { |
| "epoch": 17, |
| "selection_score": 0.5826073175265529, |
| "held_choice_macro": 0.5207894736842105, |
| "macro_nll": 1.5221892253719584, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7869565217391304, |
| "nll": 0.8957058449639802 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5707547169811321, |
| "nll": 2.281843843924606 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.9, |
| "nll": 0.33590104000057863 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8823529411764706, |
| "nll": 0.5906203307248862 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.45, |
| "nll": 1.8350471680052578 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.25, |
| "nll": 4.459872342599556 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.6666666666666666, |
| "nll": 0.6472956269669036 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.575, |
| "nll": 1.1312276057898998 |
| } |
| } |
| }, |
| "mean-pointer-54": { |
| "epoch": 10, |
| "selection_score": 0.5596496996968123, |
| "held_choice_macro": 0.5357894736842105, |
| "macro_nll": 1.265137420782477, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.782608695652174, |
| "nll": 0.8111066133210167 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5849056603773585, |
| "nll": 1.7987122467374714 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.75, |
| "nll": 0.6093474811718806 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.7352941176470589, |
| "nll": 0.5761815706928781 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.45, |
| "nll": 1.3209631159901618 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 2.8080272178631276 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5166666666666667, |
| "nll": 0.7050159197300673 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.45, |
| "nll": 1.491745200753212 |
| } |
| } |
| }, |
| "qv4": { |
| "epoch": 2, |
| "selection_score": 0.5625868058894038, |
| "held_choice_macro": 0.47684210526315784, |
| "macro_nll": 1.9873767214783178, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7608695652173914, |
| "nll": 1.4296370394480777 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5188679245283019, |
| "nll": 3.724907393212323 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.85, |
| "nll": 1.1244477602188225 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.7941176470588235, |
| "nll": 0.7729704237772256 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 2.262365103792399 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.55, |
| "nll": 4.569536149702321 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.5083333333333333, |
| "nll": 1.0399302201888834 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.675, |
| "nll": 0.9752196814864874 |
| } |
| } |
| }, |
| "all8": { |
| "epoch": 0, |
| "selection_score": 0.5799733508964227, |
| "held_choice_macro": 0.6407894736842105, |
| "macro_nll": 1.101507178779253, |
| "sources": { |
| "massive_seen": { |
| "questions": 230, |
| "accuracy": 0.7478260869565218, |
| "nll": 0.6880949599410245 |
| }, |
| "massive_unseen": { |
| "questions": 212, |
| "accuracy": 0.5990566037735849, |
| "nll": 1.309541177606421 |
| }, |
| "agnews": { |
| "questions": 20, |
| "accuracy": 0.65, |
| "nll": 0.8385712005198002 |
| }, |
| "agnews_yn": { |
| "questions": 34, |
| "accuracy": 0.8529411764705882, |
| "nll": 0.4153115503928241 |
| }, |
| "sst5": { |
| "questions": 20, |
| "accuracy": 0.4, |
| "nll": 1.4946038603782654 |
| }, |
| "mnli": { |
| "questions": 20, |
| "accuracy": 0.2, |
| "nll": 1.775769367814064 |
| }, |
| "regression_native_threshold": { |
| "questions": 120, |
| "accuracy": 0.48333333333333334, |
| "nll": 0.695085575679938 |
| }, |
| "regression_native_score": { |
| "questions": 40, |
| "accuracy": 0.3, |
| "nll": 1.5950797379016877 |
| } |
| } |
| } |
| }, |
| "dataset_sha256": "5c8b20981248dad24cfa79d7493264db49af04923207b41855697cb1cbf23a5e", |
| "checkpoint_sha256": "5ad9d0fbc6196733389bc9beb64af27f5ee5d0f55e11ff9c773c29023f71e155", |
| "test_opened": false, |
| "tie_break": "NLL rounded to four decimals, then prefer frozen over an equivalent adapter" |
| } |
|
|
