reflex-2048 / calibration_report.json
a3ro-dev's picture
reflex-2048 v0.1: 164k mean, no search, 14.5M params
b77d5ed verified
Raw History Blame Contribute Delete
7.85 kB
{
"ckpt": "/root/runs/final/reflex.pt",
"step": 25915,
"d3": {
"n": 200000,
"acc": 0.9109599590301514,
"ece": 0.010334860303325798,
"classwise_ece": 0.005813934361734425,
"nll": 0.20451036095619202,
"brier": 0.12394937127828598,
"mean_conf": 0.9212714433670044,
"reliability": [
{
"lo": 0.20000001788139343,
"conf": 0.2616492807865143,
"acc": 0.0,
"n": 1
},
{
"lo": 0.2666666805744171,
"conf": 0.3161606788635254,
"acc": 0.1666666716337204,
"n": 12
},
{
"lo": 0.3333333432674408,
"conf": 0.3777371644973755,
"acc": 0.38291141390800476,
"n": 316
},
{
"lo": 0.40000003576278687,
"conf": 0.43759623169898987,
"acc": 0.43310657143592834,
"n": 882
},
{
"lo": 0.46666669845581055,
"conf": 0.5100187063217163,
"acc": 0.5065382122993469,
"n": 4359
},
{
"lo": 0.5333333015441895,
"conf": 0.5668291449546814,
"acc": 0.566933810710907,
"n": 6738
},
{
"lo": 0.5999999642372131,
"conf": 0.6327977180480957,
"acc": 0.6009564995765686,
"n": 6691
},
{
"lo": 0.6666666269302368,
"conf": 0.7004483938217163,
"acc": 0.6584302186965942,
"n": 6880
},
{
"lo": 0.7333333492279053,
"conf": 0.7675895094871521,
"acc": 0.7328316569328308,
"n": 7441
},
{
"lo": 0.800000011920929,
"conf": 0.8342754244804382,
"acc": 0.7924944758415222,
"n": 9060
},
{
"lo": 0.8666666746139526,
"conf": 0.9024046659469604,
"acc": 0.8668531179428101,
"n": 12873
},
{
"lo": 0.9333333373069763,
"conf": 0.9931983351707458,
"acc": 0.9901137351989746,
"n": 144747
}
],
"acc_by_stage": {
"<2048": {
"acc": 0.9084683060646057,
"n": 11974
},
"2048-4096": {
"acc": 0.9051666259765625,
"n": 32826
},
"8192": {
"acc": 0.9059008359909058,
"n": 39554
},
"16384": {
"acc": 0.9102802276611328,
"n": 60689
},
">=32768": {
"acc": 0.9193551540374756,
"n": 54957
}
}
},
"d4": {
"n": 200000,
"acc": 0.9362399578094482,
"ece": 0.005877783937850665,
"classwise_ece": 0.0037371709357283678,
"nll": 0.15541012585163116,
"brier": 0.09056096524000168,
"mean_conf": 0.930762767791748,
"reliability": [
{
"lo": 0.2666666805744171,
"conf": 0.3104710280895233,
"acc": 0.3333333432674408,
"n": 9
},
{
"lo": 0.3333333432674408,
"conf": 0.3777979910373688,
"acc": 0.32258063554763794,
"n": 248
},
{
"lo": 0.40000003576278687,
"conf": 0.43695154786109924,
"acc": 0.4291091561317444,
"n": 797
},
{
"lo": 0.46666669845581055,
"conf": 0.510391354560852,
"acc": 0.509703516960144,
"n": 3710
},
{
"lo": 0.5333333015441895,
"conf": 0.56622713804245,
"acc": 0.5708888173103333,
"n": 5558
},
{
"lo": 0.5999999642372131,
"conf": 0.6333951950073242,
"acc": 0.6303596496582031,
"n": 5784
},
{
"lo": 0.6666666269302368,
"conf": 0.7004299759864807,
"acc": 0.7268487811088562,
"n": 6004
},
{
"lo": 0.7333333492279053,
"conf": 0.7674108147621155,
"acc": 0.7979346513748169,
"n": 6488
},
{
"lo": 0.800000011920929,
"conf": 0.8349699974060059,
"acc": 0.864363968372345,
"n": 8309
},
{
"lo": 0.8666666746139526,
"conf": 0.903027355670929,
"acc": 0.9281561374664307,
"n": 12555
},
{
"lo": 0.9333333373069763,
"conf": 0.9933969974517822,
"acc": 0.9946790933609009,
"n": 150538
}
],
"acc_by_stage": {
"<2048": {
"acc": 0.8899039030075073,
"n": 8429
},
"2048-4096": {
"acc": 0.9120897650718689,
"n": 24525
},
"8192": {
"acc": 0.9290807843208313,
"n": 31275
},
"16384": {
"acc": 0.9418529868125916,
"n": 58782
},
">=32768": {
"acc": 0.9476288557052612,
"n": 76989
}
}
},
"dag": {
"n": 23621,
"acc": 0.44477373361587524,
"ece": 0.21562865744482934,
"classwise_ece": 0.11597429518194752,
"nll": 2.2459492683410645,
"brier": 0.7532422542572021,
"mean_conf": 0.6604024171829224,
"reliability": [
{
"lo": 0.20000001788139343,
"conf": 0.2614020109176636,
"acc": 0.25,
"n": 12
},
{
"lo": 0.2666666805744171,
"conf": 0.31140047311782837,
"acc": 0.24229075014591217,
"n": 681
},
{
"lo": 0.3333333432674408,
"conf": 0.3702302873134613,
"acc": 0.31361210346221924,
"n": 2248
},
{
"lo": 0.40000003576278687,
"conf": 0.43376410007476807,
"acc": 0.3314346671104431,
"n": 2809
},
{
"lo": 0.46666669845581055,
"conf": 0.5007346272468567,
"acc": 0.37642714381217957,
"n": 2978
},
{
"lo": 0.5333333015441895,
"conf": 0.5658344626426697,
"acc": 0.39647752046585083,
"n": 2555
},
{
"lo": 0.5999999642372131,
"conf": 0.6301718950271606,
"acc": 0.44249120354652405,
"n": 1991
},
{
"lo": 0.6666666269302368,
"conf": 0.6996316313743591,
"acc": 0.4875076115131378,
"n": 1641
},
{
"lo": 0.7333333492279053,
"conf": 0.76542067527771,
"acc": 0.46928200125694275,
"n": 1351
},
{
"lo": 0.800000011920929,
"conf": 0.8332225680351257,
"acc": 0.5105690956115723,
"n": 1230
},
{
"lo": 0.8666666746139526,
"conf": 0.9017775058746338,
"acc": 0.5570417046546936,
"n": 1271
},
{
"lo": 0.9333333373069763,
"conf": 0.9865418672561646,
"acc": 0.6009476780891418,
"n": 4854
}
],
"acc_by_stage": {
"<2048": {
"acc": 0.4209636151790619,
"n": 18327
},
"2048-4096": {
"acc": 0.521415650844574,
"n": 4973
},
"8192": {
"acc": 0.6168224215507507,
"n": 321
}
}
},
"dageval": {
"n": 199505,
"acc": 0.6101701855659485,
"ece": 0.27116093451079476,
"classwise_ece": 0.14033046305396282,
"nll": 3.204610824584961,
"brier": 0.6528977155685425,
"mean_conf": 0.8813311457633972,
"reliability": [
{
"lo": 0.20000001788139343,
"conf": 0.2623162865638733,
"acc": 0.0,
"n": 3
},
{
"lo": 0.2666666805744171,
"conf": 0.31312525272369385,
"acc": 0.1818181723356247,
"n": 110
},
{
"lo": 0.3333333432674408,
"conf": 0.3742755949497223,
"acc": 0.2713567912578583,
"n": 1194
},
{
"lo": 0.40000003576278687,
"conf": 0.4368848204612732,
"acc": 0.29916611313819885,
"n": 2878
},
{
"lo": 0.46666669845581055,
"conf": 0.506822943687439,
"acc": 0.34820833802223206,
"n": 7507
},
{
"lo": 0.5333333015441895,
"conf": 0.5662879943847656,
"acc": 0.3780185878276825,
"n": 9690
},
{
"lo": 0.5999999642372131,
"conf": 0.6332005262374878,
"acc": 0.4069898724555969,
"n": 9757
},
{
"lo": 0.6666666269302368,
"conf": 0.7001890540122986,
"acc": 0.44430702924728394,
"n": 9705
},
{
"lo": 0.7333333492279053,
"conf": 0.7671082615852356,
"acc": 0.49206504225730896,
"n": 10271
},
{
"lo": 0.800000011920929,
"conf": 0.834148108959198,
"acc": 0.5464140176773071,
"n": 11559
},
{
"lo": 0.8666666746139526,
"conf": 0.9023152589797974,
"acc": 0.5913816690444946,
"n": 15293
},
{
"lo": 0.9333333373069763,
"conf": 0.9915003776550293,
"acc": 0.7039197683334351,
"n": 121538
}
],
"acc_by_stage": {
"<2048": {
"acc": 0.4269912540912628,
"n": 18895
},
"2048-4096": {
"acc": 0.572346031665802,
"n": 66956
},
"8192": {
"acc": 0.6439301371574402,
"n": 58379
},
"16384": {
"acc": 0.6899833083152771,
"n": 49178
},
">=32768": {
"acc": 0.6262096166610718,
"n": 6097
}
}
}
}