[ { "checkpoint": "checkpoint-38646", "id_acc": 0.9930946180555555, "ood_acc": 0.870721875, "total_acc": 0.9286879111842106, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.28550694444445, "ood_avg_resp_len": 379.828709375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 747714, "answer_accuracy": 0.9234137883338602, "avg_response_len": 281.5405074296554, "resp_tokens_sum": 227971232, "pass_at_k": { "pass@1": 0.9234137883338602, "pass@2": 0.9435549933283615, "pass@4": 0.9572698152890534, "pass@8": 0.9667895172243828, "pass@16": 0.9736588935734707, "pass@32": 0.9790904995462497, "pass@64": 0.9834172709208217, "pass@128": 0.9867214669617452 }, "per_op_pass_at_k": { "10": { "pass@1": 0.954484697640118, "pass@2": 0.9629707860079438, "pass@4": 0.9709464638269774, "pass@8": 0.9782191819839355, "pass@16": 0.984593484638989, "pass@32": 0.9892985145351701, "pass@64": 0.9930035863666121, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9980902777777778, "pass@2": 0.9997808867641542, "pass@4": 0.9999950205629058, "pass@8": 0.9999999974305525, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9998830464071856, "pass@2": 0.9999977898533642, "pass@4": 0.9999999997193464, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9991910133136095, "pass@2": 0.9999253803056423, "pass@4": 0.9999992966855474, "pass@8": 0.9999999999707041, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9977964743589743, "pass@2": 0.9990386192711489, "pass@4": 0.9997245203123646, "pass@8": 0.9999788364839716, "pass@16": 0.9999999148409177, "pass@32": 0.9999999999998113, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9962367957746479, "pass@2": 0.9972319646223802, "pass@4": 0.9975970072402921, "pass@8": 0.9979708642982112, "pass@16": 0.9985716335094632, "pass@32": 0.9993491975474896, "pass@64": 0.9999188200066541, "pass@128": 1.0 }, "11": { "pass@1": 0.9682852337110481, "pass@2": 0.977452822823493, "pass@4": 0.9842284054082472, "pass@8": 0.9891870968251745, "pass@16": 0.9931120473781875, "pass@32": 0.9965309204049758, "pass@64": 0.9991380576749798, "pass@128": 1.0 }, "12": { "pass@1": 0.9647775423728814, "pass@2": 0.9793001245607014, "pass@4": 0.9885038096403199, "pass@8": 0.9930489687895803, "pass@16": 0.9957842683111874, "pass@32": 0.9978504659297166, "pass@64": 0.9992600866784821, "pass@128": 1.0 }, "13": { "pass@1": 0.9408797021943573, "pass@2": 0.9649356379433759, "pass@4": 0.9808571087704949, "pass@8": 0.9881229406714634, "pass@16": 0.99055864455524, "pass@32": 0.9913680560096522, "pass@64": 0.9921630087220765, "pass@128": 0.9937304075235109 }, "14": { "pass@1": 0.9360640576323987, "pass@2": 0.9562122856722352, "pass@4": 0.9707606814919159, "pass@8": 0.9804801905602375, "pass@16": 0.9866715910082771, "pass@32": 0.9909012287981317, "pass@64": 0.9945214531172304, "pass@128": 0.9968847352024922 }, "15": { "pass@1": 0.9660404624277457, "pass@2": 0.9819844198033775, "pass@4": 0.989337811097312, "pass@8": 0.9927511350152156, "pass@16": 0.9945051007958996, "pass@32": 0.9954700452049894, "pass@64": 0.996392965878996, "pass@128": 0.9971098265895953 }, "16": { "pass@1": 0.8782762096774194, "pass@2": 0.9086745142240289, "pass@4": 0.9307146733674418, "pass@8": 0.9456327971297461, "pass@16": 0.9552193362938507, "pass@32": 0.9633343766463572, "pass@64": 0.9715414297733572, "pass@128": 0.9806451612903225 }, "17": { "pass@1": 0.8934248554913294, "pass@2": 0.9237297175822676, "pass@4": 0.9422482222236664, "pass@8": 0.9540871391786083, "pass@16": 0.9620315216397886, "pass@32": 0.9683743703109818, "pass@64": 0.9725996832538545, "pass@128": 0.9739884393063584 }, "18": { "pass@1": 0.8005357142857142, "pass@2": 0.8531492547806522, "pass@4": 0.8860468795864797, "pass@8": 0.9091291155842222, "pass@16": 0.9275554499538204, "pass@32": 0.9442031158786668, "pass@64": 0.9575396478067062, "pass@128": 0.9657142857142857 }, "19": { "pass@1": 0.6647428678678678, "pass@2": 0.7474041462722564, "pass@4": 0.8052284390689899, "pass@8": 0.8453281707689558, "pass@16": 0.8732022792173314, "pass@32": 0.8947298964394246, "pass@64": 0.9117678285429259, "pass@128": 0.924924924924925 }, "20": { "pass@1": 0.5938203828828829, "pass@2": 0.6811821664183864, "pass@4": 0.7462476041846112, "pass@8": 0.7985336624704658, "pass@16": 0.8402692553045812, "pass@32": 0.8729885399962004, "pass@64": 0.8982468595231101, "pass@128": 0.918918918918919 } } }, "teachers_in_school": { "count": 821120, "correct": 765697, "answer_accuracy": 0.9325031664068589, "avg_response_len": 291.1401585639127, "resp_tokens_sum": 239061007, "pass_at_k": { "pass@1": 0.9325031664068589, "pass@2": 0.9518363817271153, "pass@4": 0.9646200336464147, "pass@8": 0.9733466730553039, "pass@16": 0.9793728174194902, "pass@32": 0.983754479786111, "pass@64": 0.9872180069196503, "pass@128": 0.9900233826968043 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9433713573619632, "pass@2": 0.9535782540698516, "pass@4": 0.9618546829460732, "pass@8": 0.968359742151891, "pass@16": 0.9741511694787096, "pass@32": 0.9805862212217205, "pass@64": 0.9881224160445984, "pass@128": 0.9938650306748467 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9983901515151515, "pass@2": 0.999460898353615, "pass@4": 0.9999225821204167, "pass@8": 0.9999984049427112, "pass@16": 0.9999999996635676, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999791974852071, "pass@2": 0.9999868960536739, "pass@4": 0.9999999650561431, "pass@8": 0.9999999999999813, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9935914855072464, "pass@2": 0.9972134257674314, "pass@4": 0.9991986126734157, "pass@8": 0.9998860811105759, "pass@16": 0.9999969943734291, "pass@32": 0.9999999992176973, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9999328438395415, "pass@2": 0.99999894242267, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9981484826589595, "pass@2": 0.9991423353511447, "pass@4": 0.9997656553768929, "pass@8": 0.9999830971478822, "pass@16": 0.9999999407613452, "pass@32": 0.9999999999999076, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9978213028169014, "pass@2": 0.998696212709327, "pass@4": 0.9994122448602376, "pass@8": 0.9998848700630322, "pass@16": 0.9999964254002791, "pass@32": 0.9999999987973975, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9976609281437125, "pass@2": 0.9988149930454053, "pass@4": 0.9995776498184733, "pass@8": 0.9999453661202221, "pass@16": 0.9999993082440582, "pass@32": 0.9999999999733661, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9791274472891566, "pass@2": 0.9844943257281095, "pass@4": 0.9885222114554959, "pass@8": 0.9915842526776513, "pass@16": 0.9936573596958077, "pass@32": 0.9946784629745986, "pass@64": 0.9954818436151764, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9556720219435737, "pass@2": 0.9739207939426853, "pass@4": 0.9861632180381843, "pass@8": 0.9932283805113136, "pass@16": 0.9967175800031935, "pass@32": 0.9986257066354457, "pass@64": 0.9997926928260641, "pass@128": 1.0 }, "13": { "pass@1": 0.9692752100840336, "pass@2": 0.9822620977524867, "pass@4": 0.9899195350056031, "pass@8": 0.9950258375460176, "pass@16": 0.9984359616586164, "pass@32": 0.9998247453476256, "pass@64": 0.9999989636921891, "pass@128": 1.0 }, "14": { "pass@1": 0.9544956140350878, "pass@2": 0.9714041706957683, "pass@4": 0.9850686652107085, "pass@8": 0.9941410138686707, "pass@16": 0.9982549357238066, "pass@32": 0.9997482243275714, "pass@64": 0.9999976075223646, "pass@128": 1.0 }, "15": { "pass@1": 0.962405303030303, "pass@2": 0.9801527827487472, "pass@4": 0.9897256777561895, "pass@8": 0.9949554819709671, "pass@16": 0.9975806792718727, "pass@32": 0.9989957428151091, "pass@64": 0.999819329898887, "pass@128": 1.0 }, "16": { "pass@1": 0.9157540954415955, "pass@2": 0.9383107460125177, "pass@4": 0.9528681665860143, "pass@8": 0.9618255529076115, "pass@16": 0.9666002487818389, "pass@32": 0.9691287175972142, "pass@64": 0.9713401996282022, "pass@128": 0.9743589743589743 }, "17": { "pass@1": 0.8866459627329193, "pass@2": 0.9140307869125057, "pass@4": 0.9320510835796145, "pass@8": 0.9470251241563546, "pass@16": 0.958828933724438, "pass@32": 0.9669829599693496, "pass@64": 0.9726542282720683, "pass@128": 0.9782608695652174 }, "18": { "pass@1": 0.8592830882352941, "pass@2": 0.9078276980083376, "pass@4": 0.9323671411477975, "pass@8": 0.9477004466579106, "pass@16": 0.9584144161429146, "pass@32": 0.9654229587125491, "pass@64": 0.9701825020374101, "pass@128": 0.9735294117647059 }, "19": { "pass@1": 0.6903366815476191, "pass@2": 0.7712544232752157, "pass@4": 0.8257569664841, "pass@8": 0.8643543757323078, "pass@16": 0.8929841094671906, "pass@32": 0.914312749232494, "pass@64": 0.9313348502509535, "pass@128": 0.9464285714285714 }, "20": { "pass@1": 0.6061938622754491, "pass@2": 0.7069592360554483, "pass@4": 0.779909319718268, "pass@8": 0.8312956919046384, "pass@16": 0.8691291374054729, "pass@32": 0.9006392983887458, "pass@64": 0.926877074969234, "pass@128": 0.9461077844311377 } } }, "movie_festival_awards": { "count": 801152, "correct": 745158, "answer_accuracy": 0.930108144272248, "avg_response_len": 311.89763365753316, "resp_tokens_sum": 249877413, "pass_at_k": { "pass@1": 0.930108144272248, "pass@2": 0.9506625813159659, "pass@4": 0.9639841486302464, "pass@8": 0.9728187372779841, "pass@16": 0.9789563417603896, "pass@32": 0.9835196674940303, "pass@64": 0.9868039392795732, "pass@128": 0.9892954146029718 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9640391791044776, "pass@2": 0.9751006287460335, "pass@4": 0.9840680005484389, "pass@8": 0.989525217666148, "pass@16": 0.9922932987785924, "pass@32": 0.9943696315299635, "pass@64": 0.9970265954859788, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9998857821637427, "pass@2": 0.9999978415527007, "pass@4": 0.9999999997259115, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9920266544117647, "pass@2": 0.9942518961324689, "pass@4": 0.9955414982134585, "pass@8": 0.9967116901009395, "pass@16": 0.998155080969405, "pass@32": 0.9994198334387632, "pass@64": 0.9999589349376307, "pass@128": 1.0 }, "6": { "pass@1": 0.9999260646687698, "pass@2": 0.9999988356640751, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9976513053797469, "pass@2": 0.9984395245689226, "pass@4": 0.9992420280930073, "pass@8": 0.9998284242652762, "pass@16": 0.9999927222555876, "pass@32": 0.9999999948561573, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9929147897897898, "pass@2": 0.9970771706746117, "pass@4": 0.9993902071362702, "pass@8": 0.9999651367971397, "pass@16": 0.9999998583663093, "pass@32": 0.9999999999994196, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9923884646302251, "pass@2": 0.9939457427146365, "pass@4": 0.994643188391644, "pass@8": 0.995451850487162, "pass@16": 0.9964930745316808, "pass@32": 0.9974718962737498, "pass@64": 0.9983913004056533, "pass@128": 1.0 }, "11": { "pass@1": 0.9703125, "pass@2": 0.9826138920134985, "pass@4": 0.9905110777819439, "pass@8": 0.9952361673715535, "pass@16": 0.9982486278795625, "pass@32": 0.9997469296363813, "pass@64": 0.9999980432330696, "pass@128": 1.0 }, "12": { "pass@1": 0.9693950688073395, "pass@2": 0.9864906149919334, "pass@4": 0.9964843422668495, "pass@8": 0.999587940904666, "pass@16": 0.9999840639525752, "pass@32": 0.9999999728189605, "pass@64": 0.9999999999999997, "pass@128": 1.0 }, "13": { "pass@1": 0.9416232638888888, "pass@2": 0.9657007934772044, "pass@4": 0.9807125428765847, "pass@8": 0.9889479333237797, "pass@16": 0.9935783281704044, "pass@32": 0.9965795260843454, "pass@64": 0.9983950138936485, "pass@128": 1.0 }, "14": { "pass@1": 0.9647626112759644, "pass@2": 0.9797271840463564, "pass@4": 0.988019856104782, "pass@8": 0.992716483465879, "pass@16": 0.9955701659139958, "pass@32": 0.9973444053304108, "pass@64": 0.9984967309084293, "pass@128": 1.0 }, "15": { "pass@1": 0.9497010030864198, "pass@2": 0.9660535597112858, "pass@4": 0.9750617191369597, "pass@8": 0.9813197913201593, "pass@16": 0.9856122764931645, "pass@32": 0.9887173282223192, "pass@64": 0.991433089630488, "pass@128": 0.9938271604938271 }, "16": { "pass@1": 0.8941970870206489, "pass@2": 0.9212493177014375, "pass@4": 0.9393702139555561, "pass@8": 0.9519523737165061, "pass@16": 0.9612349386592156, "pass@32": 0.9691753292432682, "pass@64": 0.9747878703756769, "pass@128": 0.976401179941003 }, "17": { "pass@1": 0.8594455948795181, "pass@2": 0.897720433900958, "pass@4": 0.922174053073938, "pass@8": 0.9382293341864976, "pass@16": 0.9493378639982005, "pass@32": 0.959124887388815, "pass@64": 0.9681289020289342, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8485887096774194, "pass@2": 0.903131746888494, "pass@4": 0.9370043603420539, "pass@8": 0.9578532946585311, "pass@16": 0.9702169368171149, "pass@32": 0.9777537348282815, "pass@64": 0.9826360654372754, "pass@128": 0.9870967741935484 }, "19": { "pass@1": 0.7076331193353474, "pass@2": 0.7887924625924787, "pass@4": 0.8411277290300948, "pass@8": 0.878659443062788, "pass@16": 0.9053831900865986, "pass@32": 0.9237568606242907, "pass@64": 0.9352225250892545, "pass@128": 0.9425981873111783 }, "20": { "pass@1": 0.626196509009009, "pass@2": 0.7125315522215124, "pass@4": 0.7734252413268156, "pass@8": 0.8190036582565382, "pass@16": 0.8555370910100033, "pass@32": 0.8847167374444287, "pass@64": 0.9059201413232556, "pass@128": 0.9219219219219219 } } } } } ]