| [ |
| { |
| "checkpoint": "checkpoint-38646", |
| "id_acc": 0.9930946180555555, |
| "ood_acc": 0.870721875, |
| "total_acc": 0.9286879111842106, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.28550694444445, |
| "ood_avg_resp_len": 379.828709375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 747714, |
| "answer_accuracy": 0.9234137883338602, |
| "avg_response_len": 281.5405074296554, |
| "resp_tokens_sum": 227971232, |
| "pass_at_k": { |
| "pass@1": 0.9234137883338602, |
| "pass@2": 0.9435549933283615, |
| "pass@4": 0.9572698152890534, |
| "pass@8": 0.9667895172243828, |
| "pass@16": 0.9736588935734707, |
| "pass@32": 0.9790904995462497, |
| "pass@64": 0.9834172709208217, |
| "pass@128": 0.9867214669617452 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.954484697640118, |
| "pass@2": 0.9629707860079438, |
| "pass@4": 0.9709464638269774, |
| "pass@8": 0.9782191819839355, |
| "pass@16": 0.984593484638989, |
| "pass@32": 0.9892985145351701, |
| "pass@64": 0.9930035863666121, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9980902777777778, |
| "pass@2": 0.9997808867641542, |
| "pass@4": 0.9999950205629058, |
| "pass@8": 0.9999999974305525, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9998830464071856, |
| "pass@2": 0.9999977898533642, |
| "pass@4": 0.9999999997193464, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9991910133136095, |
| "pass@2": 0.9999253803056423, |
| "pass@4": 0.9999992966855474, |
| "pass@8": 0.9999999999707041, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9977964743589743, |
| "pass@2": 0.9990386192711489, |
| "pass@4": 0.9997245203123646, |
| "pass@8": 0.9999788364839716, |
| "pass@16": 0.9999999148409177, |
| "pass@32": 0.9999999999998113, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9962367957746479, |
| "pass@2": 0.9972319646223802, |
| "pass@4": 0.9975970072402921, |
| "pass@8": 0.9979708642982112, |
| "pass@16": 0.9985716335094632, |
| "pass@32": 0.9993491975474896, |
| "pass@64": 0.9999188200066541, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9682852337110481, |
| "pass@2": 0.977452822823493, |
| "pass@4": 0.9842284054082472, |
| "pass@8": 0.9891870968251745, |
| "pass@16": 0.9931120473781875, |
| "pass@32": 0.9965309204049758, |
| "pass@64": 0.9991380576749798, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9647775423728814, |
| "pass@2": 0.9793001245607014, |
| "pass@4": 0.9885038096403199, |
| "pass@8": 0.9930489687895803, |
| "pass@16": 0.9957842683111874, |
| "pass@32": 0.9978504659297166, |
| "pass@64": 0.9992600866784821, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9408797021943573, |
| "pass@2": 0.9649356379433759, |
| "pass@4": 0.9808571087704949, |
| "pass@8": 0.9881229406714634, |
| "pass@16": 0.99055864455524, |
| "pass@32": 0.9913680560096522, |
| "pass@64": 0.9921630087220765, |
| "pass@128": 0.9937304075235109 |
| }, |
| "14": { |
| "pass@1": 0.9360640576323987, |
| "pass@2": 0.9562122856722352, |
| "pass@4": 0.9707606814919159, |
| "pass@8": 0.9804801905602375, |
| "pass@16": 0.9866715910082771, |
| "pass@32": 0.9909012287981317, |
| "pass@64": 0.9945214531172304, |
| "pass@128": 0.9968847352024922 |
| }, |
| "15": { |
| "pass@1": 0.9660404624277457, |
| "pass@2": 0.9819844198033775, |
| "pass@4": 0.989337811097312, |
| "pass@8": 0.9927511350152156, |
| "pass@16": 0.9945051007958996, |
| "pass@32": 0.9954700452049894, |
| "pass@64": 0.996392965878996, |
| "pass@128": 0.9971098265895953 |
| }, |
| "16": { |
| "pass@1": 0.8782762096774194, |
| "pass@2": 0.9086745142240289, |
| "pass@4": 0.9307146733674418, |
| "pass@8": 0.9456327971297461, |
| "pass@16": 0.9552193362938507, |
| "pass@32": 0.9633343766463572, |
| "pass@64": 0.9715414297733572, |
| "pass@128": 0.9806451612903225 |
| }, |
| "17": { |
| "pass@1": 0.8934248554913294, |
| "pass@2": 0.9237297175822676, |
| "pass@4": 0.9422482222236664, |
| "pass@8": 0.9540871391786083, |
| "pass@16": 0.9620315216397886, |
| "pass@32": 0.9683743703109818, |
| "pass@64": 0.9725996832538545, |
| "pass@128": 0.9739884393063584 |
| }, |
| "18": { |
| "pass@1": 0.8005357142857142, |
| "pass@2": 0.8531492547806522, |
| "pass@4": 0.8860468795864797, |
| "pass@8": 0.9091291155842222, |
| "pass@16": 0.9275554499538204, |
| "pass@32": 0.9442031158786668, |
| "pass@64": 0.9575396478067062, |
| "pass@128": 0.9657142857142857 |
| }, |
| "19": { |
| "pass@1": 0.6647428678678678, |
| "pass@2": 0.7474041462722564, |
| "pass@4": 0.8052284390689899, |
| "pass@8": 0.8453281707689558, |
| "pass@16": 0.8732022792173314, |
| "pass@32": 0.8947298964394246, |
| "pass@64": 0.9117678285429259, |
| "pass@128": 0.924924924924925 |
| }, |
| "20": { |
| "pass@1": 0.5938203828828829, |
| "pass@2": 0.6811821664183864, |
| "pass@4": 0.7462476041846112, |
| "pass@8": 0.7985336624704658, |
| "pass@16": 0.8402692553045812, |
| "pass@32": 0.8729885399962004, |
| "pass@64": 0.8982468595231101, |
| "pass@128": 0.918918918918919 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 765697, |
| "answer_accuracy": 0.9325031664068589, |
| "avg_response_len": 291.1401585639127, |
| "resp_tokens_sum": 239061007, |
| "pass_at_k": { |
| "pass@1": 0.9325031664068589, |
| "pass@2": 0.9518363817271153, |
| "pass@4": 0.9646200336464147, |
| "pass@8": 0.9733466730553039, |
| "pass@16": 0.9793728174194902, |
| "pass@32": 0.983754479786111, |
| "pass@64": 0.9872180069196503, |
| "pass@128": 0.9900233826968043 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9433713573619632, |
| "pass@2": 0.9535782540698516, |
| "pass@4": 0.9618546829460732, |
| "pass@8": 0.968359742151891, |
| "pass@16": 0.9741511694787096, |
| "pass@32": 0.9805862212217205, |
| "pass@64": 0.9881224160445984, |
| "pass@128": 0.9938650306748467 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9983901515151515, |
| "pass@2": 0.999460898353615, |
| "pass@4": 0.9999225821204167, |
| "pass@8": 0.9999984049427112, |
| "pass@16": 0.9999999996635676, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999791974852071, |
| "pass@2": 0.9999868960536739, |
| "pass@4": 0.9999999650561431, |
| "pass@8": 0.9999999999999813, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9935914855072464, |
| "pass@2": 0.9972134257674314, |
| "pass@4": 0.9991986126734157, |
| "pass@8": 0.9998860811105759, |
| "pass@16": 0.9999969943734291, |
| "pass@32": 0.9999999992176973, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999328438395415, |
| "pass@2": 0.99999894242267, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9981484826589595, |
| "pass@2": 0.9991423353511447, |
| "pass@4": 0.9997656553768929, |
| "pass@8": 0.9999830971478822, |
| "pass@16": 0.9999999407613452, |
| "pass@32": 0.9999999999999076, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9978213028169014, |
| "pass@2": 0.998696212709327, |
| "pass@4": 0.9994122448602376, |
| "pass@8": 0.9998848700630322, |
| "pass@16": 0.9999964254002791, |
| "pass@32": 0.9999999987973975, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9976609281437125, |
| "pass@2": 0.9988149930454053, |
| "pass@4": 0.9995776498184733, |
| "pass@8": 0.9999453661202221, |
| "pass@16": 0.9999993082440582, |
| "pass@32": 0.9999999999733661, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9791274472891566, |
| "pass@2": 0.9844943257281095, |
| "pass@4": 0.9885222114554959, |
| "pass@8": 0.9915842526776513, |
| "pass@16": 0.9936573596958077, |
| "pass@32": 0.9946784629745986, |
| "pass@64": 0.9954818436151764, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9556720219435737, |
| "pass@2": 0.9739207939426853, |
| "pass@4": 0.9861632180381843, |
| "pass@8": 0.9932283805113136, |
| "pass@16": 0.9967175800031935, |
| "pass@32": 0.9986257066354457, |
| "pass@64": 0.9997926928260641, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9692752100840336, |
| "pass@2": 0.9822620977524867, |
| "pass@4": 0.9899195350056031, |
| "pass@8": 0.9950258375460176, |
| "pass@16": 0.9984359616586164, |
| "pass@32": 0.9998247453476256, |
| "pass@64": 0.9999989636921891, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9544956140350878, |
| "pass@2": 0.9714041706957683, |
| "pass@4": 0.9850686652107085, |
| "pass@8": 0.9941410138686707, |
| "pass@16": 0.9982549357238066, |
| "pass@32": 0.9997482243275714, |
| "pass@64": 0.9999976075223646, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.962405303030303, |
| "pass@2": 0.9801527827487472, |
| "pass@4": 0.9897256777561895, |
| "pass@8": 0.9949554819709671, |
| "pass@16": 0.9975806792718727, |
| "pass@32": 0.9989957428151091, |
| "pass@64": 0.999819329898887, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.9157540954415955, |
| "pass@2": 0.9383107460125177, |
| "pass@4": 0.9528681665860143, |
| "pass@8": 0.9618255529076115, |
| "pass@16": 0.9666002487818389, |
| "pass@32": 0.9691287175972142, |
| "pass@64": 0.9713401996282022, |
| "pass@128": 0.9743589743589743 |
| }, |
| "17": { |
| "pass@1": 0.8866459627329193, |
| "pass@2": 0.9140307869125057, |
| "pass@4": 0.9320510835796145, |
| "pass@8": 0.9470251241563546, |
| "pass@16": 0.958828933724438, |
| "pass@32": 0.9669829599693496, |
| "pass@64": 0.9726542282720683, |
| "pass@128": 0.9782608695652174 |
| }, |
| "18": { |
| "pass@1": 0.8592830882352941, |
| "pass@2": 0.9078276980083376, |
| "pass@4": 0.9323671411477975, |
| "pass@8": 0.9477004466579106, |
| "pass@16": 0.9584144161429146, |
| "pass@32": 0.9654229587125491, |
| "pass@64": 0.9701825020374101, |
| "pass@128": 0.9735294117647059 |
| }, |
| "19": { |
| "pass@1": 0.6903366815476191, |
| "pass@2": 0.7712544232752157, |
| "pass@4": 0.8257569664841, |
| "pass@8": 0.8643543757323078, |
| "pass@16": 0.8929841094671906, |
| "pass@32": 0.914312749232494, |
| "pass@64": 0.9313348502509535, |
| "pass@128": 0.9464285714285714 |
| }, |
| "20": { |
| "pass@1": 0.6061938622754491, |
| "pass@2": 0.7069592360554483, |
| "pass@4": 0.779909319718268, |
| "pass@8": 0.8312956919046384, |
| "pass@16": 0.8691291374054729, |
| "pass@32": 0.9006392983887458, |
| "pass@64": 0.926877074969234, |
| "pass@128": 0.9461077844311377 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 745158, |
| "answer_accuracy": 0.930108144272248, |
| "avg_response_len": 311.89763365753316, |
| "resp_tokens_sum": 249877413, |
| "pass_at_k": { |
| "pass@1": 0.930108144272248, |
| "pass@2": 0.9506625813159659, |
| "pass@4": 0.9639841486302464, |
| "pass@8": 0.9728187372779841, |
| "pass@16": 0.9789563417603896, |
| "pass@32": 0.9835196674940303, |
| "pass@64": 0.9868039392795732, |
| "pass@128": 0.9892954146029718 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9640391791044776, |
| "pass@2": 0.9751006287460335, |
| "pass@4": 0.9840680005484389, |
| "pass@8": 0.989525217666148, |
| "pass@16": 0.9922932987785924, |
| "pass@32": 0.9943696315299635, |
| "pass@64": 0.9970265954859788, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9998857821637427, |
| "pass@2": 0.9999978415527007, |
| "pass@4": 0.9999999997259115, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9920266544117647, |
| "pass@2": 0.9942518961324689, |
| "pass@4": 0.9955414982134585, |
| "pass@8": 0.9967116901009395, |
| "pass@16": 0.998155080969405, |
| "pass@32": 0.9994198334387632, |
| "pass@64": 0.9999589349376307, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999260646687698, |
| "pass@2": 0.9999988356640751, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9976513053797469, |
| "pass@2": 0.9984395245689226, |
| "pass@4": 0.9992420280930073, |
| "pass@8": 0.9998284242652762, |
| "pass@16": 0.9999927222555876, |
| "pass@32": 0.9999999948561573, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9929147897897898, |
| "pass@2": 0.9970771706746117, |
| "pass@4": 0.9993902071362702, |
| "pass@8": 0.9999651367971397, |
| "pass@16": 0.9999998583663093, |
| "pass@32": 0.9999999999994196, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9923884646302251, |
| "pass@2": 0.9939457427146365, |
| "pass@4": 0.994643188391644, |
| "pass@8": 0.995451850487162, |
| "pass@16": 0.9964930745316808, |
| "pass@32": 0.9974718962737498, |
| "pass@64": 0.9983913004056533, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9703125, |
| "pass@2": 0.9826138920134985, |
| "pass@4": 0.9905110777819439, |
| "pass@8": 0.9952361673715535, |
| "pass@16": 0.9982486278795625, |
| "pass@32": 0.9997469296363813, |
| "pass@64": 0.9999980432330696, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9693950688073395, |
| "pass@2": 0.9864906149919334, |
| "pass@4": 0.9964843422668495, |
| "pass@8": 0.999587940904666, |
| "pass@16": 0.9999840639525752, |
| "pass@32": 0.9999999728189605, |
| "pass@64": 0.9999999999999997, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9416232638888888, |
| "pass@2": 0.9657007934772044, |
| "pass@4": 0.9807125428765847, |
| "pass@8": 0.9889479333237797, |
| "pass@16": 0.9935783281704044, |
| "pass@32": 0.9965795260843454, |
| "pass@64": 0.9983950138936485, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9647626112759644, |
| "pass@2": 0.9797271840463564, |
| "pass@4": 0.988019856104782, |
| "pass@8": 0.992716483465879, |
| "pass@16": 0.9955701659139958, |
| "pass@32": 0.9973444053304108, |
| "pass@64": 0.9984967309084293, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9497010030864198, |
| "pass@2": 0.9660535597112858, |
| "pass@4": 0.9750617191369597, |
| "pass@8": 0.9813197913201593, |
| "pass@16": 0.9856122764931645, |
| "pass@32": 0.9887173282223192, |
| "pass@64": 0.991433089630488, |
| "pass@128": 0.9938271604938271 |
| }, |
| "16": { |
| "pass@1": 0.8941970870206489, |
| "pass@2": 0.9212493177014375, |
| "pass@4": 0.9393702139555561, |
| "pass@8": 0.9519523737165061, |
| "pass@16": 0.9612349386592156, |
| "pass@32": 0.9691753292432682, |
| "pass@64": 0.9747878703756769, |
| "pass@128": 0.976401179941003 |
| }, |
| "17": { |
| "pass@1": 0.8594455948795181, |
| "pass@2": 0.897720433900958, |
| "pass@4": 0.922174053073938, |
| "pass@8": 0.9382293341864976, |
| "pass@16": 0.9493378639982005, |
| "pass@32": 0.959124887388815, |
| "pass@64": 0.9681289020289342, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8485887096774194, |
| "pass@2": 0.903131746888494, |
| "pass@4": 0.9370043603420539, |
| "pass@8": 0.9578532946585311, |
| "pass@16": 0.9702169368171149, |
| "pass@32": 0.9777537348282815, |
| "pass@64": 0.9826360654372754, |
| "pass@128": 0.9870967741935484 |
| }, |
| "19": { |
| "pass@1": 0.7076331193353474, |
| "pass@2": 0.7887924625924787, |
| "pass@4": 0.8411277290300948, |
| "pass@8": 0.878659443062788, |
| "pass@16": 0.9053831900865986, |
| "pass@32": 0.9237568606242907, |
| "pass@64": 0.9352225250892545, |
| "pass@128": 0.9425981873111783 |
| }, |
| "20": { |
| "pass@1": 0.626196509009009, |
| "pass@2": 0.7125315522215124, |
| "pass@4": 0.7734252413268156, |
| "pass@8": 0.8190036582565382, |
| "pass@16": 0.8555370910100033, |
| "pass@32": 0.8847167374444287, |
| "pass@64": 0.9059201413232556, |
| "pass@128": 0.9219219219219219 |
| } |
| } |
| } |
| } |
| } |
| ] |
