[ { "checkpoint": "checkpoint-38646", "id_acc": 0.9932248263888889, "ood_acc": 0.83598125, "total_acc": 0.9104650493421053, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.3281736111111, "ood_avg_resp_len": 378.03591796875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 735827, "answer_accuracy": 0.9087335500316156, "avg_response_len": 280.81283097533986, "resp_tokens_sum": 227382012, "pass_at_k": { "pass@1": 0.9087335500316156, "pass@2": 0.9306044600025811, "pass@4": 0.9464375050603238, "pass@8": 0.9578451948086137, "pass@16": 0.9661519899120048, "pass@32": 0.9723665816849368, "pass@64": 0.9774353954902267, "pass@128": 0.9818210559595321 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9643482669616519, "pass@2": 0.9724997386941677, "pass@4": 0.9787036025917114, "pass@8": 0.9837184467724707, "pass@16": 0.9876176547055514, "pass@32": 0.990048743242238, "pass@64": 0.9918928791392225, "pass@128": 0.9941002949852508 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998805428134556, "pass@2": 0.9999977425413568, "pass@4": 0.9999999997133384, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9967757936507936, "pass@2": 0.9988044463192103, "pass@4": 0.9997930303354938, "pass@8": 0.9999940728357846, "pass@16": 0.999999996928095, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9981521332335329, "pass@2": 0.999078737210618, "pass@4": 0.9997273029868273, "pass@8": 0.9999777200738017, "pass@16": 0.9999998973084382, "pass@32": 0.999999999999679, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.998959874260355, "pass@2": 0.9998143607603783, "pass@4": 0.9999912155026479, "pass@8": 0.9999999836752375, "pass@16": 0.9999999999999905, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9960687099358975, "pass@2": 0.9985512252675145, "pass@4": 0.9996649123787412, "pass@8": 0.9999730867256917, "pass@16": 0.9999998586577866, "pass@32": 0.9999999999993815, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9940801056338028, "pass@2": 0.9950447765332151, "pass@4": 0.9956578631896364, "pass@8": 0.996375867480201, "pass@16": 0.9969724240620078, "pass@32": 0.9971729386969103, "pass@64": 0.9971830959162251, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.9711623583569405, "pass@2": 0.9805191998840088, "pass@4": 0.986303336286222, "pass@8": 0.989850641477996, "pass@16": 0.99240868699634, "pass@32": 0.994293234140082, "pass@64": 0.9956686613210934, "pass@128": 0.9971671388101983 }, "12": { "pass@1": 0.9658368644067796, "pass@2": 0.979869405889942, "pass@4": 0.9899045508612269, "pass@8": 0.9949535165900368, "pass@16": 0.9969359713595375, "pass@32": 0.9982218320963844, "pass@64": 0.9992952648675736, "pass@128": 1.0 }, "13": { "pass@1": 0.9498187695924765, "pass@2": 0.9693292893639082, "pass@4": 0.9815207164543299, "pass@8": 0.9885514952344922, "pass@16": 0.993030758542598, "pass@32": 0.9957866897580077, "pass@64": 0.9968002718717178, "pass@128": 0.9968652037617555 }, "14": { "pass@1": 0.9244548286604362, "pass@2": 0.9464950971373904, "pass@4": 0.962845547343965, "pass@8": 0.9744775229835171, "pass@16": 0.9831241464032126, "pass@32": 0.9892894037314296, "pass@64": 0.9925585252418944, "pass@128": 0.9937694704049844 }, "15": { "pass@1": 0.9763367052023122, "pass@2": 0.9890843327568163, "pass@4": 0.995195050131017, "pass@8": 0.9987062171803248, "pass@16": 0.9998947926322176, "pass@32": 0.9999995163740522, "pass@64": 0.9999999999997273, "pass@128": 1.0 }, "16": { "pass@1": 0.8670362903225807, "pass@2": 0.9042973710947418, "pass@4": 0.9324377792493678, "pass@8": 0.9511663793058849, "pass@16": 0.9632457765868611, "pass@32": 0.972380137786845, "pass@64": 0.9803184368887754, "pass@128": 0.9870967741935484 }, "17": { "pass@1": 0.8796739523121387, "pass@2": 0.9130583354194169, "pass@4": 0.9346565702950422, "pass@8": 0.949022076996675, "pass@16": 0.9577801252538675, "pass@32": 0.9642535207391926, "pass@64": 0.9711417638827387, "pass@128": 0.976878612716763 }, "18": { "pass@1": 0.7535491071428572, "pass@2": 0.8142751687289084, "pass@4": 0.854199288124698, "pass@8": 0.8827058936508151, "pass@16": 0.9034225204407647, "pass@32": 0.9183155907983432, "pass@64": 0.9304605804851537, "pass@128": 0.9428571428571428 }, "19": { "pass@1": 0.5902073948948949, "pass@2": 0.6823130955758907, "pass@4": 0.7493216109810598, "pass@8": 0.7968110404034479, "pass@16": 0.8325350334716398, "pass@32": 0.859961150550141, "pass@64": 0.8827473329655091, "pass@128": 0.9009009009009009 }, "20": { "pass@1": 0.4485735735735736, "pass@2": 0.5452467723629141, "pass@4": 0.6280490980294128, "pass@8": 0.6976146762314849, "pass@16": 0.7542370813489142, "pass@32": 0.799241057750153, "pass@64": 0.8368248092112288, "pass@128": 0.8708708708708709 } } }, "teachers_in_school": { "count": 821120, "correct": 751099, "answer_accuracy": 0.9147250097427904, "avg_response_len": 290.0505114964926, "resp_tokens_sum": 238166276, "pass_at_k": { "pass@1": 0.9147250097427904, "pass@2": 0.9368544058892373, "pass@4": 0.9526741754464987, "pass@8": 0.9638412119281327, "pass@16": 0.9716894634319847, "pass@32": 0.9774218920691676, "pass@64": 0.9817877251624777, "pass@128": 0.9851909586905689 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9476131134969326, "pass@2": 0.9578519002705184, "pass@4": 0.9660574474778076, "pass@8": 0.9719991236070697, "pass@16": 0.9763583249372905, "pass@32": 0.980656765738378, "pass@64": 0.9852951841811889, "pass@128": 0.99079754601227 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9982717803030303, "pass@2": 0.9992185635886424, "pass@4": 0.9998076788696866, "pass@8": 0.9999893019044129, "pass@16": 0.9999999789617312, "pass@32": 0.9999999999999926, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999768860946746, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9955842391304348, "pass@2": 0.9970807657195024, "pass@4": 0.9974464667460045, "pass@8": 0.9977680565376104, "pass@16": 0.9983124881128989, "pass@32": 0.999097406302473, "pass@64": 0.9998273650576287, "pass@128": 1.0 }, "6": { "pass@1": 0.9994627507163324, "pass@2": 0.9999027028856349, "pass@4": 0.9999971459513118, "pass@8": 0.9999999985260104, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9984420158959537, "pass@2": 0.9991658037868099, "pass@4": 0.9997657894822396, "pass@8": 0.999983097148883, "pass@16": 0.9999999407613452, "pass@32": 0.9999999999999076, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9972711267605634, "pass@2": 0.9988802401020296, "pass@4": 0.9997421626874106, "pass@8": 0.9999832931204404, "pass@16": 0.9999999422581453, "pass@32": 0.99999999999991, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9965147829341318, "pass@2": 0.9973931320430004, "pass@4": 0.9978456220292824, "pass@8": 0.998468082786387, "pass@16": 0.9992530482905866, "pass@32": 0.999851122735485, "pass@64": 0.9999980007921191, "pass@128": 1.0 }, "11": { "pass@1": 0.9768213478915663, "pass@2": 0.982426507209942, "pass@4": 0.9875867705919291, "pass@8": 0.9916699109297928, "pass@16": 0.9941198985224184, "pass@32": 0.9952745122110707, "pass@64": 0.9962408632343975, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.965492750783699, "pass@2": 0.9797927517093281, "pass@4": 0.9877445484870819, "pass@8": 0.9923678770900078, "pass@16": 0.9955227193860775, "pass@32": 0.9973880317416322, "pass@64": 0.9984300583339356, "pass@128": 1.0 }, "13": { "pass@1": 0.9623599439775911, "pass@2": 0.9779697804318578, "pass@4": 0.9872410336838148, "pass@8": 0.9920194344744313, "pass@16": 0.9942491977157041, "pass@32": 0.9950794358980206, "pass@64": 0.9957983155432268, "pass@128": 0.9971988795518207 }, "14": { "pass@1": 0.9557291666666666, "pass@2": 0.9763930618870013, "pass@4": 0.9898799393496869, "pass@8": 0.9971921128374414, "pass@16": 0.9996866455629769, "pass@32": 0.9999957420781709, "pass@64": 0.9999999999150341, "pass@128": 1.0 }, "15": { "pass@1": 0.9709043560606061, "pass@2": 0.9843362264376043, "pass@4": 0.9913319070911589, "pass@8": 0.9957115636967746, "pass@16": 0.9984277498659098, "pass@32": 0.9997408383719552, "pass@64": 0.9999955357422065, "pass@128": 1.0 }, "16": { "pass@1": 0.9107015669515669, "pass@2": 0.9377124133521775, "pass@4": 0.9564160948898481, "pass@8": 0.9692630161138946, "pass@16": 0.9776624723599159, "pass@32": 0.9832541934776856, "pass@64": 0.9874615955124632, "pass@128": 0.9914529914529915 }, "17": { "pass@1": 0.8672360248447205, "pass@2": 0.9112839750330124, "pass@4": 0.9387636921937549, "pass@8": 0.9558194693071742, "pass@16": 0.9657225480978907, "pass@32": 0.9716417279156446, "pass@64": 0.9759516660247486, "pass@128": 0.9782608695652174 }, "18": { "pass@1": 0.7839613970588235, "pass@2": 0.8523347035664656, "pass@4": 0.8947464600013231, "pass@8": 0.9214377697528552, "pass@16": 0.9388117402083257, "pass@32": 0.951005420422579, "pass@64": 0.9607119551856902, "pass@128": 0.9705882352941176 }, "19": { "pass@1": 0.5933547247023809, "pass@2": 0.6915281830005624, "pass@4": 0.766301561356169, "pass@8": 0.8222817559953494, "pass@16": 0.8649021620408612, "pass@32": 0.8975421565193381, "pass@64": 0.9221817078073432, "pass@128": 0.9404761904761905 }, "20": { "pass@1": 0.4492889221556886, "pass@2": 0.5484648321467304, "pass@4": 0.6333525578202419, "pass@8": 0.7015890174857582, "pass@16": 0.7546736568184237, "pass@32": 0.796908044139665, "pass@64": 0.829107778372377, "pass@128": 0.8502994011976048 } } }, "movie_festival_awards": { "count": 801152, "correct": 727325, "answer_accuracy": 0.9078489475155775, "avg_response_len": 310.94691519212336, "resp_tokens_sum": 249115743, "pass_at_k": { "pass@1": 0.9078489475155775, "pass@2": 0.9314184621389209, "pass@4": 0.9479047528888704, "pass@8": 0.959567070014469, "pass@16": 0.9676303174059113, "pass@32": 0.9731903126693562, "pass@64": 0.9775192818766577, "pass@128": 0.9816264579006231 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9642024253731343, "pass@2": 0.9741138059701494, "pass@4": 0.9819364163434796, "pass@8": 0.9881400872944226, "pass@16": 0.9925379015562921, "pass@32": 0.9955510391342886, "pass@64": 0.997754923360079, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9912913602941177, "pass@2": 0.9954405974988421, "pass@4": 0.9979701380158362, "pass@8": 0.9993477897991202, "pass@16": 0.9999126470598457, "pass@32": 0.999998726614258, "pass@64": 0.999999999987784, "pass@128": 1.0 }, "6": { "pass@1": 0.9999507097791798, "pass@2": 0.999999611888025, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9933742088607594, "pass@2": 0.9947567558307585, "pass@4": 0.9957734057965857, "pass@8": 0.9969989697228501, "pass@16": 0.9982483505948174, "pass@32": 0.9992648536542246, "pass@64": 0.9999088007454459, "pass@128": 1.0 }, "8": { "pass@1": 0.9931493993993994, "pass@2": 0.9975064877869996, "pass@4": 0.999478633301468, "pass@8": 0.9999675398986609, "pass@16": 0.9999998926590724, "pass@32": 0.9999999999998089, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9930667202572347, "pass@2": 0.9967814011190724, "pass@4": 0.9985162387257863, "pass@8": 0.9994198002264989, "pass@16": 0.9999085821695214, "pass@32": 0.999998609246854, "pass@64": 0.9999999999866449, "pass@128": 1.0 }, "11": { "pass@1": 0.963640873015873, "pass@2": 0.9752730127484065, "pass@4": 0.9833094119187484, "pass@8": 0.9885337281484384, "pass@16": 0.9929572101573484, "pass@32": 0.9967672908887808, "pass@64": 0.9991148007644227, "pass@128": 1.0 }, "12": { "pass@1": 0.9675793195718655, "pass@2": 0.9831443087962632, "pass@4": 0.9927234121537561, "pass@8": 0.9962327927192676, "pass@16": 0.9969067136543522, "pass@32": 0.9969418396245668, "pass@64": 0.9969418960244639, "pass@128": 0.9969418960244648 }, "13": { "pass@1": 0.953125, "pass@2": 0.9724694243948675, "pass@4": 0.9839528441467964, "pass@8": 0.9914051147315478, "pass@16": 0.9962751879157801, "pass@32": 0.9989514073218897, "pass@64": 0.9999069142994649, "pass@128": 1.0 }, "14": { "pass@1": 0.9672663204747775, "pass@2": 0.9819023224841703, "pass@4": 0.9891655304029129, "pass@8": 0.9926525053330164, "pass@16": 0.9941520139636356, "pass@32": 0.9947959873034944, "pass@64": 0.9955489586054439, "pass@128": 0.9970326409495549 }, "15": { "pass@1": 0.9480372299382716, "pass@2": 0.9670028768105379, "pass@4": 0.9780621108704005, "pass@8": 0.9842947803211456, "pass@16": 0.9879716058879758, "pass@32": 0.9901270313479683, "pass@64": 0.9919070327284573, "pass@128": 0.9938271604938271 }, "16": { "pass@1": 0.882973820058997, "pass@2": 0.9149514842171274, "pass@4": 0.9361787777080962, "pass@8": 0.9505317245196182, "pass@16": 0.95963740909398, "pass@32": 0.9664879570651409, "pass@64": 0.9730817511351202, "pass@128": 0.9793510324483776 }, "17": { "pass@1": 0.8233951430722891, "pass@2": 0.8709084616023144, "pass@4": 0.900955382553837, "pass@8": 0.920503593025735, "pass@16": 0.9343316545762004, "pass@32": 0.9442573518899536, "pass@64": 0.9519856298182777, "pass@128": 0.9608433734939759 }, "18": { "pass@1": 0.7824344758064516, "pass@2": 0.8496388430276858, "pass@4": 0.8933080521184847, "pass@8": 0.9212389467048724, "pass@16": 0.9380265394631339, "pass@32": 0.9476456964449388, "pass@64": 0.9534177140834851, "pass@128": 0.9580645161290322 }, "19": { "pass@1": 0.5686367069486404, "pass@2": 0.6704094012417636, "pass@4": 0.7465739156094909, "pass@8": 0.8053303353516325, "pass@16": 0.8499870368405819, "pass@32": 0.8823623062494177, "pass@64": 0.907953968913336, "pass@128": 0.9305135951661632 }, "20": { "pass@1": 0.4545561186186186, "pass@2": 0.5521487284528623, "pass@4": 0.6327929909662195, "pass@8": 0.6981138753732286, "pass@16": 0.7452411820752879, "pass@32": 0.7785809400350618, "pass@64": 0.8062433242479462, "pass@128": 0.8348348348348348 } } } } } ]