| [ |
| { |
| "checkpoint": "checkpoint-38646", |
| "id_acc": 0.9932248263888889, |
| "ood_acc": 0.83598125, |
| "total_acc": 0.9104650493421053, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.3281736111111, |
| "ood_avg_resp_len": 378.03591796875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 735827, |
| "answer_accuracy": 0.9087335500316156, |
| "avg_response_len": 280.81283097533986, |
| "resp_tokens_sum": 227382012, |
| "pass_at_k": { |
| "pass@1": 0.9087335500316156, |
| "pass@2": 0.9306044600025811, |
| "pass@4": 0.9464375050603238, |
| "pass@8": 0.9578451948086137, |
| "pass@16": 0.9661519899120048, |
| "pass@32": 0.9723665816849368, |
| "pass@64": 0.9774353954902267, |
| "pass@128": 0.9818210559595321 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9643482669616519, |
| "pass@2": 0.9724997386941677, |
| "pass@4": 0.9787036025917114, |
| "pass@8": 0.9837184467724707, |
| "pass@16": 0.9876176547055514, |
| "pass@32": 0.990048743242238, |
| "pass@64": 0.9918928791392225, |
| "pass@128": 0.9941002949852508 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9998805428134556, |
| "pass@2": 0.9999977425413568, |
| "pass@4": 0.9999999997133384, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9967757936507936, |
| "pass@2": 0.9988044463192103, |
| "pass@4": 0.9997930303354938, |
| "pass@8": 0.9999940728357846, |
| "pass@16": 0.999999996928095, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9981521332335329, |
| "pass@2": 0.999078737210618, |
| "pass@4": 0.9997273029868273, |
| "pass@8": 0.9999777200738017, |
| "pass@16": 0.9999998973084382, |
| "pass@32": 0.999999999999679, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.998959874260355, |
| "pass@2": 0.9998143607603783, |
| "pass@4": 0.9999912155026479, |
| "pass@8": 0.9999999836752375, |
| "pass@16": 0.9999999999999905, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9960687099358975, |
| "pass@2": 0.9985512252675145, |
| "pass@4": 0.9996649123787412, |
| "pass@8": 0.9999730867256917, |
| "pass@16": 0.9999998586577866, |
| "pass@32": 0.9999999999993815, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9940801056338028, |
| "pass@2": 0.9950447765332151, |
| "pass@4": 0.9956578631896364, |
| "pass@8": 0.996375867480201, |
| "pass@16": 0.9969724240620078, |
| "pass@32": 0.9971729386969103, |
| "pass@64": 0.9971830959162251, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.9711623583569405, |
| "pass@2": 0.9805191998840088, |
| "pass@4": 0.986303336286222, |
| "pass@8": 0.989850641477996, |
| "pass@16": 0.99240868699634, |
| "pass@32": 0.994293234140082, |
| "pass@64": 0.9956686613210934, |
| "pass@128": 0.9971671388101983 |
| }, |
| "12": { |
| "pass@1": 0.9658368644067796, |
| "pass@2": 0.979869405889942, |
| "pass@4": 0.9899045508612269, |
| "pass@8": 0.9949535165900368, |
| "pass@16": 0.9969359713595375, |
| "pass@32": 0.9982218320963844, |
| "pass@64": 0.9992952648675736, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9498187695924765, |
| "pass@2": 0.9693292893639082, |
| "pass@4": 0.9815207164543299, |
| "pass@8": 0.9885514952344922, |
| "pass@16": 0.993030758542598, |
| "pass@32": 0.9957866897580077, |
| "pass@64": 0.9968002718717178, |
| "pass@128": 0.9968652037617555 |
| }, |
| "14": { |
| "pass@1": 0.9244548286604362, |
| "pass@2": 0.9464950971373904, |
| "pass@4": 0.962845547343965, |
| "pass@8": 0.9744775229835171, |
| "pass@16": 0.9831241464032126, |
| "pass@32": 0.9892894037314296, |
| "pass@64": 0.9925585252418944, |
| "pass@128": 0.9937694704049844 |
| }, |
| "15": { |
| "pass@1": 0.9763367052023122, |
| "pass@2": 0.9890843327568163, |
| "pass@4": 0.995195050131017, |
| "pass@8": 0.9987062171803248, |
| "pass@16": 0.9998947926322176, |
| "pass@32": 0.9999995163740522, |
| "pass@64": 0.9999999999997273, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8670362903225807, |
| "pass@2": 0.9042973710947418, |
| "pass@4": 0.9324377792493678, |
| "pass@8": 0.9511663793058849, |
| "pass@16": 0.9632457765868611, |
| "pass@32": 0.972380137786845, |
| "pass@64": 0.9803184368887754, |
| "pass@128": 0.9870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.8796739523121387, |
| "pass@2": 0.9130583354194169, |
| "pass@4": 0.9346565702950422, |
| "pass@8": 0.949022076996675, |
| "pass@16": 0.9577801252538675, |
| "pass@32": 0.9642535207391926, |
| "pass@64": 0.9711417638827387, |
| "pass@128": 0.976878612716763 |
| }, |
| "18": { |
| "pass@1": 0.7535491071428572, |
| "pass@2": 0.8142751687289084, |
| "pass@4": 0.854199288124698, |
| "pass@8": 0.8827058936508151, |
| "pass@16": 0.9034225204407647, |
| "pass@32": 0.9183155907983432, |
| "pass@64": 0.9304605804851537, |
| "pass@128": 0.9428571428571428 |
| }, |
| "19": { |
| "pass@1": 0.5902073948948949, |
| "pass@2": 0.6823130955758907, |
| "pass@4": 0.7493216109810598, |
| "pass@8": 0.7968110404034479, |
| "pass@16": 0.8325350334716398, |
| "pass@32": 0.859961150550141, |
| "pass@64": 0.8827473329655091, |
| "pass@128": 0.9009009009009009 |
| }, |
| "20": { |
| "pass@1": 0.4485735735735736, |
| "pass@2": 0.5452467723629141, |
| "pass@4": 0.6280490980294128, |
| "pass@8": 0.6976146762314849, |
| "pass@16": 0.7542370813489142, |
| "pass@32": 0.799241057750153, |
| "pass@64": 0.8368248092112288, |
| "pass@128": 0.8708708708708709 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 751099, |
| "answer_accuracy": 0.9147250097427904, |
| "avg_response_len": 290.0505114964926, |
| "resp_tokens_sum": 238166276, |
| "pass_at_k": { |
| "pass@1": 0.9147250097427904, |
| "pass@2": 0.9368544058892373, |
| "pass@4": 0.9526741754464987, |
| "pass@8": 0.9638412119281327, |
| "pass@16": 0.9716894634319847, |
| "pass@32": 0.9774218920691676, |
| "pass@64": 0.9817877251624777, |
| "pass@128": 0.9851909586905689 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9476131134969326, |
| "pass@2": 0.9578519002705184, |
| "pass@4": 0.9660574474778076, |
| "pass@8": 0.9719991236070697, |
| "pass@16": 0.9763583249372905, |
| "pass@32": 0.980656765738378, |
| "pass@64": 0.9852951841811889, |
| "pass@128": 0.99079754601227 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9982717803030303, |
| "pass@2": 0.9992185635886424, |
| "pass@4": 0.9998076788696866, |
| "pass@8": 0.9999893019044129, |
| "pass@16": 0.9999999789617312, |
| "pass@32": 0.9999999999999926, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999768860946746, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9955842391304348, |
| "pass@2": 0.9970807657195024, |
| "pass@4": 0.9974464667460045, |
| "pass@8": 0.9977680565376104, |
| "pass@16": 0.9983124881128989, |
| "pass@32": 0.999097406302473, |
| "pass@64": 0.9998273650576287, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9994627507163324, |
| "pass@2": 0.9999027028856349, |
| "pass@4": 0.9999971459513118, |
| "pass@8": 0.9999999985260104, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9984420158959537, |
| "pass@2": 0.9991658037868099, |
| "pass@4": 0.9997657894822396, |
| "pass@8": 0.999983097148883, |
| "pass@16": 0.9999999407613452, |
| "pass@32": 0.9999999999999076, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9972711267605634, |
| "pass@2": 0.9988802401020296, |
| "pass@4": 0.9997421626874106, |
| "pass@8": 0.9999832931204404, |
| "pass@16": 0.9999999422581453, |
| "pass@32": 0.99999999999991, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9965147829341318, |
| "pass@2": 0.9973931320430004, |
| "pass@4": 0.9978456220292824, |
| "pass@8": 0.998468082786387, |
| "pass@16": 0.9992530482905866, |
| "pass@32": 0.999851122735485, |
| "pass@64": 0.9999980007921191, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9768213478915663, |
| "pass@2": 0.982426507209942, |
| "pass@4": 0.9875867705919291, |
| "pass@8": 0.9916699109297928, |
| "pass@16": 0.9941198985224184, |
| "pass@32": 0.9952745122110707, |
| "pass@64": 0.9962408632343975, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.965492750783699, |
| "pass@2": 0.9797927517093281, |
| "pass@4": 0.9877445484870819, |
| "pass@8": 0.9923678770900078, |
| "pass@16": 0.9955227193860775, |
| "pass@32": 0.9973880317416322, |
| "pass@64": 0.9984300583339356, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9623599439775911, |
| "pass@2": 0.9779697804318578, |
| "pass@4": 0.9872410336838148, |
| "pass@8": 0.9920194344744313, |
| "pass@16": 0.9942491977157041, |
| "pass@32": 0.9950794358980206, |
| "pass@64": 0.9957983155432268, |
| "pass@128": 0.9971988795518207 |
| }, |
| "14": { |
| "pass@1": 0.9557291666666666, |
| "pass@2": 0.9763930618870013, |
| "pass@4": 0.9898799393496869, |
| "pass@8": 0.9971921128374414, |
| "pass@16": 0.9996866455629769, |
| "pass@32": 0.9999957420781709, |
| "pass@64": 0.9999999999150341, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9709043560606061, |
| "pass@2": 0.9843362264376043, |
| "pass@4": 0.9913319070911589, |
| "pass@8": 0.9957115636967746, |
| "pass@16": 0.9984277498659098, |
| "pass@32": 0.9997408383719552, |
| "pass@64": 0.9999955357422065, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.9107015669515669, |
| "pass@2": 0.9377124133521775, |
| "pass@4": 0.9564160948898481, |
| "pass@8": 0.9692630161138946, |
| "pass@16": 0.9776624723599159, |
| "pass@32": 0.9832541934776856, |
| "pass@64": 0.9874615955124632, |
| "pass@128": 0.9914529914529915 |
| }, |
| "17": { |
| "pass@1": 0.8672360248447205, |
| "pass@2": 0.9112839750330124, |
| "pass@4": 0.9387636921937549, |
| "pass@8": 0.9558194693071742, |
| "pass@16": 0.9657225480978907, |
| "pass@32": 0.9716417279156446, |
| "pass@64": 0.9759516660247486, |
| "pass@128": 0.9782608695652174 |
| }, |
| "18": { |
| "pass@1": 0.7839613970588235, |
| "pass@2": 0.8523347035664656, |
| "pass@4": 0.8947464600013231, |
| "pass@8": 0.9214377697528552, |
| "pass@16": 0.9388117402083257, |
| "pass@32": 0.951005420422579, |
| "pass@64": 0.9607119551856902, |
| "pass@128": 0.9705882352941176 |
| }, |
| "19": { |
| "pass@1": 0.5933547247023809, |
| "pass@2": 0.6915281830005624, |
| "pass@4": 0.766301561356169, |
| "pass@8": 0.8222817559953494, |
| "pass@16": 0.8649021620408612, |
| "pass@32": 0.8975421565193381, |
| "pass@64": 0.9221817078073432, |
| "pass@128": 0.9404761904761905 |
| }, |
| "20": { |
| "pass@1": 0.4492889221556886, |
| "pass@2": 0.5484648321467304, |
| "pass@4": 0.6333525578202419, |
| "pass@8": 0.7015890174857582, |
| "pass@16": 0.7546736568184237, |
| "pass@32": 0.796908044139665, |
| "pass@64": 0.829107778372377, |
| "pass@128": 0.8502994011976048 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 727325, |
| "answer_accuracy": 0.9078489475155775, |
| "avg_response_len": 310.94691519212336, |
| "resp_tokens_sum": 249115743, |
| "pass_at_k": { |
| "pass@1": 0.9078489475155775, |
| "pass@2": 0.9314184621389209, |
| "pass@4": 0.9479047528888704, |
| "pass@8": 0.959567070014469, |
| "pass@16": 0.9676303174059113, |
| "pass@32": 0.9731903126693562, |
| "pass@64": 0.9775192818766577, |
| "pass@128": 0.9816264579006231 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9642024253731343, |
| "pass@2": 0.9741138059701494, |
| "pass@4": 0.9819364163434796, |
| "pass@8": 0.9881400872944226, |
| "pass@16": 0.9925379015562921, |
| "pass@32": 0.9955510391342886, |
| "pass@64": 0.997754923360079, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9912913602941177, |
| "pass@2": 0.9954405974988421, |
| "pass@4": 0.9979701380158362, |
| "pass@8": 0.9993477897991202, |
| "pass@16": 0.9999126470598457, |
| "pass@32": 0.999998726614258, |
| "pass@64": 0.999999999987784, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999507097791798, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9933742088607594, |
| "pass@2": 0.9947567558307585, |
| "pass@4": 0.9957734057965857, |
| "pass@8": 0.9969989697228501, |
| "pass@16": 0.9982483505948174, |
| "pass@32": 0.9992648536542246, |
| "pass@64": 0.9999088007454459, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9931493993993994, |
| "pass@2": 0.9975064877869996, |
| "pass@4": 0.999478633301468, |
| "pass@8": 0.9999675398986609, |
| "pass@16": 0.9999998926590724, |
| "pass@32": 0.9999999999998089, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9930667202572347, |
| "pass@2": 0.9967814011190724, |
| "pass@4": 0.9985162387257863, |
| "pass@8": 0.9994198002264989, |
| "pass@16": 0.9999085821695214, |
| "pass@32": 0.999998609246854, |
| "pass@64": 0.9999999999866449, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.963640873015873, |
| "pass@2": 0.9752730127484065, |
| "pass@4": 0.9833094119187484, |
| "pass@8": 0.9885337281484384, |
| "pass@16": 0.9929572101573484, |
| "pass@32": 0.9967672908887808, |
| "pass@64": 0.9991148007644227, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9675793195718655, |
| "pass@2": 0.9831443087962632, |
| "pass@4": 0.9927234121537561, |
| "pass@8": 0.9962327927192676, |
| "pass@16": 0.9969067136543522, |
| "pass@32": 0.9969418396245668, |
| "pass@64": 0.9969418960244639, |
| "pass@128": 0.9969418960244648 |
| }, |
| "13": { |
| "pass@1": 0.953125, |
| "pass@2": 0.9724694243948675, |
| "pass@4": 0.9839528441467964, |
| "pass@8": 0.9914051147315478, |
| "pass@16": 0.9962751879157801, |
| "pass@32": 0.9989514073218897, |
| "pass@64": 0.9999069142994649, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9672663204747775, |
| "pass@2": 0.9819023224841703, |
| "pass@4": 0.9891655304029129, |
| "pass@8": 0.9926525053330164, |
| "pass@16": 0.9941520139636356, |
| "pass@32": 0.9947959873034944, |
| "pass@64": 0.9955489586054439, |
| "pass@128": 0.9970326409495549 |
| }, |
| "15": { |
| "pass@1": 0.9480372299382716, |
| "pass@2": 0.9670028768105379, |
| "pass@4": 0.9780621108704005, |
| "pass@8": 0.9842947803211456, |
| "pass@16": 0.9879716058879758, |
| "pass@32": 0.9901270313479683, |
| "pass@64": 0.9919070327284573, |
| "pass@128": 0.9938271604938271 |
| }, |
| "16": { |
| "pass@1": 0.882973820058997, |
| "pass@2": 0.9149514842171274, |
| "pass@4": 0.9361787777080962, |
| "pass@8": 0.9505317245196182, |
| "pass@16": 0.95963740909398, |
| "pass@32": 0.9664879570651409, |
| "pass@64": 0.9730817511351202, |
| "pass@128": 0.9793510324483776 |
| }, |
| "17": { |
| "pass@1": 0.8233951430722891, |
| "pass@2": 0.8709084616023144, |
| "pass@4": 0.900955382553837, |
| "pass@8": 0.920503593025735, |
| "pass@16": 0.9343316545762004, |
| "pass@32": 0.9442573518899536, |
| "pass@64": 0.9519856298182777, |
| "pass@128": 0.9608433734939759 |
| }, |
| "18": { |
| "pass@1": 0.7824344758064516, |
| "pass@2": 0.8496388430276858, |
| "pass@4": 0.8933080521184847, |
| "pass@8": 0.9212389467048724, |
| "pass@16": 0.9380265394631339, |
| "pass@32": 0.9476456964449388, |
| "pass@64": 0.9534177140834851, |
| "pass@128": 0.9580645161290322 |
| }, |
| "19": { |
| "pass@1": 0.5686367069486404, |
| "pass@2": 0.6704094012417636, |
| "pass@4": 0.7465739156094909, |
| "pass@8": 0.8053303353516325, |
| "pass@16": 0.8499870368405819, |
| "pass@32": 0.8823623062494177, |
| "pass@64": 0.907953968913336, |
| "pass@128": 0.9305135951661632 |
| }, |
| "20": { |
| "pass@1": 0.4545561186186186, |
| "pass@2": 0.5521487284528623, |
| "pass@4": 0.6327929909662195, |
| "pass@8": 0.6981138753732286, |
| "pass@16": 0.7452411820752879, |
| "pass@32": 0.7785809400350618, |
| "pass@64": 0.8062433242479462, |
| "pass@128": 0.8348348348348348 |
| } |
| } |
| } |
| } |
| } |
| ] |
