| [ |
| { |
| "checkpoint": "checkpoint-38796", |
| "id_acc": 0.9916449652777778, |
| "ood_acc": 0.88128359375, |
| "total_acc": 0.9335600328947369, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.28180034722223, |
| "ood_avg_resp_len": 381.877265625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 754568, |
| "answer_accuracy": 0.9318783591527031, |
| "avg_response_len": 282.8403031635315, |
| "resp_tokens_sum": 229023713, |
| "pass_at_k": { |
| "pass@1": 0.9318783591527031, |
| "pass@2": 0.9517385910166891, |
| "pass@4": 0.964830907829337, |
| "pass@8": 0.9741300581165051, |
| "pass@16": 0.9808451032894651, |
| "pass@32": 0.9856391594646161, |
| "pass@64": 0.9891717827349858, |
| "pass@128": 0.9916218779639583 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9496450958702065, |
| "pass@2": 0.9618148706245786, |
| "pass@4": 0.9709195504323019, |
| "pass@8": 0.9791190874933853, |
| "pass@16": 0.9868342218489051, |
| "pass@32": 0.993298633445245, |
| "pass@64": 0.9981549989323758, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997849770642202, |
| "pass@2": 0.9999864552481399, |
| "pass@4": 0.9999999638806617, |
| "pass@8": 0.9999999999999809, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999755859375, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9954117063492064, |
| "pass@2": 0.9981478877640296, |
| "pass@4": 0.9995556263800359, |
| "pass@8": 0.9999615639259287, |
| "pass@16": 0.9999997091342684, |
| "pass@32": 0.9999999999966475, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997427020958084, |
| "pass@2": 0.999995948064501, |
| "pass@4": 0.9999999985967324, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9956083579881657, |
| "pass@2": 0.9976401976657503, |
| "pass@4": 0.9985032811172272, |
| "pass@8": 0.9992715079029393, |
| "pass@16": 0.9998386699365346, |
| "pass@32": 0.9999946547625592, |
| "pass@64": 0.9999999995148912, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9960186298076923, |
| "pass@2": 0.9985326916767614, |
| "pass@4": 0.9996494007960544, |
| "pass@8": 0.9999753031040152, |
| "pass@16": 0.9999998899899415, |
| "pass@32": 0.9999999999996564, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9917033450704226, |
| "pass@2": 0.9929716091826549, |
| "pass@4": 0.9939165605179635, |
| "pass@8": 0.9949288254759624, |
| "pass@16": 0.9958752495136213, |
| "pass@32": 0.9970100988051338, |
| "pass@64": 0.9984237773095228, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9635047804532578, |
| "pass@2": 0.977436093328277, |
| "pass@4": 0.9865923250386901, |
| "pass@8": 0.9927103321039095, |
| "pass@16": 0.9970124444229089, |
| "pass@32": 0.9993847243796702, |
| "pass@64": 0.9999852421618102, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9450918079096046, |
| "pass@2": 0.9655720338983056, |
| "pass@4": 0.9781990243804268, |
| "pass@8": 0.9861440250122284, |
| "pass@16": 0.9923112374071664, |
| "pass@32": 0.9969355017981401, |
| "pass@64": 0.9992276493263522, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9179075235109718, |
| "pass@2": 0.9502324099671706, |
| "pass@4": 0.9729623692218725, |
| "pass@8": 0.9865352063542692, |
| "pass@16": 0.9929416594162586, |
| "pass@32": 0.9950239922483407, |
| "pass@64": 0.9960875780112477, |
| "pass@128": 0.9968652037617555 |
| }, |
| "14": { |
| "pass@1": 0.9201956775700935, |
| "pass@2": 0.9446224121225502, |
| "pass@4": 0.9626010729383125, |
| "pass@8": 0.9762203698107813, |
| "pass@16": 0.986550455857625, |
| "pass@32": 0.9938274615772782, |
| "pass@64": 0.9980046339203555, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9622696893063584, |
| "pass@2": 0.9790736226161763, |
| "pass@4": 0.9878219928975642, |
| "pass@8": 0.9938863424829786, |
| "pass@16": 0.9974854612936562, |
| "pass@32": 0.9990553826707531, |
| "pass@64": 0.9998278336678724, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8738155241935484, |
| "pass@2": 0.9108982092964184, |
| "pass@4": 0.9368475640141757, |
| "pass@8": 0.9526611840974472, |
| "pass@16": 0.9624771254488251, |
| "pass@32": 0.9706463781509711, |
| "pass@64": 0.97815086262173, |
| "pass@128": 0.9838709677419355 |
| }, |
| "17": { |
| "pass@1": 0.8999954841040463, |
| "pass@2": 0.9269427597742481, |
| "pass@4": 0.9442069857777889, |
| "pass@8": 0.9584880211486807, |
| "pass@16": 0.9699099356325508, |
| "pass@32": 0.977956592718777, |
| "pass@64": 0.9832164176739884, |
| "pass@128": 0.9884393063583815 |
| }, |
| "18": { |
| "pass@1": 0.8340625, |
| "pass@2": 0.875173298650169, |
| "pass@4": 0.9003785883014621, |
| "pass@8": 0.9191113837595276, |
| "pass@16": 0.9347451165816725, |
| "pass@32": 0.9484697556158473, |
| "pass@64": 0.9606376972907202, |
| "pass@128": 0.9685714285714285 |
| }, |
| "19": { |
| "pass@1": 0.7546218093093093, |
| "pass@2": 0.8168980988862881, |
| "pass@4": 0.8594574675913256, |
| "pass@8": 0.8903657440259747, |
| "pass@16": 0.9127998493831533, |
| "pass@32": 0.9301485120186141, |
| "pass@64": 0.9444138248825593, |
| "pass@128": 0.954954954954955 |
| }, |
| "20": { |
| "pass@1": 0.7120167042042042, |
| "pass@2": 0.7915602610484499, |
| "pass@4": 0.8442564102347557, |
| "pass@8": 0.8824961305644767, |
| "pass@16": 0.9097409929460565, |
| "pass@32": 0.927053074881089, |
| "pass@64": 0.9392787772648254, |
| "pass@128": 0.948948948948949 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 769502, |
| "answer_accuracy": 0.9371370810600156, |
| "avg_response_len": 292.1462916504287, |
| "resp_tokens_sum": 239887163, |
| "pass_at_k": { |
| "pass@1": 0.9371370810600156, |
| "pass@2": 0.9563403509552427, |
| "pass@4": 0.9692129536808676, |
| "pass@8": 0.9783119434671995, |
| "pass@16": 0.984564498023125, |
| "pass@32": 0.9886527260310264, |
| "pass@64": 0.9913362234234722, |
| "pass@128": 0.9934528448947778 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9329706671779141, |
| "pass@2": 0.9490174085792956, |
| "pass@4": 0.9641241520423445, |
| "pass@8": 0.976390759682789, |
| "pass@16": 0.9846418367918528, |
| "pass@32": 0.9893313200761069, |
| "pass@64": 0.9929169376529189, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995265151515151, |
| "pass@2": 0.9999742752326415, |
| "pass@4": 0.9999999304064264, |
| "pass@8": 0.9999999999999045, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9985207100591716, |
| "pass@2": 0.9996389134790105, |
| "pass@4": 0.9999800542683642, |
| "pass@8": 0.9999999564675484, |
| "pass@16": 0.999999999999962, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9973505434782609, |
| "pass@2": 0.9985510812507132, |
| "pass@4": 0.9993644259141521, |
| "pass@8": 0.9998696856114555, |
| "pass@16": 0.9999955044084027, |
| "pass@32": 0.9999999980554193, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9993508237822349, |
| "pass@2": 0.999962984793448, |
| "pass@4": 0.9999998469030911, |
| "pass@8": 0.9999999999990058, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9994806719653179, |
| "pass@2": 0.9999896881116016, |
| "pass@4": 0.9999999953943619, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.993155809859155, |
| "pass@2": 0.9950745813463459, |
| "pass@4": 0.9964881184746273, |
| "pass@8": 0.9976291893269761, |
| "pass@16": 0.9985375133295249, |
| "pass@32": 0.9993490121785924, |
| "pass@64": 0.9999188200065704, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9948774326347305, |
| "pass@2": 0.9968984275543402, |
| "pass@4": 0.9971880869868812, |
| "pass@8": 0.9973699249827601, |
| "pass@16": 0.9977102880852468, |
| "pass@32": 0.9983202885567448, |
| "pass@64": 0.9992573907303504, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9722797439759037, |
| "pass@2": 0.9806206864861019, |
| "pass@4": 0.987148461675724, |
| "pass@8": 0.992097089748973, |
| "pass@16": 0.9952356342084575, |
| "pass@32": 0.996694117094797, |
| "pass@64": 0.9969831820276938, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9480309561128527, |
| "pass@2": 0.9714501363759779, |
| "pass@4": 0.9865298393343465, |
| "pass@8": 0.9942286337372813, |
| "pass@16": 0.9978628049514008, |
| "pass@32": 0.9996281644114471, |
| "pass@64": 0.9999956190522251, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9407169117647058, |
| "pass@2": 0.9625553469419262, |
| "pass@4": 0.9774801687079033, |
| "pass@8": 0.9874219685475781, |
| "pass@16": 0.9934829131966962, |
| "pass@32": 0.9968338780653557, |
| "pass@64": 0.9985126025262858, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9497669956140351, |
| "pass@2": 0.9702634312980611, |
| "pass@4": 0.9843248648195289, |
| "pass@8": 0.9936793355130432, |
| "pass@16": 0.9984837564707297, |
| "pass@32": 0.999880078009628, |
| "pass@64": 0.999999559195243, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9554214015151515, |
| "pass@2": 0.9778524665950848, |
| "pass@4": 0.990172612230289, |
| "pass@8": 0.9965689152787764, |
| "pass@16": 0.9990389896488282, |
| "pass@32": 0.9998467083150427, |
| "pass@64": 0.9999979765261304, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8984820156695157, |
| "pass@2": 0.9263237628149055, |
| "pass@4": 0.9466511985147156, |
| "pass@8": 0.9628916229912796, |
| "pass@16": 0.9756373410907596, |
| "pass@32": 0.9843113854696494, |
| "pass@64": 0.9896611343512275, |
| "pass@128": 0.9943019943019943 |
| }, |
| "17": { |
| "pass@1": 0.8989955357142857, |
| "pass@2": 0.9258540372670809, |
| "pass@4": 0.9430381279009376, |
| "pass@8": 0.9567724437885808, |
| "pass@16": 0.9678264048731167, |
| "pass@32": 0.975789443612367, |
| "pass@64": 0.9804645618033802, |
| "pass@128": 0.984472049689441 |
| }, |
| "18": { |
| "pass@1": 0.8785386029411765, |
| "pass@2": 0.9263135421491432, |
| "pass@4": 0.9540938742032239, |
| "pass@8": 0.970755896754488, |
| "pass@16": 0.9808158406179858, |
| "pass@32": 0.9865832611536812, |
| "pass@64": 0.989914086501943, |
| "pass@128": 0.9911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.7576497395833334, |
| "pass@2": 0.8213040752718409, |
| "pass@4": 0.8629211197930617, |
| "pass@8": 0.8939620372022703, |
| "pass@16": 0.9169075579630253, |
| "pass@32": 0.9325509025737705, |
| "pass@64": 0.94412425958293, |
| "pass@128": 0.9553571428571429 |
| }, |
| "20": { |
| "pass@1": 0.6838978293413174, |
| "pass@2": 0.7640517439530387, |
| "pass@4": 0.8220046661457729, |
| "pass@8": 0.8655627671390692, |
| "pass@16": 0.8983598878087751, |
| "pass@32": 0.9234805889172719, |
| "pass@64": 0.9421120203018302, |
| "pass@128": 0.9550898203592815 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 746348, |
| "answer_accuracy": 0.9315935053522927, |
| "avg_response_len": 312.82036117990094, |
| "resp_tokens_sum": 250616658, |
| "pass_at_k": { |
| "pass@1": 0.9315935053522927, |
| "pass@2": 0.9525782282961234, |
| "pass@4": 0.966021727662297, |
| "pass@8": 0.975328068054194, |
| "pass@16": 0.9821933995365425, |
| "pass@32": 0.9873189930516977, |
| "pass@64": 0.9910883906699982, |
| "pass@128": 0.9936092027480428 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9567863805970149, |
| "pass@2": 0.971851495475379, |
| "pass@4": 0.9827990326768856, |
| "pass@8": 0.9908164062722622, |
| "pass@16": 0.9960200237300442, |
| "pass@32": 0.9982169750293176, |
| "pass@64": 0.9992594797136242, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9989720394736842, |
| "pass@2": 0.9996438561956071, |
| "pass@4": 0.9999591621770964, |
| "pass@8": 0.9999995591583335, |
| "pass@16": 0.9999999999797444, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.99453125, |
| "pass@2": 0.9965507758221399, |
| "pass@4": 0.9972536858444163, |
| "pass@8": 0.9975790660148303, |
| "pass@16": 0.9980363206014791, |
| "pass@32": 0.9987689957872913, |
| "pass@64": 0.9996410375173692, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999014195583596, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9945361946202531, |
| "pass@2": 0.9958628712747932, |
| "pass@4": 0.9969610167399962, |
| "pass@8": 0.997929570922725, |
| "pass@16": 0.9987857390759178, |
| "pass@32": 0.9996009663236596, |
| "pass@64": 0.9999792027771768, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9935013138138138, |
| "pass@2": 0.9976191742924027, |
| "pass@4": 0.9995169683181494, |
| "pass@8": 0.9999713902369773, |
| "pass@16": 0.9999998669378484, |
| "pass@32": 0.9999999999994204, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9927150321543409, |
| "pass@2": 0.9953299459452619, |
| "pass@4": 0.9969358378032327, |
| "pass@8": 0.997933512235023, |
| "pass@16": 0.9987673566627887, |
| "pass@32": 0.9995945511170884, |
| "pass@64": 0.9999788684166812, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9630704365079366, |
| "pass@2": 0.9776848206474192, |
| "pass@4": 0.9883574401414106, |
| "pass@8": 0.9946295131968437, |
| "pass@16": 0.997513997070458, |
| "pass@32": 0.9989501368011887, |
| "pass@64": 0.9998107266450182, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9505208333333334, |
| "pass@2": 0.9706150370825212, |
| "pass@4": 0.9850789382405178, |
| "pass@8": 0.994133402138561, |
| "pass@16": 0.9983903509266632, |
| "pass@32": 0.9997473498922584, |
| "pass@64": 0.9999954976826797, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9122058256172839, |
| "pass@2": 0.9404311114756491, |
| "pass@4": 0.9606684896448127, |
| "pass@8": 0.9762644289391496, |
| "pass@16": 0.9877580918111676, |
| "pass@32": 0.9948566354295425, |
| "pass@64": 0.9986678916722411, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.955628709198813, |
| "pass@2": 0.9747088132900301, |
| "pass@4": 0.9841755971156422, |
| "pass@8": 0.9898190798079135, |
| "pass@16": 0.9938872827202818, |
| "pass@32": 0.9970027263205429, |
| "pass@64": 0.99907716563192, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9384645061728395, |
| "pass@2": 0.9601393141829493, |
| "pass@4": 0.9720348755595365, |
| "pass@8": 0.981293627330329, |
| "pass@16": 0.9892875060551926, |
| "pass@32": 0.9937818556587116, |
| "pass@64": 0.995354403432346, |
| "pass@128": 0.9969135802469136 |
| }, |
| "16": { |
| "pass@1": 0.9030927359882006, |
| "pass@2": 0.9313183024411773, |
| "pass@4": 0.949399185168225, |
| "pass@8": 0.9638089071611805, |
| "pass@16": 0.9753050149460608, |
| "pass@32": 0.9843716916440536, |
| "pass@64": 0.9918881750232778, |
| "pass@128": 0.9970501474926253 |
| }, |
| "17": { |
| "pass@1": 0.8378200301204819, |
| "pass@2": 0.8722340147993546, |
| "pass@4": 0.8994627833606642, |
| "pass@8": 0.921389194949213, |
| "pass@16": 0.9387280323416873, |
| "pass@32": 0.953114829081259, |
| "pass@64": 0.9652418490909267, |
| "pass@128": 0.9759036144578314 |
| }, |
| "18": { |
| "pass@1": 0.8834929435483871, |
| "pass@2": 0.9286580041910086, |
| "pass@4": 0.9520407616386656, |
| "pass@8": 0.966049497344724, |
| "pass@16": 0.9765995183921604, |
| "pass@32": 0.9846246388700052, |
| "pass@64": 0.989084716853413, |
| "pass@128": 0.9903225806451613 |
| }, |
| "19": { |
| "pass@1": 0.7401812688821753, |
| "pass@2": 0.8118499030615883, |
| "pass@4": 0.8589915267389155, |
| "pass@8": 0.8922240120160714, |
| "pass@16": 0.9169443378799137, |
| "pass@32": 0.9360698853549126, |
| "pass@64": 0.9515090838923458, |
| "pass@128": 0.9607250755287009 |
| }, |
| "20": { |
| "pass@1": 0.6837227852852853, |
| "pass@2": 0.7745815451278046, |
| "pass@4": 0.8314440877885758, |
| "pass@8": 0.8682970775356633, |
| "pass@16": 0.8966446199160057, |
| "pass@32": 0.92130107609107, |
| "pass@64": 0.9418564963876911, |
| "pass@128": 0.9579579579579579 |
| } |
| } |
| } |
| } |
| } |
| ] |
