[ { "checkpoint": "checkpoint-38797", "id_acc": 0.9935989583333333, "ood_acc": 0.87967265625, "total_acc": 0.9336377467105264, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.3666579861111, "ood_avg_resp_len": 381.14332578125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 753646, "answer_accuracy": 0.930739705184951, "avg_response_len": 282.40589062006006, "resp_tokens_sum": 228671957, "pass_at_k": { "pass@1": 0.930739705184951, "pass@2": 0.9504902861518782, "pass@4": 0.96328378278217, "pass@8": 0.9720240684920053, "pass@16": 0.9780653667421171, "pass@32": 0.982247035735046, "pass@64": 0.9852938579685702, "pass@128": 0.9878280113815997 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9597852138643068, "pass@2": 0.9687187884700253, "pass@4": 0.9758312157882919, "pass@8": 0.9804771165839773, "pass@16": 0.9833711817465719, "pass@32": 0.9856575859333179, "pass@64": 0.9880229297421079, "pass@128": 0.9911504424778761 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999761085626911, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999267578125, "pass@2": 0.9999996155265748, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9988591269841269, "pass@2": 0.9998297087864016, "pass@4": 0.99999456436993, "pass@8": 0.9999999950675459, "pass@16": 0.9999999999999997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9997894835329342, "pass@2": 0.9999896859823659, "pass@4": 0.9999999803542521, "pass@8": 0.999999999999998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9998150887573964, "pass@2": 0.9999959960164002, "pass@4": 0.9999999986133391, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9969951923076923, "pass@2": 0.9986194446547546, "pass@4": 0.9994739897296492, "pass@8": 0.9999200984240966, "pass@16": 0.9999985668344405, "pass@32": 0.9999999998711727, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9940140845070422, "pass@2": 0.9958311384052345, "pass@4": 0.9975970291565667, "pass@8": 0.999027114688632, "pass@16": 0.9998061149829303, "pass@32": 0.9999927706810283, "pass@64": 0.9999999988812277, "pass@128": 1.0 }, "11": { "pass@1": 0.9672671742209632, "pass@2": 0.9771213696995386, "pass@4": 0.9849074195045728, "pass@8": 0.9911905328339699, "pass@16": 0.9959339037180583, "pass@32": 0.9988444618983644, "pass@64": 0.9999333712195624, "pass@128": 1.0 }, "12": { "pass@1": 0.9643140889830508, "pass@2": 0.9784611470705997, "pass@4": 0.9880650824367292, "pass@8": 0.9938518197176022, "pass@16": 0.9970047759063408, "pass@32": 0.9989130783745419, "pass@64": 0.9998300275998665, "pass@128": 1.0 }, "13": { "pass@1": 0.9476635971786834, "pass@2": 0.9685683453459383, "pass@4": 0.9822692721795356, "pass@8": 0.9899545292765576, "pass@16": 0.9938393300559409, "pass@32": 0.9961250032333974, "pass@64": 0.9968437006557012, "pass@128": 0.9968652037617555 }, "14": { "pass@1": 0.9345064252336449, "pass@2": 0.9570436106409598, "pass@4": 0.9730763464730461, "pass@8": 0.9835112588310111, "pass@16": 0.9902983995459784, "pass@32": 0.9947020849108975, "pass@64": 0.9978331269817698, "pass@128": 1.0 }, "15": { "pass@1": 0.9710531069364162, "pass@2": 0.985325116062082, "pass@4": 0.9926385077949068, "pass@8": 0.9958626814231997, "pass@16": 0.9968806051910766, "pass@32": 0.9970993941946336, "pass@64": 0.9971098238446818, "pass@128": 0.9971098265895953 }, "16": { "pass@1": 0.8755292338709677, "pass@2": 0.9045017621285244, "pass@4": 0.9260232168559573, "pass@8": 0.9427607291657083, "pass@16": 0.9550549739603138, "pass@32": 0.9636454926423886, "pass@64": 0.9701787434524745, "pass@128": 0.9741935483870968 }, "17": { "pass@1": 0.9177203757225434, "pass@2": 0.9430488627053847, "pass@4": 0.9555040751770186, "pass@8": 0.9635777719832805, "pass@16": 0.9682680044397302, "pass@32": 0.9713182119815579, "pass@64": 0.9747075450281047, "pass@128": 0.9797687861271677 }, "18": { "pass@1": 0.8287946428571429, "pass@2": 0.8780887935883015, "pass@4": 0.9042536249933039, "pass@8": 0.9200238424642594, "pass@16": 0.9320406770143738, "pass@32": 0.9415563598161735, "pass@64": 0.949386455264049, "pass@128": 0.9571428571428572 }, "19": { "pass@1": 0.7182338588588588, "pass@2": 0.7958574962758032, "pass@4": 0.8451148347447551, "pass@8": 0.8817489132436759, "pass@16": 0.9093400886784007, "pass@32": 0.9284539824296693, "pass@64": 0.9422262642306297, "pass@128": 0.9519519519519519 }, "20": { "pass@1": 0.6150290915915916, "pass@2": 0.7114929890520442, "pass@4": 0.7800601017215189, "pass@8": 0.8286590174245814, "pass@16": 0.8633336851559491, "pass@32": 0.8881871659264902, "pass@64": 0.9061995892225608, "pass@128": 0.9219219219219219 } } }, "teachers_in_school": { "count": 821120, "correct": 770562, "answer_accuracy": 0.9384280007794232, "avg_response_len": 292.01109094894775, "resp_tokens_sum": 239776147, "pass_at_k": { "pass@1": 0.9384280007794232, "pass@2": 0.956451549180363, "pass@4": 0.9678319202305449, "pass@8": 0.9757838381540438, "pass@16": 0.9816614862816057, "pass@32": 0.9862117376062051, "pass@64": 0.9895825026086122, "pass@128": 0.992049883086516 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9428920628834356, "pass@2": 0.9549670758659001, "pass@4": 0.9651627860167783, "pass@8": 0.9737301217465295, "pass@16": 0.9808736696825141, "pass@32": 0.9874380814030427, "pass@64": 0.993214871135638, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9990767045454545, "pass@2": 0.9998654109997613, "pass@4": 0.9999975227528377, "pass@8": 0.9999999995728038, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9996995192307693, "pass@2": 0.9999861680566557, "pass@4": 0.9999999650561431, "pass@8": 0.9999999999999813, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9968523550724637, "pass@2": 0.997473396667808, "pass@4": 0.9978423669867355, "pass@8": 0.998410038397661, "pass@16": 0.9991645323584537, "pass@32": 0.9998028560467962, "pass@64": 0.9999958123575, "pass@128": 1.0 }, "6": { "pass@1": 0.999955229226361, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9997064667630058, "pass@2": 0.9999911104410358, "pass@4": 0.9999999905178037, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9958406690140845, "pass@2": 0.9978024148829987, "pass@4": 0.999183896027081, "pass@8": 0.9998904310030774, "pass@16": 0.9999984671786155, "pass@32": 0.9999999999160536, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9956025449101796, "pass@2": 0.9974796961195719, "pass@4": 0.998901010689532, "pass@8": 0.999749379380219, "pass@16": 0.9999852183129254, "pass@32": 0.9999999733961134, "pass@64": 0.9999999999999997, "pass@128": 1.0 }, "11": { "pass@1": 0.9789156626506024, "pass@2": 0.9844413332463713, "pass@4": 0.9884038818378122, "pass@8": 0.991249363378623, "pass@16": 0.9938099352663854, "pass@32": 0.9964604084001034, "pass@64": 0.998383025431589, "pass@128": 1.0 }, "12": { "pass@1": 0.9606191222570533, "pass@2": 0.9744391460765678, "pass@4": 0.9834404432792294, "pass@8": 0.9887448737629764, "pass@16": 0.9917958042057292, "pass@32": 0.9937347694982218, "pass@64": 0.9952073750268757, "pass@128": 0.9968652037617555 }, "13": { "pass@1": 0.9641981792717087, "pass@2": 0.9810507372240239, "pass@4": 0.9900469389855678, "pass@8": 0.9942131738406755, "pass@16": 0.9966176001621115, "pass@32": 0.998631916393544, "pass@64": 0.9998381618554224, "pass@128": 1.0 }, "14": { "pass@1": 0.9547925804093568, "pass@2": 0.9742169872680388, "pass@4": 0.9865623400693333, "pass@8": 0.993446275946928, "pass@16": 0.9968301457719871, "pass@32": 0.9986083975630443, "pass@64": 0.9996411765505918, "pass@128": 1.0 }, "15": { "pass@1": 0.975, "pass@2": 0.9886505308995464, "pass@4": 0.9948184190044426, "pass@8": 0.9979302980220845, "pass@16": 0.9994682095514091, "pass@32": 0.999964528888759, "pass@64": 0.9999999629289098, "pass@128": 1.0 }, "16": { "pass@1": 0.9117699430199431, "pass@2": 0.934440336384234, "pass@4": 0.948974050786386, "pass@8": 0.9611631506970469, "pass@16": 0.9714023648928639, "pass@32": 0.9789083675308171, "pass@64": 0.9838666701804571, "pass@128": 0.9886039886039886 }, "17": { "pass@1": 0.9076814829192547, "pass@2": 0.9357003778060351, "pass@4": 0.9505333914976464, "pass@8": 0.9600694844908072, "pass@16": 0.9682785149952982, "pass@32": 0.9759899415602024, "pass@64": 0.9817656205726502, "pass@128": 0.984472049689441 }, "18": { "pass@1": 0.877734375, "pass@2": 0.9222795565076425, "pass@4": 0.9457145495599809, "pass@8": 0.9600628143280701, "pass@16": 0.9688410118307001, "pass@32": 0.9746411096104598, "pass@64": 0.9800388749808259, "pass@128": 0.9852941176470589 }, "19": { "pass@1": 0.7310732886904762, "pass@2": 0.801264954185414, "pass@4": 0.8486722577646543, "pass@8": 0.8836192387270729, "pass@16": 0.9101570834003377, "pass@32": 0.9306870627573393, "pass@64": 0.9444572773545391, "pass@128": 0.9523809523809523 }, "20": { "pass@1": 0.6313388847305389, "pass@2": 0.7228336142675276, "pass@4": 0.7860283833345681, "pass@8": 0.8339708116905777, "pass@16": 0.8714372379839609, "pass@32": 0.9009187710626614, "pass@64": 0.9239962462043453, "pass@128": 0.9431137724550899 } } }, "movie_festival_awards": { "count": 801152, "correct": 746399, "answer_accuracy": 0.9316571636842946, "avg_response_len": 312.3473984961655, "resp_tokens_sum": 250237743, "pass_at_k": { "pass@1": 0.9316571636842946, "pass@2": 0.9522591208816659, "pass@4": 0.9660390632689512, "pass@8": 0.9752605531481255, "pass@16": 0.9813263861482189, "pass@32": 0.9855967910692955, "pass@64": 0.9888762089231591, "pass@128": 0.9913724237098578 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9671641791044776, "pass@2": 0.9783556381478435, "pass@4": 0.9855664534470505, "pass@8": 0.990385174548389, "pass@16": 0.993735456511528, "pass@32": 0.9956450899133359, "pass@64": 0.9966503222335554, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9993146929824561, "pass@2": 0.999843512570797, "pass@4": 0.9999924886033983, "pass@8": 0.9999999880298076, "pass@16": 0.9999999999999953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9945082720588235, "pass@2": 0.9969256600277908, "pass@4": 0.9985720141599949, "pass@8": 0.9995595644810157, "pass@16": 0.9999494805984832, "pass@32": 0.9999995833702632, "pass@64": 0.9999999999993373, "pass@128": 1.0 }, "6": { "pass@1": 0.999728903785489, "pass@2": 0.9999786538413772, "pass@4": 0.9999999024175605, "pass@8": 0.999999999999636, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9974040743670886, "pass@2": 0.9984099347154392, "pass@4": 0.9992418159953106, "pass@8": 0.9998284242624276, "pass@16": 0.9999927222555876, "pass@32": 0.9999999948561573, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9942286036036037, "pass@2": 0.9979768153980753, "pass@4": 0.9997027497125922, "pass@8": 0.9999930237698637, "pass@16": 0.9999999974717585, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9958551045016077, "pass@2": 0.9980734296275666, "pass@4": 0.9994836422175525, "pass@8": 0.9999617290113357, "pass@16": 0.9999998358800019, "pass@32": 0.9999999999993716, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9709077380952381, "pass@2": 0.9824111829771282, "pass@4": 0.9899338889186468, "pass@8": 0.9945690319277134, "pass@16": 0.9971965257441671, "pass@32": 0.998626178234906, "pass@64": 0.9996125152017663, "pass@128": 1.0 }, "12": { "pass@1": 0.962753249235474, "pass@2": 0.9791324285439094, "pass@4": 0.9909401766292976, "pass@8": 0.9965767763240273, "pass@16": 0.9983801502774032, "pass@32": 0.9992932401756379, "pass@64": 0.9999118688144722, "pass@128": 1.0 }, "13": { "pass@1": 0.9471209490740741, "pass@2": 0.9679142212258193, "pass@4": 0.9816974689506403, "pass@8": 0.9890441820692925, "pass@16": 0.9926464233456719, "pass@32": 0.9954950306506924, "pass@64": 0.9982849094681752, "pass@128": 1.0 }, "14": { "pass@1": 0.9703727744807121, "pass@2": 0.9842764287717004, "pass@4": 0.9920016803352103, "pass@8": 0.9959930755335086, "pass@16": 0.9975902111414426, "pass@32": 0.998333788020753, "pass@64": 0.9992640014830058, "pass@128": 1.0 }, "15": { "pass@1": 0.9490981867283951, "pass@2": 0.9668741494118793, "pass@4": 0.9770455270058837, "pass@8": 0.984529369547853, "pass@16": 0.9903754117368537, "pass@32": 0.9948032445307928, "pass@64": 0.9978745376446474, "pass@128": 1.0 }, "16": { "pass@1": 0.8951880530973452, "pass@2": 0.9193018634009246, "pass@4": 0.9383454012385618, "pass@8": 0.9552424407044469, "pass@16": 0.9682220592151869, "pass@32": 0.9760900204372654, "pass@64": 0.9804815886005911, "pass@128": 0.9823008849557522 }, "17": { "pass@1": 0.8645990210843374, "pass@2": 0.8966012919789398, "pass@4": 0.9182864334164551, "pass@8": 0.9331278277256979, "pass@16": 0.9432639037778405, "pass@32": 0.9518367794011443, "pass@64": 0.9604398598478096, "pass@128": 0.9698795180722891 }, "18": { "pass@1": 0.8597782258064516, "pass@2": 0.9142490316230631, "pass@4": 0.9464857808096566, "pass@8": 0.9657417071551704, "pass@16": 0.9759620011314518, "pass@32": 0.9808473385701938, "pass@64": 0.9833911490980485, "pass@128": 0.9838709677419355 }, "19": { "pass@1": 0.7302445241691843, "pass@2": 0.8114428955444009, "pass@4": 0.8644937821216446, "pass@8": 0.9003745765960082, "pass@16": 0.9237132138775155, "pass@32": 0.9401485338678736, "pass@64": 0.9528595262393311, "pass@128": 0.9637462235649547 }, "20": { "pass@1": 0.6025713213213213, "pass@2": 0.7020823727270578, "pass@4": 0.7745611930828463, "pass@8": 0.8267509783091258, "pass@16": 0.8656985659860957, "pass@32": 0.8964622303358734, "pass@64": 0.9208544430514269, "pass@128": 0.93993993993994 } } } } } ]