| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 19.45945945945946, |
| "eval_steps": 500, |
| "global_step": 540, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.36036036036036034, |
| "grad_norm": 0.5300200581550598, |
| "learning_rate": 9.259259259259259e-07, |
| "logits/chosen": 1.7405741214752197, |
| "logits/rejected": 1.663368582725525, |
| "logps/chosen": -97.57881164550781, |
| "logps/rejected": -70.59793853759766, |
| "loss": 0.6938, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 0.0018835498485714197, |
| "rewards/margins": -0.0008800366776995361, |
| "rewards/rejected": 0.0027635858859866858, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.7207207207207207, |
| "grad_norm": 0.4883837401866913, |
| "learning_rate": 1.8518518518518519e-06, |
| "logits/chosen": 1.8280715942382812, |
| "logits/rejected": 1.8215343952178955, |
| "logps/chosen": -90.60624694824219, |
| "logps/rejected": -79.04981994628906, |
| "loss": 0.6945, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": 0.0014454321935772896, |
| "rewards/margins": -0.004174981266260147, |
| "rewards/rejected": 0.005620413459837437, |
| "step": 20 |
| }, |
| { |
| "epoch": 1.0810810810810811, |
| "grad_norm": 0.5554109215736389, |
| "learning_rate": 2.7777777777777783e-06, |
| "logits/chosen": 1.7980501651763916, |
| "logits/rejected": 1.841104507446289, |
| "logps/chosen": -80.78228759765625, |
| "logps/rejected": -85.08882141113281, |
| "loss": 0.693, |
| "rewards/accuracies": 0.4749999940395355, |
| "rewards/chosen": 0.0042568682692945, |
| "rewards/margins": -0.0038191028870642185, |
| "rewards/rejected": 0.008075973019003868, |
| "step": 30 |
| }, |
| { |
| "epoch": 1.4414414414414414, |
| "grad_norm": 0.537497341632843, |
| "learning_rate": 3.7037037037037037e-06, |
| "logits/chosen": 1.7320470809936523, |
| "logits/rejected": 1.7411377429962158, |
| "logps/chosen": -82.24813842773438, |
| "logps/rejected": -80.90709686279297, |
| "loss": 0.6952, |
| "rewards/accuracies": 0.42500001192092896, |
| "rewards/chosen": -0.00350201572291553, |
| "rewards/margins": -0.012091752141714096, |
| "rewards/rejected": 0.008589735254645348, |
| "step": 40 |
| }, |
| { |
| "epoch": 1.8018018018018018, |
| "grad_norm": 0.5023094415664673, |
| "learning_rate": 4.62962962962963e-06, |
| "logits/chosen": 1.8292573690414429, |
| "logits/rejected": 1.8632844686508179, |
| "logps/chosen": -85.98481750488281, |
| "logps/rejected": -86.14008331298828, |
| "loss": 0.6928, |
| "rewards/accuracies": 0.4124999940395355, |
| "rewards/chosen": 0.00168000184930861, |
| "rewards/margins": -0.011136507615447044, |
| "rewards/rejected": 0.01281650923192501, |
| "step": 50 |
| }, |
| { |
| "epoch": 2.1621621621621623, |
| "grad_norm": 0.4981901943683624, |
| "learning_rate": 4.998119881260576e-06, |
| "logits/chosen": 1.8536121845245361, |
| "logits/rejected": 1.7948782444000244, |
| "logps/chosen": -90.0439682006836, |
| "logps/rejected": -79.8309555053711, |
| "loss": 0.6914, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": 0.0025153260212391615, |
| "rewards/margins": -0.0021209525875747204, |
| "rewards/rejected": 0.004636278375983238, |
| "step": 60 |
| }, |
| { |
| "epoch": 2.5225225225225225, |
| "grad_norm": 0.6208468079566956, |
| "learning_rate": 4.9866405060165044e-06, |
| "logits/chosen": 1.7845121622085571, |
| "logits/rejected": 1.8721017837524414, |
| "logps/chosen": -74.6461181640625, |
| "logps/rejected": -98.91078186035156, |
| "loss": 0.694, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": 0.00022823773906566203, |
| "rewards/margins": 0.0024594543501734734, |
| "rewards/rejected": -0.00223121652379632, |
| "step": 70 |
| }, |
| { |
| "epoch": 2.8828828828828827, |
| "grad_norm": 0.5074446797370911, |
| "learning_rate": 4.964774158361991e-06, |
| "logits/chosen": 1.8522154092788696, |
| "logits/rejected": 1.8161392211914062, |
| "logps/chosen": -89.16864013671875, |
| "logps/rejected": -81.97349548339844, |
| "loss": 0.6911, |
| "rewards/accuracies": 0.5249999761581421, |
| "rewards/chosen": 0.006215038243681192, |
| "rewards/margins": 0.005521883722394705, |
| "rewards/rejected": 0.0006931538810022175, |
| "step": 80 |
| }, |
| { |
| "epoch": 3.2432432432432434, |
| "grad_norm": 0.6148263812065125, |
| "learning_rate": 4.93261217644956e-06, |
| "logits/chosen": 1.7915083169937134, |
| "logits/rejected": 1.7579513788223267, |
| "logps/chosen": -88.66618347167969, |
| "logps/rejected": -80.2238998413086, |
| "loss": 0.6918, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": 0.005774274934083223, |
| "rewards/margins": 0.016623441129922867, |
| "rewards/rejected": -0.010849165730178356, |
| "step": 90 |
| }, |
| { |
| "epoch": 3.6036036036036037, |
| "grad_norm": 0.6143240332603455, |
| "learning_rate": 4.8902889044347e-06, |
| "logits/chosen": 1.6886920928955078, |
| "logits/rejected": 1.8121178150177002, |
| "logps/chosen": -72.908203125, |
| "logps/rejected": -92.44017028808594, |
| "loss": 0.6911, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.0015121791511774063, |
| "rewards/margins": 0.0036420777905732393, |
| "rewards/rejected": -0.005154256708920002, |
| "step": 100 |
| }, |
| { |
| "epoch": 3.963963963963964, |
| "grad_norm": 0.7140340209007263, |
| "learning_rate": 4.837981131305475e-06, |
| "logits/chosen": 1.7481105327606201, |
| "logits/rejected": 1.723141074180603, |
| "logps/chosen": -80.63452911376953, |
| "logps/rejected": -72.64492797851562, |
| "loss": 0.6894, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.006246576085686684, |
| "rewards/margins": -0.0035218377597630024, |
| "rewards/rejected": -0.0027247383259236813, |
| "step": 110 |
| }, |
| { |
| "epoch": 4.324324324324325, |
| "grad_norm": 1.1645854711532593, |
| "learning_rate": 4.775907352415367e-06, |
| "logits/chosen": 1.7416290044784546, |
| "logits/rejected": 1.8237574100494385, |
| "logps/chosen": -89.01248931884766, |
| "logps/rejected": -92.31901550292969, |
| "loss": 0.6869, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.008855113759636879, |
| "rewards/margins": 0.013151508755981922, |
| "rewards/rejected": -0.022006623446941376, |
| "step": 120 |
| }, |
| { |
| "epoch": 4.684684684684685, |
| "grad_norm": 0.6638664603233337, |
| "learning_rate": 4.70432685680402e-06, |
| "logits/chosen": 1.7124770879745483, |
| "logits/rejected": 1.7777938842773438, |
| "logps/chosen": -84.87271881103516, |
| "logps/rejected": -92.4839096069336, |
| "loss": 0.6855, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.004437069408595562, |
| "rewards/margins": 0.019811339676380157, |
| "rewards/rejected": -0.024248410016298294, |
| "step": 130 |
| }, |
| { |
| "epoch": 5.045045045045045, |
| "grad_norm": 0.785169780254364, |
| "learning_rate": 4.623538644118244e-06, |
| "logits/chosen": 1.7838690280914307, |
| "logits/rejected": 1.8052574396133423, |
| "logps/chosen": -94.57842254638672, |
| "logps/rejected": -80.6390609741211, |
| "loss": 0.6836, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": -0.026147600263357162, |
| "rewards/margins": 0.010936126112937927, |
| "rewards/rejected": -0.03708372637629509, |
| "step": 140 |
| }, |
| { |
| "epoch": 5.405405405405405, |
| "grad_norm": 0.745952844619751, |
| "learning_rate": 4.533880175657419e-06, |
| "logits/chosen": 1.7925735712051392, |
| "logits/rejected": 1.7987396717071533, |
| "logps/chosen": -82.4854507446289, |
| "logps/rejected": -77.50787353515625, |
| "loss": 0.6767, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.013469865545630455, |
| "rewards/margins": 0.04023212194442749, |
| "rewards/rejected": -0.053701985627412796, |
| "step": 150 |
| }, |
| { |
| "epoch": 5.7657657657657655, |
| "grad_norm": 0.7885063290596008, |
| "learning_rate": 4.435725964760331e-06, |
| "logits/chosen": 1.7725191116333008, |
| "logits/rejected": 1.8345234394073486, |
| "logps/chosen": -78.7154541015625, |
| "logps/rejected": -80.75708770751953, |
| "loss": 0.6721, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.018351273611187935, |
| "rewards/margins": 0.05368901416659355, |
| "rewards/rejected": -0.07204028964042664, |
| "step": 160 |
| }, |
| { |
| "epoch": 6.126126126126126, |
| "grad_norm": 0.7734187245368958, |
| "learning_rate": 4.329486012421531e-06, |
| "logits/chosen": 1.827528715133667, |
| "logits/rejected": 1.7939176559448242, |
| "logps/chosen": -75.33561706542969, |
| "logps/rejected": -71.34326934814453, |
| "loss": 0.6775, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.038218818604946136, |
| "rewards/margins": 0.06159939616918564, |
| "rewards/rejected": -0.09981821477413177, |
| "step": 170 |
| }, |
| { |
| "epoch": 6.486486486486487, |
| "grad_norm": 0.7796682715415955, |
| "learning_rate": 4.215604094671835e-06, |
| "logits/chosen": 1.729288101196289, |
| "logits/rejected": 1.7497504949569702, |
| "logps/chosen": -83.02750396728516, |
| "logps/rejected": -78.53959655761719, |
| "loss": 0.6632, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.03207743167877197, |
| "rewards/margins": 0.06858544796705246, |
| "rewards/rejected": -0.10066288709640503, |
| "step": 180 |
| }, |
| { |
| "epoch": 6.846846846846847, |
| "grad_norm": 0.8168752193450928, |
| "learning_rate": 4.094555908876765e-06, |
| "logits/chosen": 1.747865080833435, |
| "logits/rejected": 1.7993282079696655, |
| "logps/chosen": -78.75782775878906, |
| "logps/rejected": -85.8946304321289, |
| "loss": 0.668, |
| "rewards/accuracies": 0.612500011920929, |
| "rewards/chosen": -0.0766468197107315, |
| "rewards/margins": 0.045263100415468216, |
| "rewards/rejected": -0.12190990149974823, |
| "step": 190 |
| }, |
| { |
| "epoch": 7.207207207207207, |
| "grad_norm": 0.9816763997077942, |
| "learning_rate": 3.966847086696045e-06, |
| "logits/chosen": 1.792106032371521, |
| "logits/rejected": 1.7798125743865967, |
| "logps/chosen": -93.53439331054688, |
| "logps/rejected": -73.8904037475586, |
| "loss": 0.6582, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.06042450666427612, |
| "rewards/margins": 0.10538975894451141, |
| "rewards/rejected": -0.16581428050994873, |
| "step": 200 |
| }, |
| { |
| "epoch": 7.5675675675675675, |
| "grad_norm": 0.9053374528884888, |
| "learning_rate": 3.833011082004229e-06, |
| "logits/chosen": 1.7648969888687134, |
| "logits/rejected": 1.7725557088851929, |
| "logps/chosen": -79.61383056640625, |
| "logps/rejected": -71.8048095703125, |
| "loss": 0.6551, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.040038011968135834, |
| "rewards/margins": 0.11104954779148102, |
| "rewards/rejected": -0.15108755230903625, |
| "step": 210 |
| }, |
| { |
| "epoch": 7.927927927927928, |
| "grad_norm": 1.1332669258117676, |
| "learning_rate": 3.693606942594873e-06, |
| "logits/chosen": 1.8605678081512451, |
| "logits/rejected": 1.91280996799469, |
| "logps/chosen": -74.39783477783203, |
| "logps/rejected": -90.66893005371094, |
| "loss": 0.6565, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.08056517690420151, |
| "rewards/margins": 0.0968993678689003, |
| "rewards/rejected": -0.1774645447731018, |
| "step": 220 |
| }, |
| { |
| "epoch": 8.288288288288289, |
| "grad_norm": 1.0496221780776978, |
| "learning_rate": 3.549216974976073e-06, |
| "logits/chosen": 1.8636070489883423, |
| "logits/rejected": 1.7996151447296143, |
| "logps/chosen": -89.9018783569336, |
| "logps/rejected": -74.14125061035156, |
| "loss": 0.6423, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.05598217993974686, |
| "rewards/margins": 0.14807265996932983, |
| "rewards/rejected": -0.2040548026561737, |
| "step": 230 |
| }, |
| { |
| "epoch": 8.64864864864865, |
| "grad_norm": 0.8506491780281067, |
| "learning_rate": 3.400444312011776e-06, |
| "logits/chosen": 1.7753232717514038, |
| "logits/rejected": 1.7647806406021118, |
| "logps/chosen": -88.50201416015625, |
| "logps/rejected": -81.80894470214844, |
| "loss": 0.6358, |
| "rewards/accuracies": 0.6625000238418579, |
| "rewards/chosen": -0.09656485170125961, |
| "rewards/margins": 0.11554913222789764, |
| "rewards/rejected": -0.21211397647857666, |
| "step": 240 |
| }, |
| { |
| "epoch": 9.00900900900901, |
| "grad_norm": 0.9554071426391602, |
| "learning_rate": 3.2479103935691047e-06, |
| "logits/chosen": 1.7528200149536133, |
| "logits/rejected": 1.7827808856964111, |
| "logps/chosen": -90.01268005371094, |
| "logps/rejected": -89.0714340209961, |
| "loss": 0.6374, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.07412372529506683, |
| "rewards/margins": 0.1321595013141632, |
| "rewards/rejected": -0.20628324151039124, |
| "step": 250 |
| }, |
| { |
| "epoch": 9.36936936936937, |
| "grad_norm": 0.9363342523574829, |
| "learning_rate": 3.092252370695298e-06, |
| "logits/chosen": 1.6990101337432861, |
| "logits/rejected": 1.8281656503677368, |
| "logps/chosen": -71.15428161621094, |
| "logps/rejected": -79.72063446044922, |
| "loss": 0.6303, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.08240491151809692, |
| "rewards/margins": 0.16954098641872406, |
| "rewards/rejected": -0.2519459128379822, |
| "step": 260 |
| }, |
| { |
| "epoch": 9.72972972972973, |
| "grad_norm": 1.023535966873169, |
| "learning_rate": 2.9341204441673267e-06, |
| "logits/chosen": 1.7418006658554077, |
| "logits/rejected": 1.7500028610229492, |
| "logps/chosen": -82.24595642089844, |
| "logps/rejected": -91.16307067871094, |
| "loss": 0.6321, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.13560767471790314, |
| "rewards/margins": 0.1204828992486, |
| "rewards/rejected": -0.25609058141708374, |
| "step": 270 |
| }, |
| { |
| "epoch": 10.09009009009009, |
| "grad_norm": 1.0393471717834473, |
| "learning_rate": 2.7741751485313295e-06, |
| "logits/chosen": 1.8437154293060303, |
| "logits/rejected": 1.8877627849578857, |
| "logps/chosen": -68.61182403564453, |
| "logps/rejected": -77.12557220458984, |
| "loss": 0.6109, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.11025551706552505, |
| "rewards/margins": 0.14969798922538757, |
| "rewards/rejected": -0.25995349884033203, |
| "step": 280 |
| }, |
| { |
| "epoch": 10.45045045045045, |
| "grad_norm": 0.9554691910743713, |
| "learning_rate": 2.6130845929767662e-06, |
| "logits/chosen": 1.7998764514923096, |
| "logits/rejected": 1.796451210975647, |
| "logps/chosen": -72.6626205444336, |
| "logps/rejected": -74.15937042236328, |
| "loss": 0.6123, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.13948099315166473, |
| "rewards/margins": 0.18039894104003906, |
| "rewards/rejected": -0.319879949092865, |
| "step": 290 |
| }, |
| { |
| "epoch": 10.81081081081081, |
| "grad_norm": 1.1667226552963257, |
| "learning_rate": 2.4515216705704396e-06, |
| "logits/chosen": 1.8437564373016357, |
| "logits/rejected": 1.9043811559677124, |
| "logps/chosen": -78.18916320800781, |
| "logps/rejected": -90.88524627685547, |
| "loss": 0.6218, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.12091793864965439, |
| "rewards/margins": 0.20490169525146484, |
| "rewards/rejected": -0.32581964135169983, |
| "step": 300 |
| }, |
| { |
| "epoch": 11.17117117117117, |
| "grad_norm": 1.2568660974502563, |
| "learning_rate": 2.290161247507733e-06, |
| "logits/chosen": 1.7295608520507812, |
| "logits/rejected": 1.8652616739273071, |
| "logps/chosen": -79.8567886352539, |
| "logps/rejected": -87.59798431396484, |
| "loss": 0.6067, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.10961383581161499, |
| "rewards/margins": 0.2835196554660797, |
| "rewards/rejected": -0.3931335210800171, |
| "step": 310 |
| }, |
| { |
| "epoch": 11.531531531531531, |
| "grad_norm": 1.115491271018982, |
| "learning_rate": 2.129677344121879e-06, |
| "logits/chosen": 1.803063988685608, |
| "logits/rejected": 1.820227861404419, |
| "logps/chosen": -74.64582061767578, |
| "logps/rejected": -72.14736938476562, |
| "loss": 0.6113, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.09987158328294754, |
| "rewards/margins": 0.2059938907623291, |
| "rewards/rejected": -0.30586546659469604, |
| "step": 320 |
| }, |
| { |
| "epoch": 11.891891891891891, |
| "grad_norm": 0.9227492213249207, |
| "learning_rate": 1.970740319426474e-06, |
| "logits/chosen": 1.7561572790145874, |
| "logits/rejected": 1.789878249168396, |
| "logps/chosen": -85.27911376953125, |
| "logps/rejected": -77.4220962524414, |
| "loss": 0.5891, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.14200571179389954, |
| "rewards/margins": 0.22272682189941406, |
| "rewards/rejected": -0.3647325336933136, |
| "step": 330 |
| }, |
| { |
| "epoch": 12.252252252252251, |
| "grad_norm": 1.541570782661438, |
| "learning_rate": 1.8140140709517467e-06, |
| "logits/chosen": 1.7364771366119385, |
| "logits/rejected": 1.7182557582855225, |
| "logps/chosen": -76.05533599853516, |
| "logps/rejected": -81.50830078125, |
| "loss": 0.5994, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.12165775150060654, |
| "rewards/margins": 0.19019995629787445, |
| "rewards/rejected": -0.3118577301502228, |
| "step": 340 |
| }, |
| { |
| "epoch": 12.612612612612612, |
| "grad_norm": 1.1125155687332153, |
| "learning_rate": 1.6601532615711452e-06, |
| "logits/chosen": 1.7223026752471924, |
| "logits/rejected": 1.8163812160491943, |
| "logps/chosen": -71.94869232177734, |
| "logps/rejected": -90.12442016601562, |
| "loss": 0.6013, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.16840626299381256, |
| "rewards/margins": 0.1776614487171173, |
| "rewards/rejected": -0.3460676968097687, |
| "step": 350 |
| }, |
| { |
| "epoch": 12.972972972972974, |
| "grad_norm": 1.0530396699905396, |
| "learning_rate": 1.509800584902108e-06, |
| "logits/chosen": 1.795819878578186, |
| "logits/rejected": 1.8446025848388672, |
| "logps/chosen": -82.24161529541016, |
| "logps/rejected": -79.51258850097656, |
| "loss": 0.5857, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.13156726956367493, |
| "rewards/margins": 0.2468404471874237, |
| "rewards/rejected": -0.37840771675109863, |
| "step": 360 |
| }, |
| { |
| "epoch": 13.333333333333334, |
| "grad_norm": 1.0298306941986084, |
| "learning_rate": 1.3635840807037487e-06, |
| "logits/chosen": 1.8054157495498657, |
| "logits/rejected": 1.8246568441390991, |
| "logps/chosen": -85.40760803222656, |
| "logps/rejected": -78.77482604980469, |
| "loss": 0.5943, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.10383795201778412, |
| "rewards/margins": 0.2493254393339157, |
| "rewards/rejected": -0.35316339135169983, |
| "step": 370 |
| }, |
| { |
| "epoch": 13.693693693693694, |
| "grad_norm": 1.0601342916488647, |
| "learning_rate": 1.2221145114853172e-06, |
| "logits/chosen": 1.6507200002670288, |
| "logits/rejected": 1.720569372177124, |
| "logps/chosen": -74.90982055664062, |
| "logps/rejected": -92.51995849609375, |
| "loss": 0.5789, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.13705268502235413, |
| "rewards/margins": 0.2740176022052765, |
| "rewards/rejected": -0.4110702872276306, |
| "step": 380 |
| }, |
| { |
| "epoch": 14.054054054054054, |
| "grad_norm": 1.1135063171386719, |
| "learning_rate": 1.085982811283654e-06, |
| "logits/chosen": 1.7133989334106445, |
| "logits/rejected": 1.7338130474090576, |
| "logps/chosen": -80.55327606201172, |
| "logps/rejected": -82.32875061035156, |
| "loss": 0.5847, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.12969203293323517, |
| "rewards/margins": 0.26957422494888306, |
| "rewards/rejected": -0.39926621317863464, |
| "step": 390 |
| }, |
| { |
| "epoch": 14.414414414414415, |
| "grad_norm": 1.0821375846862793, |
| "learning_rate": 9.557576172663577e-07, |
| "logits/chosen": 1.7035369873046875, |
| "logits/rejected": 1.7184028625488281, |
| "logps/chosen": -80.99266052246094, |
| "logps/rejected": -85.05413818359375, |
| "loss": 0.583, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.1366995871067047, |
| "rewards/margins": 0.268393874168396, |
| "rewards/rejected": -0.4050934910774231, |
| "step": 400 |
| }, |
| { |
| "epoch": 14.774774774774775, |
| "grad_norm": 1.1167497634887695, |
| "learning_rate": 8.319828944714508e-07, |
| "logits/chosen": 1.8293631076812744, |
| "logits/rejected": 1.7202396392822266, |
| "logps/chosen": -96.96324157714844, |
| "logps/rejected": -80.58685302734375, |
| "loss": 0.5768, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.14515718817710876, |
| "rewards/margins": 0.27918386459350586, |
| "rewards/rejected": -0.4243410527706146, |
| "step": 410 |
| }, |
| { |
| "epoch": 15.135135135135135, |
| "grad_norm": 1.0140223503112793, |
| "learning_rate": 7.151756636052529e-07, |
| "logits/chosen": 1.7751014232635498, |
| "logits/rejected": 1.7394100427627563, |
| "logps/chosen": -104.48625183105469, |
| "logps/rejected": -90.3105239868164, |
| "loss": 0.5804, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.14849844574928284, |
| "rewards/margins": 0.2261633574962616, |
| "rewards/rejected": -0.37466174364089966, |
| "step": 420 |
| }, |
| { |
| "epoch": 15.495495495495495, |
| "grad_norm": 1.0056337118148804, |
| "learning_rate": 6.058238413897052e-07, |
| "logits/chosen": 1.7368510961532593, |
| "logits/rejected": 1.844665288925171, |
| "logps/chosen": -88.00404357910156, |
| "logps/rejected": -103.71427917480469, |
| "loss": 0.5786, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.2264096438884735, |
| "rewards/margins": 0.2332192212343216, |
| "rewards/rejected": -0.4596289098262787, |
| "step": 430 |
| }, |
| { |
| "epoch": 15.855855855855856, |
| "grad_norm": 1.0402743816375732, |
| "learning_rate": 5.043842024802675e-07, |
| "logits/chosen": 1.835862398147583, |
| "logits/rejected": 1.8621985912322998, |
| "logps/chosen": -80.8730697631836, |
| "logps/rejected": -77.32243347167969, |
| "loss": 0.5737, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.10934928804636002, |
| "rewards/margins": 0.26070791482925415, |
| "rewards/rejected": -0.3700571656227112, |
| "step": 440 |
| }, |
| { |
| "epoch": 16.216216216216218, |
| "grad_norm": 1.0037925243377686, |
| "learning_rate": 4.1128047146765936e-07, |
| "logits/chosen": 1.6536343097686768, |
| "logits/rejected": 1.748152494430542, |
| "logps/chosen": -85.28504180908203, |
| "logps/rejected": -83.04313659667969, |
| "loss": 0.5869, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.1468530297279358, |
| "rewards/margins": 0.24461284279823303, |
| "rewards/rejected": -0.39146584272384644, |
| "step": 450 |
| }, |
| { |
| "epoch": 16.576576576576578, |
| "grad_norm": 1.0473060607910156, |
| "learning_rate": 3.269015529333805e-07, |
| "logits/chosen": 1.7145726680755615, |
| "logits/rejected": 1.7245705127716064, |
| "logps/chosen": -68.06051635742188, |
| "logps/rejected": -77.09696960449219, |
| "loss": 0.5679, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.16770952939987183, |
| "rewards/margins": 0.23443298041820526, |
| "rewards/rejected": -0.4021424651145935, |
| "step": 460 |
| }, |
| { |
| "epoch": 16.936936936936938, |
| "grad_norm": 1.0985324382781982, |
| "learning_rate": 2.515999069522676e-07, |
| "logits/chosen": 1.7093912363052368, |
| "logits/rejected": 1.6903793811798096, |
| "logps/chosen": -79.38801574707031, |
| "logps/rejected": -74.43864440917969, |
| "loss": 0.5754, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.14471781253814697, |
| "rewards/margins": 0.24466648697853088, |
| "rewards/rejected": -0.38938429951667786, |
| "step": 470 |
| }, |
| { |
| "epoch": 17.2972972972973, |
| "grad_norm": 1.1457403898239136, |
| "learning_rate": 1.8569007682777417e-07, |
| "logits/chosen": 1.6642320156097412, |
| "logits/rejected": 1.7295513153076172, |
| "logps/chosen": -80.79732513427734, |
| "logps/rejected": -80.07367706298828, |
| "loss": 0.5749, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.16041259467601776, |
| "rewards/margins": 0.26678892970085144, |
| "rewards/rejected": -0.4272015690803528, |
| "step": 480 |
| }, |
| { |
| "epoch": 17.65765765765766, |
| "grad_norm": 1.04747474193573, |
| "learning_rate": 1.2944737520980883e-07, |
| "logits/chosen": 1.699061632156372, |
| "logits/rejected": 1.837684988975525, |
| "logps/chosen": -73.55027770996094, |
| "logps/rejected": -93.25712585449219, |
| "loss": 0.5732, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.20150330662727356, |
| "rewards/margins": 0.2376147210597992, |
| "rewards/rejected": -0.43911799788475037, |
| "step": 490 |
| }, |
| { |
| "epoch": 18.01801801801802, |
| "grad_norm": 1.033074140548706, |
| "learning_rate": 8.310673408334496e-08, |
| "logits/chosen": 1.655987024307251, |
| "logits/rejected": 1.6537258625030518, |
| "logps/chosen": -103.11566162109375, |
| "logps/rejected": -96.29421997070312, |
| "loss": 0.5747, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.15963700413703918, |
| "rewards/margins": 0.30574050545692444, |
| "rewards/rejected": -0.4653775095939636, |
| "step": 500 |
| }, |
| { |
| "epoch": 18.37837837837838, |
| "grad_norm": 1.0928398370742798, |
| "learning_rate": 4.6861723431538273e-08, |
| "logits/chosen": 1.7491756677627563, |
| "logits/rejected": 1.7941337823867798, |
| "logps/chosen": -79.0124282836914, |
| "logps/rejected": -94.39396667480469, |
| "loss": 0.5745, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.14681483805179596, |
| "rewards/margins": 0.31730785965919495, |
| "rewards/rejected": -0.4641226828098297, |
| "step": 510 |
| }, |
| { |
| "epoch": 18.73873873873874, |
| "grad_norm": 1.0533796548843384, |
| "learning_rate": 2.0863742672497244e-08, |
| "logits/chosen": 1.7281211614608765, |
| "logits/rejected": 1.7764543294906616, |
| "logps/chosen": -91.92194366455078, |
| "logps/rejected": -85.10621643066406, |
| "loss": 0.5726, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.15819533169269562, |
| "rewards/margins": 0.2960740923881531, |
| "rewards/rejected": -0.4542694687843323, |
| "step": 520 |
| }, |
| { |
| "epoch": 19.0990990990991, |
| "grad_norm": 1.1859172582626343, |
| "learning_rate": 5.221388247169945e-09, |
| "logits/chosen": 1.7769553661346436, |
| "logits/rejected": 1.7326488494873047, |
| "logps/chosen": -85.6234359741211, |
| "logps/rejected": -75.64092254638672, |
| "loss": 0.5688, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.15740445256233215, |
| "rewards/margins": 0.31495219469070435, |
| "rewards/rejected": -0.4723566472530365, |
| "step": 530 |
| }, |
| { |
| "epoch": 19.45945945945946, |
| "grad_norm": 1.1295475959777832, |
| "learning_rate": 0.0, |
| "logits/chosen": 1.7790734767913818, |
| "logits/rejected": 1.8196531534194946, |
| "logps/chosen": -83.07762145996094, |
| "logps/rejected": -77.76747131347656, |
| "loss": 0.575, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.1493213176727295, |
| "rewards/margins": 0.2750301659107208, |
| "rewards/rejected": -0.42435145378112793, |
| "step": 540 |
| }, |
| { |
| "epoch": 19.45945945945946, |
| "step": 540, |
| "total_flos": 1.8293058049230766e+18, |
| "train_loss": 0.6298594880987096, |
| "train_runtime": 3921.3478, |
| "train_samples_per_second": 9.033, |
| "train_steps_per_second": 0.138 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 540, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 20, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.8293058049230766e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
