| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9880609304240429, |
| "eval_steps": 500, |
| "global_step": 75, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.013174145738987238, |
| "grad_norm": 1.846787929534912, |
| "learning_rate": 1.25e-07, |
| "logits/chosen": 9.988622665405273, |
| "logits/rejected": 10.698101997375488, |
| "logps/chosen": -0.4084107577800751, |
| "logps/rejected": -0.4250854551792145, |
| "loss": 1.3828, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.8168215155601501, |
| "rewards/margins": 0.03334939479827881, |
| "rewards/rejected": -0.850170910358429, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.026348291477974475, |
| "grad_norm": 0.7274014353752136, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": 10.208279609680176, |
| "logits/rejected": 11.06594467163086, |
| "logps/chosen": -0.4457661509513855, |
| "logps/rejected": -0.4444480836391449, |
| "loss": 1.4032, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.891532301902771, |
| "rewards/margins": -0.0026361620984971523, |
| "rewards/rejected": -0.8888961672782898, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.03952243721696171, |
| "grad_norm": 0.569585919380188, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": 10.047319412231445, |
| "logits/rejected": 11.040695190429688, |
| "logps/chosen": -0.44595056772232056, |
| "logps/rejected": -0.4271169602870941, |
| "loss": 1.4316, |
| "rewards/accuracies": 0.453125, |
| "rewards/chosen": -0.8919011354446411, |
| "rewards/margins": -0.037667226046323776, |
| "rewards/rejected": -0.8542339205741882, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.05269658295594895, |
| "grad_norm": 0.9579747915267944, |
| "learning_rate": 5e-07, |
| "logits/chosen": 9.83576774597168, |
| "logits/rejected": 10.8547945022583, |
| "logps/chosen": -0.4448290169239044, |
| "logps/rejected": -0.43586766719818115, |
| "loss": 1.4147, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.8896580338478088, |
| "rewards/margins": -0.017922835424542427, |
| "rewards/rejected": -0.8717353343963623, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.06587072869493618, |
| "grad_norm": 0.7575727701187134, |
| "learning_rate": 6.249999999999999e-07, |
| "logits/chosen": 10.199325561523438, |
| "logits/rejected": 10.956421852111816, |
| "logps/chosen": -0.40898576378822327, |
| "logps/rejected": -0.4012777507305145, |
| "loss": 1.4103, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.8179715275764465, |
| "rewards/margins": -0.015416024252772331, |
| "rewards/rejected": -0.802555501461029, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.07904487443392343, |
| "grad_norm": 1.1651052236557007, |
| "learning_rate": 7.5e-07, |
| "logits/chosen": 10.649311065673828, |
| "logits/rejected": 11.438021659851074, |
| "logps/chosen": -0.4362146258354187, |
| "logps/rejected": -0.43107232451438904, |
| "loss": 1.4097, |
| "rewards/accuracies": 0.5234375, |
| "rewards/chosen": -0.8724292516708374, |
| "rewards/margins": -0.01028449460864067, |
| "rewards/rejected": -0.8621446490287781, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.09221902017291066, |
| "grad_norm": 0.7423586249351501, |
| "learning_rate": 8.75e-07, |
| "logits/chosen": 10.016755104064941, |
| "logits/rejected": 10.720447540283203, |
| "logps/chosen": -0.40585798025131226, |
| "logps/rejected": -0.41537052392959595, |
| "loss": 1.3836, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.8117159605026245, |
| "rewards/margins": 0.0190250426530838, |
| "rewards/rejected": -0.8307410478591919, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.1053931659118979, |
| "grad_norm": 0.6278178095817566, |
| "learning_rate": 1e-06, |
| "logits/chosen": 10.216046333312988, |
| "logits/rejected": 11.114391326904297, |
| "logps/chosen": -0.4205312728881836, |
| "logps/rejected": -0.3968205451965332, |
| "loss": 1.4332, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.8410625457763672, |
| "rewards/margins": -0.04742136597633362, |
| "rewards/rejected": -0.7936410903930664, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.11856731165088513, |
| "grad_norm": 0.7120531797409058, |
| "learning_rate": 9.994504457428556e-07, |
| "logits/chosen": 10.18065071105957, |
| "logits/rejected": 11.15275764465332, |
| "logps/chosen": -0.4227794110774994, |
| "logps/rejected": -0.4355461895465851, |
| "loss": 1.3879, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": -0.8455588221549988, |
| "rewards/margins": 0.02553354948759079, |
| "rewards/rejected": -0.8710923790931702, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.13174145738987236, |
| "grad_norm": 1.1985653638839722, |
| "learning_rate": 9.97802991010949e-07, |
| "logits/chosen": 10.042179107666016, |
| "logits/rejected": 10.866905212402344, |
| "logps/chosen": -0.38143518567085266, |
| "logps/rejected": -0.383830189704895, |
| "loss": 1.3934, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.7628703713417053, |
| "rewards/margins": 0.004790017381310463, |
| "rewards/rejected": -0.76766037940979, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.14491560312885962, |
| "grad_norm": 0.9829190969467163, |
| "learning_rate": 9.950612572673255e-07, |
| "logits/chosen": 9.96993637084961, |
| "logits/rejected": 11.156689643859863, |
| "logps/chosen": -0.4371436536312103, |
| "logps/rejected": -0.4104335606098175, |
| "loss": 1.4388, |
| "rewards/accuracies": 0.421875, |
| "rewards/chosen": -0.8742873072624207, |
| "rewards/margins": -0.05342020094394684, |
| "rewards/rejected": -0.820867121219635, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.15808974886784685, |
| "grad_norm": 0.771833062171936, |
| "learning_rate": 9.912312714377879e-07, |
| "logits/chosen": 9.883522033691406, |
| "logits/rejected": 10.568523406982422, |
| "logps/chosen": -0.4180852770805359, |
| "logps/rejected": -0.44434836506843567, |
| "loss": 1.3634, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.8361705541610718, |
| "rewards/margins": 0.05252611264586449, |
| "rewards/rejected": -0.8886967301368713, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.17126389460683408, |
| "grad_norm": 0.5779662728309631, |
| "learning_rate": 9.863214526624063e-07, |
| "logits/chosen": 9.908037185668945, |
| "logits/rejected": 10.980252265930176, |
| "logps/chosen": -0.4335843622684479, |
| "logps/rejected": -0.4193909764289856, |
| "loss": 1.4188, |
| "rewards/accuracies": 0.4609375, |
| "rewards/chosen": -0.8671687245368958, |
| "rewards/margins": -0.028386807069182396, |
| "rewards/rejected": -0.8387819528579712, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.1844380403458213, |
| "grad_norm": 0.755972146987915, |
| "learning_rate": 9.8034259378842e-07, |
| "logits/chosen": 10.47681999206543, |
| "logits/rejected": 11.599574089050293, |
| "logps/chosen": -0.40058645606040955, |
| "logps/rejected": -0.41547441482543945, |
| "loss": 1.3776, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.8011729121208191, |
| "rewards/margins": 0.02977588400244713, |
| "rewards/rejected": -0.8309488296508789, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.19761218608480857, |
| "grad_norm": 1.2080482244491577, |
| "learning_rate": 9.73307837645217e-07, |
| "logits/chosen": 9.785813331604004, |
| "logits/rejected": 10.71661376953125, |
| "logps/chosen": -0.4059077799320221, |
| "logps/rejected": -0.3964956998825073, |
| "loss": 1.4115, |
| "rewards/accuracies": 0.4453125, |
| "rewards/chosen": -0.8118155598640442, |
| "rewards/margins": -0.018824119120836258, |
| "rewards/rejected": -0.7929913997650146, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.2107863318237958, |
| "grad_norm": 1.0530983209609985, |
| "learning_rate": 9.652326481535433e-07, |
| "logits/chosen": 10.606256484985352, |
| "logits/rejected": 11.221076965332031, |
| "logps/chosen": -0.43960413336753845, |
| "logps/rejected": -0.43556928634643555, |
| "loss": 1.4054, |
| "rewards/accuracies": 0.4453125, |
| "rewards/chosen": -0.8792082667350769, |
| "rewards/margins": -0.008069731295108795, |
| "rewards/rejected": -0.8711385726928711, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.22396047756278303, |
| "grad_norm": 0.8931649327278137, |
| "learning_rate": 9.561347763324483e-07, |
| "logits/chosen": 10.093276977539062, |
| "logits/rejected": 10.721864700317383, |
| "logps/chosen": -0.46352270245552063, |
| "logps/rejected": -0.47664591670036316, |
| "loss": 1.398, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.9270454049110413, |
| "rewards/margins": 0.026246393099427223, |
| "rewards/rejected": -0.9532918334007263, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.23713462330177026, |
| "grad_norm": 0.5987163782119751, |
| "learning_rate": 9.460342212786932e-07, |
| "logits/chosen": 10.340713500976562, |
| "logits/rejected": 11.121424674987793, |
| "logps/chosen": -0.4333910644054413, |
| "logps/rejected": -0.4210299551486969, |
| "loss": 1.4189, |
| "rewards/accuracies": 0.5390625, |
| "rewards/chosen": -0.8667821288108826, |
| "rewards/margins": -0.024722211062908173, |
| "rewards/rejected": -0.8420599102973938, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.2503087690407575, |
| "grad_norm": 0.7226627469062805, |
| "learning_rate": 9.349531862043951e-07, |
| "logits/chosen": 10.257181167602539, |
| "logits/rejected": 10.92556095123291, |
| "logps/chosen": -0.4293927550315857, |
| "logps/rejected": -0.4481368064880371, |
| "loss": 1.3801, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.8587855100631714, |
| "rewards/margins": 0.03748808801174164, |
| "rewards/rejected": -0.8962736129760742, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.2634829147797447, |
| "grad_norm": 0.7743024826049805, |
| "learning_rate": 9.229160296295487e-07, |
| "logits/chosen": 10.641219139099121, |
| "logits/rejected": 11.451465606689453, |
| "logps/chosen": -0.40963226556777954, |
| "logps/rejected": -0.42823317646980286, |
| "loss": 1.3718, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.8192645311355591, |
| "rewards/margins": 0.03720181807875633, |
| "rewards/rejected": -0.8564663529396057, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.276657060518732, |
| "grad_norm": 1.043879747390747, |
| "learning_rate": 9.099492118367122e-07, |
| "logits/chosen": 9.668081283569336, |
| "logits/rejected": 10.759317398071289, |
| "logps/chosen": -0.41088593006134033, |
| "logps/rejected": -0.44134727120399475, |
| "loss": 1.3653, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.8217718601226807, |
| "rewards/margins": 0.06092273071408272, |
| "rewards/rejected": -0.8826945424079895, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.28983120625771924, |
| "grad_norm": 1.8495320081710815, |
| "learning_rate": 8.960812367055646e-07, |
| "logits/chosen": 10.094249725341797, |
| "logits/rejected": 10.969901084899902, |
| "logps/chosen": -0.43626973032951355, |
| "logps/rejected": -0.44013574719429016, |
| "loss": 1.4045, |
| "rewards/accuracies": 0.4609375, |
| "rewards/chosen": -0.8725394606590271, |
| "rewards/margins": 0.00773210171610117, |
| "rewards/rejected": -0.8802714943885803, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.3030053519967065, |
| "grad_norm": 0.4824877977371216, |
| "learning_rate": 8.813425890551909e-07, |
| "logits/chosen": 10.012961387634277, |
| "logits/rejected": 11.028541564941406, |
| "logps/chosen": -0.4449196755886078, |
| "logps/rejected": -0.43216580152511597, |
| "loss": 1.424, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.8898393511772156, |
| "rewards/margins": -0.025507748126983643, |
| "rewards/rejected": -0.8643316030502319, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.3161794977356937, |
| "grad_norm": 1.0434041023254395, |
| "learning_rate": 8.657656676318345e-07, |
| "logits/chosen": 10.089515686035156, |
| "logits/rejected": 10.921834945678711, |
| "logps/chosen": -0.4228805899620056, |
| "logps/rejected": -0.44924959540367126, |
| "loss": 1.3654, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.8457611799240112, |
| "rewards/margins": 0.05273808538913727, |
| "rewards/rejected": -0.8984991908073425, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.32935364347468093, |
| "grad_norm": 0.8496569991111755, |
| "learning_rate": 8.493847138894208e-07, |
| "logits/chosen": 10.220051765441895, |
| "logits/rejected": 10.924245834350586, |
| "logps/chosen": -0.4278189241886139, |
| "logps/rejected": -0.45087143778800964, |
| "loss": 1.3771, |
| "rewards/accuracies": 0.5234375, |
| "rewards/chosen": -0.8556378483772278, |
| "rewards/margins": 0.046104975044727325, |
| "rewards/rejected": -0.9017428755760193, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.34252778921366817, |
| "grad_norm": 1.1707934141159058, |
| "learning_rate": 8.322357367194108e-07, |
| "logits/chosen": 10.33618450164795, |
| "logits/rejected": 11.42149829864502, |
| "logps/chosen": -0.4403349757194519, |
| "logps/rejected": -0.4506159722805023, |
| "loss": 1.3831, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": -0.8806699514389038, |
| "rewards/margins": 0.020561927929520607, |
| "rewards/rejected": -0.9012319445610046, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.3557019349526554, |
| "grad_norm": 0.9727609753608704, |
| "learning_rate": 8.143564332954425e-07, |
| "logits/chosen": 9.973875999450684, |
| "logits/rejected": 11.174227714538574, |
| "logps/chosen": -0.48110640048980713, |
| "logps/rejected": -0.46264228224754333, |
| "loss": 1.4375, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.9622128009796143, |
| "rewards/margins": -0.03692828118801117, |
| "rewards/rejected": -0.9252845644950867, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.3688760806916426, |
| "grad_norm": 0.4669955372810364, |
| "learning_rate": 7.957861062067612e-07, |
| "logits/chosen": 9.494720458984375, |
| "logits/rejected": 10.64975357055664, |
| "logps/chosen": -0.4049557149410248, |
| "logps/rejected": -0.41160064935684204, |
| "loss": 1.3949, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.8099114298820496, |
| "rewards/margins": 0.01328977383673191, |
| "rewards/rejected": -0.8232012987136841, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.3820502264306299, |
| "grad_norm": 0.6599685549736023, |
| "learning_rate": 7.765655770625996e-07, |
| "logits/chosen": 9.987813949584961, |
| "logits/rejected": 11.066561698913574, |
| "logps/chosen": -0.4204758107662201, |
| "logps/rejected": -0.4215814769268036, |
| "loss": 1.4024, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.8409516215324402, |
| "rewards/margins": 0.002211294136941433, |
| "rewards/rejected": -0.8431629538536072, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.39522437216961714, |
| "grad_norm": 0.4469655156135559, |
| "learning_rate": 7.567370967574209e-07, |
| "logits/chosen": 10.159306526184082, |
| "logits/rejected": 11.529789924621582, |
| "logps/chosen": -0.44582635164260864, |
| "logps/rejected": -0.4426492154598236, |
| "loss": 1.4042, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.8916527032852173, |
| "rewards/margins": -0.006354227662086487, |
| "rewards/rejected": -0.8852984309196472, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.4083985179086044, |
| "grad_norm": 0.46628981828689575, |
| "learning_rate": 7.363442525942826e-07, |
| "logits/chosen": 9.731393814086914, |
| "logits/rejected": 10.743117332458496, |
| "logps/chosen": -0.4495592713356018, |
| "logps/rejected": -0.41725438833236694, |
| "loss": 1.4528, |
| "rewards/accuracies": 0.453125, |
| "rewards/chosen": -0.8991185426712036, |
| "rewards/margins": -0.06460972875356674, |
| "rewards/rejected": -0.8345087766647339, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.4215726636475916, |
| "grad_norm": 0.6232314109802246, |
| "learning_rate": 7.154318724704851e-07, |
| "logits/chosen": 9.956490516662598, |
| "logits/rejected": 11.324994087219238, |
| "logps/chosen": -0.42140334844589233, |
| "logps/rejected": -0.3955790102481842, |
| "loss": 1.4406, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": -0.8428066968917847, |
| "rewards/margins": -0.05164865404367447, |
| "rewards/rejected": -0.7911580204963684, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.43474680938657884, |
| "grad_norm": 1.1314888000488281, |
| "learning_rate": 6.940459263361248e-07, |
| "logits/chosen": 9.897272109985352, |
| "logits/rejected": 10.727909088134766, |
| "logps/chosen": -0.38281309604644775, |
| "logps/rejected": -0.4067748188972473, |
| "loss": 1.3728, |
| "rewards/accuracies": 0.5390625, |
| "rewards/chosen": -0.7656261920928955, |
| "rewards/margins": 0.04792340472340584, |
| "rewards/rejected": -0.8135496377944946, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.44792095512556607, |
| "grad_norm": 0.9336860179901123, |
| "learning_rate": 6.722334251421664e-07, |
| "logits/chosen": 10.045162200927734, |
| "logits/rejected": 10.88832950592041, |
| "logps/chosen": -0.424130380153656, |
| "logps/rejected": -0.44961968064308167, |
| "loss": 1.3642, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.848260760307312, |
| "rewards/margins": 0.050978533923625946, |
| "rewards/rejected": -0.8992393612861633, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.4610951008645533, |
| "grad_norm": 0.6091923117637634, |
| "learning_rate": 6.500423175001703e-07, |
| "logits/chosen": 10.366498947143555, |
| "logits/rejected": 11.302865028381348, |
| "logps/chosen": -0.38435739278793335, |
| "logps/rejected": -0.40687644481658936, |
| "loss": 1.3686, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.7687147855758667, |
| "rewards/margins": 0.04503808170557022, |
| "rewards/rejected": -0.8137528896331787, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.47426924660354053, |
| "grad_norm": 1.436123251914978, |
| "learning_rate": 6.275213842808382e-07, |
| "logits/chosen": 9.993330955505371, |
| "logits/rejected": 10.635811805725098, |
| "logps/chosen": -0.41794323921203613, |
| "logps/rejected": -0.416820764541626, |
| "loss": 1.4028, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.8358864784240723, |
| "rewards/margins": -0.0022448617964982986, |
| "rewards/rejected": -0.833641529083252, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.4874433923425278, |
| "grad_norm": 0.4129350483417511, |
| "learning_rate": 6.047201313830723e-07, |
| "logits/chosen": 10.213558197021484, |
| "logits/rejected": 11.014687538146973, |
| "logps/chosen": -0.4101608991622925, |
| "logps/rejected": -0.40530481934547424, |
| "loss": 1.409, |
| "rewards/accuracies": 0.4609375, |
| "rewards/chosen": -0.820321798324585, |
| "rewards/margins": -0.00971211027354002, |
| "rewards/rejected": -0.8106096386909485, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.500617538081515, |
| "grad_norm": 1.0162944793701172, |
| "learning_rate": 5.816886809092651e-07, |
| "logits/chosen": 9.751138687133789, |
| "logits/rejected": 10.836737632751465, |
| "logps/chosen": -0.4359847605228424, |
| "logps/rejected": -0.42434683442115784, |
| "loss": 1.4272, |
| "rewards/accuracies": 0.4765625, |
| "rewards/chosen": -0.8719695210456848, |
| "rewards/margins": -0.02327585220336914, |
| "rewards/rejected": -0.8486936688423157, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.5137916838205022, |
| "grad_norm": 1.3473665714263916, |
| "learning_rate": 5.584776609860413e-07, |
| "logits/chosen": 10.200762748718262, |
| "logits/rejected": 10.96191692352295, |
| "logps/chosen": -0.40851840376853943, |
| "logps/rejected": -0.38918375968933105, |
| "loss": 1.4273, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.8170368075370789, |
| "rewards/margins": -0.03866923972964287, |
| "rewards/rejected": -0.7783675193786621, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.5269658295594895, |
| "grad_norm": 3.9734771251678467, |
| "learning_rate": 5.351380944726465e-07, |
| "logits/chosen": 10.102203369140625, |
| "logits/rejected": 11.253116607666016, |
| "logps/chosen": -0.40019893646240234, |
| "logps/rejected": -0.40543556213378906, |
| "loss": 1.3934, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.8003978729248047, |
| "rewards/margins": 0.010473249480128288, |
| "rewards/rejected": -0.8108711242675781, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.5401399752984768, |
| "grad_norm": 1.0061744451522827, |
| "learning_rate": 5.117212868016303e-07, |
| "logits/chosen": 10.42828369140625, |
| "logits/rejected": 11.038559913635254, |
| "logps/chosen": -0.39824292063713074, |
| "logps/rejected": -0.41535234451293945, |
| "loss": 1.3724, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.7964858412742615, |
| "rewards/margins": 0.03421883285045624, |
| "rewards/rejected": -0.8307046890258789, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.553314121037464, |
| "grad_norm": 1.7842886447906494, |
| "learning_rate": 4.882787131983697e-07, |
| "logits/chosen": 9.360157012939453, |
| "logits/rejected": 10.43077278137207, |
| "logps/chosen": -0.3668302893638611, |
| "logps/rejected": -0.3597845435142517, |
| "loss": 1.4068, |
| "rewards/accuracies": 0.4765625, |
| "rewards/chosen": -0.7336605787277222, |
| "rewards/margins": -0.014091495424509048, |
| "rewards/rejected": -0.7195690870285034, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.5664882667764513, |
| "grad_norm": 0.7694165706634521, |
| "learning_rate": 4.648619055273537e-07, |
| "logits/chosen": 9.549002647399902, |
| "logits/rejected": 10.662269592285156, |
| "logps/chosen": -0.3708702325820923, |
| "logps/rejected": -0.3596724271774292, |
| "loss": 1.4136, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.7417404651641846, |
| "rewards/margins": -0.022395702078938484, |
| "rewards/rejected": -0.7193448543548584, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.5796624125154385, |
| "grad_norm": 0.31465020775794983, |
| "learning_rate": 4.4152233901395875e-07, |
| "logits/chosen": 9.66249942779541, |
| "logits/rejected": 10.4733247756958, |
| "logps/chosen": -0.3584354817867279, |
| "logps/rejected": -0.3858814835548401, |
| "loss": 1.3559, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.7168709635734558, |
| "rewards/margins": 0.05489201098680496, |
| "rewards/rejected": -0.7717629671096802, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.5928365582544257, |
| "grad_norm": 0.9510380029678345, |
| "learning_rate": 4.183113190907348e-07, |
| "logits/chosen": 9.850863456726074, |
| "logits/rejected": 10.66943359375, |
| "logps/chosen": -0.371550589799881, |
| "logps/rejected": -0.37234315276145935, |
| "loss": 1.3984, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.743101179599762, |
| "rewards/margins": 0.0015851231291890144, |
| "rewards/rejected": -0.7446863055229187, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.606010703993413, |
| "grad_norm": 0.7652634382247925, |
| "learning_rate": 3.9527986861692785e-07, |
| "logits/chosen": 9.94843864440918, |
| "logits/rejected": 10.614623069763184, |
| "logps/chosen": -0.3755650222301483, |
| "logps/rejected": -0.3979741632938385, |
| "loss": 1.3633, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.7511300444602966, |
| "rewards/margins": 0.044818248599767685, |
| "rewards/rejected": -0.795948326587677, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.6191848497324002, |
| "grad_norm": 0.40572136640548706, |
| "learning_rate": 3.724786157191618e-07, |
| "logits/chosen": 9.853185653686523, |
| "logits/rejected": 11.088766098022461, |
| "logps/chosen": -0.3570285439491272, |
| "logps/rejected": -0.3574231266975403, |
| "loss": 1.3937, |
| "rewards/accuracies": 0.4609375, |
| "rewards/chosen": -0.7140570878982544, |
| "rewards/margins": 0.0007891724817454815, |
| "rewards/rejected": -0.7148462533950806, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.6323589954713874, |
| "grad_norm": 0.8276376724243164, |
| "learning_rate": 3.499576824998297e-07, |
| "logits/chosen": 10.390812873840332, |
| "logits/rejected": 11.2942476272583, |
| "logps/chosen": -0.3813317120075226, |
| "logps/rejected": -0.40014880895614624, |
| "loss": 1.37, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.7626634240150452, |
| "rewards/margins": 0.03763421252369881, |
| "rewards/rejected": -0.8002976179122925, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.6455331412103746, |
| "grad_norm": 0.6045619249343872, |
| "learning_rate": 3.2776657485783356e-07, |
| "logits/chosen": 10.388471603393555, |
| "logits/rejected": 11.256227493286133, |
| "logps/chosen": -0.3872194290161133, |
| "logps/rejected": -0.37491628527641296, |
| "loss": 1.4175, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.7744388580322266, |
| "rewards/margins": -0.024606185033917427, |
| "rewards/rejected": -0.7498325705528259, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.6587072869493619, |
| "grad_norm": 0.45040056109428406, |
| "learning_rate": 3.0595407366387506e-07, |
| "logits/chosen": 10.052522659301758, |
| "logits/rejected": 10.916872024536133, |
| "logps/chosen": -0.35351964831352234, |
| "logps/rejected": -0.3540777862071991, |
| "loss": 1.3928, |
| "rewards/accuracies": 0.5234375, |
| "rewards/chosen": -0.7070392966270447, |
| "rewards/margins": 0.0011163135059177876, |
| "rewards/rejected": -0.7081555724143982, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.6718814326883491, |
| "grad_norm": 0.24182955920696259, |
| "learning_rate": 2.845681275295148e-07, |
| "logits/chosen": 10.124942779541016, |
| "logits/rejected": 10.639265060424805, |
| "logps/chosen": -0.3642256557941437, |
| "logps/rejected": -0.3618934750556946, |
| "loss": 1.4, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.7284513115882874, |
| "rewards/margins": -0.004664354957640171, |
| "rewards/rejected": -0.7237869501113892, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.6850555784273363, |
| "grad_norm": 0.7590335011482239, |
| "learning_rate": 2.636557474057173e-07, |
| "logits/chosen": 9.573650360107422, |
| "logits/rejected": 10.71945858001709, |
| "logps/chosen": -0.35338151454925537, |
| "logps/rejected": -0.3880979120731354, |
| "loss": 1.3474, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.7067630290985107, |
| "rewards/margins": 0.06943274289369583, |
| "rewards/rejected": -0.7761958241462708, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.6982297241663236, |
| "grad_norm": 0.630469799041748, |
| "learning_rate": 2.432629032425789e-07, |
| "logits/chosen": 10.021524429321289, |
| "logits/rejected": 10.909788131713867, |
| "logps/chosen": -0.3418756127357483, |
| "logps/rejected": -0.3625137209892273, |
| "loss": 1.3635, |
| "rewards/accuracies": 0.5234375, |
| "rewards/chosen": -0.6837512254714966, |
| "rewards/margins": 0.0412762388586998, |
| "rewards/rejected": -0.7250274419784546, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.7114038699053108, |
| "grad_norm": 0.6511030197143555, |
| "learning_rate": 2.2343442293740028e-07, |
| "logits/chosen": 10.15503978729248, |
| "logits/rejected": 11.120055198669434, |
| "logps/chosen": -0.39185085892677307, |
| "logps/rejected": -0.4090447425842285, |
| "loss": 1.376, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.7837017178535461, |
| "rewards/margins": 0.03438780829310417, |
| "rewards/rejected": -0.818089485168457, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.724578015644298, |
| "grad_norm": 0.2991220951080322, |
| "learning_rate": 2.0421389379323877e-07, |
| "logits/chosen": 10.134223937988281, |
| "logits/rejected": 10.953475952148438, |
| "logps/chosen": -0.37127768993377686, |
| "logps/rejected": -0.3803029954433441, |
| "loss": 1.384, |
| "rewards/accuracies": 0.5234375, |
| "rewards/chosen": -0.7425553798675537, |
| "rewards/margins": 0.018050534650683403, |
| "rewards/rejected": -0.7606059908866882, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.7377521613832853, |
| "grad_norm": 0.5964541435241699, |
| "learning_rate": 1.8564356670455767e-07, |
| "logits/chosen": 9.78146743774414, |
| "logits/rejected": 10.964405059814453, |
| "logps/chosen": -0.3709033727645874, |
| "logps/rejected": -0.3880802094936371, |
| "loss": 1.3702, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.7418067455291748, |
| "rewards/margins": 0.03435374051332474, |
| "rewards/rejected": -0.7761604189872742, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.7509263071222725, |
| "grad_norm": 0.3852439224720001, |
| "learning_rate": 1.6776426328058919e-07, |
| "logits/chosen": 10.553987503051758, |
| "logits/rejected": 11.33293342590332, |
| "logps/chosen": -0.36016741394996643, |
| "logps/rejected": -0.3851068317890167, |
| "loss": 1.3576, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.7203348278999329, |
| "rewards/margins": 0.04987882822751999, |
| "rewards/rejected": -0.7702136635780334, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.7641004528612598, |
| "grad_norm": 1.2566592693328857, |
| "learning_rate": 1.5061528611057915e-07, |
| "logits/chosen": 9.90965461730957, |
| "logits/rejected": 10.495996475219727, |
| "logps/chosen": -0.36674946546554565, |
| "logps/rejected": -0.4073812961578369, |
| "loss": 1.3422, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.7334989309310913, |
| "rewards/margins": 0.08126369118690491, |
| "rewards/rejected": -0.8147625923156738, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.7772745986002471, |
| "grad_norm": 1.2612890005111694, |
| "learning_rate": 1.3423433236816562e-07, |
| "logits/chosen": 10.17950439453125, |
| "logits/rejected": 11.087965965270996, |
| "logps/chosen": -0.374392569065094, |
| "logps/rejected": -0.3923323154449463, |
| "loss": 1.3704, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.748785138130188, |
| "rewards/margins": 0.03587958961725235, |
| "rewards/rejected": -0.7846646308898926, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.7904487443392343, |
| "grad_norm": 0.5846283435821533, |
| "learning_rate": 1.1865741094480908e-07, |
| "logits/chosen": 9.906171798706055, |
| "logits/rejected": 10.999700546264648, |
| "logps/chosen": -0.352775901556015, |
| "logps/rejected": -0.365261971950531, |
| "loss": 1.381, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.70555180311203, |
| "rewards/margins": 0.02497219666838646, |
| "rewards/rejected": -0.730523943901062, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.8036228900782215, |
| "grad_norm": 0.859355092048645, |
| "learning_rate": 1.0391876329443533e-07, |
| "logits/chosen": 9.959291458129883, |
| "logits/rejected": 10.919910430908203, |
| "logps/chosen": -0.3664035499095917, |
| "logps/rejected": -0.37674611806869507, |
| "loss": 1.3823, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.7328070998191833, |
| "rewards/margins": 0.020685214549303055, |
| "rewards/rejected": -0.7534922361373901, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.8167970358172087, |
| "grad_norm": 1.1612812280654907, |
| "learning_rate": 9.00507881632877e-08, |
| "logits/chosen": 10.075271606445312, |
| "logits/rejected": 10.889825820922852, |
| "logps/chosen": -0.364788293838501, |
| "logps/rejected": -0.36973661184310913, |
| "loss": 1.3907, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.729576587677002, |
| "rewards/margins": 0.009896687231957912, |
| "rewards/rejected": -0.7394732236862183, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.829971181556196, |
| "grad_norm": 0.7830009460449219, |
| "learning_rate": 7.708397037045128e-08, |
| "logits/chosen": 10.215938568115234, |
| "logits/rejected": 11.319831848144531, |
| "logps/chosen": -0.3826553523540497, |
| "logps/rejected": -0.3706996440887451, |
| "loss": 1.4167, |
| "rewards/accuracies": 0.453125, |
| "rewards/chosen": -0.7653107047080994, |
| "rewards/margins": -0.02391139790415764, |
| "rewards/rejected": -0.7413992881774902, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.8431453272951832, |
| "grad_norm": 0.5214446783065796, |
| "learning_rate": 6.504681379560489e-08, |
| "logits/chosen": 10.213878631591797, |
| "logits/rejected": 10.867339134216309, |
| "logps/chosen": -0.3824314773082733, |
| "logps/rejected": -0.4005742073059082, |
| "loss": 1.3685, |
| "rewards/accuracies": 0.5703125, |
| "rewards/chosen": -0.7648629546165466, |
| "rewards/margins": 0.0362853966653347, |
| "rewards/rejected": -0.8011484146118164, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.8563194730341704, |
| "grad_norm": 1.422054409980774, |
| "learning_rate": 5.396577872130675e-08, |
| "logits/chosen": 9.750934600830078, |
| "logits/rejected": 11.052637100219727, |
| "logps/chosen": -0.38261115550994873, |
| "logps/rejected": -0.37236595153808594, |
| "loss": 1.4104, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.7652223110198975, |
| "rewards/margins": -0.020490389317274094, |
| "rewards/rejected": -0.7447319030761719, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.8694936187731577, |
| "grad_norm": 0.519874095916748, |
| "learning_rate": 4.3865223667551686e-08, |
| "logits/chosen": 9.77448844909668, |
| "logits/rejected": 10.812127113342285, |
| "logps/chosen": -0.3549169898033142, |
| "logps/rejected": -0.3985462188720703, |
| "loss": 1.3386, |
| "rewards/accuracies": 0.5703125, |
| "rewards/chosen": -0.7098339796066284, |
| "rewards/margins": 0.08725835382938385, |
| "rewards/rejected": -0.7970924377441406, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.8826677645121449, |
| "grad_norm": 0.9944124221801758, |
| "learning_rate": 3.476735184645674e-08, |
| "logits/chosen": 10.103015899658203, |
| "logits/rejected": 11.05565071105957, |
| "logps/chosen": -0.36293864250183105, |
| "logps/rejected": -0.36475422978401184, |
| "loss": 1.3952, |
| "rewards/accuracies": 0.5703125, |
| "rewards/chosen": -0.7258772850036621, |
| "rewards/margins": 0.003631273517385125, |
| "rewards/rejected": -0.7295084595680237, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.8958419102511321, |
| "grad_norm": 0.4005211591720581, |
| "learning_rate": 2.6692162354782943e-08, |
| "logits/chosen": 10.448604583740234, |
| "logits/rejected": 10.800128936767578, |
| "logps/chosen": -0.36172616481781006, |
| "logps/rejected": -0.37695637345314026, |
| "loss": 1.3713, |
| "rewards/accuracies": 0.5703125, |
| "rewards/chosen": -0.7234523296356201, |
| "rewards/margins": 0.03046037256717682, |
| "rewards/rejected": -0.7539127469062805, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.9090160559901194, |
| "grad_norm": 1.8271591663360596, |
| "learning_rate": 1.9657406211579962e-08, |
| "logits/chosen": 10.343523979187012, |
| "logits/rejected": 11.240467071533203, |
| "logps/chosen": -0.3876397907733917, |
| "logps/rejected": -0.40994149446487427, |
| "loss": 1.3668, |
| "rewards/accuracies": 0.5703125, |
| "rewards/chosen": -0.7752795815467834, |
| "rewards/margins": 0.04460335522890091, |
| "rewards/rejected": -0.8198829889297485, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.9221902017291066, |
| "grad_norm": 1.0028115510940552, |
| "learning_rate": 1.3678547337593494e-08, |
| "logits/chosen": 10.036100387573242, |
| "logits/rejected": 10.593299865722656, |
| "logps/chosen": -0.3563886284828186, |
| "logps/rejected": -0.3926778733730316, |
| "loss": 1.3456, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.7127772569656372, |
| "rewards/margins": 0.07257851213216782, |
| "rewards/rejected": -0.7853557467460632, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.9353643474680938, |
| "grad_norm": 0.5587558150291443, |
| "learning_rate": 8.768728562211946e-09, |
| "logits/chosen": 10.08999252319336, |
| "logits/rejected": 11.311429023742676, |
| "logps/chosen": -0.39482226967811584, |
| "logps/rejected": -0.39989325404167175, |
| "loss": 1.3952, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.7896445393562317, |
| "rewards/margins": 0.010141978971660137, |
| "rewards/rejected": -0.7997865080833435, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.9485384932070811, |
| "grad_norm": 0.5898178815841675, |
| "learning_rate": 4.938742732674528e-09, |
| "logits/chosen": 10.197044372558594, |
| "logits/rejected": 11.203747749328613, |
| "logps/chosen": -0.3625889718532562, |
| "logps/rejected": -0.3807837665081024, |
| "loss": 1.3685, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.7251779437065125, |
| "rewards/margins": 0.03638959676027298, |
| "rewards/rejected": -0.7615675330162048, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.9617126389460683, |
| "grad_norm": 0.7782201766967773, |
| "learning_rate": 2.1970089890509524e-09, |
| "logits/chosen": 9.960372924804688, |
| "logits/rejected": 10.8052978515625, |
| "logps/chosen": -0.3511280119419098, |
| "logps/rejected": -0.3762291669845581, |
| "loss": 1.3584, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.7022560238838196, |
| "rewards/margins": 0.05020223930478096, |
| "rewards/rejected": -0.7524583339691162, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.9748867846850556, |
| "grad_norm": 1.1336719989776611, |
| "learning_rate": 5.495542571443135e-10, |
| "logits/chosen": 10.240190505981445, |
| "logits/rejected": 11.07278823852539, |
| "logps/chosen": -0.36002400517463684, |
| "logps/rejected": -0.36638352274894714, |
| "loss": 1.3849, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.7200480103492737, |
| "rewards/margins": 0.012718938291072845, |
| "rewards/rejected": -0.7327670454978943, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.9880609304240429, |
| "grad_norm": 0.6144753098487854, |
| "learning_rate": 0.0, |
| "logits/chosen": 10.015645027160645, |
| "logits/rejected": 10.800317764282227, |
| "logps/chosen": -0.35483843088150024, |
| "logps/rejected": -0.3783097267150879, |
| "loss": 1.3612, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.7096768617630005, |
| "rewards/margins": 0.04694262892007828, |
| "rewards/rejected": -0.7566194534301758, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.9880609304240429, |
| "step": 75, |
| "total_flos": 0.0, |
| "train_loss": 1.3904367176691692, |
| "train_runtime": 8038.3927, |
| "train_samples_per_second": 1.209, |
| "train_steps_per_second": 0.009 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 75, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 12, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
